Merge pull request 'mesh-delivery-builder, mesh-delivery-indexer: upstream artifacts by pinned digest; newest entry only; no git tag (design 47 H2 H-A-486, H-A-487, H-A-489, H-A-492, H-A-494)' (#336) from fix/upstream-release into main
This commit was merged in pull request #336.
This commit is contained in:
@@ -5,9 +5,12 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -54,3 +57,14 @@ func TestABuildSeatsStepsAreReadInItsHoldersState(t *testing.T) {
|
||||
t.Errorf("a seat with no holder reads %v; want an error naming it", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-487: a build key's time is its entry's own write time, read with the key and no history.
|
||||
func TestABuildKeysTimeIsItsEntrysOwnWriteTime(t *testing.T) {
|
||||
at := time.Date(2026, 11, 22, 20, 45, 0, 0, time.UTC)
|
||||
raw, rev, got, err := entryOf(func(key string) (*stdio.StateEntry, error) {
|
||||
return &stdio.StateEntry{Key: key, Value: json.RawMessage(`{}`), Revision: 4, At: at}, nil
|
||||
})("k")
|
||||
if err != nil || string(raw) != "{}" || rev != 4 || !got.Equal(at) {
|
||||
t.Errorf("entry = %s rev %d at %v (%v); want the entry's own write time %v", raw, rev, got, err, at)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +430,7 @@ func main() {
|
||||
return a, json.Unmarshal(raw, &a)
|
||||
}
|
||||
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve)
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds").Get), Asked: read(stdio.State("asked")),
|
||||
Newest: newestVia(stdio.Ask), Steps: r}))
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
as := stdio.State(versions.ActedState)
|
||||
@@ -481,13 +481,14 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// entryOf reads a key with its revision and when it was written.
|
||||
func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
// entryOf reads a key with its revision and when it was written, the entry's own write time: no key's history is read
|
||||
// (design 47 H2 H-A-487).
|
||||
func entryOf(get func(key string) (*stdio.StateEntry, error)) func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
return func(key string) (json.RawMessage, uint64, time.Time, error) {
|
||||
e, err := s.Get(key)
|
||||
e, err := get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, time.Time{}, err
|
||||
}
|
||||
return e.Value, e.Revision, written(s, key, e.Revision), nil
|
||||
return e.Value, e.Revision, e.At, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,6 +233,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
b.PackageWait = time.Duration(packageWait) * time.Minute
|
||||
// A version's digests are recorded under the builder's own bound, never the build's (H-A-492); it is record-bound-seconds.
|
||||
recordBound, err := numberFile(getenv("MESH_BUILDER_RECORD_BOUND_FILE"), int(build.DefaultRecordBound/time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.RecordBound = time.Duration(recordBound) * time.Second
|
||||
// Before it waits for an unpinned package, the build reads what the Go package registry seat holds of it (H-A-194).
|
||||
b.PublishedVersions = publishedVersionsVia(calls.Call)
|
||||
waitsBucket := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.WaitsState), "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
|
||||
@@ -254,11 +260,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
// A package's number reserved is said as the seat mesh-builds' version-reserved, so the indexer re-pins at the reservation (§805).
|
||||
reserved.names = versions.NamesFrom(versions.EntriesOf(branches))
|
||||
reserved.r.Names = reserved.names // a Go package's module path names its tag's directory (§796)
|
||||
reserved.sayReserved = sayReservedOn(js, cred)
|
||||
b.Reserved = reserved
|
||||
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
|
||||
reserved.withdraw = b.Withdraw
|
||||
reserved.recordAgain = b.RecordAgain // a done build's digests its end did not record are recorded at the start (H-A-492)
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
@@ -298,8 +304,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
reserved.pause = publishSet.Pause // a version-released whose tag the forge did not write waits the pause (H-A-484 addendum)
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+16
-16
@@ -289,7 +289,8 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
var tagsWritten []string
|
||||
call := func(tool string, body any) (json.RawMessage, error) {
|
||||
switch tool {
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796)
|
||||
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit, never read (H-A-494)
|
||||
tagsWritten = append(tagsWritten, "read "+body.(gitseat.TagsInput).Prefix)
|
||||
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
|
||||
case "seat:" + gitseat.Seat + ".tag":
|
||||
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
|
||||
@@ -360,15 +361,14 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
|
||||
}
|
||||
|
||||
// Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another
|
||||
// commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags <directory>/v<version> at
|
||||
// the build's commit, and the module's version-released clears the finding.
|
||||
rv.r.Names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
// Design 47 H2 H-A-494: no git tag is read before a reserve nor written at a release: m4's 0.2.0, which the seat git would
|
||||
// answer tagged at another commit, is reserved, no finding raised, and its version-released writes no tag.
|
||||
// Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it
|
||||
// (sayReservedOn), and waited on by the seat and the event's name.
|
||||
rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
rv.names = func(context.Context, string, string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/m4", true, nil
|
||||
}
|
||||
rv.sayReserved = sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
|
||||
said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -383,18 +383,18 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
if err := said.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib",
|
||||
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4",
|
||||
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
|
||||
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised)
|
||||
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.0" || v4.Tag != "" || raised {
|
||||
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.0 naming no tag, no tag read and no finding", v4, err, elsewhere, raised)
|
||||
}
|
||||
select {
|
||||
case m := <-reservedCh:
|
||||
ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data)
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" ||
|
||||
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.0" ||
|
||||
ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule {
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.0, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved)
|
||||
@@ -420,11 +420,11 @@ func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
}
|
||||
return m
|
||||
}
|
||||
if !rv.heard(ctx, next("m4", "0.2.1")) {
|
||||
if !rv.heard(ctx, next("m4", "0.2.0")) {
|
||||
t.Fatal("the reservations did not take version-released")
|
||||
}
|
||||
if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) {
|
||||
t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten)
|
||||
if _, standing, _ := rv.r.Of(ctx, "m4", "0.2.0"); standing || len(tagsWritten) != 0 {
|
||||
t.Errorf("after version-released m4 0.2.0 standing %v, tags read or written %v: want it released and no tag read or written", standing, tagsWritten)
|
||||
}
|
||||
|
||||
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
@@ -57,11 +56,9 @@ type reservations struct {
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands
|
||||
// it back at once.
|
||||
pause time.Duration
|
||||
// recordAgain records the digests of a done build of module at fp its end did not record, saying so (design 47 H2 H-A-492);
|
||||
// nil records none.
|
||||
recordAgain func(ctx context.Context, module, fp, when string) (string, error)
|
||||
// rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none.
|
||||
rebuild func(ctx context.Context, module, fp, why string) error
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
@@ -120,7 +117,7 @@ func sayReservedOn(js jetstream.JetStream, cred process.Account) func(context.Co
|
||||
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
|
||||
// verb versions through call (design 47 G §348), the only source numbering reads them from.
|
||||
func newReservations(store versions.ReserveStore, call toolCaller, bounds func() jetstream.KeyValue, found *findings, wake func(context.Context), logf func(string, ...any)) *reservations {
|
||||
rv := &reservations{call: call, bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv := &reservations{bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf}
|
||||
rv.r = &versions.Reserver{Store: store, Filled: rv.filled.Load, ReleasedOf: func(_ context.Context, module string) ([]string, error) {
|
||||
raw, err := call("seat:"+meshcatalog.Seat+".versions", meshcatalog.VersionsInput{Module: module})
|
||||
if err != nil {
|
||||
@@ -137,8 +134,6 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
|
||||
return out, nil
|
||||
}}
|
||||
rv.r.Took, rv.r.Logf, rv.r.Reserved = rv.took, logf, rv.reservedSaid
|
||||
// A module's tags are read before it reserves, and a number tagged at another commit skipped (design 47 H2 §796).
|
||||
rv.r.Tags, rv.r.TaggedElsewhere = tagsVia(call), rv.taggedElsewhere
|
||||
return rv
|
||||
}
|
||||
|
||||
@@ -313,15 +308,9 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err)
|
||||
break
|
||||
}
|
||||
if refused, err := rv.release(ctx, ev); err != nil {
|
||||
if err := rv.release(ctx, ev); err != nil {
|
||||
rv.logf("%v; heard again", err)
|
||||
// A tag the seat git's holder did not write is handed back with the publish pause, so the deliveries the consumer
|
||||
// allows span the holder's outage (design 47 H2 H-A-484 addendum).
|
||||
nak := m.Nak
|
||||
if refused && rv.pause > 0 {
|
||||
nak = func() error { return m.NakWithDelay(rv.pause) }
|
||||
}
|
||||
if nakErr := nak(); nakErr != nil {
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
@@ -336,16 +325,12 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369, §370): its tag written, its
|
||||
// reservation released, its bound and publish item ended and its findings cleared. refused says the tag was not written (the
|
||||
// seat git did not write it, or the reservation could not be read to tag it); the reservation then stands.
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) (refused bool, err error) {
|
||||
if err := rv.tagged(ctx, ev); err != nil {
|
||||
return true, err
|
||||
}
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369): its reservation released,
|
||||
// its bound and publish item ended and its findings cleared. No git tag is written (design 47 H2 H-A-494).
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
for _, k := range gone {
|
||||
rv.dropBound(ctx, k)
|
||||
@@ -355,20 +340,29 @@ func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionRelea
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
return false, nil
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) // one raised before H-A-494
|
||||
return nil
|
||||
}
|
||||
|
||||
// releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions
|
||||
// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one
|
||||
// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended.
|
||||
// said while the builder was away, is taken all the same, its reservation released and its publish item ended; no tag is written
|
||||
// (design 47 H2 H-A-494).
|
||||
// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released.
|
||||
// Before that, a standing reservation with no digests recorded whose build key is done has them recorded from the done key
|
||||
// (design 47 H2 H-A-492), so it is held against the released versions too.
|
||||
func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
if rv.recordedAtStart(ctx, all) {
|
||||
if all, err = rv.r.All(ctx); err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
read := map[string][]string{}
|
||||
for _, v := range builtOf(all) {
|
||||
if versions.NotStanding(v, true) != "" {
|
||||
@@ -384,7 +378,7 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
if !slices.Contains(released, v.Version) {
|
||||
continue
|
||||
}
|
||||
if _, err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
if err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err)
|
||||
continue
|
||||
}
|
||||
@@ -392,6 +386,26 @@ func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// recordedAtStart records the digests of every standing reservation that has none recorded and whose build key is done (design 47
|
||||
// H2 H-A-492): a record its build's end lost. It answers whether one was recorded; one that cannot be is said.
|
||||
func (rv *reservations) recordedAtStart(ctx context.Context, all []versions.ReservedVersion) bool {
|
||||
if rv.recordAgain == nil {
|
||||
return false
|
||||
}
|
||||
recorded := false
|
||||
for _, v := range all {
|
||||
if !v.BuiltAt.IsZero() || v.Fingerprint == "" || versions.NotStanding(v, true) != "" {
|
||||
continue
|
||||
}
|
||||
version, err := rv.recordAgain(ctx, v.Module, v.Fingerprint, "at the builder's start")
|
||||
if err != nil {
|
||||
rv.logf("%v", err)
|
||||
}
|
||||
recorded = recorded || version != ""
|
||||
}
|
||||
return recorded
|
||||
}
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
|
||||
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
|
||||
// nothing.
|
||||
@@ -451,33 +465,6 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
|
||||
return fmt.Sprintf("the reservation of %s %s is dropped (%s, %s)", module, series, by, why), nil
|
||||
}
|
||||
|
||||
// tagged writes the git tag of a released version (design 47 G §370): on the catalog's version-released, through the seat
|
||||
// git's verb tag, at the commit the reservation of that version was made from. A version with no reservation (a superseded
|
||||
// build's version is never released, and the key of a version released by another builder is gone) or one that names no
|
||||
// tag (a module with no Go package) is tagged nowhere. A tag the forge could not write is an error, the message heard again.
|
||||
func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleasedEvent) error {
|
||||
res, found, err := rv.r.Of(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the reservation of %s %s cannot be read to tag it: %w", ev.Module, ev.Version, err)
|
||||
}
|
||||
if !found || res.Tag == "" || res.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
raw, err := rv.call("seat:"+git.Seat+".tag", git.TagInput{Repository: res.Repository, Commit: res.Commit, Tag: res.Tag})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %w", res.Tag, ev.Module, err)
|
||||
}
|
||||
var a git.TagAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return fmt.Errorf("the git seat answered what is no tag answer for %s: %w", res.Tag, err)
|
||||
}
|
||||
if a.Error != "" || !a.Performed {
|
||||
return fmt.Errorf("the tag %s of %s is not written: %s (%s)", res.Tag, ev.Module, a.Error, a.ErrorKind)
|
||||
}
|
||||
rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit)
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing, from the reservations the published read (design 47 H H-A-280: no second read):
|
||||
// what each holds of digests, and how many stand at that version by the index by version (its one key, no reservation read).
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome, read []versions.ReadReservation) string {
|
||||
@@ -549,19 +536,6 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) {
|
||||
needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."})
|
||||
}
|
||||
|
||||
// taggedElsewhere raises version-tagged-elsewhere about module (design 47 H2 §796): numbers whose git tag stands at another
|
||||
// commit were skipped at reserving; a tag is never moved. Cleared at the module's next version-released.
|
||||
func (rv *reservations) taggedElsewhere(ctx context.Context, module string, skipped []string) {
|
||||
why := "the tags of " + strings.Join(skipped, ", ") + " stand at another commit, so the reserve skipped them"
|
||||
rv.logf("%s: %s", module, why)
|
||||
rv.found.raise(ctx, finding{kind: versions.VersionTaggedElsewhere, scope: meshconditionsclient.ScopeModule, id: module,
|
||||
summary: module + ": " + why,
|
||||
headline: "A number whose tag stands at another commit is not used",
|
||||
explanation: "The repository holds a tag of " + module + " at a commit other than the build's: " + why + ". A tag is never moved.",
|
||||
resolved: "The catalog says version-released for " + module + ".",
|
||||
needs: "builder_reserved shows the reservation; the seat git's tags shows each tag with its commit."})
|
||||
}
|
||||
|
||||
// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275,
|
||||
// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap
|
||||
// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a
|
||||
|
||||
@@ -156,123 +156,117 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom: design 47 G §370 — the git tag go/vMAJOR.MINOR.PATCH is written through
|
||||
// the seat git's tag on the catalog's version-released, at the commit the version's reservation was made from; a module with no
|
||||
// Go package, and a version never released (a superseded build's), are tagged nowhere; a tag the forge did not write leaves
|
||||
// version-released to be heard again with the reservation standing.
|
||||
func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
// TestAReleasedVersionWritesNoTag: design 47 H2 H-A-494 — the builder writes no git tag at a release, and numbers a series from the
|
||||
// catalog's versions and the registries' alone: the catalog's version-released releases the reservation with no call of the seat
|
||||
// git, a reservation written before (one naming a tag) included; the start's take-in of a version the catalog holds released
|
||||
// (H-A-484 c) the same; and a reserve reads no git tag, a tag standing at another commit no reason to skip a number.
|
||||
func TestAReleasedVersionWritesNoTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var tags []git.TagInput
|
||||
refuse := ""
|
||||
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
|
||||
var gitCalls []string // every call of the seat git: none is made
|
||||
var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers
|
||||
call := func(addr string, body any) (json.RawMessage, error) {
|
||||
switch addr {
|
||||
case "seat:git.tags":
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true})
|
||||
case "seat:git.tag":
|
||||
default:
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
if strings.HasPrefix(addr, "seat:git.") {
|
||||
gitCalls = append(gitCalls, addr)
|
||||
if addr == "seat:git.tags" { // a tag standing at another commit, as the forge may still hold one
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}}})
|
||||
}
|
||||
return json.Marshal(git.TagAnswer{Performed: true})
|
||||
}
|
||||
tags = append(tags, body.(git.TagInput))
|
||||
return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse})
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
}
|
||||
fw := &memFindings{vals: map[string][]byte{}}
|
||||
found := &findings{module: builds.BuilderModule, w: fw, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
store := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
rv := newReservations(store, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
// Design 47 H2 §796: the tag names the Go module's directory, read from its module path: mesh-sdk's `go` here.
|
||||
rv.r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib" || artifact == "pkg", nil
|
||||
rv.names = func(_ context.Context, _, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib", nil
|
||||
}
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true, GoArtifact: "lib"}
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := rv.Reserve(ctx, "prog", "2.0", []string{"prog:bin"}, versions.Source{Repository: "o/p", Commit: "beef"}); err != nil {
|
||||
t.Fatal(err)
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", Fingerprint: "fpA"}
|
||||
// oldRecord rewrites a reservation as one written before H-A-494, naming the tag its release was to be written as.
|
||||
oldRecord := func(module, series, tag string) {
|
||||
key := versions.ReservedKey(module, series)
|
||||
var v versions.ReservedVersion
|
||||
if err := json.Unmarshal(store.vals[key], &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v.Tag = tag
|
||||
store.vals[key], _ = json.Marshal(v)
|
||||
}
|
||||
release := func(module, version string) *wakeMsg {
|
||||
m := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"` + module + `","version":"` + version + `","source":1,"artifacts":[]}`)}
|
||||
rv.heard(ctx, m)
|
||||
return m
|
||||
}
|
||||
if m := release("prog", "2.0.0"); !m.acked || len(tags) != 0 {
|
||||
t.Fatalf("a module with no Go package: acked %v, tags %v; want none written", m.acked, tags)
|
||||
// The release path: a Go package's version-released releases it, acknowledged, with no tag written.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil || v.Tag != "" {
|
||||
t.Fatalf("reserved %+v (%v), want a reservation naming no tag", v, err)
|
||||
}
|
||||
refuse = "forge down"
|
||||
if m := release("pkg", "0.1.0"); !m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked)
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
if m := release("pkg", "0.1.0"); !m.acked || m.naked {
|
||||
t.Fatalf("the release: acked %v naked %v; want it acknowledged", m.acked, m.naked)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: with a publish pause set, a refused tag hands the version-released back after the pause, so
|
||||
// the deliveries its consumer allows span the holder's outage.
|
||||
rv.pause = 30 * time.Second
|
||||
if m := release("pkg", "0.1.0"); m.delayed != 30*time.Second || m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write, with the pause: delayed %v naked %v acked %v; want handed back after 30s", m.delayed, m.naked, m.acked)
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the release called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
rv.pause = 0
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("the reservation went with an unwritten tag: %v", all)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 {
|
||||
t.Fatalf("the released version still stands: %v", all)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: at the builder's start, a built reservation the catalog holds released is tagged and released
|
||||
// as its version-released would (a version-released the bus gave up on), and one the catalog does not hold, or not built,
|
||||
// stands.
|
||||
refuse = ""
|
||||
tags = nil
|
||||
// The start check (H-A-484 c): a built reservation the catalog holds released is released at the start, no tag written; one
|
||||
// not built, or not released, stands.
|
||||
// Design 47 H2 H-A-492: at the start, a standing reservation with no digests recorded whose build key is done has them
|
||||
// recorded (the builder's RecordAgain), and is then held against the released versions as a built one is.
|
||||
if _, err := rv.Reserve(ctx, "rec", "0.1", []string{"rec:bin"}, versions.Source{Repository: "o/q", Commit: "abc", Fingerprint: "fpR"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var recordedAt []string
|
||||
rv.recordAgain = func(ctx context.Context, module, fp, when string) (string, error) {
|
||||
recordedAt = append(recordedAt, module+" "+fp+" "+when)
|
||||
_, err := rv.r.Built(ctx, module, "0.1.0", "rec:bin", "sha256:bb", nil) // its done key's digest
|
||||
return "0.1.0", err
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "rec", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if _, standing, _ := rv.r.Of(ctx, "rec", "0.1.0"); standing || !slices.Equal(recordedAt, []string{"rec fpR at the builder's start"}) {
|
||||
t.Fatalf("at the start, rec 0.1.0 unrecorded and released: standing %v, recorded %q; want it recorded once at the start and released", standing, recordedAt)
|
||||
}
|
||||
catalog, rv.recordAgain = nil, nil
|
||||
var ended []string
|
||||
rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) }
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", Fingerprint: "fpB"}); err != nil {
|
||||
t.Fatal(err) // 0.1.0 again: the stub catalog holds none released
|
||||
}
|
||||
oldRecord("pkg", "0.1", "go/v0.1.0")
|
||||
rv.releasedAtStart(ctx) // built nothing yet: nothing taken
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with no build done: reservations %v, tags %+v; want the reservation standing, no tag", all, tags)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with no build done: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.releasedAtStart(ctx) // the catalog holds none released: it stands
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v, tags %+v; want the reservation standing", all, tags)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v; want the reservation standing", all)
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, tags %+v, items ended %v; want it tagged, released and its item ended", all, tags, ended)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, items ended %v; want it released and its item ended", all, ended)
|
||||
}
|
||||
if len(gitCalls) != 0 {
|
||||
t.Errorf("the start check called the seat git: %v; want no tag written", gitCalls)
|
||||
gitCalls = nil
|
||||
}
|
||||
catalog, rv.released = nil, nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // the version is reserved again for the rows below
|
||||
t.Fatal(err)
|
||||
}
|
||||
tags = nil
|
||||
if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) {
|
||||
t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags)
|
||||
}
|
||||
tags = nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true, GoArtifact: "lib"}); err != nil || v.Version != "0.1.1" { // build B supersedes A
|
||||
t.Fatalf("second build reserved %q (%v)", v.Version, err)
|
||||
}
|
||||
release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key
|
||||
if len(tags) != 0 {
|
||||
t.Fatalf("a superseded build's version was tagged: %+v", tags)
|
||||
}
|
||||
if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" {
|
||||
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
|
||||
}
|
||||
// Design 47 H2 §796: a tag standing at another commit makes the next number and the finding version-tagged-elsewhere, which
|
||||
// clears at the module's next version-released.
|
||||
standing = []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}, {Tag: "go/v0.2.1", Commit: "elsewhere"}}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.2" || v.Tag != "go/v0.2.2" {
|
||||
t.Fatalf("reserved %+v (%v) with 0.2.0 and 0.2.1 tagged elsewhere, want 0.2.2 tagged go/v0.2.2", v, err)
|
||||
// Numbering reads no tag: the seat git would answer go/v0.2.0 at another commit, and 0.2.0 is reserved all the same.
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.0" || v.Tag != "" || len(gitCalls) != 0 {
|
||||
t.Errorf("reserved %+v (%v), git calls %v; want 0.2.0 naming no tag, no git tag read", v, err, gitCalls)
|
||||
}
|
||||
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "pkg", versions.VersionTaggedElsewhere)
|
||||
if got, err := decodeFinding(fw.vals[key]); err != nil || !strings.Contains(got.Summary, "0.2.0") || !strings.Contains(got.Summary, "0.2.1") {
|
||||
t.Fatalf("the finding %s = %+v (%v), want it naming 0.2.0 and 0.2.1", key, got, err)
|
||||
if fw.vals[key] != nil {
|
||||
t.Errorf("a reserve raised %s", key)
|
||||
}
|
||||
if release("pkg", "0.2.2"); fw.vals[key] != nil {
|
||||
t.Errorf("version-released of pkg left %s standing", key)
|
||||
}
|
||||
standing = nil
|
||||
// Design 47 H2 §805, H-A-420 (2): a package's number newly reserved is said as version-reserved, its path the package's name
|
||||
// and its revision the indexer's `<branch>.<n>`; a reservation kept as it stood, an artifact that is no package and a
|
||||
// reservation naming no catalog revision say none.
|
||||
@@ -281,7 +275,7 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
return "git.example/o/r/" + artifact, module == "lib2" && artifact == "pkg", nil
|
||||
}
|
||||
rv.sayReserved = func(_ context.Context, ev meshbuilds.VersionReservedEvent) error { said = append(said, ev); return nil }
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", GoPackage: true, GoArtifact: "pkg", Branch: "main", CatalogRevision: 158}
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", Branch: "main", CatalogRevision: 158}
|
||||
for range 2 {
|
||||
if _, err := rv.Reserve(ctx, "lib2", "0.3", []string{"lib2:pkg", "lib2:bin"}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
// `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and
|
||||
// since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch
|
||||
// whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle.
|
||||
// **Each key's time is its entry's own write time** (design 47 H2 H-A-487): the history of a key is read only for the one key a
|
||||
// fingerprint names, when its entry does not say when it was written, and a listing cut at MostBuilds says so in its note.
|
||||
|
||||
// BuildTool is the builder's read tool of a module's build keys.
|
||||
const BuildTool = "builder_build"
|
||||
@@ -56,14 +58,16 @@ type BuildAnswer struct {
|
||||
Builds []BuildRow `json:"builds"`
|
||||
// Newest is the fingerprint of the module's newest catalog graph entry on each watched branch.
|
||||
Newest map[string]string `json:"newest-entries,omitempty"`
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint.
|
||||
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint, and that the listing
|
||||
// was cut at MostBuilds (H-A-487).
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// BuildReader reads what builder_build answers.
|
||||
type BuildReader struct {
|
||||
Builds StateRead
|
||||
// Entry is a key's value with its revision and when it was written; nil leaves both out.
|
||||
// Entry is a key's value with its revision and when it was written, the entry's own write time (H-A-487); nil leaves both
|
||||
// out. Builds.Written, the key's history, is read only for the key a fingerprint names when Entry says no time.
|
||||
Entry func(key string) (json.RawMessage, uint64, time.Time, error)
|
||||
// Asked is the builder's state asked, which holds the asks the operator was asked.
|
||||
Asked StateRead
|
||||
@@ -96,6 +100,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
if raw == nil {
|
||||
continue
|
||||
}
|
||||
if at.IsZero() && fingerprint != "" && r.Builds.Written != nil {
|
||||
at = r.Builds.Written(key) // one key's history, never every key's (H-A-487)
|
||||
}
|
||||
v, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s holds what is not a build: %w", key, err)
|
||||
@@ -118,7 +125,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
}
|
||||
return rows[i].Key > rows[j].Key
|
||||
})
|
||||
var cut string
|
||||
if len(rows) > MostBuilds {
|
||||
cut = fmt.Sprintf("%d of %s's %d build keys are answered, the newest by write time; name a fingerprint for another", MostBuilds, module, len(rows))
|
||||
rows = rows[:MostBuilds]
|
||||
}
|
||||
asks := r.openAsks()
|
||||
@@ -149,6 +158,9 @@ func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
|
||||
out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ")
|
||||
}
|
||||
}
|
||||
if cut != "" {
|
||||
out.Note = strings.TrimPrefix(out.Note+"; "+cut, "; ")
|
||||
}
|
||||
out.Builds = append(out.Builds, rows...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package build
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -66,6 +67,31 @@ func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) {
|
||||
if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) {
|
||||
t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err)
|
||||
}
|
||||
|
||||
// H-A-487: a module of 100 keys is read by each entry's own write time, no key's history read, and the listing is cut at the
|
||||
// 20 newest, saying so; a fingerprint's key with no time in its entry has its one history read.
|
||||
many, at := map[string][]byte{}, map[string]time.Time{}
|
||||
base := time.Date(2026, 11, 22, 20, 0, 0, 0, time.UTC)
|
||||
for i := range 100 {
|
||||
k, _ := builds.BuilderBuildKey("big", fmt.Sprintf("src2:%064x", i))
|
||||
many[k], at[k] = done, base.Add(time.Duration(i)*time.Minute)
|
||||
}
|
||||
histories := 0
|
||||
st := stateOf(many)
|
||||
st.Written = func(string) time.Time { histories++; return base }
|
||||
big := &BuildReader{Builds: st, Entry: func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, at[k], nil }}
|
||||
a, err = big.Build("big", "")
|
||||
if err != nil || len(a.Builds) != MostBuilds || !a.Builds[0].Since.Equal(base.Add(99*time.Minute)) || histories != 0 ||
|
||||
!strings.Contains(a.Note, "20 of big's 100 build keys are answered") {
|
||||
t.Fatalf("100 keys: %d rows, newest %v, %d histories read, note %q; want the 20 newest, no history, the cut said", len(a.Builds), a.Builds[0].Since, histories, a.Note)
|
||||
}
|
||||
big.Entry = func(k string) (json.RawMessage, uint64, time.Time, error) { return many[k], 1, time.Time{}, nil }
|
||||
if a, err := big.Build("big", ""); err != nil || len(a.Builds) != MostBuilds || histories != 0 {
|
||||
t.Fatalf("100 keys with no time in their entries: %d rows (%v), %d histories read; want no history read for a listing", len(a.Builds), err, histories)
|
||||
}
|
||||
if one, err := big.Build("big", fmt.Sprintf("src2:%064x", 7)); err != nil || len(one.Builds) != 1 || !one.Builds[0].Since.Equal(base) || histories != 1 {
|
||||
t.Errorf("one fingerprint with no time in its entry: %+v (%v), %d histories; want its one history read", one, err, histories)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it.
|
||||
|
||||
@@ -170,6 +170,7 @@ func Unreferenced(b builds.BuilderBuild) bool {
|
||||
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
||||
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
|
||||
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
|
||||
// - a failed key withdrawn (withdrawnPrefix) is final for its fingerprint, taken again only by a person's build again (H-A-489);
|
||||
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
|
||||
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
|
||||
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
|
||||
@@ -229,6 +230,8 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, withdrawnPrefix) && !opt.ByHand:
|
||||
return nil, nil // withdrawn is final for its fingerprint: no look builds it again (H-A-489)
|
||||
case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent:
|
||||
return nil, nil
|
||||
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
|
||||
|
||||
@@ -106,11 +106,19 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return nil
|
||||
}
|
||||
raw, _, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key cannot be read, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
cur, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || cur.State != builds.BuildDone {
|
||||
if err != nil {
|
||||
b.logf("%s: the build key holds what is no build, so it is taken as not done: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if cur.State != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
@@ -122,6 +130,57 @@ func (b *Builder) doneDigests(ctx context.Context, module, fp string) map[string
|
||||
return out
|
||||
}
|
||||
|
||||
// DefaultRecordBound is the bound of the record of a reservation's digests, its setting record-bound-seconds' default (design 47
|
||||
// H2 H-A-492).
|
||||
const DefaultRecordBound = time.Minute
|
||||
|
||||
// recordContext is the context a reservation's digests are recorded under: ctx's values with no cancel of its own, bounded by
|
||||
// RecordBound (design 47 H2 H-A-492), so a takeover or an end cancelling the build cannot cancel the record.
|
||||
func (b *Builder) recordContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
bound := b.RecordBound
|
||||
if bound <= 0 {
|
||||
bound = DefaultRecordBound
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), bound)
|
||||
}
|
||||
|
||||
// RecordAgain records the digests of fp's done build under its standing reservation when its end did not record them (design 47
|
||||
// H2 H-A-492): at the builder's look of the key and at its start (when says which), said each time. It answers the version
|
||||
// recorded, "" when nothing was to be recorded: the key is not done, no reservation stands for fp, or it is recorded already.
|
||||
func (b *Builder) RecordAgain(ctx context.Context, module, fp, when string) (string, error) {
|
||||
b.init()
|
||||
if b.Reserved == nil {
|
||||
return "", nil
|
||||
}
|
||||
digests := b.doneDigests(ctx, module, fp)
|
||||
if digests == nil {
|
||||
return "", nil
|
||||
}
|
||||
res, ok, err := b.Reserved.ReservedFor(ctx, module, fp)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s at %s: whether its done build's digests are recorded cannot be read %s: %w", module, fp, when, err)
|
||||
}
|
||||
if !ok || !res.BuiltAt.IsZero() || len(res.Artifacts) == 0 || versions.NotStanding(res, true) != "" {
|
||||
return "", nil
|
||||
}
|
||||
refs := map[string]string{}
|
||||
for ref := range res.Artifacts {
|
||||
_, artifact, _ := strings.Cut(ref, ":")
|
||||
d, have := digests[artifact]
|
||||
if !have {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded, and the done key holds no digest of %s to record %s", module, fp, res.Version, ref, when)
|
||||
}
|
||||
refs[ref] = d
|
||||
}
|
||||
rec, stop := b.recordContext(ctx)
|
||||
defer stop()
|
||||
if err := b.Reserved.Built(rec, module, res.Version, refs, nil); err != nil {
|
||||
return "", fmt.Errorf("%s at %s: the digests of %s are not recorded %s either: %w", module, fp, res.Version, when, err)
|
||||
}
|
||||
b.logf("%s at %s: the digests of %s were not recorded at its build's end: recorded %s from the done key", module, fp, res.Version, when)
|
||||
return res.Version, nil
|
||||
}
|
||||
|
||||
// PackageNamed looks at a module again when the indexer's packages state names a name for one of its packages
|
||||
// (versions.NamedModule): a done trunk build that left the package unnumbered, its name not known at its claim, is
|
||||
// numbered then with the done build's digests, and version.numbered says them for its build module's late publish.
|
||||
@@ -199,8 +258,8 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
|
||||
}
|
||||
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
|
||||
for _, t := range plan.Tasks {
|
||||
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
|
||||
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
|
||||
if t.Kind == catalogue.ArtifactUpstream {
|
||||
src.Upstream = append(src.Upstream, module+":"+t.Artifact) // copied by its pin, never awaited (H-A-486)
|
||||
}
|
||||
}
|
||||
res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src)
|
||||
|
||||
@@ -32,6 +32,8 @@ type fakeReservations struct {
|
||||
next string // the version Reserve answers when set, else the series' .5
|
||||
fail error // the reservation cannot be made (the catalog's versions cannot be read)
|
||||
tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244)
|
||||
held map[string]versions.ReservedVersion // by fingerprint, the reservation ReservedFor answers as it stands (H-A-492)
|
||||
lost error // the record of the digests fails, as a store refusing it would
|
||||
}
|
||||
|
||||
func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
@@ -46,6 +48,9 @@ func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, e
|
||||
func (f *fakeReservations) Filled() bool { return f.filled }
|
||||
|
||||
func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
if v, ok := f.held[fp]; ok {
|
||||
return v, true, nil
|
||||
}
|
||||
for _, s := range f.sources {
|
||||
if s.Fingerprint == fp {
|
||||
return versions.ReservedVersion{Version: "held"}, true, nil
|
||||
@@ -65,7 +70,13 @@ func (f *fakeReservations) Reserve(_ context.Context, module, series string, art
|
||||
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
func (f *fakeReservations) Built(ctx context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err // the state's write runs under ctx: a cancelled one records nothing
|
||||
}
|
||||
if f.lost != nil {
|
||||
return f.lost
|
||||
}
|
||||
f.deps = deps
|
||||
f.built = map[string]string{version: ""}
|
||||
for ref, d := range digests {
|
||||
@@ -118,6 +129,13 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved for the record and named as upstream, so its publish is never awaited.
|
||||
up := &fakeReservations{filled: true}
|
||||
plan := Plan{Series: "0.1", Tasks: []Task{{Artifact: "server", Kind: catalogue.ArtifactUpstream}, {Artifact: "tools", Kind: catalogue.ArtifactBundle}}}
|
||||
if _, ok := (&Builder{Reserved: up}).reserve(context.Background(), "nats", meshcatalog.ModulesByModuleValue{Fp2: "f"}, plan, Origin{}, false); !ok ||
|
||||
!slices.Equal(up.reserved[0], []string{"nats:server", "nats:tools"}) || !slices.Equal(up.sources[0].Upstream, []string{"nats:server"}) {
|
||||
t.Errorf("an upstream artifact reserved %v from %+v, want both refs reserved and nats:server named upstream", up.reserved, up.sources)
|
||||
}
|
||||
r.consider(t, "modules/m")
|
||||
if len(res.reserved) != 1 {
|
||||
t.Errorf("a done key reserved again: %v", res.reserved)
|
||||
@@ -146,6 +164,97 @@ func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound: design 47 H2 H-A-492 — the record of a reservation's digests after a
|
||||
// build ends runs under the builder's own bound, so a build context cancelled at its end still records; a done build whose
|
||||
// digests are not recorded is recorded at the builder's next look, said.
|
||||
func TestAVersionsDigestsAreRecordedUnderTheBuildersOwnBound(t *testing.T) {
|
||||
manifest := catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
t.Run("a build's context cancelled at its end still records", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
r.b.Ended = func(context.Context, string) { cancel() } // a takeover or the builder's end, once the key is written
|
||||
r.b.Module(ctx, "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
|
||||
t.Fatalf("build = %+v, want done", v)
|
||||
}
|
||||
if res.built["m:tool"] != digest {
|
||||
t.Errorf("recorded %v, want the digest of m:tool under 0.1.5 though the build's context ended", res.built)
|
||||
}
|
||||
})
|
||||
t.Run("the record's bound is the setting record-bound-seconds, never the build's", func(t *testing.T) {
|
||||
build, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, c := range []struct {
|
||||
bound, want time.Duration
|
||||
}{{0, DefaultRecordBound}, {5 * time.Second, 5 * time.Second}} {
|
||||
rec, stop := (&Builder{RecordBound: c.bound}).recordContext(build)
|
||||
deadline, has := rec.Deadline()
|
||||
stop()
|
||||
if left := time.Until(deadline); !has || left > c.want || left < c.want-time.Second {
|
||||
t.Errorf("RecordBound %v: the record's context has deadline %v (%v left, %v), want %v left of its own", c.bound, deadline, left, has, c.want)
|
||||
}
|
||||
}
|
||||
rec, stop := (&Builder{}).recordContext(build)
|
||||
defer stop()
|
||||
if rec.Err() != nil {
|
||||
t.Errorf("the record's context ended with the build's: %v", rec.Err())
|
||||
}
|
||||
})
|
||||
t.Run("a build key that cannot be read is said, not taken silently", func(t *testing.T) {
|
||||
var journal []string
|
||||
b := &Builder{Claimer: Claimer{Store: unreadable{newMem()}}, Logf: func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }}
|
||||
if d := b.doneDigests(context.Background(), "m", fp1); d != nil {
|
||||
t.Errorf("an unreadable key answered digests %v", d)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "cannot be read") && strings.Contains(l, "bus timeout")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying the build key cannot be read, with why", journal)
|
||||
}
|
||||
})
|
||||
t.Run("a done key whose digests are not recorded is recorded at the next look", func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = manifest
|
||||
res := &fakeReservations{filled: true, lost: errors.New("context canceled")}
|
||||
r.b.Reserved = res
|
||||
var journal []string
|
||||
r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
|
||||
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone || res.built != nil {
|
||||
t.Fatalf("build %s recorded %v, want done and nothing recorded", v.State, res.built)
|
||||
}
|
||||
res.lost = nil
|
||||
res.held = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": ""}}}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built["m:tool"] != digest || len(res.built) != 2 {
|
||||
t.Errorf("recorded %v at the next look, want the done key's digest of m:tool under 0.1.5", res.built)
|
||||
}
|
||||
if !slices.ContainsFunc(journal, func(l string) bool {
|
||||
return strings.Contains(l, "0.1.5 were not recorded") && strings.Contains(l, "recorded at a look")
|
||||
}) {
|
||||
t.Errorf("journal %q, want a line saying 0.1.5's digests were recorded at a look", journal)
|
||||
}
|
||||
// Recorded once: a reservation whose build ended recorded is left.
|
||||
res.built = nil
|
||||
res.held[fp1] = versions.ReservedVersion{Module: "m", Series: "0.1", Version: "0.1.5", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": digest}, BuiltAt: time.Unix(1, 0)}
|
||||
r.consider(t, "modules/m")
|
||||
if res.built != nil {
|
||||
t.Errorf("a recorded reservation was recorded again: %v", res.built)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// unreadable is a builds bucket whose reads fail, as one on a bus that does not answer.
|
||||
type unreadable struct{ *memStore }
|
||||
|
||||
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
|
||||
return nil, 0, false, errors.New("bus timeout")
|
||||
}
|
||||
|
||||
// TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through
|
||||
// reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why.
|
||||
func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) {
|
||||
@@ -236,7 +345,7 @@ func TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith(t *testing.T) {
|
||||
|
||||
// TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion: design 47 G §370 — a module whose one artifact is
|
||||
// a Go package (mesh-wireguard's go-overlay) is planned on the Go build seat, its ask carries the reserved version and names
|
||||
// the seat go-package-registry the package is published and tagged through, and the build is done; a build reserving none
|
||||
// the seat go-package-registry the package is published through, and the build is done; a build reserving none
|
||||
// names no seat.
|
||||
func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *testing.T) {
|
||||
goPackage := catalogue.Artifact{Name: "go-overlay", Kind: catalogue.ArtifactPackage, Language: "go", Paths: []string{"go"}}
|
||||
@@ -265,10 +374,6 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
|
||||
len(asked[0].Build.Artifacts) != 1 || asked[0].Build.Artifacts[0] != (buildstep.AskArtifact{Artifact: "go-overlay", Seat: c.seat}) {
|
||||
t.Fatalf("asks %+v, want one Go ask with version %q naming go-overlay on seat %q", asked, c.version, c.seat)
|
||||
}
|
||||
// Design 47 H2 §796: the reservation names the Go package's artifact, whose module path names its tag's directory.
|
||||
if c.version != "" && (len(res.sources) != 1 || !res.sources[0].GoPackage || res.sources[0].GoArtifact != "go-overlay") {
|
||||
t.Errorf("reserved from %+v, want one source naming the Go package go-overlay", res.sources)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,9 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun,
|
||||
}
|
||||
}
|
||||
// Each asked fingerprint is recorded as the branch's, so its build reserves its version (H-A-155) as one the picture
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded.
|
||||
// took in does; what the picture heard stays as it is, so the newer fingerprint's build is not superseded. Once the picture
|
||||
// heard a newer entry of the module on the first branch, H-A-489 overrides H-A-205: the older fingerprint is refused
|
||||
// superseded (buildEntry), never built or reserved.
|
||||
b.mu.Lock()
|
||||
for _, e := range run.Entries {
|
||||
key, err := builds.BuilderBuildKey(e.Module, e.Fingerprint)
|
||||
|
||||
@@ -127,6 +127,9 @@ type Builder struct {
|
||||
// package-wait-minutes (H-A-165); DefaultPackageWait when 0. Only a registry seat's published heard (PackagePublished)
|
||||
// gives a pin (H-A-192).
|
||||
PackageWait time.Duration
|
||||
// RecordBound bounds the record of a reservation's digests after a build ends, its setting record-bound-seconds (design 47
|
||||
// H2 H-A-492): the builder's own bound, never the build's context; DefaultRecordBound when 0.
|
||||
RecordBound time.Duration
|
||||
// PublishedVersions asks the Go package registry seat's verb versions for one package path, its versions newest first
|
||||
// (H-A-194): read once before a build waits for that package. Nil reads none.
|
||||
PublishedVersions func(ctx context.Context, path string) ([]string, error)
|
||||
@@ -177,6 +180,8 @@ type Builder struct {
|
||||
// publishedPins are the newest version a registry seat said published, by package path and series (PackagePublished):
|
||||
// a package with no pin waits for one (H-A-192).
|
||||
publishedPins map[string]map[string]string
|
||||
// supersededSaid are the build keys refused as superseded (H-A-489), so each is said once.
|
||||
supersededSaid sync.Map
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
@@ -310,11 +315,49 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
||||
if superseded != nil {
|
||||
// A newer fingerprint of the module: the build of the older one in flight is taken out, and its asks are withdrawn
|
||||
// from the build seats (ADR 0664 §4).
|
||||
superseded(buildrun.Withdrawn{Why: "the build of " + key + " at " + old.Fp2 + " was superseded by the fingerprint " + m.Fp2})
|
||||
superseded(buildrun.Withdrawn{Why: supersededWhy(key, old.Fp2, m.Fp2)})
|
||||
}
|
||||
b.consider(ctx, key, m)
|
||||
}
|
||||
|
||||
// supersededWhy is the words of a build withdrawn because a newer fingerprint of its module superseded it.
|
||||
func supersededWhy(module, fp, newer string) string {
|
||||
return "the build of " + module + " at " + fp + " was superseded by the fingerprint " + newer
|
||||
}
|
||||
|
||||
// withdrawnPrefix opens the error of a key whose build was withdrawn (H-A-225, ADR 0664 §4). It is final for its fingerprint
|
||||
// (H-A-489): no look builds it again; a person's build again does.
|
||||
const withdrawnPrefix = "withdrawn: "
|
||||
|
||||
// supersededBy is the fingerprint that supersedes fp, a fingerprint of module whose build key is key, when fp is of its
|
||||
// registration's first watched branch and is no longer the newest entry heard on that branch (H-A-489); "" otherwise. A key no
|
||||
// entry named is of the first branch, as originOf has it.
|
||||
func (b *Builder) supersededBy(module, key, fp string) string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
o, ok := b.origin[key]
|
||||
if !ok {
|
||||
o = Origin{First: true}
|
||||
}
|
||||
if !o.First {
|
||||
return ""
|
||||
}
|
||||
newest, heard := b.heard[o.Branch+"\x00"+module]
|
||||
if !heard || newest.Fp2 == "" || newest.Fp2 == fp {
|
||||
return ""
|
||||
}
|
||||
return newest.Fp2
|
||||
}
|
||||
|
||||
// saySuperseded says, once per key, that the build of fp is refused as superseded by newer (H-A-489), and gives the words.
|
||||
func (b *Builder) saySuperseded(module, key, fp, newer string) string {
|
||||
why := "is not built: superseded by " + newer + ", the newest entry heard on its first branch"
|
||||
if _, said := b.supersededSaid.LoadOrStore(key, true); !said {
|
||||
b.logf("%s at %s: %s", module, fp, why)
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
|
||||
// directory, commit; no source stamp, design 47 H H-A-48), the toolchains its fingerprint was computed with, the bases it is
|
||||
// built against, its packages and needs, and the named paths of its artifacts when every artifact names them.
|
||||
@@ -637,6 +680,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
whyEnded = fmt.Sprintf(format, args...)
|
||||
b.logf("%s at %s: %s", module, m.Fp2, whyEnded)
|
||||
}
|
||||
// A fingerprint no longer the newest entry heard on its first branch is neither reserved for nor built (H-A-489): two
|
||||
// fingerprints of one module never take the series from each other.
|
||||
if newer := b.supersededBy(module, key, m.Fp2); newer != "" {
|
||||
whyEnded = b.saySuperseded(module, key, m.Fp2, newer)
|
||||
return
|
||||
}
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
var noModule *NoBuildModule
|
||||
if errors.As(planErr, &noModule) {
|
||||
@@ -680,6 +729,12 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
||||
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||
if trunk && b.originOf(key).First && planErr == nil && !nothing && !reserving {
|
||||
// A done build whose digests its end did not record is recorded at the next look (design 47 H2 H-A-492).
|
||||
if _, err := b.RecordAgain(ctx, module, m.Fp2, "at a look"); err != nil {
|
||||
b.logf("%v", err)
|
||||
}
|
||||
}
|
||||
if reserving && b.waitFill(module, m) {
|
||||
end("is not built yet: wait-held, the catalog has not said versions-filled")
|
||||
return
|
||||
@@ -727,7 +782,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
}
|
||||
var reserved versions.ReservedVersion
|
||||
hasReserved := false
|
||||
hasReserved, supersededNow := false, ""
|
||||
if rebuild && cl.Build.State == builds.BuildBuilding {
|
||||
// Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased
|
||||
// and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's
|
||||
@@ -737,10 +792,16 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
reserving = true
|
||||
} else if reserving && cl.Build.State != builds.BuildDone {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
// Looked at again just before it reserves (H-A-489): a newer entry heard since the claim is not taken over.
|
||||
if supersededNow = b.supersededBy(module, key, m.Fp2); supersededNow == "" {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
|
||||
}
|
||||
}
|
||||
var result builds.BuilderBuild
|
||||
if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
if supersededNow != "" {
|
||||
// Ended as the newer entry's take-in withdraws a build in flight: final for this fingerprint.
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + supersededWhy(module, m.Fp2, supersededNow)})}
|
||||
} else if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
|
||||
// A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would
|
||||
// stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again.
|
||||
result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be")
|
||||
@@ -765,7 +826,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
var w buildrun.Withdrawn
|
||||
if ctx.Err() == nil && errors.As(context.Cause(run), &w) {
|
||||
// Taken out, not stopped: the key says so, and the asks are withdrawn (the asker dropped them).
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{"withdrawn: " + w.Why})}
|
||||
result = builds.BuilderBuild{State: builds.BuildFailed, Error: keyError([]string{withdrawnPrefix + w.Why})}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
@@ -807,8 +868,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
for _, a := range result.Artifacts {
|
||||
digests[module+":"+a.Name] = a.Digest
|
||||
}
|
||||
if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
|
||||
// Under the builder's own bound, never the build's context: a takeover or an end that cancels it no longer leaves a passed
|
||||
// build unrecorded, so unreleased (design 47 H2 H-A-492).
|
||||
rec, stop := b.recordContext(ctx)
|
||||
err := b.Reserved.Built(rec, module, reserved.Version, digests, b.takeDependencies(key))
|
||||
stop()
|
||||
if err != nil {
|
||||
b.logf("%s: the digests of %s are not recorded: %v; recorded again at the builder's next look or start", cl.Key, reserved.Version, err)
|
||||
}
|
||||
}
|
||||
if result.State == builds.BuildFailed {
|
||||
|
||||
@@ -333,8 +333,9 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
asked, release := make(chan struct{}), make(chan struct{})
|
||||
answer := r.answer
|
||||
var once sync.Once
|
||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
close(asked)
|
||||
once.Do(func() { close(asked) })
|
||||
<-release
|
||||
return answer(a)
|
||||
}
|
||||
@@ -356,6 +357,85 @@ func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") {
|
||||
t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v)
|
||||
}
|
||||
// H-A-489: withdrawn is final for the fingerprint: a later look builds it not again; a person's build again does.
|
||||
n, rev := len(r.asks.Asks()), r.st.revs[r.key]
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); len(r.asks.Asks()) != n || r.st.revs[r.key] != rev || !strings.HasPrefix(v.Error, "withdrawn: ") {
|
||||
t.Errorf("a later look built the withdrawn key again: %d asks (was %d), the key %+v", len(r.asks.Asks()), n, v)
|
||||
}
|
||||
if cl, err := r.b.Claimer.Claim(context.Background(), "m", fp1, mdigest('a'), ClaimOptions{ByHand: true}); err != nil || cl == nil {
|
||||
t.Errorf("a person's build again of the withdrawn key was not claimed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489 (review round 1): a newer entry of the first branch heard between the older build's claim and its reservation is
|
||||
// not taken over: the older key ends failed withdrawn with no reservation, and the newer reserves.
|
||||
func TestANewerEntryHeardBeforeTheReservationIsNotTakenOver(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
newer, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp2, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.before = func() { r.b.Module(context.Background(), "m", newer) } // at the older key's claim
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.HasPrefix(v.Error, "withdrawn: ") || !strings.Contains(v.Error, "superseded by the fingerprint "+fp2) {
|
||||
t.Errorf("the older key = %+v, want failed withdrawn, superseded by %s", v, fp2)
|
||||
}
|
||||
if len(res.sources) != 1 || res.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("reserved %+v, want one reservation, the newer's", res.sources)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-489: a fingerprint of the first branch that is no longer its newest entry heard is neither reserved for nor built, and
|
||||
// said once; the again loop that looks at it later builds nothing.
|
||||
func TestAnOlderFingerprintAskedAfterANewerEntryIsRefused(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
var mu sync.Mutex
|
||||
var said []string
|
||||
r.b.Logf = func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
r.bm.fails["ace/build_go_build"] = "go build of tool: boom"
|
||||
older := meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp2: fp1, Manifest: mdigest('a'),
|
||||
Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)}
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
delete(r.bm.fails, "ace/build_go_build")
|
||||
newer := older
|
||||
newer.Fp2 = fp2
|
||||
data, err := meshcatalogclient.EncodeModule(newer)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
r.b.Wait()
|
||||
rev := r.st.revs[r.key]
|
||||
for range 2 {
|
||||
r.b.consider(context.Background(), "m", older) // as consider's again loop or a step's look takes it
|
||||
r.b.Wait()
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || r.st.revs[r.key] != rev {
|
||||
t.Errorf("the older key = %+v at revision %d (was %d), want it left as it failed", v, r.st.revs[r.key], rev)
|
||||
}
|
||||
k2, _ := builds.BuilderBuildKey("m", fp2)
|
||||
if v := valueOf(t, r.st, k2); v.State != builds.BuildDone {
|
||||
t.Errorf("the newer key = %+v, want done", v)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range said {
|
||||
if strings.Contains(l, fp1+": is not built: superseded by "+fp2) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("superseded said %d times, want once: %q", n, said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
|
||||
|
||||
@@ -142,6 +142,10 @@ func (b *Builder) buildEntry(ctx context.Context, pipeline string, e meshbuilds.
|
||||
}
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, failedWords, "existing build")
|
||||
}
|
||||
if newer := b.supersededBy(entry.Module, key, entry.Fp); newer != "" {
|
||||
// Refused, not waited on (H-A-489): no look builds a superseded fingerprint, so its end would never come.
|
||||
return answered(meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s %s", entry.Module, entry.Fp, b.saySuperseded(entry.Module, key, entry.Fp, newer)), "refused")
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalogclient.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalogclient.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
b.consider(ctx, entry.Module, m)
|
||||
@@ -232,7 +236,7 @@ func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed str
|
||||
switch {
|
||||
case v.State == builds.BuildDone && !Unpublished(v, b.Seats):
|
||||
return true, "", true
|
||||
case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error):
|
||||
case v.State == builds.BuildFailed && (builds.FailedUntilChanged(v.Error) || strings.HasPrefix(v.Error, withdrawnPrefix)):
|
||||
return false, key + ": " + v.Error, true
|
||||
}
|
||||
return false, "", false
|
||||
|
||||
@@ -74,6 +74,21 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
|
||||
if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" {
|
||||
t.Errorf("a done build: %+v", end)
|
||||
}
|
||||
// H-A-489 (review round 1): a withdrawn key whose fingerprint is still the newest entry is answered failed at once, naming
|
||||
// withdrawn: no look builds it again, so its end is never waited for.
|
||||
rw := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"})
|
||||
rw.b.StepEntry = r.b.StepEntry
|
||||
withdrawn, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Manifest: mdigest('a'), Error: withdrawnPrefix + "its reservation 0.1.3 was taken over"})
|
||||
if _, err := rw.st.Create(t.Context(), rw.key, withdrawn); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rw.b.Module(t.Context(), "m", moduleValue(t, "modules/m")) // the newest entry heard is fp1
|
||||
rw.b.Wait()
|
||||
bounded, cancel := context.WithTimeout(t.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if end := rw.b.BuildStep(bounded, "ask1", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries}); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "withdrawn: ") {
|
||||
t.Errorf("a withdrawn key asked: %+v, want failed at once naming withdrawn", end)
|
||||
}
|
||||
}
|
||||
|
||||
// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97).
|
||||
@@ -434,9 +449,10 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, whose entry of m moved
|
||||
// on, while the older pipeline asks the build of revision 45. Its kept catalog revision is read by number and the build runs;
|
||||
// a kept revision naming another fingerprint fails the step, naming it.
|
||||
// H-A-205: two pipelines of consecutive revisions run at once; the picture holds source revision 46, while the older pipeline
|
||||
// asks the build of revision 45. Its kept catalog revision is read by number and the build runs while the picture never heard
|
||||
// a newer entry of m (a restart); once it heard one on the first branch, H-A-489 overrides H-A-205 and the older ask is refused
|
||||
// superseded. A kept revision naming another fingerprint fails the step, naming it.
|
||||
func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
var held atomic.Int64
|
||||
held.Store(46)
|
||||
@@ -460,6 +476,24 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
if len(res.sources) != 1 || res.sources[0].Revision != 45 || res.sources[0].CatalogRevision != 47 {
|
||||
t.Errorf("reserved from %+v; want one version reserved from source revision 45 at catalog revision 47 (H-A-218)", res.sources)
|
||||
}
|
||||
// H-A-489: once the picture heard the newer entry of m on the first branch, the older ask is refused superseded and
|
||||
// nothing is built or reserved for it; the newer entry's build keeps the series.
|
||||
rn, resn, entryn := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
rn.b.RevisionWait = time.Nanosecond
|
||||
rn.b.FirstBranch = r.b.FirstBranch
|
||||
rn.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return 45, map[string]string{"m": fp1}, nil
|
||||
}
|
||||
entryn.Fp = fp2
|
||||
rn.b.Entry(context.Background(), Origin{Branch: "main", Source: 46, First: true}, entryn)
|
||||
rn.b.Wait()
|
||||
if endn := rn.b.BuildStep(t.Context(), "a1", run); endn.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(endn.Said, "superseded by "+fp2) {
|
||||
t.Errorf("the older ask after the newer entry: end = %+v; want failed, superseded by %s", endn, fp2)
|
||||
}
|
||||
rn.b.Wait()
|
||||
if _, built := rn.st.vals[rn.key]; built || len(resn.sources) != 1 || resn.sources[0].Fingerprint != fp2 {
|
||||
t.Errorf("built %v, reserved %+v; want nothing built for the older, one reservation, the newer's", built, resn.sources)
|
||||
}
|
||||
r2, _, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r2.b.RevisionWait = time.Nanosecond
|
||||
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
|
||||
@@ -10,10 +10,8 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -42,8 +40,8 @@ const (
|
||||
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
|
||||
// H H-A-275), scope module.
|
||||
VersionHeldElsewhere = "version-held-elsewhere"
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47
|
||||
// H2 §796), scope module; cleared at the module's next version-released.
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stood at another commit, which a reserve skipped before no
|
||||
// tag was read (design 47 H2 H-A-494); one raised before is cleared at the module's next version-released.
|
||||
VersionTaggedElsewhere = "version-tagged-elsewhere"
|
||||
)
|
||||
|
||||
@@ -55,6 +53,10 @@ type ReservedVersion struct {
|
||||
Version string `json:"version"` // MAJOR.MINOR.PATCH
|
||||
// Artifacts are the module's artifacts at the version: the digest once built, empty before.
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
// Upstream are the artifacts of kind upstream, by reference (design 47 H2 H-A-486): kept in Artifacts with their digest for the
|
||||
// record, never awaited, as design 51 copies an upstream image by its pinned digest and no registry seat says it published at
|
||||
// a version. A reservation written before carries none.
|
||||
Upstream []string `json:"upstream,omitempty"`
|
||||
// BuiltAt is when the build ended with every digest known; the publish's bound counts from it.
|
||||
BuiltAt time.Time `json:"built-at,omitzero"`
|
||||
// Published are the artifacts a registry seat said published at this version with the built digest.
|
||||
@@ -63,8 +65,8 @@ type ReservedVersion struct {
|
||||
// was kept.
|
||||
PublishedAt map[string]time.Time `json:"published-at,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go
|
||||
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known.
|
||||
// Repository and Commit are what the build was made from. Tag is kept for a reservation written before design 47 H2 H-A-494:
|
||||
// no git tag is written at a release any more, and a new reservation names none.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
@@ -110,22 +112,19 @@ type KeptPublished struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
|
||||
// written at, and whether the module has a Go package, whose version is tagged.
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit, kept on the reservation.
|
||||
type Source struct {
|
||||
Repository, Commit string
|
||||
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
|
||||
Fingerprint string
|
||||
GoPackage bool
|
||||
// GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in
|
||||
// (design 47 H2 §796).
|
||||
GoArtifact string
|
||||
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
|
||||
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
|
||||
Branch string
|
||||
Revision int64
|
||||
// CatalogRevision is the catalog revision number the build's step ask named (H-A-218), 0 when none did.
|
||||
CatalogRevision int64
|
||||
// Upstream are the artifacts of kind upstream among those reserved, by reference (H-A-486), kept on the reservation.
|
||||
Upstream []string
|
||||
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
|
||||
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
|
||||
Rebuild bool
|
||||
@@ -171,92 +170,6 @@ type Reserver struct {
|
||||
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
|
||||
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
|
||||
HeldElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
// Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a
|
||||
// name not known, tags the version nowhere, said.
|
||||
Names PackageNames
|
||||
// Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47
|
||||
// H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil
|
||||
// reads none.
|
||||
Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error)
|
||||
// TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil
|
||||
// is told none.
|
||||
TaggedElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
|
||||
tagsMu sync.Mutex
|
||||
// tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read
|
||||
// under: a module's tags are read once per catalog revision it reserves under (§796).
|
||||
tagsRead map[string]tagsAt
|
||||
}
|
||||
|
||||
// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names.
|
||||
type tagsAt struct {
|
||||
revision, prefix string
|
||||
commits map[int]string
|
||||
}
|
||||
|
||||
// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when
|
||||
// no name or no directory of the repository is known.
|
||||
func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) {
|
||||
if r.Names == nil {
|
||||
return "", false
|
||||
}
|
||||
path, known, err := r.Names(ctx, module, src.GoArtifact)
|
||||
switch {
|
||||
case err != nil:
|
||||
r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err))
|
||||
return "", false
|
||||
case !known:
|
||||
r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact))
|
||||
return "", false
|
||||
}
|
||||
dir, ok := GoDirectory(path, src.Repository)
|
||||
if !ok {
|
||||
r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository))
|
||||
}
|
||||
return dir, ok
|
||||
}
|
||||
|
||||
// taggedAt are the patches of series whose tag `<directory>/v<series>.<patch>` stands, each with its commit, read once per
|
||||
// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the
|
||||
// reservation goes on; any other failure refuses the reservation, naming it.
|
||||
func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) {
|
||||
if r.Tags == nil {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := TagPrefix(dir) + series + "."
|
||||
revision := ""
|
||||
if src.CatalogRevision > 0 {
|
||||
revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision)
|
||||
r.tagsMu.Lock()
|
||||
got, ok := r.tagsRead[module]
|
||||
r.tagsMu.Unlock()
|
||||
if ok && got.revision == revision && got.prefix == prefix {
|
||||
return got.commits, nil
|
||||
}
|
||||
}
|
||||
tags, away, err := r.Tags(ctx, src.Repository, prefix)
|
||||
if err != nil && away {
|
||||
r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err))
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err)
|
||||
}
|
||||
commits := map[int]string{}
|
||||
for _, t := range tags {
|
||||
if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) {
|
||||
commits[p] = t.Commit
|
||||
}
|
||||
}
|
||||
if revision != "" {
|
||||
r.tagsMu.Lock()
|
||||
if r.tagsRead == nil {
|
||||
r.tagsRead = map[string]tagsAt{}
|
||||
}
|
||||
r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits}
|
||||
r.tagsMu.Unlock()
|
||||
}
|
||||
return commits, nil
|
||||
}
|
||||
|
||||
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
|
||||
@@ -277,19 +190,15 @@ func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint
|
||||
return out
|
||||
}
|
||||
|
||||
// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the
|
||||
// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796).
|
||||
func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) {
|
||||
var skipped, elsewhere []string
|
||||
// nextFree is the first patch after after that no registry holds, with the held numbers it skipped (H-A-275). No git tag is
|
||||
// read: a series is numbered from the catalog's versions and the registries' alone (design 47 H2 H-A-494).
|
||||
func nextFree(series string, after int, held map[int]bool) (int, []string) {
|
||||
var skipped []string
|
||||
for p := after + 1; ; p++ {
|
||||
switch c, ok := tagged[p]; {
|
||||
case held[p]:
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
case ok && c != commit:
|
||||
elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c))
|
||||
default:
|
||||
return p, skipped, elsewhere
|
||||
if !held[p] {
|
||||
return p, skipped
|
||||
}
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,15 +251,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
|
||||
dir, tagged := "", false
|
||||
var tags map[int]string
|
||||
if src.GoPackage {
|
||||
if dir, tagged = r.goDirectory(ctx, module, src); tagged {
|
||||
if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil {
|
||||
return ReservedVersion{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil {
|
||||
@@ -368,15 +268,13 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
next = p
|
||||
}
|
||||
}
|
||||
patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit)
|
||||
patch, skipped := nextFree(series, next, held)
|
||||
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
|
||||
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
|
||||
if tagged {
|
||||
out.Tag = GoTagOf(dir, out.Version)
|
||||
}
|
||||
for _, a := range artifacts {
|
||||
out.Artifacts[a] = ""
|
||||
}
|
||||
out.Upstream = slices.Clone(src.Upstream)
|
||||
if found {
|
||||
out.Withdrawn = maps.Clone(cur.Withdrawn)
|
||||
if cur.Fingerprint != src.Fingerprint {
|
||||
@@ -410,9 +308,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
if len(skipped) > 0 && r.HeldElsewhere != nil {
|
||||
r.HeldElsewhere(ctx, module, skipped)
|
||||
}
|
||||
if len(elsewhere) > 0 && r.TaggedElsewhere != nil {
|
||||
r.TaggedElsewhere(ctx, module, elsewhere)
|
||||
}
|
||||
if found && r.Took != nil {
|
||||
r.Took(ctx, cur, out)
|
||||
}
|
||||
@@ -1134,11 +1029,11 @@ const (
|
||||
ReservedVerb = meshbuilds.VerbReserved
|
||||
)
|
||||
|
||||
// Missing are the artifacts of v no registry seat said published, sorted.
|
||||
// Missing are the artifacts of v no registry seat said published, sorted; an upstream one is never awaited (H-A-486).
|
||||
func (v ReservedVersion) Missing() []string {
|
||||
var out []string
|
||||
for a := range v.Artifacts {
|
||||
if !v.Published[a] {
|
||||
if !v.Published[a] && !slices.Contains(v.Upstream, a) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
)
|
||||
|
||||
@@ -171,6 +169,23 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) {
|
||||
t.Errorf("missing = %v, want m:b", got)
|
||||
}
|
||||
// H-A-486: an upstream artifact is reserved and recorded with its digest, but never awaited: once the others are published
|
||||
// nothing is missing.
|
||||
up, err := r.Reserve(ctx, "u", "0.1", []string{"u:server", "u:tools"}, Source{Upstream: []string{"u:server"}})
|
||||
if err != nil || !reflect.DeepEqual(up.Upstream, []string{"u:server"}) {
|
||||
t.Fatalf("reserved %+v (%v), want u:server kept as upstream", up, err)
|
||||
}
|
||||
for _, a := range []string{"u:server", "u:tools"} {
|
||||
if _, err := r.Built(ctx, "u", up.Version, a, "sha256:"+a, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if marked, _, err := r.Published(ctx, "u-tools", up.Version, "sha256:u:tools"); err != nil || len(marked) != 1 {
|
||||
t.Fatalf("the published of u:tools marked %v (%v)", marked, err)
|
||||
}
|
||||
if cur, _, _ := r.At(ctx, ReservedKey("u", "0.1")); len(cur.Missing()) != 0 || cur.Artifacts["u:server"] != "sha256:u:server" {
|
||||
t.Errorf("missing = %v with artifacts %v, want none missing and the upstream digest kept", cur.Missing(), cur.Artifacts)
|
||||
}
|
||||
if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") {
|
||||
t.Errorf("the bound key does not name the reservation back: %q %v", key, ok)
|
||||
}
|
||||
@@ -399,7 +414,7 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
return held, readErr
|
||||
},
|
||||
HeldElsewhere: func(_ context.Context, _ string, s []string) { skipped = append(skipped, s...) }}
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true})
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1"})
|
||||
if err != nil || v.Version != "0.1.39" || len(skipped) != 2 {
|
||||
t.Fatalf("reserved %q (%v), skipped %v: want 0.1.39 past the two numbers the registry holds", v.Version, err, skipped)
|
||||
}
|
||||
@@ -413,9 +428,9 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
}
|
||||
held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"}
|
||||
// The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse.
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true})
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", Repository: "o/r", Commit: "c1", Rebuild: true})
|
||||
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != "" {
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (no Go package name known)", again, err)
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (design 47 H2 H-A-494)", again, err)
|
||||
}
|
||||
if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 {
|
||||
t.Errorf("the index by version still names the withdrawn number: %v", at)
|
||||
@@ -428,50 +443,4 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" {
|
||||
t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err)
|
||||
}
|
||||
// Design 47 H2 §796: a Go package's tags are read before it reserves, once per catalog revision; a number whose tag
|
||||
// `<directory>/v<version>` stands at another commit is skipped and said, one at the build's own commit is not; the version's
|
||||
// tag names the Go module's directory; a read met away lets the reservation go on, any other failure refuses it.
|
||||
r.HeldNumbers = nil
|
||||
var tagReads int
|
||||
var tagsErr error
|
||||
var tagsAway bool
|
||||
var elsewhere []string
|
||||
tags := []git.Tag{{Tag: "modules/nt/v0.3.0", Commit: "c-other"}, {Tag: "modules/nt/v0.3.1", Commit: "c-mine"}, {Tag: "modules/nt/v0.3.2", Commit: "c-other2"}}
|
||||
r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/nt", module == "node-tools" && artifact == "lib", nil
|
||||
}
|
||||
r.Tags = func(_ context.Context, repository, prefix string) ([]git.Tag, bool, error) {
|
||||
tagReads++
|
||||
if repository != "o/r" || prefix != "modules/nt/v0.3." {
|
||||
t.Errorf("tags read of %s %q, want o/r modules/nt/v0.3.", repository, prefix)
|
||||
}
|
||||
return tags, tagsAway, tagsErr
|
||||
}
|
||||
r.TaggedElsewhere = func(_ context.Context, _ string, s []string) { elsewhere = append(elsewhere, s...) }
|
||||
at := Source{Fingerprint: "fp3", GoPackage: true, GoArtifact: "lib", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 7}
|
||||
v3, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at)
|
||||
if err != nil || v3.Version != "0.3.1" || v3.Tag != "modules/nt/v0.3.1" || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.0") {
|
||||
t.Fatalf("reserved %+v (%v), tagged elsewhere %v: want 0.3.1 (0.3.0 tagged at another commit, 0.3.1 at the build's own), tag modules/nt/v0.3.1", v3, err, elsewhere)
|
||||
}
|
||||
at.Fingerprint, at.Commit, elsewhere = "fp4", "c-new", nil
|
||||
if v4, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v4.Version != "0.3.3" || tagReads != 1 || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.2") {
|
||||
t.Fatalf("reserved %+v (%v) after %d reads, tagged elsewhere %v: want 0.3.3 past 0.3.2's tag, the tags read once for revision main.7", v4, err, tagReads, elsewhere)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsErr, tagsAway = 8, "fp5", errors.New("the forge is away"), true
|
||||
if v5, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v5.Version != "0.3.4" || tagReads != 2 {
|
||||
t.Fatalf("with the tags away reserved %+v (%v) after %d reads, want 0.3.4 and the reservation going on", v5, err, tagReads)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsAway = 9, "fp6", false
|
||||
if _, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err == nil || !strings.Contains(err.Error(), "tags") {
|
||||
t.Errorf("a tags read refused not away reserved anyway (%v), want the reservation refused naming the tags", err)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/novox/mesh-sdk/go", "https://git.example/novox/mesh-sdk.git"); !ok || d != "go" || GoTagOf(d, "0.1.9") != "go/v0.1.9" {
|
||||
t.Errorf("mesh-sdk's Go package: directory %q (%v), want go and the tag go/v0.1.9", d, ok)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/o/r", "o/r"); !ok || d != "" || GoTagOf(d, "1.0.0") != "v1.0.0" {
|
||||
t.Errorf("a root Go module: directory %q (%v), want the root and the tag v1.0.0", d, ok)
|
||||
}
|
||||
if _, ok := GoDirectory("git.example/o/rr/x", "o/r"); ok {
|
||||
t.Error("a module path of another repository was given a directory")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// GoTagOf is the git tag a Go package's released version is written as (design 47 H2 §796): `<directory>/v<version>`, the
|
||||
// directory of its Go module in the repository, which the Go toolchain's direct fallback reads; `v<version>` at the root. The
|
||||
// `go/v<version>` tags go from new versions.
|
||||
func GoTagOf(directory, version string) string { return TagPrefix(directory) + version }
|
||||
|
||||
// GoDirectory is the directory of the Go module path modulePath in repository (`<owner>/<name>`, or a clone address naming
|
||||
// it): the part of the path after the repository's own, as the Go toolchain reads a module kept in a subdirectory; empty at
|
||||
// the root. ok is false when the path is not one of that repository.
|
||||
func GoDirectory(modulePath, repository string) (string, bool) {
|
||||
repo := "/" + strings.ToLower(trunk.RepositoryName(repository))
|
||||
lower := strings.ToLower(modulePath)
|
||||
for from := 0; ; {
|
||||
i := strings.Index(lower[from:], repo)
|
||||
if i < 0 || repo == "/" {
|
||||
return "", false
|
||||
}
|
||||
end := from + i + len(repo)
|
||||
if end == len(lower) || lower[end] == '/' {
|
||||
return strings.Trim(modulePath[end:], "/"), true
|
||||
}
|
||||
from = end
|
||||
}
|
||||
}
|
||||
@@ -152,7 +152,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
// Design 51 *The floor*: the floor's three sources, the registry seats' versions, git.tags and indexer_requirements.
|
||||
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
|
||||
if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
|
||||
"seat:git.tag", "seat:git.tags",
|
||||
"seat:git.tags", // the floor's source alone: no tag is written at a release (design 47 H2 H-A-494)
|
||||
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
|
||||
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
|
||||
@@ -199,15 +199,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 13 ||
|
||||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
|
||||
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
|
||||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
|
||||
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
|
||||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
// H-A-492: the record of a version's digests runs under the builder's own bound, a setting of 60 s by default
|
||||
m.Config["record-bound-seconds"].Default != float64(60) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), record-bound-seconds of 60 (H-A-492), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
}
|
||||
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
|
||||
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
@@ -266,7 +268,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("health = %v, want kind unit alone", p["health"])
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
if len(env) != 9 || env["MESH_BUILDER_RECORD_BOUND_FILE"] != "${dir:state}/record-bound-seconds" || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
|
||||
}
|
||||
for k, v := range env {
|
||||
@@ -274,14 +276,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v)
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" {
|
||||
found = true
|
||||
for _, setting := range []string{"toolchain-wait-minutes", "record-bound-seconds"} { // H-A-492: the record's bound too
|
||||
found := false
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "file" && r["path"] == "${dir:state}/"+setting && r["content"] == "${config:"+setting+"}" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:%s}: A-71", setting)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71")
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "settings" {
|
||||
|
||||
@@ -91,7 +91,6 @@
|
||||
"invokes": [
|
||||
"seat:container-image-registry.remove",
|
||||
"seat:container-image-registry.versions",
|
||||
"seat:git.tag",
|
||||
"seat:git.tags",
|
||||
"seat:go-package-registry.remove",
|
||||
"seat:go-package-registry.versions",
|
||||
@@ -183,6 +182,14 @@
|
||||
"trusted": false,
|
||||
"default": 10
|
||||
},
|
||||
"record-bound-seconds": {
|
||||
"type": "number",
|
||||
"description": "seconds the builder gives the record of a version's digests after its build ends, under its own bound and never the build's (design 47 H2 H-A-492).",
|
||||
"minimum": 1,
|
||||
"maximum": 3600,
|
||||
"trusted": false,
|
||||
"default": 60
|
||||
},
|
||||
"ask-taken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).",
|
||||
@@ -310,6 +317,14 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:package-wait-minutes}"
|
||||
},
|
||||
{
|
||||
"id": "record-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/record-bound-seconds",
|
||||
"owner": "mesh-delivery-builder",
|
||||
"mode": "0600",
|
||||
"content": "${config:record-bound-seconds}"
|
||||
},
|
||||
{
|
||||
"id": "publish-pause",
|
||||
"type": "file",
|
||||
@@ -361,6 +376,7 @@
|
||||
"MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes",
|
||||
"MESH_BUILDER_REVISION_WAIT_FILE": "${dir:state}/revision-wait-minutes",
|
||||
"MESH_BUILDER_PACKAGE_WAIT_FILE": "${dir:state}/package-wait-minutes",
|
||||
"MESH_BUILDER_RECORD_BOUND_FILE": "${dir:state}/record-bound-seconds",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause",
|
||||
"MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max",
|
||||
"MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries",
|
||||
|
||||
@@ -140,7 +140,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
|
||||
}
|
||||
var want []string
|
||||
for _, a := range entry.Artifacts {
|
||||
want = append(want, a.Ref)
|
||||
if a.Kind != catalogue.ArtifactUpstream { // an upstream copy is released by its pinned digest, never published (H-A-486)
|
||||
want = append(want, a.Ref)
|
||||
}
|
||||
}
|
||||
key := PendingPrefix + keys.EscapeName(p.Module) + "." + keys.EscapeName(p.Version)
|
||||
var have published
|
||||
@@ -211,7 +213,15 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
late := len(kept) > 0 && compareVersions(kept[0].Version, p.Version) > 0
|
||||
at := ix.now().UTC()
|
||||
v := meshcatalog.VersionEntry{V: int64(keys.Version), Module: p.Module, Version: p.Version, Source: have.Source, At: &at, Late: late}
|
||||
pins, err := upstreamPins(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if a.Kind == catalogue.ArtifactUpstream {
|
||||
v.Artifacts = append(v.Artifacts, meshcatalog.VersionArtifact{Artifact: a.Ref, Digest: pins[a.Ref]})
|
||||
continue
|
||||
}
|
||||
v.Artifacts = append(v.Artifacts, have.Artifacts[a.Ref])
|
||||
}
|
||||
slices.SortFunc(v.Artifacts, func(a, b meshcatalog.VersionArtifact) int { return strings.Compare(a.Artifact, b.Artifact) })
|
||||
@@ -255,6 +265,46 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
return ix.forgetPublished(ctx, pendingKey)
|
||||
}
|
||||
|
||||
// upstreamPins are the pinned digests of the entry's upstream artifacts, by reference, read from its manifest's from
|
||||
// (`<image>@sha256:<digest>`): design 51 copies an upstream image by that digest, so a release names it (H-A-486). An upstream
|
||||
// artifact whose manifest names no pin refuses the release, naming it.
|
||||
func upstreamPins(entry meshcatalog.CatalogEntry) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
if !slices.ContainsFunc(entry.Artifacts, func(a meshcatalog.Artifact) bool { return a.Kind == catalogue.ArtifactUpstream }) {
|
||||
return out, nil
|
||||
}
|
||||
raw, err := json.Marshal(entry.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("the manifest of %s's entry does not read, so its upstream artifacts' pins are not known: %w", entry.Module, err)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
continue
|
||||
}
|
||||
if _, d, ok := strings.Cut(a.From, "@"); ok && strings.HasPrefix(d, "sha256:") {
|
||||
out[ArtifactRefOf(entry.Module, a.Name)] = d
|
||||
}
|
||||
}
|
||||
for _, a := range entry.Artifacts {
|
||||
if _, ok := out[a.Ref]; a.Kind == catalogue.ArtifactUpstream && !ok {
|
||||
return nil, fmt.Errorf("%s is an upstream artifact whose manifest names no pinned digest (from <image>@sha256:<digest>), so %s releases no version", a.Ref, entry.Module)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// trimVersions deletes the module's oldest versions beyond the kept 100, never its highest of any MAJOR.MINOR.
|
||||
func (ix *Indexer) trimVersions(ctx context.Context, module string) error {
|
||||
kept, err := ModuleVersions(ctx, ix.Versions, module) // newest first
|
||||
|
||||
@@ -83,6 +83,36 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
if compareVersions(oldest, "0.1.5") < 0 {
|
||||
t.Errorf("the oldest kept is %s: the oldest were not the ones deleted", oldest)
|
||||
}
|
||||
|
||||
// H-A-486: an upstream artifact is copied by its pinned digest and never published, so the version is released once every
|
||||
// other artifact is, the upstream one named with the pin its manifest gives; one whose manifest names no pin releases none.
|
||||
pin := "sha256:" + strings.Repeat("ab", 32)
|
||||
nats := func(from string) map[string]meshcatalog.CatalogEntry {
|
||||
return map[string]meshcatalog.CatalogEntry{"nats": {Module: "nats", Manifest: map[string]any{"build": map[string]any{"artifacts": []any{
|
||||
map[string]any{"name": "server", "kind": "upstream", "from": from},
|
||||
map[string]any{"name": "tools", "kind": "bundle", "from": "."}}}},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Kind: "bundle", Language: "go"}}}}
|
||||
}
|
||||
for i, c := range []struct {
|
||||
from, version string
|
||||
released bool
|
||||
}{{"nats:2.12-alpine@" + pin, "0.1.1", true}, {"nats:2.12-alpine", "0.1.2", false}} {
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: int64(4 + i), Kind: "trunk"}, nats(c.from))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = ix.HearPackagePublished(ctx, meshbuilds.PackagePublished{Module: "nats", Artifact: "nats:tools", Digest: "sha256:tools", Version: c.version, Source: 9, Branch: "main"})
|
||||
got, _ := ModuleVersions(ctx, ix.Versions, "nats")
|
||||
switch {
|
||||
case c.released && (err != nil || len(got) != 1 || len(got[0].Artifacts) != 2 || got[0].Artifacts[0] != (meshcatalog.VersionArtifact{Artifact: "nats:server", Digest: pin})):
|
||||
t.Fatalf("pinned %q: released %+v (%v); want the server named with its pin", c.from, got, err)
|
||||
case !c.released && (err == nil || !strings.Contains(err.Error(), "no pinned digest") || len(got) != 1):
|
||||
t.Fatalf("unpinned %q: released %+v (%v); want a refusal naming the pin", c.from, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or
|
||||
|
||||
Reference in New Issue
Block a user