diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index b05427b8..bfc658dc 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -434,7 +434,7 @@ func main() { Newest: newestVia(stdio.Ask), Steps: r})) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState) - tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + tools = append(tools, republishTools(readerOf(qs), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) ws := stdio.State(build.WaitsState) wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := ws.Get(key) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go index 9b48df0f..a406d3a1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -28,7 +28,7 @@ func readerOf(s stdio.KeptState) versions.Reader { // republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: // it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer // with the versions it queued. -func republishTools(queue, acted versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { +func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, @@ -60,6 +60,6 @@ func republishTools(queue, acted versions.Reader, ask queueAsker, now func() tim }}, {Name: versions.PublishQueueTool, Description: versions.PublishQueueDescription, Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted) }}, + Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted, reasks) }}, } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go index 983a54f4..a690371c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go @@ -13,7 +13,7 @@ import ( "mesh-delivery-builder/internal/versions" ) -// reservedTools are builder_reserved and builder_reserved_drop (design 47 G section 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to +// reservedTools are builder_reserved, builder_reserved_drop and builder_reserve_done (A-455) (design 47 G section 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to // reasks for the builder, which owns the reservations, to answer. // // The seat's verb reserved is the builder's process's (design 47 G A-115, A-127), not the bundle's. @@ -51,5 +51,32 @@ func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, act } return map[string]any{"key": key, "answer": a.Answer}, nil }}, + {Name: versions.ReserveDoneTool, Description: versions.ReserveDoneDescription, + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module"}, + "fp": map[string]any{"type": "string", "description": "the fingerprint of its done trunk build"}, + "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewReserveDone(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("reserved a version for the done build of %s at %s by hand", str(args, "module"), str(args, "fp")), + Why: str(args, "why"), Cause: versions.ReserveDoneTool, Verb: "reserve-done"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was reserved: %w", err) + } + raw, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if a.Refused { + return nil, errors.New(a.Answer) + } + return map[string]any{"key": key, "answer": a.Answer}, nil + }}, } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go index e782acf2..a65a854f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/waits_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "strings" "testing" "time" @@ -44,3 +45,56 @@ func TestWaitsToolsListTheWaitsAndReleaseAfterTheHandAct(t *testing.T) { t.Fatalf("release: %v, %d recorded, asked %+v", err, recorded, asked) } } + +// Design 47 G A-455: builder_reserve_done records the hand act first and only then asks the builder through reasks; the builder's +// answer (the version and its queued publish) is the tool's answer, its refusal the tool's error, and an act that was not +// recorded asks nothing. +func TestReserveDoneRecordsTheHandActFirstAndAnswersTheBuilders(t *testing.T) { + var order []string + var asked []build.ReAsk + reply := func(refuse bool) func(string, []byte) (json.RawMessage, error) { + return func(key string, value []byte) (json.RawMessage, error) { + order = append(order, "asked") + a, _ := build.DecodeReAsk(value) + asked = append(asked, a) + a.Answered, a.Answer, a.Refused = time.Now(), "reserved 0.1.13 for node-tools and queued 1 versions to publish: node-tools@0.1.13", refuse + if refuse { + a.Answer = "the build key is not done" + } + return json.Marshal(a) + } + } + record := func(fail bool) meshhandactsclient.Recorder { + return meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) { + order = append(order, "recorded") + if fail { + return nil, errors.New("no responders") + } + return json.RawMessage(`{}`), nil + }} + } + run := func(ask func(string, []byte) (json.RawMessage, error), acts meshhandactsclient.Recorder, args map[string]any) (any, error) { + for _, tl := range reservedTools(versions.Reader{}, ask, time.Now, acts) { + if tl.Name == versions.ReserveDoneTool { + return tl.Run(args) + } + } + t.Fatal("the bundle serves no builder_reserve_done") + return nil, nil + } + args := map[string]any{"module": "node-tools", "fp": "src2:7df8", "why": "its build ended done with no version"} + out, err := run(reply(false), record(false), args) + if err != nil || len(order) != 2 || order[0] != "recorded" || order[1] != "asked" || asked[0].Mode != build.ModeReserveDone || asked[0].Name != "node-tools" || asked[0].FP != "src2:7df8" { + t.Fatalf("answer %v, %v; order %v, asked %+v", out, err, order, asked) + } + if ans := out.(map[string]any)["answer"].(string); !strings.Contains(ans, "reserved 0.1.13") || !strings.Contains(ans, "queued 1 versions") { + t.Errorf("the answer %q names no version and queued publish", ans) + } + if _, err := run(reply(true), record(false), args); err == nil || !strings.Contains(err.Error(), "not done") { + t.Errorf("the builder's refusal is not the tool's error: %v", err) + } + order = nil + if _, err := run(reply(false), record(true), args); err == nil || len(order) != 1 || order[0] != "recorded" { + t.Errorf("an unrecorded act was asked of the builder: %v, %v", err, order) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index ff60d11f..d813037b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -234,18 +234,36 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider} // A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G // §588): its states publish-queue and acted, bound once the bus holds them. - republishMax, err := republishMaxOf(getenv("MESH_BUILDER_REPUBLISH_MAX_FILE")) - if err != nil { - return err - } queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf) actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf) republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}}, - Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, Entry: entryVia(calls.Call), Send: sendVia(calls.Call), - Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf, + Entry: entryVia(calls.Call), Send: sendVia(calls.Call), + Unreleased: reserved.unreleasedWords, Logf: logger.Printf, Holders: holders} // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a) + publishSet, err := publishSettingsOf(getenv) + if err != nil { + return err + } + releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept) + if err != nil { + return err + } + reasksKV := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.ReAsksState), "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf) + // A version's publish is the builder's own work until its release is written (design 47 G A-457a): an item of the re-ask state. + items := &publishItems{reasks: reasksKV.KV, set: publishSet, now: time.Now, logf: logger.Printf, + reserved: func(ctx context.Context, module, version string) (versions.ReservedVersion, bool, error) { + return reserved.r.Of(ctx, module, version) + }, + republish: republish.Republish, endedWith: reserved.unreleased, + allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { + all, err := reserved.r.All(ctx) + return builtOf(all), err + }} + reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) }) + wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept) acts.Republish = republish.Republish - reserved.publishAtEnd = republish.Ended + // builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does. + acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf) again := &republishing{rp: republish} // The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel. decided := &decisions{subject: oc.SubjectDecided(cred.Module)} @@ -387,9 +405,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} events := func(m jetstream.Msg) { - if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them - return - } if decided.heard(m) { // the operator's answer to an ask about a key (§568) return } @@ -462,8 +477,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as // every other input. The builder asks again by itself without it. // A state not on the bus yet at this start is watched once it appears (issue 1790). - process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.ReAsksState), "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) { - eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, acts, kv, e, logger.Printf) } + reasksKV.Ready(func(kv jetstream.KeyValue) { + eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, acts, items, kv, e, logger.Printf) } if rw, err := settle(ctx, kv, eachReAsk); err != nil { process.NotWatched(logger.Printf, builds.StateBucket(cred.Module, build.ReAsksState), err, "builder_ask_again is not heard until the builder starts again") } else { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go new file mode 100644 index 00000000..ea2de17e --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem.go @@ -0,0 +1,216 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588): each reservation of a +// done build is a publish item in the builder's re-ask state, the mechanism builder_reserve_done and publish again use. It is +// asked when the build ends and at the builder's start; each try queues the publish and writes the item again with one more try +// and a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue. + +// publishSettings are the settings publish-pause, publish-pause-max and publish-tries. +type publishSettings struct { + Pause, PauseMax time.Duration + Tries int +} + +// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 8 tries. +var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 8} + +// pauseAfter is the pause after the try-th try: the setting doubled per try, up to the maximum. +func (s publishSettings) pauseAfter(try int) time.Duration { + d := s.Pause + for i := 1; i < try && d < s.PauseMax; i++ { + d *= 2 + } + return min(d, s.PauseMax) +} + +// publishItems asks, tries and ends the publish items. +type publishItems struct { + reasks func() jetstream.KeyValue // the re-ask state, nil until it is bound + reserved func(ctx context.Context, module, version string) (versions.ReservedVersion, bool, error) + republish func(ctx context.Context, module, version string) ([]string, error) + endedWith func(ctx context.Context, v versions.ReservedVersion, why string) // raises the finding version-unreleased + allDone func(ctx context.Context) ([]versions.ReservedVersion, error) + set publishSettings + now func() time.Time + after func(time.Duration) <-chan time.Time + logf func(string, ...any) +} + +func (p *publishItems) wait(ctx context.Context, d time.Duration) bool { + if d <= 0 { + return ctx.Err() == nil + } + after := p.after + if after == nil { + after = time.After + } + select { + case <-after(d): + return true + case <-ctx.Done(): + return false + } +} + +// Ask writes the publish item of a version whose build is done, when none stands: its first try is now. +func (p *publishItems) Ask(ctx context.Context, module, version string) error { + kv := p.reasks() + if kv == nil { + return errors.New("the re-ask state is not on the bus yet") + } + raw, err := json.Marshal(build.ReAsk{V: 1, Why: "the build ended with its version unreleased", By: "the builder", At: p.now().UTC(), Mode: build.ModePublish, Module: module, Version: version, NextTry: p.now().UTC()}) + if err != nil { + return err + } + if _, err := kv.Create(ctx, versions.PublishItemKey(module, version), raw); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { + return fmt.Errorf("the publish item of %s %s cannot be written: %w", module, version, err) + } + return nil +} + +// AskAllDone asks the publish item of every reservation whose build is done and that has no release: at the builder's start, so +// what stood before this change moves too, whatever its artifacts. +func (p *publishItems) AskAllDone(ctx context.Context) { + vs, err := p.allDone(ctx) + if err != nil { + p.logf("the reservations of done builds are not asked to publish at the start: %v", err) + return + } + for _, v := range vs { + if err := p.Ask(ctx, v.Module, v.Version); err != nil { + p.logf("%v", err) + } + } +} + +// Done deletes a version's publish item: its release is written. +func (p *publishItems) Done(ctx context.Context, module, version string) { + kv := p.reasks() + if kv == nil { + return + } + if err := kv.Delete(ctx, versions.PublishItemKey(module, version)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + p.logf("the publish item of %s %s is not deleted: %v", module, version, err) + } +} + +// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with +// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released, +// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written. +func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) { + if !p.wait(ctx, r.NextTry.Sub(p.now())) { + return + } + cur, err := kv.Get(ctx, e.Key()) + if err != nil || cur.Revision() != e.Revision() { + return // deleted by the release, or written again + } + v, held, err := p.reserved(ctx, r.Module, r.Version) + if err != nil { + p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err) + return + } + if !held { // dropped, taken over or released: never because its artifacts are published (A-456b) + _ = kv.Delete(ctx, e.Key()) + return + } + if r.Tries >= p.set.Tries { + r.Answer = fmt.Sprintf("the publish of %s %s was tried %d times and the version is not released; builder_publish_again tries it again", r.Module, r.Version, r.Tries) + r.Answered = p.now().UTC() + p.logf("%s: %s", e.Key(), r.Answer) + if p.endedWith != nil { + p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries") + } + } else { + r.Tries++ + if _, err := p.republish(ctx, r.Module, r.Version); err != nil { + p.logf("%s %s: try %d queued no publish: %v", r.Module, r.Version, r.Tries, err) + } + r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries)) + } + data, err := json.Marshal(r) + if err != nil { + return + } + if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.logf("%s: the publish item could not be written: %v", e.Key(), err) + } +} + +// numberFile reads a setting's number from the file the manifest declares: no file or an empty one is def; a content that is no +// number of at least one is an error naming the file. +func numberFile(path string, def int) (int, error) { + if path == "" { + return def, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return def, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return def, nil + } + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return 0, fmt.Errorf("%s holds %q, which is not a number of at least one", path, v) + } + return n, nil +} + +// publishSettingsOf reads publish-pause (seconds), publish-pause-max (minutes) and publish-tries from their files. +func publishSettingsOf(getenv func(string) string) (publishSettings, error) { + s := defaultPublishSettings + pause, err := numberFile(getenv("MESH_BUILDER_PUBLISH_PAUSE_FILE"), int(s.Pause/time.Second)) + if err != nil { + return s, err + } + pmax, err := numberFile(getenv("MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"), int(s.PauseMax/time.Minute)) + if err != nil { + return s, err + } + tries, err := numberFile(getenv("MESH_BUILDER_PUBLISH_TRIES_FILE"), s.Tries) + if err != nil { + return s, err + } + return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil +} + +// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a +// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module. +func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) { + rv.built, rv.released = items.Ask, items.Done + rv.r.ReleasedKept, rv.r.Releases = kept, releases +} + +// builtOf are the reservations whose build is done, whatever their artifacts' state: each is asked its publish item at the start +// (design 47 G A-456b), which goes only with its release. +func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion { + var out []versions.ReservedVersion + for _, v := range all { + if !v.BuiltAt.IsZero() { + out = append(out, v) + } + } + return out +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go new file mode 100644 index 00000000..f0a41838 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go @@ -0,0 +1,276 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// itemKV is the re-ask state in memory: one revision per key. +type itemKV struct { + jetstream.KeyValue + vals map[string][]byte + revs map[string]uint64 + rev uint64 +} + +type itemEntry struct { + jetstream.KeyValueEntry + key string + value []byte + rev uint64 +} + +func (e itemEntry) Key() string { return e.key } +func (e itemEntry) Value() []byte { return e.value } +func (e itemEntry) Revision() uint64 { return e.rev } +func (e itemEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +func (k *itemKV) entry(key string) jetstream.KeyValueEntry { + return itemEntry{key: key, value: k.vals[key], rev: k.revs[key]} +} +func (k *itemKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + if _, ok := k.vals[key]; !ok { + return nil, jetstream.ErrKeyNotFound + } + return k.entry(key), nil +} +func (k *itemKV) Create(_ context.Context, key string, v []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { + if _, ok := k.vals[key]; ok { + return 0, jetstream.ErrKeyExists + } + k.rev++ + k.vals[key], k.revs[key] = v, k.rev + return k.rev, nil +} +func (k *itemKV) Update(_ context.Context, key string, v []byte, rev uint64) (uint64, error) { + if k.revs[key] != rev { + return 0, jetstream.ErrKeyExists + } + k.rev++ + k.vals[key], k.revs[key] = v, k.rev + return k.rev, nil +} +func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { + delete(k.vals, key) + return nil +} + +func decodeItem(t *testing.T, raw []byte) build.ReAsk { + t.Helper() + r, err := build.DecodeReAsk(raw) + if err != nil { + t.Fatal(err) + } + return r +} + +// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a — a version's publish item is asked once, each try queues +// the publish and writes the item again with one more try and the pause doubled up to its maximum, the release deletes it, and +// after the setting's tries it ends with its words, raises the condition and stays; a version no longer reserved has it deleted. +func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { + ctx := context.Background() + kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} + now := time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC) + var waited []time.Duration + var queued, ended []string + reserved := true + v := versions.ReservedVersion{Module: "m", Version: "0.1.2", Artifacts: map[string]string{"m:a": "sha256:a"}, BuiltAt: now} + p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: 30 * time.Second, PauseMax: 2 * time.Minute, Tries: 4}, + now: func() time.Time { return now }, logf: t.Logf, + after: func(d time.Duration) <-chan time.Time { + waited = append(waited, d) + now = now.Add(d) + c := make(chan time.Time, 1) + c <- now + return c + }, + reserved: func(_ context.Context, _, ver string) (versions.ReservedVersion, bool, error) { + if ver == "0.3.0" { // every artifact published, its release not written yet + return versions.ReservedVersion{Module: "m", Version: ver, Artifacts: map[string]string{"m:a": "sha256:a"}, Published: map[string]bool{"m:a": true}, BuiltAt: now}, true, nil + } + return v, reserved, nil + }, + republish: func(_ context.Context, m, ver string) ([]string, error) { + queued = append(queued, m+" "+ver) + return nil, errors.New("no holder") + }, + endedWith: func(_ context.Context, v versions.ReservedVersion, why string) { + ended = append(ended, v.Version+": "+why) + }} + key := versions.PublishItemKey("m", "0.1.2") + if err := p.Ask(ctx, "m", "0.1.2"); err != nil || p.Ask(ctx, "m", "0.1.2") != nil || len(kv.vals) != 1 { + t.Fatalf("asked: %v, %d items, want one item however often it is asked", err, len(kv.vals)) + } + for i := 0; i < 5; i++ { + e, _ := kv.Get(ctx, key) + p.Try(ctx, kv, e, decodeItem(t, e.Value())) + } + got := decodeItem(t, kv.vals[key]) + if len(queued) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") { + t.Fatalf("queued %v, item %+v, ended %v: want four tries, then the end said with its words and the condition", queued, got, ended) + } + if want := []time.Duration{30 * time.Second, time.Minute, 2 * time.Minute, 2 * time.Minute}; len(waited) != len(want) || waited[0] != want[0] || waited[1] != want[1] || waited[2] != want[2] || waited[3] != want[3] { + t.Errorf("pauses before the tries and the end = %v, want %v (doubled, capped)", waited, want) + } + // An item whose artifacts are all published stays until its release is written, and is tried (A-456b). + queued = nil + _ = p.Ask(ctx, "m", "0.3.0") + pk := versions.PublishItemKey("m", "0.3.0") + pe, _ := kv.Get(ctx, pk) + p.Try(ctx, kv, pe, decodeItem(t, pe.Value())) + if _, ok := kv.vals[pk]; !ok || len(queued) != 1 || decodeItem(t, kv.vals[pk]).Tries != 1 { + t.Errorf("an item whose artifacts are published was deleted or not tried: %v, queued %v", kv.vals, queued) + } + kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} + // At the start every reservation of a done build with no release is asked, one item each. + kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} + p.allDone = func(context.Context) ([]versions.ReservedVersion, error) { + published := versions.ReservedVersion{Module: "n", Version: "0.2.0", Artifacts: map[string]string{"n:a": "sha256:a"}, Published: map[string]bool{"n:a": true}, BuiltAt: now} + return builtOf([]versions.ReservedVersion{v, published, {Module: "o", Version: "0.1.0"}}), nil // the last build has not ended + } + p.AskAllDone(ctx) + if len(kv.vals) != 2 { + t.Errorf("the start asked %d items, want one per done unreleased reservation", len(kv.vals)) + } + // A released version's item is deleted; a version no longer reserved has it deleted at its next try. + p.Done(ctx, "m", "0.1.2") + p.Done(ctx, "n", "0.2.0") + if len(kv.vals) != 0 { + t.Errorf("the release left %v", kv.vals) + } + _ = p.Ask(ctx, "m", "0.1.2") + reserved = false + e, _ := kv.Get(ctx, key) + p.Try(ctx, kv, e, decodeItem(t, e.Value())) + if len(kv.vals) != 0 { + t.Errorf("a dropped reservation left its item %v", kv.vals) + } +} + +// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a — the three settings, 30 s, 30 min and 8 tries unless set, and a +// content that is no number is refused naming its file. +func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) { + dir := t.TempDir() + write := func(name, v string) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(v), 0o600); err != nil { + t.Fatal(err) + } + return p + } + env := map[string]string{} + got, err := publishSettingsOf(func(k string) string { return env[k] }) + if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 8 { + t.Fatalf("defaults = %+v (%v)", got, err) + } + env["MESH_BUILDER_PUBLISH_PAUSE_FILE"], env["MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"], env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("p", "5\n"), write("m", "2"), write("t", "3") + if got, err = publishSettingsOf(func(k string) string { return env[k] }); err != nil || got != (publishSettings{5 * time.Second, 2 * time.Minute, 3}) { + t.Errorf("set = %+v (%v)", got, err) + } + env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("bad", "many") + if _, err = publishSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "bad") { + t.Errorf("a content that is no number: %v", err) + } +} + +// TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch: design 47 G A-457a, A-456b — a mode-publish value of the re-ask state +// reaches the item's try through reAsked; a done build gets an item; the catalog's version-released, a drop and a takeover delete it +// and the release is kept by module. +func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) { + ctx := context.Background() + kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}} + st := &memFindings{vals: map[string][]byte{}} + found := &findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} + call := func(string, any) (json.RawMessage, error) { + return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) + } + rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf) + rv.filled.Store(true) + tried := make(chan string, 4) + items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: time.Now, logf: t.Logf, + reserved: func(ctx context.Context, m, v string) (versions.ReservedVersion, bool, error) { + return rv.r.Of(ctx, m, v) + }, + republish: func(_ context.Context, m, v string) ([]string, error) { tried <- m + " " + v; return nil, nil }} + releases := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}} + wirePublishItems(rv, items, releases, 2) + done := func(fp string) versions.ReservedVersion { + v, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: fp}) + if err != nil { + t.Fatal(err) + } + if err := rv.Built(ctx, "m", v.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil { + t.Fatal(err) + } + return v + } + first := done("src2:one") + key := versions.PublishItemKey("m", first.Version) + if _, ok := kv.vals[key]; !ok { + t.Fatalf("a done build has no publish item: %v", kv.vals) + } + // A build whose artifacts are all published still gets its item, until its release is written (A-456b). + if _, _, err := rv.r.Published(ctx, versions.NameOf("m", "a"), first.Version, "sha256:a"); err != nil { + t.Fatal(err) + } + items.Done(ctx, "m", first.Version) + if err := rv.Built(ctx, "m", first.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil { + t.Fatal(err) + } + if _, ok := kv.vals[key]; !ok { + t.Fatalf("a build with every artifact published has no item: %v", kv.vals) + } + // The watch hands the item to its try, which queues the publish. + e, _ := kv.Get(ctx, key) + reAsked(ctx, nil, versions.Acts{}, items, kv, e, t.Logf) + select { + case got := <-tried: + if got != "m "+first.Version { + t.Errorf("tried %q", got) + } + case <-time.After(10 * time.Second): + t.Fatal("a mode-publish value was not handed to the item's try") + } + // A takeover by another build deletes the older's item and asks the newer's. + second := done("src2:two") + if _, ok := kv.vals[key]; ok { + t.Errorf("the taken-over reservation kept its item") + } + secondKey := versions.PublishItemKey("m", second.Version) + if _, ok := kv.vals[secondKey]; !ok { + t.Fatalf("the newer build has no item: %v", kv.vals) + } + // version-released deletes the item and keeps the release. + msg := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"` + second.Version + `","source":1,"artifacts":[]}`)} + if !rv.heard(ctx, msg) || !msg.acked { + t.Fatal("version-released was not taken in") + } + if _, ok := kv.vals[secondKey]; ok { + t.Errorf("a released version kept its publish item") + } + if ok, _ := rv.ReleasedFor(ctx, "m", "src2:two"); !ok { + t.Errorf("the release is not kept by fingerprint") + } + // A drop deletes the item too. + third := done("src2:three") + if _, err := rv.dropped(ctx, "m", "0.1", "op", "never", time.Unix(0, 0)); err != nil { + t.Fatal(err) + } + if _, ok := kv.vals[versions.PublishItemKey("m", third.Version)]; ok { + t.Errorf("a dropped reservation kept its item") + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index c8d4f0b2..04990fa2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -33,7 +33,7 @@ const buildAgainBound = 15 * time.Second // reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the // key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value // answered already (this builder's own write) or deleted is left. -func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { +func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publishItems, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) { if e.Operation() != jetstream.KeyValuePut { return } @@ -45,6 +45,13 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream if r.Answer != "" { return } + if r.Mode == build.ModePublish { + // A version's publish item (design 47 G A-457a): its try waits for its pause, so it is made beside the watch. + if items != nil { + go items.Try(ctx, kv, e, r) + } + return + } if r.Mode == build.ModeBuild { // The answer comes when the claim and the asks are written, so it is made beside the watch, not in it. go builtAgain(ctx, b, kv, e, r, logf) @@ -58,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream released(ctx, b, kv, e, r, logf) return } - if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop { + if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone { // A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch. go versionActed(ctx, acts, kv, e, r, logf) return @@ -201,6 +208,12 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them" } } + } else if r.Mode == build.ModeReserveDone { + if acts.ReserveDone == nil { + err = errors.New("this builder keeps no reservations") + } else { + r.Answer, err = acts.ReserveDone(ctx, r.Name, r.FP, r.By, r.Why, r.At) + } } else if r.Mode == build.ModeReservedDrop { if acts.DropReserved == nil { err = errors.New("this builder keeps no reservations") diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go index 83ed02ae..e723ca63 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go @@ -3,17 +3,11 @@ package main import ( "context" "encoding/json" - "errors" "fmt" - "io/fs" - "os" - "strconv" - "strings" "sync/atomic" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -77,32 +71,8 @@ func publishBody(p versions.Publish) gopackageregistry.PublishInput { return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version} } -// republishMaxOf reads the setting republish-max-versions from the file the manifest declares (design 47 G §633, A-71's -// form): no file or an empty one is the default; a content that is no positive number is an error naming the file. -func republishMaxOf(path string) (int, error) { - if path == "" { - return versions.DefaultRepublishMax, nil - } - raw, err := os.ReadFile(path) - if errors.Is(err, fs.ErrNotExist) { - return versions.DefaultRepublishMax, nil - } - if err != nil { - return 0, err - } - v := strings.TrimSpace(string(raw)) - if v == "" { - return versions.DefaultRepublishMax, nil - } - n, err := strconv.Atoi(v) - if err != nil || n < 1 { - return 0, fmt.Errorf("%s holds %q, which is not a number of versions", path, v) - } - return n, nil -} - -// republishing starts the publish again once both its states are bound: the queue is taken up again and the last running -// of every node's module is read (§556), acted on where its sequence is newer than the kept one. +// republishing starts the publish again once both its states are bound: the queue is taken up again (design 47 G §588). A version's +// publish is asked when its build ends and at the start (A-457a), never waited on a running or manifest-reads-changed. type republishing struct { rp *versions.Republisher ready atomic.Int32 @@ -116,32 +86,5 @@ func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf f if err := g.rp.Resume(ctx); err != nil { logf("the queued publishes are not taken up again: %v", err) } - err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(subject string, data []byte, seq uint64) { - if _, err := g.rp.Heard(ctx, subject, data, seq); err != nil { - logf("the running %s is not acted on at the start: %v", subject, err) - } - }) - if err != nil { - logf("the running of the node-engine seat is not read for the publish again at the start: %v", err) - } } } - -// heard hands a message of the events consumer to the publish again: one of its subjects whose state cannot take it is left -// for the bus to give again (Nak); every message goes on to the other parts. -func (g *republishing) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) { - meta, err := m.Metadata() - if err != nil { - return false - } - handled, err := g.rp.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream) - if handled && err != nil { - logf("%v; heard again", err) - if nakErr := m.Nak(); nakErr != nil { - logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) - } - givenUp(m, "its event", err, logf) - return true - } - return false -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go index 641cb9ae..d78396ca 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "errors" - "os" - "path/filepath" "strings" "testing" "time" @@ -59,18 +57,6 @@ func TestRepublishSend(t *testing.T) { t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want) } } - dir := t.TempDir() - for content, want := range map[string]int{"": 50, "7": 7} { - path := filepath.Join(dir, "max") - _ = os.WriteFile(path, []byte(content), 0o600) - if got, err := republishMaxOf(path); err != nil || got != want { - t.Errorf("republishMaxOf(%q) = %d, %v, want %d", content, got, err, want) - } - } - _ = os.WriteFile(filepath.Join(dir, "bad"), []byte("0"), 0o600) - if _, err := republishMaxOf(filepath.Join(dir, "bad")); err == nil { - t.Error("a setting of 0 is accepted") - } if _, _, err := build.NewRepublish("", "0.1.3", "why", "me", versionsNow()); err == nil { t.Error("a version with no module is accepted") } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 75ef30f4..8923151c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -40,19 +40,20 @@ func (s awaitedReserved) Delete(ctx context.Context, key string) error { // reservations is the builder's reserving: the Reserver, the fill the catalog said, the bounds of a built reservation's // publish and the findings. type reservations struct { - r *versions.Reserver - filled atomic.Bool - bounds func() jetstream.KeyValue - found *findings - bound time.Duration - wake func(context.Context) - logf func(string, ...any) + r *versions.Reserver + // built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none. + built func(ctx context.Context, module, version string) error + released func(ctx context.Context, module, version string) + filled atomic.Bool + bounds func() jetstream.KeyValue + found *findings + bound time.Duration + wake func(context.Context) + logf func(string, ...any) // call asks a seat's verb: the git seat's tag (design 47 G §370). call toolCaller // say says package-published on the seat mesh-builds; nil says none. say func(context.Context, meshbuilds.PackagePublished) error - // publishAtEnd asks the tagged publish of a reservation whose build has ended (design 47 G A-473); nil asks none. - publishAtEnd func(ctx context.Context, module, version string) ([]string, error) } // newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's @@ -74,11 +75,41 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func() } return out, nil }} + rv.r.Took = rv.took return rv } +// took is a build's takeover of the reservation of an older build never released (design 47 G A-455): the older version is +// dropped, its bound ended and its finding cleared (the newer build's reservation has its own), and the builder's log says so. +func (rv *reservations) took(ctx context.Context, older, newer versions.ReservedVersion) { + if older.Fingerprint == newer.Fingerprint { + rv.logf("%s: the build of %s reserves again: %s takes the place of %s", newer.Module, newer.Fingerprint, newer.Version, older.Version) + } else { + rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version) + } + if rv.released != nil { + rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b) + } + rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series)) + rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module) +} + var _ build.Reservations = (*reservations)(nil) +// ReservedFor is the reservation made for the build of fingerprint. +func (rv *reservations) ReservedFor(ctx context.Context, module, fingerprint string) (versions.ReservedVersion, bool, error) { + all, err := rv.r.All(ctx) + if err != nil { + return versions.ReservedVersion{}, false, err + } + for _, v := range all { + if v.Module == module && v.Fingerprint == fingerprint { + return v, true, nil + } + } + return versions.ReservedVersion{}, false, nil +} + func (rv *reservations) Filled() bool { return rv.filled.Load() } func (rv *reservations) Reserve(ctx context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) { @@ -98,6 +129,11 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges return err } rv.arm(ctx, cur) + if rv.built != nil { + if err := rv.built(ctx, module, version); err != nil { + rv.logf("%v", err) + } + } // A published heard before the digests were recorded marks its artifact now, said as one heard after (design 47 G A-415). marked, unmatched, err := rv.r.Kept(ctx, module, version) if err != nil { @@ -105,16 +141,14 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges } for _, h := range unmatched { rv.logf("a published of %s@%s kept on %s %s marks no artifact: its digest %s is none of the built digests %v", h.Name, h.Version, module, version, h.Digest, cur.Artifacts) + // A build whose recorded digest differs from what the holder holds at its version fails loudly at once: the version is taken + // and no artifact is marked for it, so it is never released (design 47 G A-490j). + rv.unreleased(ctx, cur, fmt.Sprintf("the holder holds %s@%s under digest %s, which no artifact of the build carries (%v): the version is taken and is never released", + h.Name, h.Version, h.Digest, cur.Artifacts)) } for _, m := range marked { rv.sayPublished(ctx, m, m.Reserved.Artifacts[m.Artifact]) } - // The build's end asks the tagged publish itself, at the holder of each artifact's registry seat (design 47 G A-473). - if rv.publishAtEnd != nil { - if _, err := rv.publishAtEnd(ctx, module, version); err != nil { - rv.logf("the publish of %s %s is not asked at the build's end: %v", module, version, err) - } - } return nil } @@ -215,6 +249,9 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { for _, k := range gone { rv.dropBound(ctx, k) } + if rv.released != nil { + rv.released(ctx, ev.Module, ev.Version) + } rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) default: return false @@ -257,10 +294,14 @@ func (rv *reservations) sayPublished(ctx context.Context, m versions.Marked, dig // dropped is builder_reserved_drop: the reservation is deleted and its bound ended. func (rv *reservations) dropped(ctx context.Context, module, series, by, why string, at time.Time) (string, error) { + cur, held, _ := rv.r.At(ctx, versions.ReservedKey(module, series)) ok, err := rv.r.Drop(ctx, module, series) if err != nil { return "", err } + if ok && held && rv.released != nil { + rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b) + } if !ok { return "", fmt.Errorf("%s holds no reservation of %s", module, series) } @@ -318,3 +359,27 @@ func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string { } return strings.Join(out, "; ") } + +// reserveDoneAct is builder_reserve_done's act (design 47 G A-455): the next version of the series is reserved for a done trunk build +// that reserved none, and its publish is queued as builder_publish_again queues it. Its answer says the version and the queued +// publishes; a reservation whose publish could not be queued is said so, with the tool that queues it. +func reserveDoneAct(reserve func(ctx context.Context, module, fp string) (versions.ReservedVersion, error), + republish func(ctx context.Context, module, version string) ([]string, error), logf func(string, ...any)) func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error) { + return func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error) { + res, err := reserve(ctx, module, fp) + if err != nil { + return "", err + } + logf("%s at %s: version %s is reserved by hand (%s, %s) for the done build", module, fp, res.Version, by, why) + queued, err := republish(ctx, module, res.Version) + if err != nil { + return "", fmt.Errorf("%s is reserved for %s at %s, but its publish is not queued (builder_publish_again queues it): %w", res.Version, module, fp, err) + } + return fmt.Sprintf("reserved %s for %s at %s and queued %d versions to publish: %s; builder_publish_queue shows them", res.Version, module, fp, len(queued), strings.Join(queued, ", ")), nil + } +} + +// ReleasedFor says whether a version was released for the build of fingerprint (design 47 G A-456a). +func (rv *reservations) ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) { + return rv.r.ReleasedFor(ctx, module, fingerprint) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 28218fad..11075cbd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "slices" "strings" "testing" "time" @@ -16,6 +17,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" ) @@ -89,6 +91,9 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { } rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found, func(context.Context) { woke++ }, t.Logf) + var items []string // design 47 G A-457a: the end of a build asks its publish, the release ends it + rv.built = func(_ context.Context, m, v string) error { items = append(items, "asked "+m+" "+v); return nil } + rv.released = func(_ context.Context, m, v string) { items = append(items, "ended "+m+" "+v) } if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err == nil || rv.Filled() { t.Fatalf("a version was reserved before versions-filled (%v)", err) } @@ -102,16 +107,13 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { if err != nil || res.Version != "1.4.3" { t.Fatalf("reserved %q (%v), want 1.4.3 after the catalog's 1.4.2", res.Version, err) } - var asked []string - rv.publishAtEnd = func(_ context.Context, module, version string) ([]string, error) { - asked = append(asked, module+" "+version) - return nil, nil - } + items = nil if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil { t.Fatal(err) } - if len(asked) != 1 || asked[0] != "m 1.4.3" { - t.Fatalf("the build's end asked the publish of %v, want m 1.4.3 once (A-473)", asked) + // The build's end asks the publish item once, the one mechanism (design 47 G A-457a, A-473). + if len(items) != 1 || items[0] != "asked m 1.4.3" { + t.Fatalf("the build's end asked %v, want the publish item of m 1.4.3 once (A-457a, A-473)", items) } boundKey := versions.ReservedBoundKey("m", "1.4") if !bounds.keys[boundKey] { @@ -136,6 +138,9 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { if !rv.heard(ctx, released) || !released.acked { t.Fatalf("version-released was not taken in (acked %v)", released.acked) } + if strings.Join(items, ", ") != "asked m 1.4.3, ended m 1.4.3" { + t.Errorf("publish items %v, want the build's end to ask the publish and the release to end it", items) + } if all, _ := rv.r.All(ctx); len(all) != 0 || bounds.keys[boundKey] || len(st.vals) != 0 { t.Errorf("after version-released: reservations %v, bounds %v, findings %v; want none", all, bounds.keys, st.vals) } @@ -216,13 +221,14 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { // is said in the log. func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing.T) { ctx := context.Background() + fs := &memFindings{vals: map[string][]byte{}} var logs []string logf := func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) } none := func(string, any) (json.RawMessage, error) { return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) } rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, - &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, logf) + &findings{module: builds.BuilderModule, w: fs, logf: t.Logf, now: time.Now}, nil, logf) var said []meshbuilds.PackagePublished rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil } rv.filled.Store(true) @@ -244,9 +250,137 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing if log := strings.Join(logs, "\n"); !strings.Contains(log, "is kept on") || !strings.Contains(log, "m-x@"+res.Version+" kept on m "+res.Version+" marks no artifact") { t.Fatalf("log = %q, want the published kept and the one never matching said", logs) } + // A-490j: the one that never matched is raised at once, naming the version taken and the holder's digest. + raised := "" + for _, v := range fs.vals { + raised += string(v) + } + if !strings.Contains(raised, "sha256:x") || !strings.Contains(raised, "is taken and is never released") { + t.Fatalf("no finding raised for the digest the holder holds: %s", raised) + } // Taken in once: a published of m:a heard again says nothing more. rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"}) if len(said) != 1 { t.Fatalf("package-published said again: %+v", said) } } + +// TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo: design 47 G A-455 — a build that reserves while the series is +// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal. +func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T) { + ctx := context.Background() + st := &memFindings{vals: map[string][]byte{}} + found := &findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} + bounds := &boundsKV{keys: map[string]bool{}} + var journal []string + call := func(string, any) (json.RawMessage, error) { + return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"}) + } + rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found, + nil, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }) + rv.filled.Store(true) + older, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-old"}) + if err != nil { + t.Fatal(err) + } + if err := rv.Built(ctx, "m", older.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil { + t.Fatal(err) + } + rv.unreleased(ctx, older, "nothing within the bound") + if !bounds.keys[versions.ReservedBoundKey("m", "0.1")] || len(st.vals) != 1 { + t.Fatalf("the older reservation's bound %v or finding %v is not there", bounds.keys, st.vals) + } + newer, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-new"}) + if err != nil || newer.Version != "0.1.1" { + t.Fatalf("reserved %q (%v), want 0.1.1", newer.Version, err) + } + if len(bounds.keys) != 0 || len(st.vals) != 0 { + t.Errorf("after the takeover: bounds %v, findings %v; want none", bounds.keys, st.vals) + } + if !slices.ContainsFunc(journal, func(l string) bool { + return strings.Contains(l, "0.1.0") && strings.Contains(l, "dropped") && strings.Contains(l, "0.1.1") + }) { + t.Errorf("journal %q does not say the reservation of 0.1.0 was dropped for 0.1.1", journal) + } + if got, ok, _ := rv.ReservedFor(ctx, "m", "fp-new"); !ok || got.Version != "0.1.1" { + t.Errorf("reserved for fp-new: %+v %v", got, ok) + } +} + +// ansKV records the value versionActed writes back to a reasks key. +type ansKV struct { + jetstream.KeyValue + wrote chan []byte +} + +func (a *ansKV) Update(_ context.Context, _ string, v []byte, _ uint64) (uint64, error) { + a.wrote <- v + return 2, nil +} + +type ansEntry struct { + jetstream.KeyValueEntry + key string + value []byte +} + +func (e ansEntry) Key() string { return e.key } +func (e ansEntry) Revision() uint64 { return 1 } +func (e ansEntry) Value() []byte { return e.value } +func (e ansEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +// TestAReserveDoneIsAnsweredWithTheVersionAndItsQueuedPublish: design 47 G A-455 — the reasks value of builder_reserve_done reaches +// the act, which reserves for the done build and queues the publish as publish again does, and its answer names both; a +// refusal of the reserving queues nothing and is the answer; a publish that cannot be queued says the version is reserved. +func TestAReserveDoneIsAnsweredWithTheVersionAndItsQueuedPublish(t *testing.T) { + ctx := context.Background() + var queued []string + var logged []string + logf := func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } + reserve := func(_ context.Context, module, fp string) (versions.ReservedVersion, error) { + if fp == "refused" { + return versions.ReservedVersion{}, fmt.Errorf("the build key of %s at %s is not done", module, fp) + } + return versions.ReservedVersion{Module: module, Version: "0.1.13", Fingerprint: fp}, nil + } + republish := func(_ context.Context, module, version string) ([]string, error) { + if module == "nope" { + return nil, fmt.Errorf("the queue is full") + } + queued = append(queued, module+"@"+version) + return []string{module + "@" + version}, nil + } + acts := versions.Acts{ReserveDone: reserveDoneAct(reserve, republish, logf)} + for _, c := range []struct { + module, fp, want string + refused bool + }{ + {"node-tools", "src2:7df8", "reserved 0.1.13 for node-tools at src2:7df8 and queued 1 versions to publish: node-tools@0.1.13", false}, + {"node-tools", "refused", "not done", true}, + {"nope", "src2:1", "0.1.13 is reserved for nope at src2:1, but its publish is not queued", true}, + } { + value, err := json.Marshal(build.ReAsk{V: 1, Why: "w", By: "me", Mode: build.ModeReserveDone, Name: c.module, FP: c.fp}) + if err != nil { + t.Fatal(err) + } + kv := &ansKV{wrote: make(chan []byte, 1)} + // reAsked sends a reserve-done to the version acts (b is not asked), and the answer is written back to the key. + reAsked(ctx, nil, acts, nil, kv, ansEntry{key: "reserve-done." + c.module, value: value}, logf) + var wrote []byte + select { + case wrote = <-kv.wrote: + case <-time.After(10 * time.Second): + t.Fatalf("%s at %s: no answer was written to the key", c.module, c.fp) + } + var got build.ReAsk + if err := json.Unmarshal(wrote, &got); err != nil || got.Refused != c.refused || !strings.Contains(got.Answer, c.want) || got.Answered.IsZero() { + t.Errorf("%s at %s: written %s (%v), want refused=%v containing %q", c.module, c.fp, wrote, err, c.refused, c.want) + } + } + if !slices.Equal(queued, []string{"node-tools@0.1.13"}) { + t.Errorf("queued %v, want only node-tools@0.1.13", queued) + } + if !slices.ContainsFunc(logged, func(l string) bool { return strings.Contains(l, "0.1.13 is reserved by hand") }) { + t.Errorf("the reserving by hand is not said in the log: %q", logged) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/buildagain.go b/modules/mesh-delivery-builder/internal/build/buildagain.go index e72384b9..d80ac955 100644 --- a/modules/mesh-delivery-builder/internal/build/buildagain.go +++ b/modules/mesh-delivery-builder/internal/build/buildagain.go @@ -88,7 +88,7 @@ func (b *Builder) handEnded(key, why string) { // BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a // new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key // left building that no look of this builder holds is taken as it stands and its asks put if they never were (issue 1966). -// A key done or queued is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again). +// A key done (unless its reserved version is not released) or queued is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again). // ctx is the builder's own context: the build it starts lives on it. func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult { b.init() @@ -109,7 +109,11 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo } switch { case cur.State == builds.BuildDone: - return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)} + // A done key whose version is not released is built again, so the build's own publish runs (A-490); one released, or + // that reserves no version, has nothing to do. + if why := b.rebuildRefused(ctx, module, fp); why != "" { + return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do (%s)", key, why)} + } case cur.State == builds.BuildQueued: return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)} case cur.State == builds.BuildBuilding: @@ -196,3 +200,29 @@ func boolInt(b bool) int { } return 0 } + +// rebuildRefused says why a done key of module at fp is not built again by hand: empty when its build is the first watched +// branch's trunk build and its reserved version is not released (a publish refused leaves no copy to publish, A-490). +func (b *Builder) rebuildRefused(ctx context.Context, module, fp string) string { + if b.Reserved == nil { + return "this builder reserves no version" + } + key, err := builds.BuilderBuildKey(module, fp) + if err != nil { + return err.Error() + } + b.mu.Lock() + trunk, first := b.trunkFP[key], b.origin[key].First || b.origin[key] == (Origin{}) + b.mu.Unlock() + if !trunk || !first { + return "only the first watched branch's trunk build reserves a version" + } + released, err := b.Reserved.ReleasedFor(ctx, module, fp) + if err != nil { + return fmt.Sprintf("whether its version is released cannot be read: %v", err) + } + if released { + return "its version is released" + } + return "" +} diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index bfda68d4..20b189c2 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -54,6 +54,9 @@ type ClaimOptions struct { // ByHand is a person's hand act (BuildAgain): a key failed on a transport fault is taken again only by it, or by a builder // that has just started (TakeOver), never by the builder's own retake (ADR 0664 §3). ByHand bool + // Rebuild takes a done key again as a new revision and builds it whole: a person's BuildAgain of a key whose reserved version + // is not released (design 47 G A-490: a package whose build's publish was refused is built again, never published from a copy). + Rebuild bool // ByEvent is the builder's retake on an event that says a transport fault's cause passed (design 47 G §568): it takes a key // failed on a transport fault, as a new revision that counts one more of EventBuilds. ByEvent bool @@ -189,6 +192,11 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string } switch { case cur.State == builds.BuildDone: + if opt.Rebuild { + again := building + again.Step = "built again by hand: its reserved version is not released" + return c.write(ctx, key, again, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) + } if Unreferenced(cur) { return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) } diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 15b1f888..9200129a 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -2,6 +2,7 @@ package build import ( "context" + "errors" "fmt" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -81,6 +82,15 @@ func (b *Builder) number(ctx context.Context, module string, m meshcatalog.Modul b.logf("%s at %s: %v", module, m.Fp2, fmt.Errorf("not numbered, the versions cannot be used: %w", err)) return versions.Numbering{}, true } + if done != nil && len(got.Versions) > 0 && b.Reserved != nil { + // A late look numbered a done build: G §349 retires numbering at a look, so one with no reservation and no release is + // said aloud with the recovery that reserves it (A-455), never taken in silently. + _, reserved, rerr := b.Reserved.ReservedFor(ctx, module, m.Fp2) + released, derr := b.Reserved.ReleasedFor(ctx, module, m.Fp2) + if rerr == nil && derr == nil && !reserved && !released { + b.logf("%s at %s: a late look numbered this done build (%v) and no version is reserved for it: builder_reserve_done reserves one and queues its publish", module, m.Fp2, got.Versions) + } + } if got.Refused != "" { b.logf("%s at %s: %s", module, m.Fp2, got.Refused) return got, true @@ -139,6 +149,10 @@ type Reservations interface { // Built records the digests of the artifacts built under the reserved version, by reference, and what each resolved and was // built with (design 47 G §466, §673). Built(ctx context.Context, module, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error + // ReservedFor is the reservation made for the build of fingerprint, and whether there is one. + ReservedFor(ctx context.Context, module, fingerprint string) (versions.ReservedVersion, bool, error) + // ReleasedFor says whether a version was released for the build of fingerprint, read from the builder's own record (A-456a). + ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) } // reserves says whether a look at a trunk build is one that reserves: the build has a series of the form MAJOR.MINOR @@ -155,6 +169,11 @@ func (b *Builder) reserves(ctx context.Context, module string, m meshcatalog.Mod return b.doneDigests(ctx, module, m.Fp2) == nil } +// reservesVersion says whether a plan's build reserves a version: it has artifacts and a MAJOR.MINOR series (§368). +func (b *Builder) reservesVersion(plan Plan) bool { + return len(plan.Tasks) > 0 && strings.Count(plan.Series, ".") == 1 +} + // waitFill holds a build that would reserve until the catalog says versions-filled, saying wait-held naming the fill once // (design 47 G §369, §372). It answers whether the build waits. func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bool { @@ -173,13 +192,13 @@ func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bo } // reserve reserves the version of the build that starts; a reservation that cannot be made is said and the build goes on -// unreserved, as an unnumbered build does. -func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, o Origin) (versions.ReservedVersion, bool) { +// unreserved: the caller fails the key, which a look takes again (A-490). +func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, o Origin, rebuild bool) (versions.ReservedVersion, bool) { refs := make([]string, 0, len(plan.Tasks)) for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source} + src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, Rebuild: rebuild} for _, t := range plan.Tasks { src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") } @@ -191,3 +210,70 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu b.logf("%s: version %s is reserved for this build", module, res.Version) return res, true } + +// ReserveDone is builder_reserve_done (design 47 G A-455): a done trunk build of the first watched branch that reserved no +// version (its key is done, so a look never reserves for it, and a done key cannot be built again) reserves the next version of +// its series now, with the digests the done key holds, as a build that started would, and its build ends with every digest +// recorded. It answers the reserved version; the caller queues its publish (versions.Republisher). A fingerprint whose key is not +// done, whose entry no watched branch carries now, that is no first branch's, or that holds a reservation already is refused, saying which. +func (b *Builder) ReserveDone(ctx context.Context, module, fp string) (versions.ReservedVersion, error) { + b.init() + if b.Reserved == nil { + return versions.ReservedVersion{}, errors.New("this builder reserves no versions") + } + key, err := builds.BuilderBuildKey(module, fp) + if err != nil { + return versions.ReservedVersion{}, err + } + b.mu.Lock() + var m meshcatalog.ModulesByModuleValue + found := false + for hk, v := range b.heard { + if strings.HasSuffix(hk, "\x00"+module) && v.Fp2 == fp { + m, found = v, true + } + } + trunk, first := b.trunkFP[key], b.origin[key].First + b.mu.Unlock() + switch { + case !found: + return versions.ReservedVersion{}, fmt.Errorf("no watched branch carries an entry of %s at %s now, so there is nothing to reserve it from", module, fp) + case !trunk || !first: + return versions.ReservedVersion{}, fmt.Errorf("%s at %s is not the first watched branch's build: only that one reserves a version (§371)", module, fp) + } + digests := b.doneDigests(ctx, module, fp) + if digests == nil { + return versions.ReservedVersion{}, fmt.Errorf("the build key of %s at %s is not done", module, fp) + } + plan, planErr, _ := b.planOf(ctx, m.Manifest) + switch { + case planErr != nil: + return versions.ReservedVersion{}, fmt.Errorf("the plan of %s at %s cannot be read: %w", module, fp, planErr) + case len(plan.Tasks) == 0 || strings.Count(plan.Series, ".") != 1: + return versions.ReservedVersion{}, fmt.Errorf("%s at %s builds nothing or has no MAJOR.MINOR series (%q): it reserves no version", module, fp, plan.Series) + } + if cur, ok, err := b.Reserved.ReservedFor(ctx, module, fp); err != nil { + return versions.ReservedVersion{}, err + } else if ok { + return versions.ReservedVersion{}, fmt.Errorf("%s at %s holds the reservation of %s already", module, fp, cur.Version) + } + if !b.Reserved.Filled() { + return versions.ReservedVersion{}, versions.ErrNotFilled + } + res, ok := b.reserve(ctx, module, m, plan, b.originOf(key), false) + if !ok { + return versions.ReservedVersion{}, fmt.Errorf("no version could be reserved for %s at %s (the builder's journal says why)", module, fp) + } + refs := map[string]string{} + for _, t := range plan.Tasks { + d, have := digests[t.Artifact] + if !have { + return res, fmt.Errorf("%s is reserved for %s at %s, but the done key holds no digest of %s", res.Version, module, fp, t.Artifact) + } + refs[module+":"+t.Artifact] = d + } + if err := b.Reserved.Built(ctx, module, res.Version, refs, nil); err != nil { + return res, fmt.Errorf("%s is reserved for %s at %s, but its digests are not recorded: %w", res.Version, module, fp, err) + } + return res, nil +} diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go index d0979f55..dca68ef6 100644 --- a/modules/mesh-delivery-builder/internal/build/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -3,6 +3,7 @@ package build import ( "context" "encoding/json" + "errors" "fmt" "reflect" "slices" @@ -22,17 +23,39 @@ import ( // fakeReservations answers a fill and keeps what the builder reserved and recorded. type fakeReservations struct { filled bool + released map[string]bool reserved [][]string series []string built map[string]string sources []versions.Source deps map[string][]meshbuilds.Dependency + next string // the version Reserve answers when set, else the series' .5 + fail error // the reservation cannot be made (the catalog's versions cannot be read) +} + +func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, error) { + return f.released[fp], nil } func (f *fakeReservations) Filled() bool { return f.filled } +func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) { + for _, s := range f.sources { + if s.Fingerprint == fp { + return versions.ReservedVersion{Version: "held"}, true, nil + } + } + return versions.ReservedVersion{}, false, nil +} + func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) { + if f.fail != nil { + return versions.ReservedVersion{}, f.fail + } f.reserved, f.series, f.sources = append(f.reserved, artifacts), append(f.series, series), append(f.sources, src) + if f.next != "" { + return versions.ReservedVersion{Module: module, Series: series, Version: f.next}, nil + } return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil } @@ -238,3 +261,128 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t }) } } + +// TestAReservingTrunkBuildNeverEndsDoneWithoutItsVersion: design 47 G A-490 (a package whose build's publish was refused is built +// again, never published from a copy; the kind met three times on 2026-11-19: a trunk build ended done with "no version is +// reserved") — a reservation that cannot be made fails the key with why and builds nothing, so a look or builder_build_again takes +// it again; and a done key whose version is not released is built again by hand under a reserved version, a released one is refused. +func TestAReservingTrunkBuildNeverEndsDoneWithoutItsVersion(t *testing.T) { + setup := func(t *testing.T, res *fakeReservations) *rig { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + r.b.Reserved = res + return r + } + t.Run("a reservation that cannot be made fails the key and asks nothing", func(t *testing.T) { + r := setup(t, &fakeReservations{filled: true, fail: errors.New("the catalog's versions cannot be read")}) + v := r.consider(t, "modules/m") + if v.State != builds.BuildFailed || !strings.Contains(v.Error, "no version is reserved") || len(r.asks.Asks()) != 0 { + t.Fatalf("key %+v with %d asks; want failed saying no version is reserved, nothing asked", v, len(r.asks.Asks())) + } + }) + for _, c := range []struct { + name string + released bool + standing bool // a reservation of 0.1.0 stands, and the catalog now answers 0.1.16 released + refuse string + }{ + {name: "a standing 0.1.0 reservation is not kept: the rebuild reserves afresh, 0.1.17 above the released 0.1.16", standing: true}, + {name: "a done key whose version is not released is built again under a reserved version"}, + {name: "a done key whose version is released is refused", released: true, refuse: "its version is released"}, + } { + t.Run(c.name, func(t *testing.T) { + r := setup(t, nil) + r.b.Reserved = nil + if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { + t.Fatalf("first build = %+v, want done", v) + } + res := &fakeReservations{filled: true, released: map[string]bool{fp1: c.released}} + if c.standing { + res.sources, res.next = []versions.Source{{Fingerprint: fp1}}, "0.1.17" // ReservedFor answers a held reservation + } + r.b.Reserved = res + asked := len(r.asks.Asks()) + got := r.b.BuildAgain(context.Background(), "m", fp1, "its publish was refused", "shanks.lead", 2*time.Second) + r.b.Wait() + if c.refuse != "" { + if !strings.Contains(got.Refused, c.refuse) || len(r.asks.Asks()) != asked { + t.Fatalf("answer %+v; want refused with %q and nothing asked", got, c.refuse) + } + return + } + if got.Refused != "" || len(r.asks.Asks()) == asked || len(res.reserved) != 1 { + t.Fatalf("answer %+v with %d new asks, reserved %v; want built again under one reserved version", got, len(r.asks.Asks())-asked, res.reserved) + } + want := "0.1.5" + if c.standing { + want = "0.1.17" + } + if _, ok := res.built[want]; !ok || len(res.reserved) != 1 { + t.Fatalf("built %v after %d reservations; want the digests recorded under the freshly reserved %s so the build's end asks its publish", res.built, len(res.reserved), want) + } + }) + } +} + +// realReserve is the real versions.Reserver over a map, answering the builder's Reservations as the running builder's does. +type realReserve struct { + *versions.Reserver + reservations map[string][]byte +} + +func (r *realReserve) Filled() bool { return true } +func (r *realReserve) ReleasedFor(context.Context, string, string) (bool, error) { + return false, nil +} +func (r *realReserve) ReservedFor(ctx context.Context, module, fp string) (versions.ReservedVersion, bool, error) { + all, err := r.All(ctx) + for _, v := range all { + if v.Module == module && v.Fingerprint == fp { + return v, true, err + } + } + return versions.ReservedVersion{}, false, err +} +func (r *realReserve) Built(ctx context.Context, module, version string, digests map[string]string, _ map[string][]meshbuilds.Dependency) error { + for ref, d := range digests { + if _, err := r.Reserver.Built(ctx, module, version, ref, d, nil); err != nil { + return err + } + } + return nil +} + +type reserveMap struct{ *memStore } + +func (m reserveMap) KeysOf(ctx context.Context, _ string) ([]string, error) { return m.Keys(ctx) } +func (m reserveMap) Delete(_ context.Context, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.vals, key) + return nil +} + +// TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber: design 47 G A-490b, at the build, through the real +// reserver — a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again +// under 0.1.17 (never 0.1.31), the standing reservation taken over. +func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { + t.Fatalf("first build = %+v, want done", v) + } + store := reserveMap{newMem()} + real := &realReserve{Reserver: &versions.Reserver{Store: store, Filled: func() bool { return true }, + ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"0.1.16"}, nil }}} + raw, _ := json.Marshal(versions.ReservedVersion{V: 1, Module: "m", Series: "0.1", Version: "0.1.30", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": "d"}}) + if _, err := store.Create(context.Background(), versions.ReservedKey("m", "0.1"), raw); err != nil { + t.Fatal(err) + } + r.b.Reserved = real + got := r.b.BuildAgain(context.Background(), "m", fp1, "its publish was refused", "shanks.lead", 2*time.Second) + r.b.Wait() + cur, ok, err := real.Of(context.Background(), "m", "0.1.17") + if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 { + t.Fatalf("answer %+v; reservation of 0.1.17 = %+v (%v, %v); want built again under 0.1.17, the catalog's highest 0.1.16 plus one", got, cur, ok, err) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/reservedone_test.go b/modules/mesh-delivery-builder/internal/build/reservedone_test.go new file mode 100644 index 00000000..693b48cb --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/reservedone_test.go @@ -0,0 +1,56 @@ +package build + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// TestADoneTrunkBuildThatReservedNoneIsGivenAVersionByHand: design 47 G A-455 — builder_reserve_done reserves the series' next +// version for a done key that reserved none, with the digests the key holds; a key not done, a fingerprint no entry carries and +// a build that holds a reservation already are refused; the value it writes is a reasks value of its own key. +func TestADoneTrunkBuildThatReservedNoneIsGivenAVersionByHand(t *testing.T) { + ctx := context.Background() + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + res := &fakeReservations{filled: true} + r.b.Reserved = res + if _, err := r.b.ReserveDone(ctx, "m", fp1); err == nil { + t.Fatal("a version was reserved for a key that is not built") + } + // The build ran while the builder reserved nothing (as the live key did): done, with no reservation. + r.b.Reserved = nil + if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { + t.Fatalf("build = %+v, want done", v) + } + r.b.Reserved = res + if _, err := r.b.ReserveDone(ctx, "m", "src2:"+strings.Repeat("9", 64)); err == nil || !strings.Contains(err.Error(), "no watched branch carries") { + t.Errorf("a fingerprint no entry carries: %v", err) + } + got, err := r.b.ReserveDone(ctx, "m", fp1) + if err != nil || got.Version != "0.1.5" || !slices.Equal(res.series, []string{"0.1"}) || !slices.Equal(res.reserved[0], []string{"m:tool"}) { + t.Fatalf("reserved %+v (%v) in %v for %v, want 0.1.5 of m:tool in 0.1", got, err, res.series, res.reserved) + } + if res.built["m:tool"] != digest || res.sources[0].Fingerprint != fp1 { + t.Errorf("recorded %v from %+v, want the done key's digest for the entry %s", res.built, res.sources, fp1) + } + if _, err := r.b.ReserveDone(ctx, "m", fp1); err == nil || !strings.Contains(err.Error(), "reservation") { + t.Errorf("a build that holds a reservation was given another: %v", err) + } + at := time.Unix(0, 0) + key, _, err := NewReserveDone("m", fp1, "why", "op", at) + if err != nil || !strings.HasPrefix(key, "reserve-done.") { + t.Errorf("key %q (%v)", key, err) + } + for _, args := range [][3]string{{"", fp1, "why"}, {"m", "", "why"}, {"m", fp1, " "}} { + if _, _, err := NewReserveDone(args[0], args[1], args[2], "op", at); err == nil { + t.Errorf("%q made a value", args) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 9212971b..9b923a6a 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -617,7 +617,10 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // A key built again by hand is taken over as it stands too: one left building by a look that ended without its write // (issue 1966) is the hand tool's to finish. - cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished, + // A done key a person builds again whose reserved version is not released is built whole, under a version reserved afresh (A-490b). + rebuild := byHand && trunk && planErr == nil && !nothing && b.Reserved != nil && b.doneDigests(ctx, module, m.Fp2) != nil && + b.rebuildRefused(ctx, module, m.Fp2) == "" && b.reservesVersion(plan) + cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{Rebuild: rebuild, TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { end("the claim was refused: %v", err) @@ -655,11 +658,22 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } var reserved versions.ReservedVersion hasReserved := false - if reserving && cl.Build.State != builds.BuildDone { - reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key)) + if rebuild && cl.Build.State == builds.BuildBuilding { + // Reserved afresh, above the highest the catalog's versions answers, taking over the older unreleased reservation of this + // fingerprint (A-455), never its standing number kept (A-490b); the build's own publish then runs. + if b.Reserved.Filled() { + reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), true) + } + reserving = true + } else if reserving && cl.Build.State != builds.BuildDone { + reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false) } var result builds.BuilderBuild - if numbering.Refused != "" && cl.Build.State != builds.BuildDone { + if reserving && !hasReserved && cl.Build.State != builds.BuildDone { + // A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would + // stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again. + result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be") + } else if numbering.Refused != "" && cl.Build.State != builds.BuildDone { // Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key // fails with why, and is taken again at the next look. result = failed("%s", numbering.Refused) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index d3d5c0c6..41f4d44f 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -468,9 +468,19 @@ func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The t.Errorf("a named package's done build: numbered %d more times with %v, %d image builds; want once with img %s and no build", nums.numbered-looks, nums.done, len(r.bm.callsOf("build_image_build")), digest) } + // A late look that numbers a done build with no reservation is said aloud, naming builder_reserve_done (G §349, A-455). + late := &logged{} + r.b.Logf, r.b.Reserved = late.logf, &fakeReservations{filled: true} + r.b.PackageNamed(context.Background(), "m") + r.b.Wait() + if !late.has("late look numbered", "builder_reserve_done") { + t.Errorf("a late look numbered a done build with no reservation and said %v", late.lines) + } + r.b.Reserved = nil + looks = nums.numbered r.b.PackageNamed(context.Background(), "other") r.b.Wait() - if nums.numbered != looks+1 { + if nums.numbered != looks { t.Errorf("a module the builder does not know was looked at") } // D-B5 (ADR 0666 §15; design 47, test 70): a build whose published packages cannot be read is refused, naming the diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index df195edb..ebfa5729 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -67,7 +67,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc defer b.dropEnd(key, ended) if done, failedWords, found := b.keyEnd(ctx, key); found { if done { - return meshbuilds.StepEndedOutcomePassed, "" + return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, failedWords } @@ -77,7 +77,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc select { case log := <-ended: if log.Outcome == builds.BuildDone { - return meshbuilds.StepEndedOutcomePassed, "" + return b.passedWithItsVersion(ctx, entry) } return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error) case <-ctx.Done(): @@ -85,6 +85,46 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc } } +// passedWithItsVersion is the verdict of a build that ended done (design 47 G A-456): a build that reserves a version (a build +// of the first watched branch's trunk, with a MAJOR.MINOR series and artifacts) passes only when a version is reserved for its +// fingerprint; one that ended done with none fails loudly, naming builder_reserve_done, so no pipeline says passed for a build +// that can never be released. A build that reserves none (a candidate's, a series of before G) passes as it did. +func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.CatalogEntry) (outcome, said string) { + if b.Reserved == nil { + return meshbuilds.StepEndedOutcomePassed, "" + } + key, err := builds.BuilderBuildKey(entry.Module, entry.Fp) + if err != nil { + return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error() + } + b.mu.Lock() + reserving := b.trunkFP[key] && b.origin[key].First + b.mu.Unlock() + if !reserving { + return meshbuilds.StepEndedOutcomePassed, "" + } + plan, planErr, _ := b.planOf(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest)) + switch { + case planErr != nil: + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the plan cannot be read to judge its version: %v", entry.Module, entry.Fp, planErr) + case len(plan.Tasks) == 0 || strings.Count(plan.Series, ".") != 1: + return meshbuilds.StepEndedOutcomePassed, "" + } + _, held, err := b.Reserved.ReservedFor(ctx, entry.Module, entry.Fp) + switch { + case err != nil: + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the reservations cannot be read: %v", entry.Module, entry.Fp, err) + case !held: + if released, err := b.Reserved.ReleasedFor(ctx, entry.Module, entry.Fp); err != nil { + return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the releases cannot be read: %v", entry.Module, entry.Fp, err) + } else if released { + return meshbuilds.StepEndedOutcomePassed, "" // a released build counts as reserved (A-456a) + } + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s: built at %s, but no version is reserved for it; builder_reserve_done reserves one for the done build", entry.Module, entry.Fp) + } + return meshbuilds.StepEndedOutcomePassed, "" +} + // keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended. func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) { raw, _, ok, err := b.Claimer.Store.Get(ctx, key) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 225ff77e..0cd9c551 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -11,6 +11,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-builder/internal/versions" ) // The runner's build step ask is the builder's one entry point for a candidate (design 47 G A-97): each entry is read by its @@ -74,3 +76,48 @@ func TestAStepAskIsWokenByItsBuildsEnd(t *testing.T) { t.Errorf("waits left: %v", r.b.stepWaits) } } + +// A build step passes a build that reserves a version only when a version is reserved for its fingerprint (design 47 G A-456): +// a trunk build that ended done with none fails naming builder_reserve_done, and passes once one is reserved for it; a build +// that reserves none (a manifest of before G, version "1") passes as it did. +func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) { + for _, c := range []struct { + name, version string + reserve, released, unreadable bool + want, said string + }{ + {"no version reserved", "0.1", false, false, false, meshbuilds.StepEndedOutcomeFailed, "built at " + fp1 + ", but no version is reserved for it"}, + {"a version reserved", "0.1", true, false, false, meshbuilds.StepEndedOutcomePassed, ""}, + {"a released build whose reservation is gone", "0.1", false, true, false, meshbuilds.StepEndedOutcomePassed, ""}, + {"the plan cannot be read", "0.1", false, false, true, meshbuilds.StepEndedOutcomeCouldNotRun, "plan cannot be read"}, + {"a series of before G", "1", false, false, false, meshbuilds.StepEndedOutcomePassed, ""}, + } { + t.Run(c.name, func(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: c.version, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + res := &fakeReservations{filled: true, released: map[string]bool{fp1: c.released}} + if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { // built while the builder reserved nothing, as the live key was + t.Fatalf("build = %+v, want done", v) + } + r.b.Reserved = res + if c.reserve { + res.sources = append(res.sources, versions.Source{Fingerprint: fp1}) + } + r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) { + return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: "c1", Fp: fp1, ManifestDigest: digestOf(c.unreadable), Path: "modules/m"}, nil + } + end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}) + if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && !strings.Contains(end.Said, "builder_reserve_done")) { + t.Errorf("end = %+v, want %s saying %q", end, c.want, c.said) + } + }) + } +} + +// digestOf is the manifest digest a step's entry names: the rig's manifest, or one no store holds, whose plan cannot be read. +func digestOf(unreadable bool) string { + if unreadable { + return mdigest('z') + } + return mdigest('a') +} diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 9f156410..2682685e 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -61,8 +61,15 @@ type ReAsk struct { Name string `json:"name,omitempty"` Version string `json:"version,omitempty"` Queued []string `json:"queued,omitempty"` + // Tries and NextTry are a ModePublish value's (design 47 G A-457a): the publishes tried so far, and when the next is. + Tries int `json:"tries,omitempty"` + NextTry time.Time `json:"next-try,omitzero"` } +// ModePublish is the Mode of a reasks value that is a version's publish item (design 47 G A-457a): Module and Version name it, +// Tries and NextTry say where it stands; it is written by the builder itself, when a build ends and at its start. +const ModePublish = "publish" + // ModeRepublish is the Mode of a reasks value of builder_publish_again (design 47 G §588): Module and Version, each // optional, name what to publish again; the builder answers with the versions it queued (Queued). const ModeRepublish = "republish" @@ -137,6 +144,24 @@ func NewReservedDrop(module, series, why, by string, at time.Time) (string, []by return "reserved." + versions.ReservedKey(module, series), data, err } +// ModeReserveDone is the Mode of a reasks value of builder_reserve_done (design 47 G A-455): Name is the module and FP the +// fingerprint of a done trunk build that reserved no version. +const ModeReserveDone = "reserve-done" + +// NewReserveDone is the value builder_reserve_done writes: the key `reserve-done..` (never a build key) and +// the value. +func NewReserveDone(module, fp, why, by string, at time.Time) (string, []byte, error) { + module, fp = strings.TrimSpace(module), strings.TrimSpace(fp) + if module == "" || fp == "" { + return "", nil, errors.New("module and fp are required") + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why a version is reserved for the done build (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeReserveDone, Name: module, FP: fp}) + return "reserve-done." + versions.EscapeToken(module) + "." + versions.EscapeToken(fp), data, err +} + // ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again. const ModeBuild = "build" diff --git a/modules/mesh-delivery-builder/internal/versions/again.go b/modules/mesh-delivery-builder/internal/versions/again.go index 4bc81b89..5f9d2f61 100644 --- a/modules/mesh-delivery-builder/internal/versions/again.go +++ b/modules/mesh-delivery-builder/internal/versions/again.go @@ -24,6 +24,9 @@ type Acts struct { Republish func(ctx context.Context, module, version string) ([]string, error) // DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not. DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error) + // ReserveDone reserves the next version for a done trunk build that reserved none and queues its publish (builder_reserve_done): + // its answer, or why not. + ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error) // DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not. DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error) } diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 824a9ce9..bf68e038 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -5,22 +5,20 @@ import ( "encoding/json" "errors" "fmt" - "slices" "sort" "strings" "sync" "time" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) -// **A reserved version left unpublished is published again by the builder, on an event or by a person's tool** (design 47 -// G §588, ADR 0674): on the node-engine seat's `running` of a module whose catalog graph entry claims the registry seat -// of the missing artifacts' kind, and on its `manifest-reads-changed` with `why` `returned`, at each registry seat's holder -// the seat-holders view names (A-416, A-416a: known from the start, not from the running heard). Each message is acted on once, by its stream sequence (state `acted`); each holder has one queue of versions, -// kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no -// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. +// **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the +// publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and +// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has one queue of +// versions, kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no +// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a +// node-engine `running` or `manifest-reads-changed` any more. const ( // PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB. @@ -31,8 +29,6 @@ const ( ActedMiB = 1 // PublishQueueTool is builder_publish_queue. PublishQueueTool = "builder_publish_queue" - // DefaultRepublishMax is the setting republish-max-versions unless set (§633). - DefaultRepublishMax = 50 // QueueFull is what version-unreleased says of a version the queue's cap refused. QueueFull = "queue full" // NotStoredWords opens what a refused artifact is said as. @@ -106,17 +102,14 @@ type Publish struct { type Republisher struct { Reserved *Reserver Queue QueueStore - Acted ReserveStore // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) // Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words. Send func(ctx context.Context, p Publish) error // Unreleased says version-unreleased about a version with the holder's words. Unreleased func(ctx context.Context, v ReservedVersion, why string) - // Max is the setting republish-max-versions: the most versions one event queues. - Max int - Now func() time.Time - Logf func(string, ...any) + Now func() time.Time + Logf func(string, ...any) // Go starts the sending of a holder's queue; nil starts a goroutine. Go func(func()) // Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat @@ -159,97 +152,6 @@ func (r *Republisher) logf(f string, a ...any) { // ActedKey is the acted key of a subject. func ActedKey(subject string) string { return EscapeToken(subject) } -// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq: it acts -// on it only when its sequence is newer than the one kept for its subject, and keeps it after. A subject that is neither is -// not its; an error leaves the message not acted on, so it is read again. -func (r *Republisher) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) { - running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*")) - changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".") - if !running && !changed { - return false, nil - } - key := ActedKey(subject) - raw, rev, found, err := r.Acted.Get(ctx, key) - if err != nil { - return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err) - } - if found { - var a Acted - if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq { - return true, nil - } - } - if running { - err = r.running(ctx, data) - } else { - err = r.returned(ctx, data) - } - if err != nil { - return true, err - } - val, _ := json.Marshal(Acted{Sequence: seq}) - if found { - _, err = r.Acted.Update(ctx, key, val, rev) - } else { - _, err = r.Acted.Create(ctx, key, val) - } - if err != nil && !errors.Is(err, ErrConflict) { - return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err) - } - return true, nil -} - -// registrySeatsOf are the registry seats the module of the catalog entry fingerprint claims, by their publish verb. -func (r *Republisher) registrySeatsOf(ctx context.Context, fingerprint string) ([]string, error) { - e, err := r.Entry(ctx, fingerprint) - if err != nil { - return nil, err - } - var out []string - for _, seat := range RegistrySeats { - if slices.Contains(e.Serves, seat+"."+VerbPublish) { - out = append(out, seat) - } - } - return out, nil -} - -// running takes a running in: one of a module whose entry claims a registry seat queues the versions waiting on it (§588). It -// is not how a holder is known: Holders answers that (design 47 G A-416). -func (r *Republisher) running(ctx context.Context, data []byte) error { - var ev nodeengine.Running - if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" { - r.logf("a running that does not read is not acted on: %v", err) - return nil - } - if ev.Stopped || ev.Fingerprint == "" { - return nil - } - seats, err := r.registrySeatsOf(ctx, ev.Fingerprint) - if err != nil { - return fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err) - } - if len(seats) == 0 { - return nil - } - _, err = r.queueEvent(ctx, map[string][]string{ev.Module: seats}) - return err -} - -// heldSeats are the registry seats' holders as Holders answers them: holder module -> the registry seats it holds. -func (r *Republisher) heldSeats() map[string][]string { - held := map[string][]string{} - if r.Holders == nil { - return held - } - for _, seat := range RegistrySeats { - for _, h := range r.Holders(seat) { - held[h] = append(held[h], seat) - } - } - return held -} - // holdersFn answers a seat's holders from Holders. func (r *Republisher) holdersFn(seat string) []string { if r.Holders == nil { @@ -258,59 +160,12 @@ func (r *Republisher) holdersFn(seat string) []string { return r.Holders(seat) } -// returned takes a manifest-reads-changed in: only one with why returned acts, for every registry seat's holder Holders -// answers (design 47 G A-416a); a version queued at a holder already is not queued again. -func (r *Republisher) returned(ctx context.Context, data []byte) error { - var ev nodeengine.ManifestReadsChanged - if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" { - r.logf("a manifest-reads-changed that does not read is not acted on: %v", err) - return nil - } - if ev.Why != nodeengine.ManifestReadsChangedWhyReturned { - return nil - } - holders := r.heldSeats() - if len(holders) == 0 { - return nil - } - _, err := r.queueEvent(ctx, holders) - return err -} - // view is a reserved version with the artifacts of it a holder would publish. type view struct { v ReservedVersion arts map[string]meshcatalog.Artifact // reference -> its catalog artifact } -// queueEvent queues, for each holder (module -> the seats it claims), the unreleased versions whose missing artifacts -// that holder's seats take: newest keys first, at most Max versions in all (one key per module and MAJOR.MINOR is the -// reservation's own). -func (r *Republisher) queueEvent(ctx context.Context, holders map[string][]string) ([]string, error) { - over, err := r.Reserved.Overdue(ctx, nil) - if err != nil { - return nil, err - } - var vs []ReservedVersion - for _, o := range over { - if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok { - vs = append(vs, v) - } - } - return r.queue(ctx, vs, func(seat string) []string { return holdersOf(holders, seat) }, r.Max) -} - -func holdersOf(holders map[string][]string, seat string) []string { - var out []string - for m, seats := range holders { - if slices.Contains(seats, seat) { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - // queue queues vs newest first, at most max of them (0 is no limit), each artifact to the first holder holdersOf names for // its seat. It answers the versions queued, ` `. func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf func(seat string) []string, max int) ([]string, error) { @@ -444,28 +299,39 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([] return r.queue(ctx, vs, r.holdersFn, 0) } -// Ended queues a reserved version whose build has just ended (built-at set, every artifact's digest known) at the holder of -// each artifact's registry seat, as a holder's running does, and drains it (design 47 G A-473). A version with no built-at, or -// whose artifacts are all published, queues nothing. A holder that refuses or does not answer leaves the version in -// version-unreleased with its words (send says it); a seat with no holder leaves it for the holder's running. -func (r *Republisher) Ended(ctx context.Context, module, version string) ([]string, error) { - v, ok, err := r.Reserved.Of(ctx, module, version) - if err != nil || !ok || v.BuiltAt.IsZero() || len(v.Missing()) == 0 { - return nil, err - } - return r.queue(ctx, []ReservedVersion{v}, r.holdersFn, 0) -} - // Queued are the queued versions, per holder in the order they are sent, and the acted keys with their sequences: // builder_publish_queue. type Queued struct { Queue []QueuedPublish `json:"queue"` Acted map[string]uint64 `json:"acted"` + // Items are the publish items (design 47 G A-457a), each with its tries and its next try; an item that ended says why. + Items []PublishItem `json:"items"` +} + +// PublishItemKeyPrefix opens the key of a publish item in the builder's re-ask state. +const PublishItemKeyPrefix = "publish." + +// PublishItem is the builder's own record of a version's publish (design 47 G A-457a): a value of the re-ask state, its tries and +// its next try. It is asked when the build ends and at the builder's start, rewritten with one more try and a longer pause after +// each, and deleted when the version is released; after the setting's number of tries it ends, saying so (Answer), and stays. +type PublishItem struct { + Module string `json:"module"` + Version string `json:"version"` + Tries int `json:"tries"` + NextTry time.Time `json:"next-try"` + Why string `json:"why,omitempty"` + Answer string `json:"answer,omitempty"` + Answered time.Time `json:"answered,omitzero"` +} + +// PublishItemKey is the key of module's version's publish item. +func PublishItemKey(module, version string) string { + return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version) } // AnswerQueue is builder_publish_queue over the two sections as the tools bundle reads them. -func AnswerQueue(queue, acted Reader) (Queued, error) { - out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}} +func AnswerQueue(queue, acted, reasks Reader) (Queued, error) { + out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}} keys, err := queue.Keys() if err != nil { return out, fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) @@ -500,18 +366,37 @@ func AnswerQueue(queue, acted Reader) (Queued, error) { out.Acted[k] = a.Sequence } } + rkeys, err := reasks.Keys() + if err != nil { + return out, fmt.Errorf("the re-ask state cannot be listed: %w", err) + } + for _, k := range rkeys { + if !strings.HasPrefix(k, PublishItemKeyPrefix) { + continue + } + raw, ok, err := reasks.Get(k) + if err != nil || !ok { + continue + } + var it PublishItem + if json.Unmarshal(raw, &it) == nil { + out.Items = append(out.Items, it) + } + } + sort.SliceStable(out.Items, func(i, j int) bool { return out.Items[i].NextTry.Before(out.Items[j].NextTry) }) return out, nil } // PublishQueueDescription is builder_publish_queue's description. const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588): per registry " + "seat holder, in the order they are sent, each with its artifacts' digests, when it was queued and whether its publish has begun; and the " + - "keys of the state " + ActedState + ", the last stream sequence the builder acted on for each running and manifest-reads-changed subject. Read-only." + "keys of the state " + ActedState + ", the last stream sequence the builder acted on for each subject; and the publish items (design 47 G " + + "A-457a), each version's tries and next try, and why one ended after its tries. Read-only." // PublishAgainDescription is builder_publish_again's description. const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " + "version's missing artifacts; with module alone, each reserved version of that module whose artifacts are not all published; with neither, every " + - "version the finding version-unreleased names, newest first. It queues them as the builder does on a registry holder's running, and answers at once " + + "version the finding version-unreleased names, newest first. It queues them as the builder's publish items do, and answers at once " + "with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " + "recorded in the hand-act log." diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go index 4fa1d498..72541453 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish_test.go +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" ) @@ -26,7 +25,6 @@ type republishBed struct { *Republisher reserved *reserveMem queue *queueMem - acted *reserveMem now time.Time sent []Publish said []string @@ -35,11 +33,11 @@ type republishBed struct { holders map[string][]string } -func newRepublishBed(t *testing.T, max int) *republishBed { +func newRepublishBed(t *testing.T) *republishBed { t.Helper() - b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, acted: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)} + b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)} res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} - b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Acted: b.acted, Max: max, Now: func() time.Time { return b.now }, Go: func(f func()) { f() }, + b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Now: func() time.Time { return b.now }, Go: func(f func()) { f() }, Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) { if fp == "src2:holder" { return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil @@ -82,97 +80,32 @@ func (b *republishBed) reserve(t *testing.T, module string) ReservedVersion { return v } -func running(node, module, fp string, stopped bool) (string, []byte) { - data, _ := json.Marshal(nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Stopped: stopped, At: time.Unix(1, 0)}) - return nodeengine.SubjectRunning(node, module), data -} - // TestRepublish: design 47 G §588 (A-73, A-76): a reserved version left unpublished is published again, on a registry // holder's running and on its node's returned, once per message by its stream sequence, as a publish by digest with no parts, // at most the setting's number per event, newest first; an artifact no store holds is said once and not tried again. func TestRepublish(t *testing.T) { ctx := context.Background() - t.Run("running of a holder, once per sequence", func(t *testing.T) { - b := newRepublishBed(t, 50) + t.Run("a done build is published as a digest with no parts, to its seat's holder", func(t *testing.T) { + b := newRepublishBed(t) v := b.reserve(t, "m") + b.holders = map[string][]string{SeatArtifacts: {"oci"}} if v.Fingerprint != "src2:m" { t.Fatalf("the reservation keeps the entry's fingerprint: %q", v.Fingerprint) } - subject, data := running("n1", "oci", "src2:holder", false) - for _, seq := range []uint64{5, 4} { // 4 is older than the acted 5: it starts nothing - if _, err := b.Heard(ctx, subject, data, seq); err != nil || len(b.sent) != 1 { - t.Fatalf("seq %d: sent %d (%v), want 1", seq, len(b.sent), err) - } + if q, err := b.Republish(ctx, "", ""); err != nil || len(b.sent) != 1 || fmt.Sprint(q) != "[m "+v.Version+"]" { + t.Fatalf("queued %v sent %d (%v), want 1", q, len(b.sent), err) } p := b.sent[0] if p.Seat != SeatArtifacts || p.Name != "m/prog" || p.Kind != "bundle" || p.Version != v.Version || p.Digest != "sha256:aa" || p.Fingerprint != "src2:m" || p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") { t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p) } - if _, err := b.Heard(ctx, subject, data, 5); err != nil || len(b.sent) != 1 { - t.Errorf("the same sequence read again sent %d, want still 1", len(b.sent)) - } - if _, err := b.Heard(ctx, subject, data, 6); err != nil || len(b.sent) != 2 { - t.Errorf("a newer sequence sent %d, want 2", len(b.sent)) - } if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 { t.Errorf("a key stays in the queue after its outcome: %v", keys) } }) - t.Run("a build's end queues its version at the holders, without any event (A-473)", func(t *testing.T) { - b := newRepublishBed(t, 50) - b.holders = map[string][]string{SeatArtifacts: {"oci"}} - b.now = b.now.Add(time.Minute) - v, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog", "m:lib"}, Source{Fingerprint: "src2:m"}) - if err != nil { - t.Fatal(err) - } - if _, err := b.Ended(ctx, "m", v.Version); err != nil || len(b.sent) != 0 { - t.Fatalf("a build not ended sent %d (%v)", len(b.sent), err) - } - for ref, d := range map[string]string{"m:prog": "sha256:aa", "m:lib": "sha256:bb"} { - if _, err := b.Reserved.Built(ctx, "m", v.Version, ref, d, nil); err != nil { - t.Fatal(err) - } - } - if queued, err := b.Ended(ctx, "m", v.Version); err != nil || len(queued) != 1 || len(b.sent) != 1 || b.sent[0].Name != "m/prog" || b.sent[0].Digest != "sha256:aa" { - t.Fatalf("the ended build queued %v, sent %+v (%v), want the bundle at the oci holder", queued, b.sent, err) - } - }) - t.Run("returned acts at the seat's holder, started does not", func(t *testing.T) { - b := newRepublishBed(t, 50) - b.reserve(t, "m") - b.holders = map[string][]string{SeatArtifacts: {"oci"}} - for i, why := range []string{nodeengine.ManifestReadsChangedWhyStarted, "", nodeengine.ManifestReadsChangedWhyReturned} { - ev, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "n1", Why: why}) - handled, err := b.Heard(ctx, nodeengine.SubjectManifestReadsChanged("n1"), ev, uint64(10+i)) - if !handled || err != nil { - t.Fatalf("%q: handled %v, %v", why, handled, err) - } - if want := map[string]int{"started": 0, "": 0, "returned": 1}[why]; len(b.sent) != want { - t.Errorf("why %q: sent %d, want %d", why, len(b.sent), want) - } - } - if handled, _ := b.Heard(ctx, "mesh.seat.node-engine.event.composed.n1", nil, 1); handled { - t.Error("a subject that is neither is taken as its own") - } - }) - t.Run("the setting limits an event, newest first", func(t *testing.T) { - b := newRepublishBed(t, 2) - var versions []string - for _, m := range []string{"m1", "m2", "m3"} { - versions = append(versions, b.reserve(t, m).Module) - } - subject, data := running("n1", "oci", "src2:holder", false) - if _, err := b.Heard(ctx, subject, data, 1); err != nil { - t.Fatal(err) - } - if len(b.sent) != 2 || !strings.HasPrefix(b.sent[0].Name, "m3/") || !strings.HasPrefix(b.sent[1].Name, "m2/") { - t.Errorf("sent %+v, want the two newest, m3 then m2", b.sent) - } - }) t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) { - b := newRepublishBed(t, 50) + b := newRepublishBed(t) b.reserve(t, "m1") b.reserve(t, "m2") // newest: queued before m1's entry is met entry := b.Entry @@ -182,8 +115,8 @@ func TestRepublish(t *testing.T) { } return entry(ctx, fp) } - subject, data := running("n1", "oci", "src2:holder", false) - if _, err := b.Heard(ctx, subject, data, 1); err == nil || !strings.Contains(err.Error(), "src2:m1") { + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + if _, err := b.Republish(ctx, "", ""); err == nil || !strings.Contains(err.Error(), "src2:m1") { t.Errorf("the unreadable entry is not said: %v", err) } if len(b.sent) != 1 || !strings.HasPrefix(b.sent[0].Name, "m2/") { @@ -191,39 +124,39 @@ func TestRepublish(t *testing.T) { } }) t.Run("an artifact no store holds is said once and not tried again", func(t *testing.T) { - b := newRepublishBed(t, 50) + b := newRepublishBed(t) b.reserve(t, "m") b.refuse = "oci-artifact-registry's publish of m/prog: not stored: no blob sha256:aa" - subject, data := running("n1", "oci", "src2:holder", false) - _, _ = b.Heard(ctx, subject, data, 1) - _, _ = b.Heard(ctx, subject, data, 2) + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + _, _ = b.Republish(ctx, "", "") + _, _ = b.Republish(ctx, "", "") if len(b.sent) != 1 || len(b.said) != 1 || !strings.Contains(b.said[0], "not stored: ") { t.Errorf("sent %d, said %v: want one try and one saying of not stored", len(b.sent), b.said) } }) t.Run("a refusal in other words is said each time", func(t *testing.T) { - b := newRepublishBed(t, 50) + b := newRepublishBed(t) b.reserve(t, "m") b.refuse = "the holder is full" - subject, data := running("n1", "oci", "src2:holder", false) - _, _ = b.Heard(ctx, subject, data, 1) - _, _ = b.Heard(ctx, subject, data, 2) + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + _, _ = b.Republish(ctx, "", "") + _, _ = b.Republish(ctx, "", "") if len(b.sent) != 2 || len(b.said) != 2 { t.Errorf("sent %d, said %v: want both events tried and said", len(b.sent), b.said) } }) t.Run("a queue past its cap says queue full", func(t *testing.T) { - b := newRepublishBed(t, 50) + b := newRepublishBed(t) b.reserve(t, "m") b.queue.bytes = PublishQueueMiB << 20 - subject, data := running("n1", "oci", "src2:holder", false) - _, _ = b.Heard(ctx, subject, data, 1) + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + _, _ = b.Republish(ctx, "", "") if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) { t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said) } }) t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) { - b := newRepublishBed(t, 50) + b := newRepublishBed(t) v := b.reserve(t, "m") raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true}) if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil { @@ -236,7 +169,7 @@ func TestRepublish(t *testing.T) { t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) { // Design 47 G A-416, A-416a: the holder is the one the seat-holders view names from the start, not one learnt from a // running heard since; a seat with no holder queues nothing. - b := newRepublishBed(t, 1) + b := newRepublishBed(t) b.reserve(t, "m1") b.reserve(t, "m2") if q, err := b.Republish(ctx, "", ""); err != nil || len(q) != 0 { @@ -250,12 +183,16 @@ func TestRepublish(t *testing.T) { if _, err := b.Republish(ctx, "m1", "0.1.9"); err == nil { t.Error("a version no reservation holds is not refused") } - subject, data := running("n1", "oci", "src2:holder", false) - _, _ = b.Heard(ctx, subject, data, 1) - answer, err := AnswerQueue(Reader{Keys: func() ([]string, error) { return nil, nil }}, Reader{Keys: func() ([]string, error) { return b.acted.KeysOf(ctx, AllFilter) }, - Get: func(k string) (json.RawMessage, bool, error) { v, _, ok, err := b.acted.Get(ctx, k); return v, ok, err }}) - if err != nil || answer.Acted[ActedKey(subject)] != 1 { - t.Errorf("builder_publish_queue acted = %v (%v), want the running's sequence 1", answer.Acted, err) + b.holders = map[string][]string{SeatArtifacts: {"oci"}} + _, _ = b.Republish(ctx, "", "") + items := Reader{Keys: func() ([]string, error) { return []string{"publish.m.0.1.0", "reserve-done.m.x"}, nil }, + Get: func(k string) (json.RawMessage, bool, error) { + return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil + }} + none := Reader{Keys: func() ([]string, error) { return nil, nil }} + answer, err := AnswerQueue(none, none, items) + if err != nil || len(answer.Items) != 1 || answer.Items[0].Tries != 3 || answer.Items[0].Version != "0.1.0" { + t.Errorf("builder_publish_queue items = %+v (%v), want the one publish item with its tries", answer.Items, err) } }) } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index c6216f79..0bd46450 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -28,6 +28,8 @@ const ( // ReservedTool is builder_reserved, ReservedDropTool builder_reserved_drop. ReservedTool = "builder_reserved" ReservedDropTool = "builder_reserved_drop" + // ReserveDoneTool is builder_reserve_done (design 47 G A-455). + ReserveDoneTool = "builder_reserve_done" // UnreleasedBound is the publish's bound: past it a built reservation with an artifact unpublished is the finding // version-unreleased (§369). UnreleasedBound = 10 * time.Minute @@ -89,6 +91,9 @@ type Source struct { // that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication. Branch string Revision int64 + // Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced + // and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b). + Rebuild bool } // ReservedKey is the key of module's reservation of series (MAJOR.MINOR). @@ -110,7 +115,16 @@ type Reserver struct { ReleasedOf func(ctx context.Context, module string) ([]string, error) // Filled says the catalog said versions-filled. Filled func() bool - Now func() time.Time + // Took is told when a build took over the series' reservation of an older build never released (design 47 G A-455): the + // older reservation, which no want follows any more, is gone from the key, replaced by the newer's. Nil is told none. + Took func(ctx context.Context, older, newer ReservedVersion) + // Releases is the builder's own record of the versions it saw released, by module and fingerprint (design 47 G A-456a): the + // reservation is deleted at release, so a rerun of a released build's step reads it here, never from the catalog. Nil keeps none. + Releases ReserveStore + // ReleasedKept is how many releases per module the record keeps, the oldest dropped (the setting released-kept; A-456b); 0 is + // DefaultReleasedKept. + ReleasedKept int + Now func() time.Time } func (r *Reserver) now() time.Time { @@ -157,7 +171,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts return ReservedVersion{}, err } next := high - if found { + if found && !(src.Rebuild && cur.Fingerprint == src.Fingerprint) { if _, p, err := SplitVersion(cur.Version); err == nil && p > next { next = p } @@ -182,6 +196,9 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts if err != nil { return ReservedVersion{}, fmt.Errorf("%s cannot be reserved: %w", key, err) } + if found && r.Took != nil { + r.Took(ctx, cur, out) + } return out, nil } } @@ -240,6 +257,82 @@ func allSet(m map[string]string) bool { return true } +// DefaultReleasedKept is the setting released-kept unless set (design 47 G A-456b). +const DefaultReleasedKept = 16 + +// ReleasedKey is the key of a module's record of its last releases (design 47 G A-456a, A-456b), in the state acted. +func ReleasedKey(module string) string { return "released." + EscapeToken(module) } + +// ReleasedRecord is that key's value: the module's last releases, oldest first. +type ReleasedRecord struct { + Kept []ReleasedBuild `json:"kept"` +} + +// ReleasedBuild is one release: the fingerprint of the build and its version. +type ReleasedBuild struct { + Fingerprint string `json:"fingerprint"` + Version string `json:"version"` +} + +// ReleasedFor says whether the builder saw a version released for the build of fingerprint among the module's last releases. +func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) { + if r.Releases == nil { + return false, nil + } + raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module)) + if err != nil || !found { + return false, err + } + var rec ReleasedRecord + if json.Unmarshal(raw, &rec) != nil { + return false, nil + } + for _, k := range rec.Kept { + if k.Fingerprint == fingerprint { + return true, nil + } + } + return false, nil +} + +// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept. +func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error { + n := r.ReleasedKept + if n < 1 { + n = DefaultReleasedKept + } + for { + raw, rev, found, err := r.Releases.Get(ctx, ReleasedKey(module)) + if err != nil { + return err + } + var rec ReleasedRecord + if found { + _ = json.Unmarshal(raw, &rec) + } + kept := rec.Kept[:0:0] + for _, k := range rec.Kept { + if k.Fingerprint != fingerprint { + kept = append(kept, k) + } + } + kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version}) + if len(kept) > n { + kept = kept[len(kept)-n:] + } + out, _ := json.Marshal(ReleasedRecord{Kept: kept}) + if found { + _, err = r.Releases.Update(ctx, ReleasedKey(module), out, rev) + } else { + _, err = r.Releases.Create(ctx, ReleasedKey(module), out) + } + if errors.Is(err, ErrConflict) { + continue + } + return err + } +} + // Released takes the catalog's version-released in: the key of that version is deleted, and every key of module's other // series (the manifest moved on). A key holding a newer reserved version of the same series stays. It answers the keys // deleted. @@ -261,6 +354,11 @@ func (r *Reserver) Released(ctx context.Context, module, version string) ([]stri if !found || (cur.Series == series && cur.Version != version) { continue } + if cur.Version == version && cur.Fingerprint != "" && r.Releases != nil { + if err := r.keepRelease(ctx, module, cur.Fingerprint, version); err != nil { + return gone, fmt.Errorf("the release of %s %s cannot be recorded: %w", module, version, err) + } + } if err := r.Store.Delete(ctx, k); err != nil { return gone, fmt.Errorf("%s cannot be deleted: %w", k, err) } @@ -622,6 +720,10 @@ const ( "seat said published. With module, that module's only. A key goes when the catalog says version-released for it. Read-only." ReservedDropDescription = "Drop a module's reservation of one MAJOR.MINOR by hand (design 47 G section 369), when its version will never be " + "released; the next build takes the next after the highest released. A hand act: it is recorded in the hand-act log." + ReserveDoneDescription = "Reserve a version for a done trunk build that reserved none (design 47 G A-455): given module and fp, the next version " + + "of the series is reserved for that fingerprint with the digests of the build's artifacts, and its publish is queued as builder_publish_again " + + "queues it; builds nothing. For a done key that cannot be built again. Refused when the key is not done, is no first watched branch's, or " + + "holds a reservation already. A hand act: it is recorded in the hand-act log." // ReservedVerb is the seat mesh-builds' verb served as a tool of the same name. ReservedVerb = meshbuilds.VerbReserved ) diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index 2ba984d3..3b68a10b 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -58,7 +58,14 @@ func TestReservation(t *testing.T) { if err != nil || first.Version != "0.1.20" { t.Fatalf("first build reserves %q (%v), want 0.1.20 after the highest released 0.1.19", first.Version, err) } + var tookOlder, tookNewer []string + r.Took = func(_ context.Context, older, newer ReservedVersion) { + tookOlder, tookNewer = append(tookOlder, older.Version), append(tookNewer, newer.Version) + } second, err := r.Reserve(ctx, "m", "0.1", arts, Source{}) + if len(tookOlder) != 1 || tookOlder[0] != "0.1.20" || tookNewer[0] != "0.1.21" { + t.Fatalf("takeover of the older reservation told %v by %v, want 0.1.20 by 0.1.21 (design 47 G A-455)", tookOlder, tookNewer) + } if err != nil || second.Version != "0.1.21" { t.Fatalf("second build reserves %q (%v), want 0.1.21: never the first's", second.Version, err) } @@ -166,3 +173,83 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) { t.Errorf("the bound key does not name the reservation back: %q %v", key, ok) } } + +// TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes: design 47 G A-456a — the version-released of a version records +// that the build of its fingerprint was released, so a rerun of its step finds it after the reservation is deleted. +func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) { + ctx := context.Background() + rel := &reserveMem{mem: newMem()} + r := &Reserver{Store: &reserveMem{mem: newMem()}, Releases: rel, ReleasedKept: 2, Filled: func() bool { return true }, + ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} + v, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "src2:aa"}) + if err != nil { + t.Fatal(err) + } + if ok, _ := r.ReleasedFor(ctx, "m", "src2:aa"); ok { + t.Fatal("a build not released is said released") + } + if _, err := r.Released(ctx, "m", v.Version); err != nil { + t.Fatal(err) + } + if ok, err := r.ReleasedFor(ctx, "m", "src2:aa"); !ok || err != nil { + t.Errorf("released for the fingerprint: %v (%v), want true", ok, err) + } + if ok, _ := r.ReleasedFor(ctx, "m", "src2:bb"); ok { + t.Error("another fingerprint is said released") + } + if all, _ := r.All(ctx); len(all) != 0 { + t.Errorf("reservations left: %v", all) + } + // One key per module holds the last releases, the oldest dropped (A-456b). + for _, fp := range []string{"src2:b1", "src2:b2"} { + n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp}) + if err != nil { + t.Fatal(err) + } + if _, err := r.Released(ctx, "m", n.Version); err != nil { + t.Fatal(err) + } + } + if keys, _ := rel.KeysOf(ctx, AllFilter); len(keys) != 1 { + t.Errorf("release keys %v, want one per module", keys) + } + if old, _ := r.ReleasedFor(ctx, "m", "src2:aa"); old { + t.Error("the oldest release is kept past released-kept") + } + if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok { + t.Error("the newest release is not kept") + } +} + +// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces +// its fingerprint's standing reservation (and tells Took) and numbers above the catalog's highest alone, whether the standing number +// is below it or above it; another fingerprint's standing reservation stays a floor. +func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t *testing.T) { + ctx := context.Background() + for _, c := range []struct { + name, standing, standingFp string + want string + }{ + {name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17"}, + {name: "standing above the catalog's highest", standing: "0.1.30", standingFp: "fp", want: "0.1.17"}, + {name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31"}, + } { + t.Run(c.name, func(t *testing.T) { + store := &reserveMem{mem: newMem()} + r := &Reserver{Store: store, Filled: func() bool { return true }, + ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"0.1.16"}, nil }} + raw, _ := json.Marshal(ReservedVersion{V: 1, Module: "m", Series: "0.1", Version: c.standing, Fingerprint: c.standingFp, Artifacts: map[string]string{"m:a": "d"}}) + if _, err := store.Create(ctx, ReservedKey("m", "0.1"), raw); err != nil { + t.Fatal(err) + } + var took []string + r.Took = func(_ context.Context, older, newer ReservedVersion) { + took = append(took, older.Version+">"+newer.Version) + } + got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: true}) + if err != nil || got.Version != c.want || len(took) != 1 || took[0] != c.standing+">"+c.want { + t.Fatalf("reserved %q (%v), took %v; want %s replacing %s", got.Version, err, took, c.want, c.standing) + } + }) + } +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 4d126c2d..f833d98a 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -175,7 +175,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. @@ -185,8 +185,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 2 || m.Config["republish-max-versions"].Default != float64(50) { - t.Errorf("config = %+v, want toolchain-wait-minutes of 60 and republish-max-versions of 50 (design 47 G §588, §633)", m.Config) + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 5 || + m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(8) { + t.Errorf("config = %+v, want toolchain-wait-minutes of 60, the publish settings 30 s, 30 min and 8 tries (design 47 G §588, §633, A-457a)", m.Config) } // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { @@ -239,7 +240,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("process = %v", p) } env, _ := p["env"].(map[string]any) - if len(env) != 3 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_REPUBLISH_MAX_FILE"] != "${dir:state}/republish-max-versions" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { + if len(env) != 6 || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"]) } for k, v := range env { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index aa8f9443..f98edc9c 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -29,6 +29,7 @@ "builder_reserved", "builder_release", "builder_reserved_drop", + "builder_reserve_done", "builder_seat_queue", "builder_stuck", "builder_toolchains", @@ -146,13 +147,37 @@ "trusted": false, "default": 60 }, - "republish-max-versions": { + "publish-pause": { "type": "number", - "description": "the most versions one event of a registry holder queues to publish again, newest first (design 47 G section 588); the rest wait for the next event or builder_publish_again.", + "description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).", "minimum": 1, - "maximum": 10000, + "maximum": 3600, "trusted": false, - "default": 50 + "default": 30 + }, + "publish-pause-max": { + "type": "number", + "description": "minutes, the most a pause between a version's publish tries grows to (design 47 G A-457a).", + "minimum": 1, + "maximum": 1440, + "trusted": false, + "default": 30 + }, + "released-kept": { + "type": "number", + "description": "how many of a module's last releases the builder keeps, by fingerprint, to judge a rerun of a released build's step (design 47 G A-456b); the oldest is dropped.", + "minimum": 1, + "maximum": 1000, + "trusted": false, + "default": 16 + }, + "publish-tries": { + "type": "number", + "description": "how many times the builder tries a done build's publish before it ends loudly (log and condition) and leaves it readable in builder_publish_queue (design 47 G A-457a).", + "minimum": 1, + "maximum": 100, + "trusted": false, + "default": 8 } }, "logs": [ @@ -199,12 +224,36 @@ "content": "${config:toolchain-wait-minutes}" }, { - "id": "republish-max", + "id": "publish-pause", "type": "file", - "path": "${dir:state}/republish-max-versions", + "path": "${dir:state}/publish-pause", "owner": "mesh-delivery-builder", "mode": "0600", - "content": "${config:republish-max-versions}" + "content": "${config:publish-pause}" + }, + { + "id": "publish-pause-max", + "type": "file", + "path": "${dir:state}/publish-pause-max", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:publish-pause-max}" + }, + { + "id": "released-kept", + "type": "file", + "path": "${dir:state}/released-kept", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:released-kept}" + }, + { + "id": "publish-tries", + "type": "file", + "path": "${dir:state}/publish-tries", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:publish-tries}" }, { "id": "builder", @@ -220,7 +269,10 @@ "env": { "MESH_BROKER_FILE": "${dir:state}/broker", "MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes", - "MESH_BUILDER_REPUBLISH_MAX_FILE": "${dir:state}/republish-max-versions" + "MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause", + "MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max", + "MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries", + "MESH_BUILDER_RELEASED_KEPT_FILE": "${dir:state}/released-kept" }, "user": "mesh-delivery-builder", "health": {