diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 5fe1094f3..3f0e8e7b4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -298,7 +298,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1)) + pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1)) + reserved.pause = publishSet.Pause // a version-released whose tag the forge did not write waits the pause (H-A-484 addendum) releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept) if err != nil { return err @@ -322,7 +323,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)). republish.Ended = items.Ended pq.take = items.take - reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) }) + // At the start, a built reservation the catalog holds released is taken in as its version-released would (H-A-484 addendum), + // before the publish items of the rest are asked. + reasksKV.Ready(func(jetstream.KeyValue) { + reserved.releasedAtStart(ctx) + items.AskAllDone(ctx) + }) wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept) acts.Republish = republish.Republish acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 53e946a10..5ffcead8b 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -59,6 +59,9 @@ type reservations struct { withdraw func(module, fingerprint, why string) // call asks a seat's verb: the git seat's tag (design 47 G §370). call toolCaller + // pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands + // it back at once. + pause time.Duration // rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none. rebuild func(ctx context.Context, module, fp, why string) error // say says package-published on the seat mesh-builds; nil says none. @@ -310,32 +313,20 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err) break } - if err := rv.tagged(ctx, ev); err != nil { + if refused, err := rv.release(ctx, ev); err != nil { rv.logf("%v; heard again", err) - if nakErr := m.Nak(); nakErr != nil { + // A tag the seat git's holder did not write is handed back with the publish pause, so the deliveries the consumer + // allows span the holder's outage (design 47 H2 H-A-484 addendum). + nak := m.Nak + if refused && rv.pause > 0 { + nak = func() error { return m.NakWithDelay(rv.pause) } + } + if nakErr := nak(); nakErr != nil { rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) } givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf) return true } - gone, err := rv.r.Released(ctx, ev.Module, ev.Version) - if err != nil { - rv.logf("%v; heard again", err) - if nakErr := m.Nak(); nakErr != nil { - rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) - } - givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf) - return true - } - for _, k := range gone { - rv.dropBound(ctx, k) - } - if rv.released != nil { - rv.released(ctx, ev.Module, ev.Version) - } - rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) - rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module) - rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) default: return false } @@ -345,6 +336,62 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { return true } +// release takes a released version in as the catalog's version-released says it (design 47 G §369, §370): its tag written, its +// reservation released, its bound and publish item ended and its findings cleared. refused says the tag was not written (the +// seat git did not write it, or the reservation could not be read to tag it); the reservation then stands. +func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) (refused bool, err error) { + if err := rv.tagged(ctx, ev); err != nil { + return true, err + } + gone, err := rv.r.Released(ctx, ev.Module, ev.Version) + if err != nil { + return false, err + } + for _, k := range gone { + rv.dropBound(ctx, k) + } + if rv.released != nil { + rv.released(ctx, ev.Module, ev.Version) + } + rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) + rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module) + rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module) + return false, nil +} + +// releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions +// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one +// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended. +// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released. +func (rv *reservations) releasedAtStart(ctx context.Context) { + all, err := rv.r.All(ctx) + if err != nil { + rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err) + return + } + read := map[string][]string{} + for _, v := range builtOf(all) { + if versions.NotStanding(v, true) != "" { + continue + } + released, ok := read[v.Module] + if !ok { + if released, err = rv.r.ReleasedOf(ctx, v.Module); err != nil { + rv.logf("%s: the catalog's released versions cannot be read at the start, so its built reservations wait for its next version-released: %v", v.Module, err) + } + read[v.Module] = released + } + if !slices.Contains(released, v.Version) { + continue + } + if _, err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil { + rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err) + continue + } + rv.logf("%s %s is released in the catalog: taken in at the start, as its version-released", v.Module, v.Version) + } +} + // published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told // before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told // nothing. diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go index 874e41f9f..076f709b7 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -164,14 +164,15 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { ctx := context.Background() var tags []git.TagInput refuse := "" - var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796) + var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796) + var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers call := func(addr string, body any) (json.RawMessage, error) { switch addr { case "seat:git.tags": return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true}) case "seat:git.tag": default: - return json.Marshal(meshcatalog.VersionsAnswer{}) + return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog}) } tags = append(tags, body.(git.TagInput)) return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse}) @@ -203,11 +204,44 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) { if m := release("pkg", "0.1.0"); !m.naked || m.acked { t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked) } + // Design 47 H2 H-A-484 addendum: with a publish pause set, a refused tag hands the version-released back after the pause, so + // the deliveries its consumer allows span the holder's outage. + rv.pause = 30 * time.Second + if m := release("pkg", "0.1.0"); m.delayed != 30*time.Second || m.naked || m.acked { + t.Fatalf("a tag the forge did not write, with the pause: delayed %v naked %v acked %v; want handed back after 30s", m.delayed, m.naked, m.acked) + } + rv.pause = 0 if all, _ := rv.r.All(ctx); len(all) != 1 { t.Fatalf("the reservation went with an unwritten tag: %v", all) } + // Design 47 H2 H-A-484 addendum: at the builder's start, a built reservation the catalog holds released is tagged and released + // as its version-released would (a version-released the bus gave up on), and one the catalog does not hold, or not built, + // stands. refuse = "" tags = nil + var ended []string + rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) } + rv.releasedAtStart(ctx) // built nothing yet: nothing taken + if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 { + t.Fatalf("at the start with no build done: reservations %v, tags %+v; want the reservation standing, no tag", all, tags) + } + if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil { + t.Fatal(err) + } + rv.releasedAtStart(ctx) // the catalog holds none released: it stands + if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 { + t.Fatalf("at the start with nothing released: reservations %v, tags %+v; want the reservation standing", all, tags) + } + catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}} + rv.releasedAtStart(ctx) + if all, _ := rv.r.All(ctx); len(all) != 0 || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) || !slices.Equal(ended, []string{"pkg 0.1.0"}) { + t.Fatalf("at the start with 0.1.0 released: reservations %v, tags %+v, items ended %v; want it tagged, released and its item ended", all, tags, ended) + } + catalog, rv.released = nil, nil + if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // the version is reserved again for the rows below + t.Fatal(err) + } + tags = nil if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) { t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags) }