diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 1bf8b15b8..0c07361da 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -54,7 +54,10 @@ type ReservedVersion struct { BuiltAt time.Time `json:"built-at,omitzero"` // Published are the artifacts a registry seat said published at this version with the built digest. Published map[string]bool `json:"published,omitempty"` - At time.Time `json:"at"` + // PublishedAt is when the builder heard or confirmed each artifact's publish (H-A-331); absent for a mark written before it + // was kept. + PublishedAt map[string]time.Time `json:"published-at,omitempty"` + At time.Time `json:"at"` // Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go // package's, GoTagOf): empty for a module with no Go package (design 47 G §370). Repository string `json:"repository,omitempty"` @@ -84,6 +87,17 @@ type ReservedVersion struct { Withdrawn map[string]string `json:"withdrawn,omitempty"` } +// markPublished marks artifact a published at at, the moment the builder heard or confirmed its publish (H-A-331). +func (v *ReservedVersion) markPublished(a string, at time.Time) { + if v.Published == nil { + v.Published = map[string]bool{} + } + if v.PublishedAt == nil { + v.PublishedAt = map[string]time.Time{} + } + v.Published[a], v.PublishedAt[a] = true, at.UTC() +} + // KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415). type KeptPublished struct { Name string `json:"name"` @@ -396,6 +410,8 @@ type ReleasedRecord struct { type ReleasedBuild struct { Fingerprint string `json:"fingerprint"` Version string `json:"version"` + // Digests are the published digest of each artifact, by reference (H-A-331); absent for a release kept before. + Digests map[string]string `json:"digests,omitempty"` } // ReleasedFor says whether the builder saw a version released for the build of fingerprint among the module's last releases. @@ -421,7 +437,8 @@ func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string) // PublishedOf answers the seat mesh-builds' verb published (design 47 H-A-328): whether artifact of module built at fingerprint // is published. A reservation of that fingerprint whose artifact a registry seat said published at the built digest (marked -// from the registry's record too, H-A-274) answers its version and digest; a release of that fingerprint among the module's +// from the registry's record too, H-A-274), its number not withdrawn, answers its version, digest and, where kept, the +// publish time (H-A-331); a release of that fingerprint among the module's // last releases answers published at its version, every artifact of a release being published; otherwise not published. func (r *Reserver) PublishedOf(ctx context.Context, module, artifact, fingerprint string) (meshbuilds.PublishedAnswer, error) { module, artifact, fingerprint = strings.TrimSpace(module), strings.TrimSpace(artifact), strings.TrimSpace(fingerprint) @@ -438,8 +455,13 @@ func (r *Reserver) PublishedOf(ctx context.Context, module, artifact, fingerprin if err != nil { return meshbuilds.PublishedAnswer{}, err } - if found && cur.Module == module && cur.Fingerprint == fingerprint && cur.Published[artifact] { - return meshbuilds.PublishedAnswer{Published: true, Version: cur.Version, Digest: cur.Artifacts[artifact]}, nil + // A number withdrawn by a take-over is never answered published (H-A-331). + if found && cur.Module == module && cur.Fingerprint == fingerprint && cur.Published[artifact] && !hasKey(cur.Withdrawn, cur.Version) { + a := meshbuilds.PublishedAnswer{Published: true, Version: cur.Version, Digest: cur.Artifacts[artifact]} + if at, ok := cur.PublishedAt[artifact]; ok { + a.At = &at + } + return a, nil } } if r.Releases != nil { @@ -451,7 +473,7 @@ func (r *Reserver) PublishedOf(ctx context.Context, module, artifact, fingerprin if found && json.Unmarshal(raw, &rec) == nil { for i := len(rec.Kept) - 1; i >= 0; i-- { if rec.Kept[i].Fingerprint == fingerprint { - return meshbuilds.PublishedAnswer{Published: true, Version: rec.Kept[i].Version}, nil + return meshbuilds.PublishedAnswer{Published: true, Version: rec.Kept[i].Version, Digest: rec.Kept[i].Digests[artifact]}, nil } } } @@ -460,7 +482,7 @@ func (r *Reserver) PublishedOf(ctx context.Context, module, artifact, fingerprin } // keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept. -func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error { +func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string, digests map[string]string) error { n := r.ReleasedKept if n < 1 { n = DefaultReleasedKept @@ -480,7 +502,7 @@ func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version kept = append(kept, k) } } - kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version}) + kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version, Digests: digests}) if len(kept) > n { kept = kept[len(kept)-n:] } @@ -519,7 +541,7 @@ func (r *Reserver) Released(ctx context.Context, module, version string) ([]stri continue } if cur.Version == version && cur.Fingerprint != "" && r.Releases != nil { - if err := r.keepRelease(ctx, module, cur.Fingerprint, version); err != nil { + if err := r.keepRelease(ctx, module, cur.Fingerprint, version, cur.Artifacts); err != nil { return gone, fmt.Errorf("the release of %s %s cannot be recorded: %w", module, version, err) } } @@ -735,11 +757,9 @@ func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version } matched = true if !cur.Published[a] { - cur.Published = maps.Clone(cur.Published) - if cur.Published == nil { - cur.Published = map[string]bool{} - } - cur.Published[a], changed = true, true + cur.Published, cur.PublishedAt = maps.Clone(cur.Published), maps.Clone(cur.PublishedAt) + cur.markPublished(a, r.now()) + changed = true arts = append(arts, a) } } @@ -808,10 +828,7 @@ func (r *Reserver) Kept(ctx context.Context, module, version string) ([]Marked, } matched = true if !cur.Published[a] { - if cur.Published == nil { - cur.Published = map[string]bool{} - } - cur.Published[a] = true + cur.markPublished(a, r.now()) arts = append(arts, a) } } diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go index a78f2fe0a..327ad2aa7 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve_test.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -226,8 +226,10 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) { // artifact not yet marked, and another fingerprint, are not published. func TestPublishedAnswersByTheBuildFingerprint(t *testing.T) { ctx := context.Background() - r := &Reserver{Store: &reserveMem{mem: newMem()}, Releases: &reserveMem{mem: newMem()}, ReleasedKept: 2, Filled: func() bool { return true }, - ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} + heard := time.Date(2026, 11, 21, 19, 0, 0, 0, time.UTC) + store := &reserveMem{mem: newMem()} + r := &Reserver{Store: store, Releases: &reserveMem{mem: newMem()}, ReleasedKept: 2, Filled: func() bool { return true }, + ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }, Now: func() time.Time { return heard }} v, err := r.Reserve(ctx, "mesh-grants", "0.1", []string{"mesh-grants:busgrants", "mesh-grants:grantsstate"}, Source{Fingerprint: "src2:aa"}) if err != nil { t.Fatal(err) @@ -249,8 +251,35 @@ func TestPublishedAnswersByTheBuildFingerprint(t *testing.T) { if marked, _, err := r.Published(ctx, "mesh-grants-busgrants", v.Version, "sha256:bb"); err != nil || len(marked) != 1 { t.Fatalf("the registry's published marked %v (%v)", marked, err) } - if a := ask("mesh-grants:busgrants", "src2:aa"); !a.Published || a.Version != v.Version || a.Digest != "sha256:bb" { - t.Errorf("published answered %+v, want %s at sha256:bb", a, v.Version) + // H-A-331: the mark keeps when the publish was heard, answered as at. + if a := ask("mesh-grants:busgrants", "src2:aa"); !a.Published || a.Version != v.Version || a.Digest != "sha256:bb" || a.At == nil || !a.At.Equal(heard) { + t.Errorf("published answered %+v, want %s at sha256:bb, heard %s", a, v.Version, heard) + } + // A mark written before the publish time was kept answers no time, never an invented one; a withdrawn number is never + // answered published. + key := ReservedKey("mesh-grants", "0.1") + cur, rev, _, _ := r.get(ctx, key) + older := cur + older.PublishedAt = nil + raw, _ := json.Marshal(older) + if rev, err = store.Update(ctx, key, raw, rev); err != nil { + t.Fatal(err) + } + if a := ask("mesh-grants:busgrants", "src2:aa"); !a.Published || a.At != nil { + t.Errorf("an older mark answered %+v, want published with no at", a) + } + withdrawn := older + withdrawn.Withdrawn = map[string]string{v.Version: "src2:aa"} + raw, _ = json.Marshal(withdrawn) + if rev, err = store.Update(ctx, key, raw, rev); err != nil { + t.Fatal(err) + } + if a := ask("mesh-grants:busgrants", "src2:aa"); a.Published { + t.Errorf("a withdrawn number is answered %+v, want not published", a) + } + raw, _ = json.Marshal(cur) + if _, err = store.Update(ctx, key, raw, rev); err != nil { + t.Fatal(err) } if a := ask("mesh-grants:grantsstate", "src2:aa"); a.Published { t.Errorf("an artifact not marked is answered %+v", a) @@ -264,6 +293,10 @@ func TestPublishedAnswersByTheBuildFingerprint(t *testing.T) { if a := ask("mesh-grants:grantsstate", "src2:aa"); !a.Published || a.Version != v.Version { t.Errorf("after its release the fingerprint's artifact is answered %+v, want published at %s", a, v.Version) } + // H-A-331: the kept release keeps each published digest with its version. + if a := ask("mesh-grants:busgrants", "src2:aa"); !a.Published || a.Digest != "sha256:bb" { + t.Errorf("after its release busgrants is answered %+v, want its digest sha256:bb", a) + } if _, err := r.PublishedOf(ctx, "mesh-grants", "", "src2:aa"); err == nil { t.Error("an ask naming no artifact is answered") } diff --git a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go index 6a3ad819c..10c632d2d 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshbuilds/contract.go @@ -230,11 +230,11 @@ type PublishedInput struct { // PublishedAnswer is what published answers. type PublishedAnswer struct { - // when it was published, when published + // when the builder heard or confirmed the publish, when published; absent for a mark written before the publish time was kept (design 47 H-A-331) At *time.Time `json:"at,omitempty"` - // the published artifact's digest, when published + // the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331) Digest string `json:"digest,omitempty"` - // true when the artifact built at the fingerprint is published + // true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331) Published bool `json:"published"` // the version it is published at, MAJOR.MINOR.PATCH, when published Version string `json:"version,omitempty"` diff --git a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json index bf2764921..4f68b5ebb 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-builds/seat.json @@ -102,7 +102,7 @@ "properties": { "published": { "type": "boolean", - "description": "true when the artifact built at the fingerprint is published" + "description": "true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331)" }, "version": { "type": "string", @@ -110,12 +110,12 @@ }, "digest": { "type": "string", - "description": "the published artifact's digest, when published" + "description": "the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331)" }, "at": { "type": "string", "format": "date-time", - "description": "when it was published, when published" + "description": "when the builder heard or confirmed the publish, when published; absent for a mark written before the publish time was kept (design 47 H-A-331)" } }, "required": [ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts index 86b89ad15..5ac18af13 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshbuilds/index.d.ts @@ -184,11 +184,11 @@ export interface PublishedInput { /** is what published answers. */ export interface PublishedAnswer { - /** when it was published, when published */ + /** when the builder heard or confirmed the publish, when published; absent for a mark written before the publish time was kept (design 47 H-A-331) */ "at"?: string; - /** the published artifact's digest, when published */ + /** the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331) */ "digest"?: string; - /** true when the artifact built at the fingerprint is published */ + /** true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331) */ "published": boolean; /** the version it is published at, MAJOR.MINOR.PATCH, when published */ "version"?: string; diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go index c0c2b04b4..47f81be7a 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/consumers.go @@ -43,7 +43,7 @@ func Stalled(pipelines Kept, now time.Time) ([]string, error) { // A required step waiting (an approval, a composition on its validate) is open, as the end's judgement holds it; // an optional one waiting holds nothing (A-104), and neither does one waiting on a package's publish, as a mover // with no steps waiting on one does not (H-A-326 (2)). - if open(st) || (st.Outcome == seat.StepResultOutcomeWaiting && st.Status == seat.StepResultStatusRequired && !strings.HasPrefix(st.Why, PinWait)) { + if open(st) || (st.Outcome == seat.StepResultOutcomeWaiting && st.Status == seat.StepResultStatusRequired && !pinHeld(st)) { busy = true break } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go index 75edef4d0..3e830fd04 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/order_test.go @@ -175,6 +175,10 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) { if len(asked) != before2 { t.Errorf("a placing again asked published %d more times, want each package asked once", len(asked)-before2) } + // At the pipeline's end its kept answers go. + if _, err := s2.Turn("main.7", view); err != nil || len(s2.publishedRead) != 0 { + t.Errorf("after main.7's end %d answers are kept (%v), want none", len(s2.publishedRead), err) + } } // Design 47 G §270, A-326 (H §746, H-A-83 (2)): what a build module's new version is built from is built by its running diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go index cecc84a88..8a8c16951 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/queue.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/queue.go @@ -236,6 +236,7 @@ func (s *Starter) StartQueue(view func(branch string) (Half, bool)) (int, error) // Turn takes the end of pipeline key (its pipeline-ended, a cancel alike, H-A-53): its running key is deleted and the // queue placed (PlaceQueued). func (s *Starter) Turn(key string, view func(branch string) (Half, bool)) (int, error) { + s.forgetPublished(key) // the ended pipeline's published answers go with it (H-A-328) if s.Queue == nil { return 0, nil } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 62926dc07..e9f4f5526 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -252,6 +252,17 @@ func (s *Starter) readPublished(p *seat.Pipeline, mover string, waiting []string return added } +// forgetPublished drops the published answers readPublished kept for pipeline key, at its end (H-A-328). +func (s *Starter) forgetPublished(key string) { + s.publishedMu.Lock() + defer s.publishedMu.Unlock() + for k := range s.publishedRead { + if strings.HasPrefix(k, key+"\x00") { + delete(s.publishedRead, k) + } + } +} + // holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view // seat-holders names no holder for (design 47 G §32, §535, §647 as A-466y and A-466z amend them; A-470g, A-470h): judged only // once that view's first view is read, never from the picture before it. Each such step ends `could not run` naming the seat,