diff --git a/modules/mesh-delivery-builder/internal/build/revisionwait.go b/modules/mesh-delivery-builder/internal/build/revisionwait.go index 0c4ec3d4..651d6f30 100644 --- a/modules/mesh-delivery-builder/internal/build/revisionwait.go +++ b/modules/mesh-delivery-builder/internal/build/revisionwait.go @@ -25,6 +25,12 @@ import ( // below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for // its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it // asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for. +// +// **An ask naming a revision is held by its number** (design 47 H H-A-222): when the picture does not name the ask's +// entries, the ask's catalog revision is read by number, and the ask is built when the catalog keeps it and it names every +// fingerprint asked, whatever the source the picture holds says (a re-pin keeps a later revision of the same source, H-A-211, +// H-A-213, so the source alone cannot say it). The source is compared only for an ask naming no revision; an ask whose +// revision the catalog does not answer and whose source is not below the picture's waits as before. // DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156). const DefaultRevisionWait = 2 * time.Minute @@ -73,10 +79,13 @@ func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuild return true } -// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its -// source revision) and that revision names every entry of the ask; else failed or could not run, with its words. +// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number) and that +// revision names every entry of the ask (H-A-205, H-A-222); else failed or could not run, with its words. func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) { - below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held) + below := fmt.Sprintf("revision %d of %s is not named by the builder's picture, which holds source revision %d", run.Source, run.Branch, held) + if run.Source < held { + below = fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held) + } if b.KeptRevision == nil || run.Revision <= 0 { return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by" } @@ -85,11 +94,15 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err) return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err) } + // An ask names the revision and the source of one revision (H-A-218, H-A-219): a kept revision of another source is + // not the ask's. if source != run.Source { - return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source) + b.logf("%s: %s, and the catalog's revision %d is of source revision %d, the ask names %d: the step fails, nothing is built", run.Pipeline, below, run.Revision, source, run.Source) + return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d, the ask names %d", below, run.Revision, source, run.Source) } for _, e := range run.Entries { if fps[e.Module] != e.Fingerprint { + b.logf("%s: %s, and the catalog's revision %d names %s at %q, not %s: the step fails, nothing is built", run.Pipeline, below, run.Revision, e.Module, fps[e.Module], e.Fingerprint) return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint) } } @@ -116,8 +129,17 @@ func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (ou if b.Picture == nil || run.Source <= 0 || run.Branch == "" { return "", "" } - if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) { - return b.keptRevisionHolds(ctx, run, held) + if held, ok := b.Picture(run.Branch); !b.holdsRevision(run.Branch, run.Source, run.Entries) { + below := ok && run.Source < held + if (run.Revision > 0 && b.KeptRevision != nil) || below { + // By its number (H-A-222), whatever the source the picture holds; for an ask naming none, only below the + // picture (H-A-205). A kept revision of another source or naming other fingerprints is a mismatch the wait + // cannot mend: it fails at once; only a revision the catalog does not answer is waited for. + outcome, said := b.keptRevisionHolds(ctx, run, held) + if outcome == "" || below || outcome == meshbuilds.StepEndedOutcomeFailed { + return outcome, said + } + } } bound := b.revisionWait() notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch) diff --git a/modules/mesh-delivery-builder/internal/build/step_test.go b/modules/mesh-delivery-builder/internal/build/step_test.go index 4b63b05a..f69d1aaf 100644 --- a/modules/mesh-delivery-builder/internal/build/step_test.go +++ b/modules/mesh-delivery-builder/internal/build/step_test.go @@ -356,7 +356,7 @@ func aStepAskWaitsForItsRevision(t *testing.T) { held.Store(40) asked := make(chan struct{}, 1) r, res, entry := revisionRig(t, &held, asked) - var keptReads atomic.Int64 // H-A-205: a revision above the newest is waited for, never read by number + var keptReads atomic.Int64 // H-A-205, H-A-222: a revision above the newest is read by number once; not answered, it is waited for r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { keptReads.Add(1) return 0, nil, errors.New("not asked") @@ -385,8 +385,8 @@ func aStepAskWaitsForItsRevision(t *testing.T) { t.Fatal("the ask was not woken by the entry the picture took in") } r.b.Wait() - if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 { - t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load()) + if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 1 { + t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, read by number once", end, res.sources, keptReads.Load()) } } @@ -425,6 +425,45 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) { if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") { t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end) } + // H-A-222: an ask of the very source the picture holds, whose entries the picture no longer names (a re-pin kept revision 48 + // of the same source 46 and the picture took its entries in), is held by its revision read by number, never waited for. + r3, res3, _ := revisionRig(t, &held, make(chan struct{}, 1)) + r3.b.RevisionWait = time.Nanosecond + r3.b.FirstBranch = func(branch, module string) bool { return branch == "main" && module == "m" } + var read3 []string + r3.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) { + read3 = append(read3, fmt.Sprintf("%s %d", branch, revision)) + return 46, map[string]string{"m": fp1}, nil + } + same := meshbuilds.StepRun{Pipeline: "main.48.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 48, Source: 46, + Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}} + end3 := r3.b.BuildStep(t.Context(), same) + r3.b.Wait() + if end3.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read3, []string{"main 48"}) || len(res3.sources) != 1 || res3.sources[0].CatalogRevision != 48 { + t.Errorf("end = %+v, kept revisions read %v, reserved %+v; want passed, revision 48 of main read once and reserved at it", end3, read3, res3.sources) + } + // H-A-218, H-A-222: a kept revision of another source than the ask names, or naming another fingerprint, is a mismatch: + // the step fails at once naming it, never waiting out its bound (the bound here is long, the ask is not below the picture). + for _, c := range []struct { + name string + source int64 + fp string + said string + }{ + {"another source", 45, fp1, "the catalog's revision 48 is of source revision 45, the ask names 46"}, + {"another fingerprint", 46, "src2:other", "names m at \"src2:other\""}, + } { + rc, resc, _ := revisionRig(t, &held, make(chan struct{}, 1)) + rc.b.RevisionWait = time.Hour + rc.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) { + return c.source, map[string]string{"m": c.fp}, nil + } + end := rc.b.BuildStep(t.Context(), same) + rc.b.Wait() + if end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, c.said) || len(resc.sources) != 0 { + t.Errorf("%s: end = %+v, reserved %+v; want failed at once saying %q, nothing reserved", c.name, end, resc.sources, c.said) + } + } } func aStepAskPastItsRevisionBoundFails(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index 21a2f59e..0c827489 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -93,16 +93,20 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 { t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err) } - // A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held. + // A-487c: the manifests by digest answer from the view held; the verb revision asked of the trunk's newest lists the bucket + // once for which revision is newest (H-A-222) and takes the entries of the view held at that very revision. listed = objects.n.Load() if raw, err := kept.Manifest(meshcatalogclient.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` { t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err) } + if got := objects.n.Load() - listed; got != 0 { + t.Errorf("the manifest listed the bucket %d times, want none", got) + } if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 { t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err) } - if got := objects.n.Load() - listed; got != 0 { - t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got) + if got := objects.n.Load() - listed; got != 1 { + t.Errorf("the verb revision listed the bucket %d times, want once, for the newest", got) } if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") { t.Errorf("said %q, want each read with its revision, module count and time", said) @@ -126,6 +130,16 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { if keys, _ := modules.Keys(); len(keys) != 3 { t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys) } + // H-A-222: two revisions kept that the view has not heard are answered by the verb revision from the store, whole; the + // view held stays a cache at its own revision. + keep(4, "avahi", "ca-trust", "dbus") + keep(5, "avahi", "ca-trust", "dbus", "feh") + if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision == nil || a.Revision.Revision != 5 || len(a.Entries) != 4 { + t.Errorf("revision of the newest answered %+v (%v) while the view held 3, want revision 5 whole from the store", a, err) + } + if keys, _ := modules.Keys(); len(keys) != 3 { + t.Errorf("the verb moved the view held to %v, want it at revision 3 until it hears the event", keys) + } } // A-487: the verb module reads the one module it is asked. diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index 2667fbb4..ffd7f654 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -23,17 +23,22 @@ const RevisionInvoke = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevisi // H-A-66: every account may call every verb). With no trunk revision kept yet the answer is `none-yet` alone (design 47 G ยง530), never an empty // catalog and never an error. // -// kept, when not nil, is the trunk's newest revision held in memory: the trunk's newest answers from it with no read of -// the bucket (A-487c); a candidate, a branch and an older number read the bucket. +// The trunk's newest is the newest the bucket keeps at the time of the ask, read from the store (a listing of the names and +// one object, design 47 H-A-222), never the view held in memory, which runs behind until it hears the revision. kept, when +// not nil, is that view: a cache of the entries only, used when it holds the very revision read; else the entries are read +// from the bucket. A candidate, a branch and an older number read the bucket. func Revision(ctx context.Context, o Objects, kept *KeptTrunk, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) { var out meshcatalog.RevisionAnswer - var newest meshcatalog.CatalogRevision - var ok bool var trunkEntries, branchEntries []meshcatalog.CatalogEntry - var err error - if kept != nil { - newest, trunkEntries, ok, err = kept.Newest(ctx, o) - } else { + newest, ok, err := newestOf(ctx, o, TrunkBranch) + cached := false + if err == nil && ok && kept != nil { + if held, entries, any, kerr := kept.Newest(ctx, o); kerr == nil && any && held.Revision == newest.Revision { + trunkEntries, cached = entries, true + } + } + if !cached && (err != nil || ok) { + // readNewestOf reads the entries whole, and the newest again when the pin judge removed one under the read. newest, trunkEntries, ok, err = readNewestOf(ctx, o, TrunkBranch) } if err != nil {