diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go index e6f4d2db..8aaec502 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go @@ -45,7 +45,7 @@ func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error { if m.byFp == nil { m.byFp = map[string]catalogue.Manifest{} } - m.byFp[e.ManifestDigest] = mf + m.byFp[meshcatalog.BareManifestDigest(e.ManifestDigest)] = mf return nil } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go index 40ec236c..8d9c45b1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go @@ -15,10 +15,10 @@ func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) { if err := m.keep(e); err != nil { t.Fatal(err) } - if mf, err := m.Manifest(context.Background(), "sha256:aa"); err != nil || mf.Module != "netcheck" { - t.Errorf("Manifest(sha256:aa) = %+v, %v", mf, err) + if mf, err := m.Manifest(context.Background(), "aa"); err != nil || mf.Module != "netcheck" { + t.Errorf("Manifest(aa) = %+v, %v", mf, err) } - if _, err := m.Manifest(context.Background(), "sha256:bb"); err == nil { + if _, err := m.Manifest(context.Background(), "bb"); err == nil { t.Error("Manifest of a digest no entry holds: want an error") } if err := m.keep(meshcatalog.CatalogEntry{Module: "bare"}); err != nil || len(m.byFp) != 1 { diff --git a/modules/mesh-delivery-builder/internal/build/buildagain.go b/modules/mesh-delivery-builder/internal/build/buildagain.go index 2c135769..e72384b9 100644 --- a/modules/mesh-delivery-builder/internal/build/buildagain.go +++ b/modules/mesh-delivery-builder/internal/build/buildagain.go @@ -42,6 +42,8 @@ type handWatch struct { shut bool // ended is set when the look ended before the asks were all put. ended bool + // why is what the look that wrote nothing said in the builder's log. + why string } // settle closes done once the claim is written and every lane has put its ask. Call with b.mu held. @@ -73,8 +75,9 @@ func (b *Builder) handAsk(key, language, id string) { } // handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended. -func (b *Builder) handEnded(key string) { +func (b *Builder) handEnded(key, why string) { b.hand(key, func(w *handWatch) { + w.why = why if !w.shut { w.shut, w.ended = true, true close(w.done) @@ -158,11 +161,14 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo } b.mu.Lock() res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)} - ended, shut, lanes := w.ended, w.shut, w.lanes + ended, shut, lanes, said := w.ended, w.shut, w.lanes, w.why b.mu.Unlock() switch { case res.Revision == 0 && shut: res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key) + if said != "" { + res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing: %s", key, said) + } case res.Revision == 0: res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound) case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0): diff --git a/modules/mesh-delivery-builder/internal/build/entry_test.go b/modules/mesh-delivery-builder/internal/build/entry_test.go index eba6952f..dbf0432e 100644 --- a/modules/mesh-delivery-builder/internal/build/entry_test.go +++ b/modules/mesh-delivery-builder/internal/build/entry_test.go @@ -16,7 +16,7 @@ func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) { Inputs: &meshcatalog.EntryInputs{Src: 7, Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}}, Paths: map[string][]string{"lib": {"a/b", "c"}}}} got := ModuleOf(e) - if got.Fp2 != "src2:f" || got.Manifest != "sha256:m" || got.Path != "modules/netcheck" || got.Commit != "abc" || got.Src != 7 || + if got.Fp2 != "src2:f" || got.Manifest != "m" || got.Path != "modules/netcheck" || got.Commit != "abc" || got.Src != 7 || got.Toolchains["go"].Version != "go1" || got.Packages["p"] != "1" || !reflect.DeepEqual(got.Needs, []string{"n"}) || got.Inputs2 == nil || !reflect.DeepEqual(got.Inputs2.Bases, []string{"base"}) || !reflect.DeepEqual(got.Inputs2.Paths, map[string]string{"a/b": "lib", "c": "lib"}) { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 5e0b5bea..9faaa720 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -270,7 +270,7 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog. // built against, its packages and needs, and the named paths of its artifacts. func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None, - Manifest: e.ManifestDigest, Packages: e.Packages, Needs: e.Needs} + Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs} if len(e.Bases) > 0 { m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases} } @@ -562,14 +562,20 @@ func (b *Builder) startOnce(ctx context.Context, key, module string, m meshcatal // building by this builder: it takes it again when it starts (ADR 0478 clause 5). func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) { key, _ := builds.BuilderBuildKey(module, m.Fp2) - defer b.handEnded(key) + whyEnded := "" + defer func() { b.handEnded(key, whyEnded) }() defer b.unrelease(key) + // end logs why a look wrote nothing and keeps it for the answer to a build asked again (design 47 G A-412). + end := func(format string, args ...any) { + whyEnded = fmt.Sprintf(format, args...) + b.logf("%s at %s: %s", module, m.Fp2, whyEnded) + } plan, planErr, read := b.planOf(ctx, m.Manifest) var noModule *NoBuildModule if errors.As(planErr, &noModule) { // The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again // when they do (Reconsider, at each change of either; design 47 G A-82). - b.logf("%s at %s is not built yet: %v", module, m.Fp2, planErr) + end("is not built yet: %v", planErr) return } nothing := read && planErr == nil && len(plan.Tasks) == 0 @@ -580,7 +586,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build // now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written. - b.logf("%s at %s is not built: %s", module, m.Fp2, why) + end("is not built: %s", why) return } if b.Waits != nil { @@ -599,12 +605,14 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB if trunk && planErr == nil && !nothing { var goOn bool if numbering, goOn = b.number(ctx, module, m, plan); !goOn { + end("is not built yet: it waits for its package's floor, nothing is claimed") return } } // Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371). reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) if reserving && b.waitFill(module, m) { + end("is not built yet: wait-held, the catalog has not said versions-filled") return } // A key built again by hand is taken over as it stands too: one left building by a look that ended without its write @@ -612,10 +620,11 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { - b.logf("%s: %v", module, err) + end("the claim was refused: %v", err) return } if cl == nil { + end("nothing was claimed: the key is held by a build or a look already, is done or failed for a reason not retried on this fingerprint, waits for a package, or was made by another builder first") return } b.handClaimed(key, cl.Rev) diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 1d0d2bef..b30aeab5 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -902,7 +902,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, {name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *meshcatalog.ModulesByModuleValue) { m.Manifest = "" }, key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}}, - {name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing", + {name: "build again: a toolchain that moved writes nothing and says why at once", again: true, refuse: "wrote nothing: is not built:", edit: func(m *meshcatalog.ModulesByModuleValue) { m.Toolchains = CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}) }, diff --git a/modules/mesh-delivery-builder/internal/build/step.go b/modules/mesh-delivery-builder/internal/build/step.go index f92fa35d..df195edb 100644 --- a/modules/mesh-delivery-builder/internal/build/step.go +++ b/modules/mesh-delivery-builder/internal/build/step.go @@ -71,8 +71,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc } return meshbuilds.StepEndedOutcomeFailed, failedWords } - m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path, - Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages} + m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalog.BareManifestDigest(entry.ManifestDigest), Path: entry.Path, + Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages} b.consider(ctx, entry.Module, m) select { case log := <-ended: diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/manifestdigest.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/manifestdigest.go new file mode 100644 index 00000000..b48e93ff --- /dev/null +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/manifestdigest.go @@ -0,0 +1,21 @@ +package meshcatalog + +import "strings" + +// manifestDigestPrefix is how a catalog entry's field manifest-digest begins, as every digest a record names (design 47 G §578). +const manifestDigestPrefix = "sha256:" + +// EntryManifestDigest is the entry's field manifest-digest for the module's digest bare, the 64 hex digits the module's +// value and the key of `manifests` hold: `sha256:` and the digits (design 47 G §578). A digest already prefixed stays, and +// none stays none. +func EntryManifestDigest(bare string) string { + if bare == "" || strings.HasPrefix(bare, manifestDigestPrefix) { + return bare + } + return manifestDigestPrefix + bare +} + +// BareManifestDigest is the 64 hex digits of an entry's manifest-digest: the module's value and the key of `manifests` +// (design 47 G §578). An entry taken in before G carries the digits alone, which is what its module value held (§609), and +// stays as it is. +func BareManifestDigest(field string) string { return strings.TrimPrefix(field, manifestDigestPrefix) } diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go index 24c653b1..93d43574 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_graph.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_graph.go @@ -8,7 +8,6 @@ import ( "maps" "reflect" "slices" - "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" @@ -24,24 +23,6 @@ import ( // CatalogEntryLayout is the layout version an entry and a revision are written in. const CatalogEntryLayout = 1 -// manifestDigestPrefix is how an entry's field manifest-digest begins, as every digest a record names (design 47 G §578). -const manifestDigestPrefix = "sha256:" - -// EntryManifestDigest is the entry's field manifest-digest for the module's digest bare, the 64 hex digits the module's -// value and the key of `manifests` hold: `sha256:` and the digits (design 47 G §578). A digest already prefixed stays, and -// none stays none. -func EntryManifestDigest(bare string) string { - if bare == "" || strings.HasPrefix(bare, manifestDigestPrefix) { - return bare - } - return manifestDigestPrefix + bare -} - -// BareManifestDigest is the 64 hex digits of an entry's manifest-digest: the module's value and the key of `manifests` -// (design 47 G §578). An entry taken in before G carries the digits alone, which is what its module value held (§609), and -// stays as it is. -func BareManifestDigest(field string) string { return strings.TrimPrefix(field, manifestDigestPrefix) } - // CatalogEntryOf is the catalog graph entry of one module: the module graph's node of it as a recompute computed it // (its build, provision and verb edges, the verb edges left out on a cycle, ADR 0614), with where the trunk holds it, // its manifest's digest and verdict, the main package its process runs (MainOf), the package versions it builds against @@ -49,7 +30,7 @@ func BareManifestDigest(field string) string { return strings.TrimPrefix(field, // ModuleValueOf gives the module's value back from the entry alone. The acting rules are the revision's to add. func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catalogue.Manifest, raw string, n graph.Node) meshcatalog.CatalogEntry { e := meshcatalog.CatalogEntry{V: CatalogEntryLayout, Module: module, Repository: cur.Repository, Path: cur.Path, - Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, + Commit: cur.Commit, Manifest: manifestWhole(raw), ManifestDigest: meshcatalog.EntryManifestDigest(cur.Manifest), Verdict: cur.Verdict, Fp: n.FP, None: n.None, Bases: n.Bases, Packages: maps.Clone(cur.Packages), Provides: n.Provides, Requires: n.Requires, Awaits: n.Awaits, Invokes: n.Invokes, Serves: n.Serves, LeftOut: n.LeftOut, Main: MainOf(m), Contributes: contributesOf(m)} // requires names every requirement as catalogue.Manifest.Wants lists them, optional ones and those only contributes names @@ -86,7 +67,7 @@ func CatalogEntryOf(module string, cur meshcatalog.ModulesByModuleValue, m catal // ruling on milestone 47/G finding 1; ADR 0656 §6: one form of the fact). The old line's fields (the old-form // fingerprint, the reader needs) are not kept. func ModuleValueOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue { - v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: BareManifestDigest(e.ManifestDigest), + v := meshcatalog.ModulesByModuleValue{V: keys.Version, Repository: e.Repository, Path: e.Path, Commit: e.Commit, Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Verdict: e.Verdict, Fp2: e.Fp, None2: e.None, Packages: maps.Clone(e.Packages)} if in := e.Inputs; in != nil { v.Toolchains, v.Paths, v.Problems, v.Warnings, v.Unknown = maps.Clone(in.Toolchains), maps.Clone(in.Paths), in.Problems, in.Warnings, in.Unknown