diff --git a/modules/mesh-delivery-indexer/internal/index/registration_findings.go b/modules/mesh-delivery-indexer/internal/index/registration_findings.go index ffe323a5..0837acd6 100644 --- a/modules/mesh-delivery-indexer/internal/index/registration_findings.go +++ b/modules/mesh-delivery-indexer/internal/index/registration_findings.go @@ -19,6 +19,10 @@ const ( // and kept after a Decline or an expiry (§8). SeedWaitsKind = "seed-waits" SeedWaitsID = "mesh-catalog.seed" + // SeedLeftOutKind is the finding of a module the seed leaves out because no build seat builds an artifact of it (scope + // module, id the module's name), naming the artifact and its language or kind; cleared only when a later seed finds a + // build seat for every artifact of it, or the module leaves the catalogue (A-324, A-329). + SeedLeftOutKind = "seed-left-out" ) // HistoryFill writes the finding registration-history-filling when the stream holds used bytes of its max, 80 % or more, diff --git a/modules/mesh-delivery-indexer/internal/index/seed.go b/modules/mesh-delivery-indexer/internal/index/seed.go index 0c61b385..c4116d63 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed.go +++ b/modules/mesh-delivery-indexer/internal/index/seed.go @@ -19,6 +19,9 @@ type SeedModule struct { Origin string Path string Artifacts []SeedArtifact + // LeftOut says why the seed leaves the module out: an artifact of it no build seat builds, named with its language or + // kind (design 47 G A-324, A-329). Empty: the module is seeded. + LeftOut string } // SeedArtifact is an artifact of a module and what its build seat answered `steps` with: Answered false is a seat that has not @@ -61,6 +64,10 @@ func (r *Registry) ComposeSeed(modules []SeedModule) (SeedProposal, error) { return out, nil } for _, m := range modules { + if m.LeftOut != "" { + out.Left = append(out.Left, fmt.Sprintf("%s (%s)", m.Module, m.LeftOut)) + continue + } b, err := r.Branch(m.Origin, "") if err != nil { return out, fmt.Errorf("the seat git does not answer the default branch of %s: %w", m.Origin, err) diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 06b69a88..adbe12b8 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -7,6 +7,7 @@ import ( "maps" "path" "slices" + "strings" "time" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" @@ -25,6 +26,9 @@ type StepsCaller func(ctx context.Context, seat string, in nodebuildgo.StepsInpu // ModuleManifest is the file name of a module's manifest in its directory. const ModuleManifest = "module.json" +// errNoBuildSeat says no build seat's acting rule build matches an artifact's language or kind. +var errNoBuildSeat = errors.New("no seat builds the artifact") + // buildSeatOf is the seat whose acting rule `build` has `on` matching the artifact's language or kind (design 47 G §704, // §203, §324), read from the seat list defs as BuildModulesOf reads it; none, or more than one, is said. func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (string, error) { @@ -42,7 +46,7 @@ func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (st } switch len(seats) { case 0: - return "", fmt.Errorf("no seat builds the artifact %s (language %q, kind %q)", a.Ref, a.Language, a.Kind) + return "", fmt.Errorf("%w %s (language %q, kind %q)", errNoBuildSeat, a.Ref, a.Language, a.Kind) case 1: return seats[0], nil } @@ -69,11 +73,19 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. if err != nil { return nil, err } - seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made + seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made + unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind for _, e := range entries { for _, a := range e.Artifacts { + if packedOrFetched(a) { + continue + } seat, err := buildSeatOf(defs, a) - if err != nil { + switch { + case errors.Is(err, errNoBuildSeat): + unbuilt[e.Module] = append(unbuilt[e.Module], unbuiltWords(a)) + continue + case err != nil: return nil, fmt.Errorf("%s: %w", e.Module, err) } seatOf[a.Ref] = seat @@ -84,9 +96,19 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. var out []SeedModule for _, e := range entries { m := SeedModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)} + if words, ok := unbuilt[e.Module]; ok { + m.LeftOut = "no build seat builds " + strings.Join(words, ", ") + out = append(out, m) + continue + } for _, a := range e.Artifacts { - seat := seatOf[a.Ref] name := artifactName(a.Ref) + if packedOrFetched(a) { + // The builder packs or fetches it: no steps call, and its lint, unit and integration are bypass (A-324). + m.Artifacts = append(m.Artifacts, SeedArtifact{Name: name, Answered: true}) + continue + } + seat := seatOf[a.Ref] sa := SeedArtifact{Name: name, Seat: seat} in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp, Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name), Packages: maps.Clone(e.Packages)}} @@ -109,9 +131,64 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. out = append(out, m) } ix.stepsFindings(ctx, answered, unanswered) + ix.leftOutFindings(ctx, entries, out) return out, nil } +// packedOrFetched says the artifact is one the builder packs or fetches itself, of kind archive or upstream with no +// language: it has no lint, unit or integration step, so no build seat's steps is called for it (design 47 G A-324). +func packedOrFetched(a meshcatalog.Artifact) bool { + return a.Language == "" && (a.Kind == "archive" || a.Kind == "upstream") +} + +// unbuiltWords names an artifact no build seat builds, with its language, else its kind. +func unbuiltWords(a meshcatalog.Artifact) string { + if a.Language != "" { + return fmt.Sprintf("%s (language %s)", a.Ref, a.Language) + } + return fmt.Sprintf("%s (kind %q)", a.Ref, a.Kind) +} + +// leftOutFindings writes seed-left-out for each module the seed leaves out because no build seat builds an artifact of it, +// and clears it only for a module of which a build seat now builds every artifact, or which left the catalogue; a seed +// applied without the module clears nothing (design 47 G A-324, A-329). +func (ix *Indexer) leftOutFindings(ctx context.Context, entries []meshcatalog.CatalogEntry, mods []SeedModule) { + if ix.Findings == nil { + return + } + left := map[string]bool{} + for _, m := range mods { + if m.LeftOut == "" { + continue + } + left[keyID(m.Module)] = true + f := ix.observed(mcc.ScopeModule, m.Module, SeedLeftOutKind, + fmt.Sprintf("the seed leaves out %s: %s", m.Module, m.LeftOut), + "A module is left out of the first registrations", + fmt.Sprintf("The seed registers no module with an artifact no build seat builds, since a pipeline of it could not run that artifact's steps; %s is left out: %s.", m.Module, m.LeftOut), + "indexer."+SeedLeftOutKind) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("seed: the finding %s of %s is not written: %v", SeedLeftOutKind, m.Module, err) + } + } + inCatalogue := map[string]bool{} + for _, e := range entries { + inCatalogue[keyID(e.Module)] = true + } + for _, f := range ix.Findings.Held() { + if f.Kind != SeedLeftOutKind || f.Scope != mcc.ScopeModule || left[f.ID] { + continue + } + why := "a build seat builds every artifact of it" + if !inCatalogue[f.ID] { + why = "the module left the catalogue" + } + if err := ix.Findings.Clear(ctx, f.Scope, f.ID, f.Kind, why); err != nil { + ix.logf("seed: the finding %s of %s is not cleared: %v", SeedLeftOutKind, f.ID, err) + } + } +} + // artifactName is the artifact's name in an artifact reference :. func artifactName(ref string) string { for i := len(ref) - 1; i >= 0; i-- { diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go index 65d70714..8b933ded 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps_test.go @@ -93,3 +93,73 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) { t.Errorf("the finding stands after the seat answered: %+v", w.Held()) } } + +// Design 47 G A-324, A-329: an archive and an upstream artifact, which the builder packs or fetches, are called of no build +// seat and seeded with lint, unit and integration bypass; a module with an artifact no build seat builds is left out of +// the seed, named in its ask, with the finding seed-left-out naming the artifact and its language, never the whole seed; +// the finding clears only once a build seat builds every artifact of it, or once it left the catalogue. +func TestTheSeedBypassesWhatTheBuilderPacksAndLeavesOutAModuleNoSeatBuilds(t *testing.T) { + ctx := context.Background() + w := newFindings() + ix := &Indexer{Module: testSelf, Findings: w, Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: time.Now} + entries := map[string]meshcatalog.CatalogEntry{ + "a": {Module: "a", Repository: "novox/a", Path: "a", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "a:files", Kind: "archive"}}}, + "nats": {Module: "nats", Repository: "novox/n", Path: "n", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "nats:server", Kind: "upstream"}, {Ref: "nats:tools", Language: "go"}}}, + "odd": {Module: "odd", Repository: "novox/o", Path: "o", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "odd:web", Language: "rust"}}}, + } + keep := func(n int64) { + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: n, Kind: "trunk"}, entries) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + } + keep(1) + defs := map[string]seatdef.Definition{"node-build-go": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "go"}}}} + var called []string + call := func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) { + called = append(called, seat+" "+in.Artifact) + return nodebuildgo.StepsAnswer{Steps: []nodebuildgo.DetectedStep{{Step: "unit"}}}, nil + } + mods, err := ix.SeedModulesFrom(ctx, defs, call) + if err != nil { + t.Fatal(err) + } + if strings.Join(called, "|") != "node-build-go nats.tools" { + t.Errorf("steps was called %q, want only of the go artifact", called) + } + if h := w.Held(); len(h) != 1 || h[0].Kind != SeedLeftOutKind || h[0].Scope != mcc.ScopeModule || h[0].ID != "odd" || !strings.Contains(h[0].Summary, "odd:web (language rust)") { + t.Fatalf("the findings: %+v", h) + } + seed, err := newRegRig().r.ComposeSeed(mods) + if err != nil || len(seed.Unanswered) != 0 || len(seed.Registrations) != 2 || len(seed.Left) != 1 || !strings.Contains(seed.Left[0], "odd (no build seat builds odd:web (language rust))") { + t.Fatalf("the seed: %+v, %v", seed, err) + } + bypass := meshcatalog.StepStatuses{Lint: "bypass", Unit: "bypass", Integration: "bypass"} + if a := seed.Registrations[0].Policy.Artifacts["files"]; seed.Registrations[0].Module != "a" || a != bypass { + t.Errorf("the archive's statuses are %+v in %s, want all three bypass", a, seed.Registrations[0].Module) + } + if p := seed.Registrations[1].Policy.Artifacts; p["server"] != bypass || p["tools"] != (meshcatalog.StepStatuses{Lint: "bypass", Integration: "bypass"}) { + t.Errorf("nats's statuses are %+v", p) + } + // A seed again with the language still unserved keeps the finding; once a seat builds it, it clears. + if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil || len(w.Held()) != 1 { + t.Fatalf("a second seed: %v, %+v", err, w.Held()) + } + defs["node-build-rust"] = seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "rust"}}} + if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil || len(w.Held()) != 0 { + t.Fatalf("the seed with a rust build seat: %v, the findings %+v", err, w.Held()) + } + // A module left out that leaves the catalogue clears its finding too. + delete(defs, "node-build-rust") + if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil || len(w.Held()) != 1 { + t.Fatalf("the seed without the rust seat again: %v, %+v", err, w.Held()) + } + delete(entries, "odd") + keep(2) + if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil || len(w.Held()) != 0 { + t.Fatalf("the seed after odd left the catalogue: %v, the findings %+v", err, w.Held()) + } +} diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place.go b/modules/mesh-delivery-pipeline/internal/pipeline/place.go index b71206d5..48feee34 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place.go @@ -95,9 +95,13 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog } for _, step := range buildSteps { status := Status(reg, name, step) + why := "bypassed by the module's policy" + if packedOrFetched(a) { + status, why = seat.StepResultStatusBypass, fmt.Sprintf("an artifact of kind %s has no %s step: the builder packs or fetches it", a.Kind, step) + } s := seat.StepResult{Step: step, Artifact: a.Ref, Status: status, Fingerprint: p.Fingerprint} if status == seat.StepResultStatusBypass { - s.Outcome, s.Why = seat.StepResultOutcomeBypassed, "bypassed by the module's policy" + s.Outcome, s.Why = seat.StepResultOutcomeBypassed, why steps = append(steps, s) continue } @@ -143,6 +147,13 @@ func Place(p seat.Pipeline, reg catalog.Registration, entries map[string]catalog return steps, asks } +// packedOrFetched says the artifact is one the builder packs or fetches itself, of kind archive or upstream with no language: +// it has no lint, unit or integration step, so those are bypass whatever the policy says and no build seat is asked +// (design 47 G A-324, A-325). +func packedOrFetched(a catalog.Artifact) bool { + return a.Language == "" && (a.Kind == "archive" || a.Kind == "upstream") +} + // Asked is one ask to put on a build seat's work queue. type Asked struct { Seat string diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go index c1a91d44..8f3d93eb 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/place_test.go @@ -280,18 +280,44 @@ func TestAStepAskCarriesTheKeptResultsAndTheToolchain(t *testing.T) { } } -// An image artifact, which has no language, is asked of the build seat whose acting rule build is on kind:image, the form -// the seat definitions write (design 47 G §324, §397). -func TestAnImageArtifactsStepsAreAskedOfTheSeatOnKindImage(t *testing.T) { - reg := catalog.Registration{Module: "app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "required"}}} +// An artifact without a language is placed by its kind (design 47 G §324, §397, A-324, A-325): an image is asked of the build +// seat whose acting rule build is on kind:image, the form the seat definitions write; an archive's and an upstream +// artifact's lint, unit and integration are bypassed whatever the policy says, asked of no one; an artifact whose language no +// build seat serves stays could not run, naming it (§535). +func TestAnArtifactWithoutALanguageIsPlacedByItsKind(t *testing.T) { + reg := catalog.Registration{Module: "app", Policy: catalog.Policy{Steps: catalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required"}}} entries := map[string]catalog.CatalogEntry{ - "app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:server", Kind: "image"}}}, + "app": {Module: "app", Artifacts: []catalog.Artifact{{Ref: "app:server", Kind: "image"}, {Ref: "app:files", Kind: "archive"}, + {Ref: "app:mirror", Kind: "upstream"}, {Ref: "app:web", Language: "rust"}}}, "mesh-build-image": {Module: "mesh-build-image", Acts: []catalog.ActingRule{{Seat: "node-build-image", Step: "build", On: seatdef.OnKindImage}}}, } - steps, _ := Place(seat.Pipeline{Pipeline: "main.4.app", Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner") + steps, asks := Place(seat.Pipeline{Pipeline: "main.4.app", Module: "app"}, reg, entries, time.Unix(0, 0), "the-runner") + want := map[string]struct{ outcome, seat, status string }{ + "app:server": {seat.StepResultOutcomeRunning, "node-build-image", "required"}, + "app:files": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass}, + "app:mirror": {seat.StepResultOutcomeBypassed, "", seat.StepResultStatusBypass}, + "app:web": {seat.StepResultOutcomeCouldNotRun, "", "required"}, + } + n := 0 for _, s := range steps { - if s.Artifact == "app:server" && (s.Outcome != seat.StepResultOutcomeRunning || s.Seat != "node-build-image") { - t.Errorf("the image's step %s is %s at %q (%s), want running at node-build-image", s.Step, s.Outcome, s.Seat, s.Why) + w, ok := want[s.Artifact] + if !ok { + continue + } + n++ + if s.Outcome != w.outcome || s.Seat != w.seat || s.Status != w.status { + t.Errorf("%s %s is %s/%s at %q (%s), want %s/%s at %q", s.Artifact, s.Step, s.Status, s.Outcome, s.Seat, s.Why, w.status, w.outcome, w.seat) } } + if n != 12 { + t.Errorf("placed %d build steps of the four artifacts, want 12", n) + } + for _, a := range asks { + if a.Seat != builds.Seat && a.Seat != wants.Seat && a.Seat != "node-build-image" { + t.Errorf("asked %s, want only the build, the composition and the image's build seat", a.Seat) + } + } + if len(asks) != 5 { + t.Errorf("asked %d, want the build, the image's three steps and the composition", len(asks)) + } }