diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go index 98c0465d..d918b4eb 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go @@ -38,6 +38,9 @@ func (b *builtEnds) put(data []byte) error { if e.Module == "" || e.Fingerprint == "" { return fmt.Errorf("a build's end naming no module or fingerprint: %s", data) } + if e.Outcome == "" { // an end said before A-162 is a done build's + e.Outcome = builds.BuildDone + } b.mu.Lock() defer b.mu.Unlock() if b.ends == nil { @@ -47,6 +50,14 @@ func (b *builtEnds) put(data []byte) error { return nil } +// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended. +func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) { + b.mu.Lock() + defer b.mu.Unlock() + e, ok := b.ends[b.key(module, fp)] + return e, ok +} + // partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `:`), with the // node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose // parts the version's release says (releasedVersions). diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go index d73743e2..7a86f9b3 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go @@ -86,3 +86,25 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *tes t.Error("no revision kept names a release") } } + +// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed. +func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) { + w := newWorld() + say := func(e builds.BuildEnded) { + raw, _ := json.Marshal(e) + if err := w.ends.put(raw); err != nil { + t.Fatal(err) + } + } + if failed, _ := w.buildStanding("web", "fp1"); failed { + t.Error("a build nothing was heard of failed") + } + say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"}) + if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" { + t.Errorf("failed end: failed %v why %q", failed, why) + } + say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone}) + if failed, _ := w.buildStanding("web", "fp1"); failed { + t.Error("a later done end leaves the build failed") + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 2c9f7b45..f6cfa1d5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -329,6 +329,7 @@ func (p *passer) wants(ctx context.Context) { m.Released = p.answeredWith(hold, "release") m.Kept = p.answeredWith(hold, "keep") m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant) + m.BuildFailed, m.BuildError = p.w.buildStanding(name, fp) if raw := p.w.value(grants, grant); raw != nil { if g, err := grantsstate.DecodeGrant(raw); err == nil { m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index e8398ead..48a1e5f4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -811,14 +811,14 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) { p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC)) var logged []string p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } - notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}} + notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is being built"}}} wanted := assign.Outcome{Wants: map[string]mwc.Want{"shanks": {Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}} for i, c := range []struct { before map[string]mwc.Want out assign.Outcome line string // the line said in this pass, "" for none }{ - {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"}, + {nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is being built"}, {nil, notDone, ""}, {map[string]mwc.Want{"shanks": {Removed: true}}, notDone, ""}, {nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"}, diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index e591751b..88590412 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -475,6 +475,15 @@ func (w *world) built(module, fp string) bool { return err == nil && b.State == builds.BuildDone } +// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G +// §575, A-162): the words of a hold under rule 1. +func (w *world) buildStanding(module, fp string) (failed bool, words string) { + if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed { + return true, e.Error + } + return false, "" +} + // buildNeeds are the reference forms the builder's done build of module at fp is written in (the build key's needs, ADR // 0618 §1): nil when it is not done or names none. func (w *world) buildNeeds(module, fp string) []string { diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index af04e1b6..b5f365d5 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -26,11 +26,15 @@ import ( // trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller // issued its credential for F, its manifest at F, and the nodes it is assigned to. type Module struct { - Name string - FP string - Built bool - Held bool - Granted bool + Name string + FP string + Built bool + // BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole + // (design 47 G §575, A-162): the words of a hold under rule 1. + BuildFailed bool + BuildError string + Held bool + Granted bool // GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node // that runs an older one names it (issue 1945). Nil: none. GrantedAt func(fp string) bool @@ -188,7 +192,10 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) { func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { switch { case !m.Built: - return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP)) + if m.BuildFailed { + return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError) + } + return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP)) case m.Held: return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP)) case m.MachineHeld != nil && m.MachineHeld(node): diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 5cd5b4af..28b7f3a0 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -1364,8 +1364,8 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) { }{ {"no key: nothing written", nil, map[string]mwc.Want{}}, {"a removed want: it says why", map[string]mwc.Want{"shanks": gone}, - map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}}, - {"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}, + map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}}, + {"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}, map[string]mwc.Want{}}, {"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}}, } { @@ -1658,3 +1658,23 @@ func TestHealthIsReadFromRunningAlone(t *testing.T) { t.Errorf("a provider saying unhealthy: no module unreached, %+v", out) } } + +// §575: a hold under rule 1 says how the build stands: failed with the error whole, or being built. +func TestRuleOneSaysHowTheBuildStands(t *testing.T) { + for _, c := range []struct { + name string + failed bool + err string + want string + }{ + {"a build that runs", false, "", "the build of mail at " + short(fNew) + " is being built"}, + {"a failed build", true, "transport fault: no responders at mesh-build-go", "the build of mail at " + short(fNew) + " failed: transport fault: no responders at mesh-build-go"}, + } { + m, n := ready() + m.Built, m.BuildFailed, m.BuildError = false, c.failed, c.err + held, rule, words := Holds(m, "g14", n) + if held || rule != mwc.RuleBuildDone || words != c.want { + t.Errorf("%s: held %v rule %d words %q, want %q", c.name, held, rule, words, c.want) + } + } +} diff --git a/modules/mesh-delivery-builder/builds/ended.go b/modules/mesh-delivery-builder/builds/ended.go index 0f01a3d1..23e99104 100644 --- a/modules/mesh-delivery-builder/builds/ended.go +++ b/modules/mesh-delivery-builder/builds/ended.go @@ -17,11 +17,19 @@ const ( // BuildEnded is the body of the event build.ended: the module, the fingerprint, each artifact made with its digest, the build // node that keeps a kept build (none for one not kept) and the watched branch a build of a candidate is for (none for the trunk). type BuildEnded struct { - Module string `json:"module"` - Fingerprint string `json:"fingerprint"` - Made map[string]string `json:"made,omitempty"` - KeptOn string `json:"kept-on,omitempty"` - Branch string `json:"branch,omitempty"` + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + // Outcome is done or failed (design 47 G §575, A-162): the event is said for every end that is one of these; Error is why a + // failed build failed, as its key holds it (`transport fault:` included, §568), none for a done one. + Outcome string `json:"outcome"` + Error string `json:"error,omitempty"` + // Manifest is the digest of the manifest built and Needs the reader capabilities its reference forms need (ADR 0618 §1), of a + // done build: what a reader of the end asks of the builder's key otherwise. + Manifest string `json:"manifest,omitempty"` + Needs []string `json:"needs,omitempty"` + Made map[string]string `json:"made,omitempty"` + KeptOn string `json:"kept-on,omitempty"` + Branch string `json:"branch,omitempty"` } // BuildEndedEventName is the event's name for a module and fingerprint: `build.ended..`, each escaped. @@ -29,15 +37,15 @@ func BuildEndedEventName(module, fp string) string { return BuildEndedEvent + "." + keys.EscapeName(module) + "." + keys.EscapeName(fp) } -// BuildEndedOf is the event of a log entry of a build's end: false for any entry but the end of a done build, which says no +// BuildEndedOf is the event of a log entry of a build's end: false for any entry but the end of a done or a failed build. A done end says no // digest of any part. func BuildEndedOf(e BuildLogEntry) (BuildEnded, bool) { - if e.Event != BuildEndedEvent || e.Outcome != BuildDone || e.Module == "" || e.FP == "" { + if e.Event != BuildEndedEvent || (e.Outcome != BuildDone && e.Outcome != BuildFailed) || e.Module == "" || e.FP == "" { return BuildEnded{}, false } made := map[string]string{} for _, a := range e.Made { made[a.Name] = a.Digest } - return BuildEnded{Module: e.Module, Fingerprint: e.FP, Made: made, KeptOn: e.KeptOn, Branch: e.Ref}, true + return BuildEnded{Module: e.Module, Fingerprint: e.FP, Outcome: e.Outcome, Error: e.Error, Manifest: e.Manifest, Needs: e.Needs, Made: made, KeptOn: e.KeptOn, Branch: e.Ref}, true } diff --git a/modules/mesh-delivery-builder/builds/ended_test.go b/modules/mesh-delivery-builder/builds/ended_test.go index 33e3f70f..8572b8cf 100644 --- a/modules/mesh-delivery-builder/builds/ended_test.go +++ b/modules/mesh-delivery-builder/builds/ended_test.go @@ -11,7 +11,7 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) { made := []BuiltArtifact{{Name: "program", Kind: "program", Digest: "sha256:1"}} e := BuildLogEntry{Event: BuildEndedEvent, Outcome: BuildDone, Module: "node-engine", FP: "src2:a", Ref: "cand", KeptOn: "ace", Made: made} got, ok := BuildEndedOf(e) - want := BuildEnded{Module: "node-engine", Fingerprint: "src2:a", Made: map[string]string{"program": "sha256:1"}, KeptOn: "ace", Branch: "cand"} + want := BuildEnded{Module: "node-engine", Fingerprint: "src2:a", Outcome: BuildDone, Made: map[string]string{"program": "sha256:1"}, KeptOn: "ace", Branch: "cand"} if !ok || !reflect.DeepEqual(got, want) { t.Errorf("event = %+v, %v", got, ok) } @@ -20,7 +20,7 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) { } for name, mut := range map[string]func(*BuildLogEntry){ "started": func(e *BuildLogEntry) { e.Event = BuildStartedEvent }, - "failed": func(e *BuildLogEntry) { e.Outcome = BuildFailed }, + "queued": func(e *BuildLogEntry) { e.Outcome = BuildQueued }, "no fp": func(e *BuildLogEntry) { e.FP = "" }, } { c := e @@ -30,3 +30,13 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) { } } } + +// Design 47 G §575, A-162: a failed build's end is said too, with its outcome and its error whole. +func TestAFailedBuildsEndIsSaidWithItsError(t *testing.T) { + e := BuildLogEntry{Event: BuildEndedEvent, Outcome: BuildFailed, Module: "web", FP: "src2:b", Error: "transport fault: no responders at mesh-build-go"} + got, ok := BuildEndedOf(e) + want := BuildEnded{Module: "web", Fingerprint: "src2:b", Outcome: BuildFailed, Error: e.Error, Made: map[string]string{}} + if !ok || !reflect.DeepEqual(got, want) { + t.Errorf("event = %+v, %v", got, ok) + } +} diff --git a/modules/mesh-delivery-builder/builds/log.go b/modules/mesh-delivery-builder/builds/log.go index 3eed958b..3fd4aa85 100644 --- a/modules/mesh-delivery-builder/builds/log.go +++ b/modules/mesh-delivery-builder/builds/log.go @@ -83,6 +83,9 @@ type BuildLogEntry struct { // narration names neither. Error string `json:"error,omitempty"` Step string `json:"step,omitempty"` + // Needs are, at a done build's end, the reader capabilities its reference forms need (BuilderBuild.Needs, ADR 0618 §1): the end + // event carries them (design 47 G §575, A-162). + Needs []string `json:"needs,omitempty"` // Line is a narration's words, Why what it narrates (asked, waiting, stored, refused): a build.said entry's alone. Line string `json:"line,omitempty"` Why string `json:"why,omitempty"` diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index fe5f9e56..7a264025 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -594,7 +594,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 t.Errorf("no build.ended of netcheck's build: %v", err) } else { var ev builds.BuildEnded - if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || len(ev.Made) != 2 { + if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || ev.Outcome != builds.BuildDone || len(ev.Made) != 2 { t.Errorf("build.ended = %s (%v), want netcheck at its fingerprint with two artifacts' digests", raw.Data, err) } } @@ -656,9 +656,18 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 b.fake.hang.Store(&release) third := "src2:" + strings.Repeat("4", 64) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) - if f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed)); !strings.Contains(f.Error, "did not end within") { + f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed)) + if !strings.Contains(f.Error, "did not end within") { t.Errorf("the failed build = %+v, want its bound run out", f) } + // A-162: the failed end is said as build.ended too, with its error whole. + if st, err := b.js.Stream(b.ctx, "BUILDER_BUILDS"); err != nil { + t.Fatal(err) + } else if raw, err := st.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("netcheck", third)); err != nil { + t.Errorf("no build.ended of netcheck's failed build: %v", err) + } else if ev := (builds.BuildEnded{}); json.Unmarshal(raw.Data, &ev) != nil || ev.Outcome != builds.BuildFailed || ev.Error != f.Error { + t.Errorf("build.ended = %s, want the failed outcome with the error %q", raw.Data, f.Error) + } failed := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "netcheck", BuildFailed) reg.awaitFinding(t, failed, true) b.fake.hang.Store(nil) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 2c5c1ef5..c8d4f0b2 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -161,9 +161,9 @@ func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account return nil } -// emitBuildEnded says the builder's event build.ended of a done build, on the builder's own stream (design 47 G §379, A-96): +// emitBuildEnded says the builder's event build.ended of a done or failed build (A-162), on the builder's own stream (design 47 G §379, A-96): // mesh.stream..event.build.ended.., published where the log entry of the end is written. An -// entry that is not a done build's end says none. The id makes a repeated end one event. +// entry that is not a done or failed build's end says none. The id makes a repeated end one event. func emitBuildEnded(ctx context.Context, js jetstream.JetStream, cred process.Account, e builds.BuildLogEntry) error { body, ok := builds.BuildEndedOf(e) if !ok { diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index a6a59251..bce359de 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -1208,6 +1208,7 @@ func LogEnded(module string, m meshcatalog.ModulesByModuleValue, ref, by string, e.KeptOn = result.KeptOn e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2) e.Error, e.Step = result.Error, result.Step // a failed build's reason, not only its outcome + e.Needs = append([]string(nil), result.Needs...) return e }