diff --git a/modules/mesh-delivery-builder/builds/ended.go b/modules/mesh-delivery-builder/builds/ended.go new file mode 100644 index 00000000..0f01a3d1 --- /dev/null +++ b/modules/mesh-delivery-builder/builds/ended.go @@ -0,0 +1,43 @@ +package builds + +import ( + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// The builder says each done build's end itself, as an event on its own stream (design 47 G §379, A-96): the stream +// BuildsStream carries `build.ended..` (each part escaped), published where the log entry of the end is +// written. A part that wants a build's parts or its keeping node hears it (the assigner consumes `mesh-delivery-builder.build.ended`). +const ( + // BuildsStream is the builder's stream of build ends, named by its manifest's `streams` entry. + BuildsStream = "mesh-delivery-builder-builds" + // BuildsStreamEvents is the pattern of the events that entry carries. + BuildsStreamEvents = BuildEndedEvent + ".**" +) + +// BuildEnded is the body of the event build.ended: the module, the fingerprint, each artifact made with its digest, the build +// node that keeps a kept build (none for one not kept) and the watched branch a build of a candidate is for (none for the trunk). +type BuildEnded struct { + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + Made map[string]string `json:"made,omitempty"` + KeptOn string `json:"kept-on,omitempty"` + Branch string `json:"branch,omitempty"` +} + +// BuildEndedEventName is the event's name for a module and fingerprint: `build.ended..`, each escaped. +func BuildEndedEventName(module, fp string) string { + return BuildEndedEvent + "." + keys.EscapeName(module) + "." + keys.EscapeName(fp) +} + +// BuildEndedOf is the event of a log entry of a build's end: false for any entry but the end of a done build, which says no +// digest of any part. +func BuildEndedOf(e BuildLogEntry) (BuildEnded, bool) { + if e.Event != BuildEndedEvent || e.Outcome != BuildDone || e.Module == "" || e.FP == "" { + return BuildEnded{}, false + } + made := map[string]string{} + for _, a := range e.Made { + made[a.Name] = a.Digest + } + return BuildEnded{Module: e.Module, Fingerprint: e.FP, Made: made, KeptOn: e.KeptOn, Branch: e.Ref}, true +} diff --git a/modules/mesh-delivery-builder/builds/ended_test.go b/modules/mesh-delivery-builder/builds/ended_test.go new file mode 100644 index 00000000..33e3f70f --- /dev/null +++ b/modules/mesh-delivery-builder/builds/ended_test.go @@ -0,0 +1,32 @@ +package builds + +import ( + "reflect" + "testing" +) + +// Design 47 G §379, A-96: a done build's end is the event build.ended, its subject escaped by module and fingerprint; any other +// entry is none. +func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) { + made := []BuiltArtifact{{Name: "program", Kind: "program", Digest: "sha256:1"}} + e := BuildLogEntry{Event: BuildEndedEvent, Outcome: BuildDone, Module: "node-engine", FP: "src2:a", Ref: "cand", KeptOn: "ace", Made: made} + got, ok := BuildEndedOf(e) + want := BuildEnded{Module: "node-engine", Fingerprint: "src2:a", Made: map[string]string{"program": "sha256:1"}, KeptOn: "ace", Branch: "cand"} + if !ok || !reflect.DeepEqual(got, want) { + t.Errorf("event = %+v, %v", got, ok) + } + if n := BuildEndedEventName("node-engine", "src2:a"); n != "build.ended.node-engine.src2=3aa" { + t.Errorf("event name = %q", n) + } + for name, mut := range map[string]func(*BuildLogEntry){ + "started": func(e *BuildLogEntry) { e.Event = BuildStartedEvent }, + "failed": func(e *BuildLogEntry) { e.Outcome = BuildFailed }, + "no fp": func(e *BuildLogEntry) { e.FP = "" }, + } { + c := e + mut(&c) + if _, ok := BuildEndedOf(c); ok { + t.Errorf("%s: an event", name) + } + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 7feaa294..aba095dd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -314,6 +314,10 @@ func startBedWith(t *testing.T, closed bool) *bed { "mesh.stream.mesh-delivery-builder.event.finding-cleared.*", "mesh.stream.mesh-delivery-builder.event.findings-stated.*"}}); err != nil { t.Fatal(err) } + // The builder's stream of build ends (design 47 G A-96), which it says build.ended into. + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "BUILDER_BUILDS", Subjects: []string{"mesh.stream.mesh-delivery-builder.event.build.ended.>"}}); err != nil { + t.Fatal(err) + } b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} b.fake.closed.Store(closed) b.fake.serve(t, "mesh-build-go", "n1", "n2") @@ -555,6 +559,18 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" { t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended) } + // Design 47 G §379, A-96: the end is also said as the event build.ended on the builder's stream, its body the module, the + // fingerprint and each artifact's digest. + if st, err := b.js.Stream(b.ctx, "BUILDER_BUILDS"); err != nil { + t.Fatal(err) + } else if raw, err := st.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("netcheck", fp)); err != nil { + t.Errorf("no build.ended of netcheck's build: %v", err) + } else { + var ev builds.BuildEnded + if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || len(ev.Made) != 2 { + t.Errorf("build.ended = %s (%v), want netcheck at its fingerprint with two artifacts' digests", raw.Data, err) + } + } for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} { if got := b.fake.count(tool); got != want { t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index b873b4c2..d8678236 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -253,6 +253,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil { logger.Printf("%v", err) } + if err := emitBuildEnded(context.WithoutCancel(ctx), js, cred, e); err != nil { + logger.Printf("%v", err) + } // A failed trunk build is the builder's finding build-failed, cleared by a later done one (DLV-B7). found.buildEnded(context.WithoutCancel(ctx), e) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 26da5ebb..2082cf87 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -161,6 +161,34 @@ func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account return nil } +// emitBuildEnded says the builder's event build.ended of a done build, on the builder's own stream (design 47 G §379, A-96): +// mesh.stream..event.build.ended.., published where the log entry of the end is written. An +// entry that is not a done build's end says none. The id makes a repeated end one event. +func emitBuildEnded(ctx context.Context, js jetstream.JetStream, cred process.Account, e builds.BuildLogEntry) error { + body, ok := builds.BuildEndedOf(e) + if !ok { + return nil + } + data, err := json.Marshal(body) + if err != nil { + return err + } + msg := nats.NewMsg(cred.StreamSubject(builds.BuildEndedEventName(e.Module, e.FP))) + id := "build-ended." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", cred.Module) + msg.Header.Set("x-node", cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = data + pub, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { + return fmt.Errorf("saying %s of %s at %s: %w", builds.BuildEndedEvent, e.Module, e.FP, err) + } + return nil +} + // versionActed answers a reasks value of builder_publish_again, builder_floor_again or builder_old_form's drop, a hand act said in the log. func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) { var err error diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 79789582..3a9a8c5b 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -175,9 +175,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. // Design 47 G A-30, §383: and its own findings' events, which its finding writer says. - if len(m.Streams) != 2 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 || - m.Streams[1].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[1].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { - t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, and the findings' stream", m.Streams) + if len(m.Streams) != 3 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 || + m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 || + m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { + t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days) and the findings' stream", m.Streams) } if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 1 { t.Errorf("config = %+v, want toolchain-wait-minutes of 60", m.Config) diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index ac053db9..f784dcb2 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -118,6 +118,7 @@ ], "streams": [ { "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 }, + { "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16 }, { "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 } ], "config": {