novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Builder: keep each build's start and end in its own module log build-log, and answer builder_builds_since from it, so readers and mesh-catalog read the builder's history without a bare stream (issue 1955, ADR 0571 §1, §9)

Browse Source
This commit is contained in:
jschoubben
2026-10-29 23:48:14 +01:00
parent c045057ca0
commit 17c0734fb2
11 changed files with 179 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go
+16 -2
View File
@@ -26,7 +26,7 @@ type queueAsker func(key string, value []byte) (json.RawMessage, error)
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool {
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: build.StuckTool,
@@ -94,6 +94,20 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
}},
{Name: delivery.BuildsSinceTool,
Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
"log held more than one answer carries: ask again from the last build's time. What mesh-catalog registers a build from. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
Run: func(args map[string]any) (any, error) {
at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
if err != nil {
return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
}
return build.BuildsSince(since, at)
}},
{Name: build.AnnounceAgainTool,
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
@@ -250,7 +264,7 @@ func main() {
a, err := build.DecodeReAsk(v)
return err == nil && !a.Answered.IsZero()
})
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil {
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+37
View File
@@ -23,6 +23,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -268,6 +269,12 @@ func startBedWith(t *testing.T, closed bool) *bed {
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
b.instances = mk(instancesBucket, 1)
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog)
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"},
AllowDirect: true}); err != nil {
t.Fatal(err)
}
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
b.fake.closed.Store(closed)
b.fake.serve(t, "mesh-build-go", "n1", "n2")
@@ -280,6 +287,29 @@ func startBedWith(t *testing.T, closed bool) *bed {
return b
}
// logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user.
func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry {
t.Helper()
conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"})
if err != nil {
t.Fatal(err)
}
defer conn.Close()
entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module)
if err != nil {
t.Fatal(err)
}
var out []delivery.BuildLogEntry
for _, le := range entries {
e, err := delivery.DecodeBuildLogEntry(le.Value)
if err != nil {
t.Fatal(err)
}
out = append(out, e)
}
return out
}
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
t.Helper()
@@ -406,6 +436,13 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
t.Errorf("toolchains = %+v", v.Toolchains)
}
// 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a
// reader with the log's read reads them, the end naming the manifest and what was made.
ended := b.logged(t, "netcheck")
if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent ||
ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" {
t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended)
}
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} {
if got := b.fake.count(tool); got != want {
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+3 -3
View File
@@ -130,10 +130,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
trunks.start()
b.Trunk = trunks.head
// The builder's log (ADR 0571 §1): each build's start and end, on its own stream, read by the view, the indexer, the
// assigner and mesh-catalog.
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
// indexer and the assigner through their reads, and by mesh-catalog through the tool builder_builds_since.
b.Log = func(e delivery.BuildLogEntry) {
if err := emitLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
logger.Printf("%v", err)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go
+11 -9
View File
@@ -8,6 +8,7 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
@@ -140,20 +141,21 @@ func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e
}
}
// emitLog says one entry of the builder's log on its stream `log` (ADR 0571 §1), under an id the stream keeps one of.
func emitLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error {
// appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key
// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential.
// The id makes a repeated append of one entry one entry.
func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error {
data, err := delivery.EncodeBuildLogEntry(e)
if err != nil {
return err
}
msg := nats.NewMsg(process.ModuleStreamSubject(cred.Module, e.Event))
id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano)
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", e.At.Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
key, err := delivery.BuildLogKey(e.Module)
if err != nil {
return err
}
msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key)
msg.Data = data
id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano)
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {