B6 (H-A-425): a plan no seat builds is kept in the state unbuildable and said in one line

A line per module drowned the journal at 6,500 plans; the plans live on the bus, where
builder_unbuildable reads them, and the builder says them once per start and per seat list change.

Builds: design 47 H2 §795, §820; H-A-425
This commit is contained in:
2026-11-22 01:37:32 +01:00
parent 346b37d623
commit bcff47d257
10 changed files with 359 additions and 10 deletions
@@ -446,6 +446,7 @@ func main() {
return stdio.Ask(dropVerbKey, in)
}, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, unbuildableTool(unbuildablePage(stdio.State(build.UnbuildableState)))) // the plans no seat builds (§795)
tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
// The builder's expiring keys (state bounds): what each bounds and when it ends.
tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription,
@@ -0,0 +1,70 @@
package main
import (
"encoding/json"
"errors"
"sort"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// unbuildableTool is builder_unbuildable (design 47 H2 §795, H-A-425): the builder's state unbuildable, the plans no seat builds,
// read in one page of at most 768 KB (capped says what it left out), with the summary line of the rows read.
func unbuildableTool(page func() (versions.StatePage, error)) stdio.Tool {
return stdio.Tool{Name: build.UnbuildableTool,
Description: "The modules whose artifact no build seat builds, or whose build seat no module claims, as the builder judged their plans " +
"(its state " + build.UnbuildableState + ", worked out again at every start and every change of the seat list): each module, the fingerprint " +
"judged, the artifact's language and kind, why and when; with the one summary line the builder says of them. At most 768 KB, capped " +
"saying what was left out. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
pg, err := page()
if err != nil {
return nil, err
}
rows := []build.Unbuildable{}
var unread []string
for k, raw := range pg.Values {
var u build.Unbuildable
if err := json.Unmarshal(raw, &u); err != nil || u.Module == "" {
unread = append(unread, k)
continue
}
rows = append(rows, u)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
sort.Strings(unread)
out := map[string]any{"line": build.UnbuildableLine(rows), "unbuildable": rows}
if len(unread) > 0 {
out["unreadable"] = unread
}
if pg.More {
out["capped"] = versions.StateCapped{State: build.UnbuildableState, KeysRead: len(pg.Values), StateKeys: pg.Keys, StateBytes: pg.Bytes, InfoWords: pg.InfoWords}
}
return out, nil
}}
}
// unbuildablePage is the first page of the state unbuildable; a runtime that pages nothing is read key by key.
func unbuildablePage(s stdio.KeptState) func() (versions.StatePage, error) {
return func() (versions.StatePage, error) {
pg, err := onePage(s.Page, s.Info)
if !errors.Is(err, errors.ErrUnsupported) {
return pg, err
}
keys, err := s.Keys()
if err != nil {
return versions.StatePage{}, err
}
out := versions.StatePage{Values: map[string]json.RawMessage{}}
for _, k := range keys {
if e, err := s.Get(k); err == nil && e != nil {
out.Values[k] = e.Value
}
}
return out, nil
}
}
@@ -240,6 +240,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
}, waitBound, logger.Printf)
waitsBucket.Ready(func(kv jetstream.KeyValue) { b.Waits.(*waitsKept).start(ctx, kv) })
// The plans no seat builds (design 47 H2 §795, H-A-425): kept in the state unbuildable, read by builder_unbuildable.
b.Unbuildables = unbuildableKept{b: process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.UnbuildableState), "the plans no seat builds are said, and builder_unbuildable lists none", logger.Printf)}
// The versions a build reserves at its start (design 47 G §369): its state reserved, and the catalog's versions-filled
// and version-released heard; nothing is reserved before the fill.
reservedKV := process.AwaitBucket(ctx, js, conn, own.reserved, "no version is reserved: builds are made unreserved", logger.Printf)
@@ -508,7 +510,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := serveSteps(ctx, conn, js, steps, &ackWait); err != nil {
return err
}
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf)
// A change of the seat list judges every plan no seat builds again and says them in one line (design 47 H2 §795).
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx); b.SayUnbuildable(ctx) }, logger.Printf)
half.holders = func() { seatsChanged(ctx) }
for _, pattern := range seatEventPatterns {
if err := hub.Handle(pattern, events); err != nil {
@@ -529,6 +532,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := b.Sweep(ctx, half.pictured()...); err != nil {
errs <- err
}
b.SayUnbuildable(ctx) // at the start, once the first views are in: the plans no seat builds in one line (§795)
}
branches.Changed = half.Changed(ctx)
// A catalog revision the builder hears wakes a floor whose tags or requirements were silent (design 47 G A-119).
@@ -0,0 +1,56 @@
package main
import (
"context"
"errors"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
)
// unbuildableKept is the builder's state unbuildable on the bus as build.Unbuildables (design 47 H2 §795, H-A-425).
type unbuildableKept struct{ b *process.AwaitedBucket }
func (u unbuildableKept) Put(ctx context.Context, module string, value []byte) error {
_, err := u.b.Put(ctx, module, value)
if errors.Is(err, process.ErrNotYet) {
return nil // kept, and written when the state appears
}
return err
}
// Delete removes module's key when the state holds it: a key not there is no error, and writes no marker.
func (u unbuildableKept) Delete(ctx context.Context, module string) error {
kv := u.b.KV()
if kv == nil {
return nil
}
if _, err := kv.Get(ctx, module); errors.Is(err, jetstream.ErrKeyNotFound) {
return nil
} else if err != nil {
return err
}
return kv.Delete(ctx, module)
}
func (u unbuildableKept) Keys(ctx context.Context) ([]string, error) {
kv := u.b.KV()
if kv == nil {
return nil, nil
}
l, err := kv.ListKeys(ctx)
if err != nil {
return nil, err
}
defer func() { _ = l.Stop() }()
var out []string
for k := range l.Keys() {
out = append(out, k)
}
return out, nil
}
var _ build.Unbuildables = unbuildableKept{}
@@ -82,7 +82,10 @@ func (n *NotPinned) Error() string {
// the plan is not kept, so it is made again when they are read (design 47 G A-82).
type NoBuildModule struct {
Artifact string
Err error
// Language and Kind are the artifact's, as the seat list is asked for them: what the summary of the plans no seat builds counts
// (design 47 H2 §795).
Language, Kind string
Err error
}
func (n *NoBuildModule) Error() string { return fmt.Sprintf("artifact %s: %v", n.Artifact, n.Err) }
@@ -103,7 +106,7 @@ func PlanOf(m catalogue.Manifest, seats *buildrun.BuildSeats) (Plan, error) {
task := func(a catalogue.Artifact, language, kind string) error {
bm, err := seats.For(language, kind)
if err != nil {
return &NoBuildModule{Artifact: a.Name, Err: err}
return &NoBuildModule{Artifact: a.Name, Language: language, Kind: kind, Err: err}
}
t := Task{Artifact: a.Name, Kind: a.Kind, Module: bm.Module, Language: bm.Toolchain, From: a.From}
if a.Kind != catalogue.ArtifactUpstream {
@@ -1,13 +1,16 @@
package build
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatdef"
@@ -114,9 +117,58 @@ func TestAPlanNamesTheHoldersOfTheBuildSeats(t *testing.T) {
if !errors.As(err, &none) {
t.Fatalf("with no holder: %v, want NoBuildModule", err)
}
if none.Language != "" || none.Kind != "image" {
t.Errorf("the plan no seat builds names %q %q, want the image's kind (design 47 H2 §795)", none.Language, none.Kind)
}
// Design 47 H2 §795, H-A-425: 6,500 plans no seat builds are kept in the state unbuildable and said in one summary line, with
// each language and kind and its count, never a line per module; a module whose look judges its plan so writes its key and
// no line of its own, and the module is removed once it builds.
t.Run("6,500 plans no seat builds are one line", func(t *testing.T) {
var lines []string
kept := &memUnbuildables{vals: map[string][]byte{}}
mf := &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): m, mdigest('b'): manifestOf(catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"})}}
b := &Builder{OwnModule: builds.BuilderModule, Seats: seats, Claimer: Claimer{Store: newMem(), By: "novox.builder"}, Manifests: mf, Unbuildables: kept,
Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }}
holders["node-build-go"], holders["node-build-image"] = "", ""
b.init()
for i := range 6500 {
digest := mdigest('a')
if i%5 == 0 {
digest = mdigest('b')
}
b.known[fmt.Sprintf("m%04d", i)] = meshcatalog.ModulesByModuleValue{Fp2: fp1, Manifest: digest}
}
kept.vals["gone"] = []byte(`{"module":"gone"}`) // a module that builds now
judged := b.SayUnbuildable(context.Background())
if len(judged) != 6500 || len(kept.vals) != 6500 || kept.vals["gone"] != nil {
t.Fatalf("judged %d, kept %d (gone kept: %v), want 6,500 each and gone removed", len(judged), len(kept.vals), kept.vals["gone"] != nil)
}
if len(lines) != 1 || lines[0] != "6500 "+UnbuildableSaid+": image 5200, typescript bundle 1300" {
t.Errorf("said %d lines %q, want one naming each language and kind with its count", len(lines), lines)
}
lines = nil
delete(kept.vals, "m0001")
b.try(context.Background(), "m0001", b.known["m0001"])
if kept.vals["m0001"] == nil || len(lines) != 0 {
t.Errorf("a look at a plan no seat builds kept %s and said %q, want its key kept and no line", kept.vals["m0001"], lines)
}
})
holders["node-build-go"], holders["node-build-image"] = "acme-go", "acme-img"
p, err := PlanOf(m, seats)
if err != nil || len(p.Tasks) != 2 || p.Tasks[0].Module != "acme-img" || p.Tasks[1].Module != "acme-go" || p.Tasks[1].Language != "go" || p.Tasks[0].Language != "" {
t.Fatalf("plan = %+v, %v", p, err)
}
}
// memUnbuildables is the state unbuildable in memory.
type memUnbuildables struct{ vals map[string][]byte }
func (m *memUnbuildables) Put(_ context.Context, k string, v []byte) error { m.vals[k] = v; return nil }
func (m *memUnbuildables) Delete(_ context.Context, k string) error { delete(m.vals, k); return nil }
func (m *memUnbuildables) Keys(context.Context) ([]string, error) {
out := make([]string, 0, len(m.vals))
for k := range m.vals {
out = append(out, k)
}
return out, nil
}
@@ -61,8 +61,10 @@ type Builder struct {
Toolchains ToolchainEntries
// Waits keeps the toolchain waits of design 47 G §672. Nil holds none: an entry whose fingerprint names another toolchain is
// left unbuilt, as before.
Waits Waits
Manifests Manifests
Waits Waits
// Unbuildables keeps the plans no seat builds in the builder's state unbuildable (design 47 H2 §795, H-A-425). Nil keeps none.
Unbuildables Unbuildables
Manifests Manifests
// Left, when set, is told a key a look claimed and left building because its end could not be written (issue 1966,
// novox/hq ADR 0668 §3; 1748 note 27): no look holds it then, so nothing ends it but builder_build_again or the
// builder's next start. Ended is told each key whose end was written.
@@ -639,8 +641,10 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
var noModule *NoBuildModule
if errors.As(planErr, &noModule) {
// The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again
// when they do (Reconsider, at each change of either; design 47 G A-82).
end("is not built yet: %v", planErr)
// when they do (Reconsider, at each change of either; design 47 G A-82). The plan is kept in the state unbuildable and
// said in the one summary line (SayUnbuildable), never in a line per module (design 47 H2 §795).
whyEnded = "is not built yet: " + planErr.Error()
b.judgedUnbuildable(ctx, module, m.Fp2, noModule)
return
}
nothing := read && planErr == nil && len(plan.Tasks) == 0
@@ -696,6 +700,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
return
}
b.handClaimed(key, cl.Rev)
b.buildable(ctx, module) // the module builds: no longer a plan no seat builds (design 47 H2 §795, H-A-425)
if b.Waits != nil {
b.Waits.End(ctx, module, m.Fp2, WaitPublished) // a key queued behind a package is claimed once it is published
}
@@ -0,0 +1,156 @@
package build
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
)
// **The plans no seat builds** (design 47 H2 §795, H-A-425): a module whose artifact no build seat builds, or whose build seat
// no module claims, is kept in the builder's state unbuildable, one key per module, written when its plan is judged and removed
// when the module builds; builder_unbuildable reads that state. The judgement is made again at every start and every change
// of the seat list (SayUnbuildable), and said in one summary line each time, never a line per module.
const (
// UnbuildableState is the builder's state of the plans no seat builds, one key <module>, 8 MiB.
UnbuildableState = "unbuildable"
// UnbuildableTool is builder_unbuildable, which lists that state.
UnbuildableTool = "builder_unbuildable"
// UnbuildableSaid opens the summary line.
UnbuildableSaid = "modules have an artifact no seat builds"
)
// Unbuildable is one key of the state unbuildable: the module, the fingerprint whose plan was judged, the first artifact's
// language and kind no seat builds, and why.
type Unbuildable struct {
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
Language string `json:"language,omitempty"`
Kind string `json:"kind"`
Why string `json:"why"`
At time.Time `json:"at"`
}
// Unbuildables keeps the state unbuildable: Put writes a module's key, Delete removes it (a key not there is no error) and Keys
// lists the modules it holds.
type Unbuildables interface {
Put(ctx context.Context, module string, value []byte) error
Delete(ctx context.Context, module string) error
Keys(ctx context.Context) ([]string, error)
}
// unbuildableOf is the key value of module's plan judged at fp with none.
func unbuildableOf(module, fp string, none *NoBuildModule, at time.Time) Unbuildable {
return Unbuildable{Module: module, Fingerprint: fp, Language: none.Language, Kind: none.Kind, Why: none.Error(), At: at.UTC()}
}
// judgedUnbuildable writes module's key: its plan, judged at fp, names an artifact no seat builds.
func (b *Builder) judgedUnbuildable(ctx context.Context, module, fp string, none *NoBuildModule) {
if b.Unbuildables == nil {
return
}
raw, _ := json.Marshal(unbuildableOf(module, fp, none, b.now()))
if err := b.Unbuildables.Put(ctx, module, raw); err != nil {
b.logf("%s: the plan no seat builds is not kept in the state %s: %v", module, UnbuildableState, err)
}
}
// buildable removes module's key: the module builds.
func (b *Builder) buildable(ctx context.Context, module string) {
if b.Unbuildables == nil {
return
}
if err := b.Unbuildables.Delete(ctx, module); err != nil {
b.logf("%s: its key of the state %s is not removed: %v", module, UnbuildableState, err)
}
}
// SayUnbuildable judges the plan of every module the builder knows again, keeps the ones no seat builds in the state unbuildable
// (writing the modules it does not hold yet and removing the ones that build now), and says them in one line: how many, and
// each language and kind with its count (design 47 H2 §795). It answers the modules judged so.
func (b *Builder) SayUnbuildable(ctx context.Context) []Unbuildable {
b.init()
b.mu.Lock()
known := sortedKeys(b.known)
modules := make(map[string]string, len(known)) // the fingerprint each is known at
digests := make(map[string]string, len(known))
for _, k := range known {
modules[k], digests[k] = b.known[k].Fp2, b.known[k].Manifest
}
b.mu.Unlock()
var out []Unbuildable
now := b.now()
for _, k := range known {
if modules[k] == "" || digests[k] == "" {
continue
}
_, err, _ := b.planOf(ctx, digests[k])
var none *NoBuildModule
if errors.As(err, &none) {
out = append(out, unbuildableOf(k, modules[k], none, now))
}
}
if b.Unbuildables != nil {
held := map[string]bool{}
keys, err := b.Unbuildables.Keys(ctx)
if err != nil {
b.logf("the state %s cannot be listed, so the plans no seat builds are written whole: %v", UnbuildableState, err)
}
for _, k := range keys {
held[k] = true
}
judged := map[string]bool{}
for _, u := range out {
judged[u.Module] = true
if held[u.Module] {
continue // written when its plan was judged at its look
}
raw, _ := json.Marshal(u)
if err := b.Unbuildables.Put(ctx, u.Module, raw); err != nil {
b.logf("%s: the plan no seat builds is not kept in the state %s: %v", u.Module, UnbuildableState, err)
}
}
for _, k := range keys {
if !judged[k] {
b.buildable(ctx, k)
}
}
}
b.logf("%s", UnbuildableLine(out))
return out
}
// UnbuildableLine is the one summary line of the plans no seat builds: "N modules have an artifact no seat builds:" with each
// language and kind and its count, most first (design 47 H2 §795).
func UnbuildableLine(us []Unbuildable) string {
counts := map[string]int{}
for _, u := range us {
what := u.Kind
if u.Language != "" {
what = u.Language + " " + u.Kind
}
counts[what]++
}
whats := make([]string, 0, len(counts))
for w := range counts {
whats = append(whats, w)
}
sort.Slice(whats, func(i, j int) bool {
if counts[whats[i]] != counts[whats[j]] {
return counts[whats[i]] > counts[whats[j]]
}
return whats[i] < whats[j]
})
parts := make([]string, len(whats))
for i, w := range whats {
parts[i] = fmt.Sprintf("%s %d", w, counts[w])
}
line := fmt.Sprintf("%d %s", len(us), UnbuildableSaid)
if len(parts) > 0 {
line += ": " + strings.Join(parts, ", ")
}
return line
}
@@ -115,7 +115,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}, {Name: "unbuildable", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -177,7 +177,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_ask_drop", "builder_asks", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_unbuildable", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
+3 -1
View File
@@ -37,6 +37,7 @@
"builder_toolchains",
"builder_toolchains_drop",
"builder_waits",
"builder_unbuildable",
"builder_ask_drop",
"builder_asks",
"builder_versions",
@@ -140,7 +141,8 @@
{ "name": "asked", "history": 1 },
{ "name": "reasks", "history": 1 },
{ "name": "versions", "history": 1 },
{ "name": "first-views", "history": 1 }
{ "name": "first-views", "history": 1 },
{ "name": "unbuildable", "history": 1, "max-mib": 8 }
],
"streams": [
{ "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 },