novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/47-G-builder6y' into feat/47-G-join8

Browse Source 
# Conflicts:
#	modules/mesh-delivery-builder/buildrun/asks_test.go
#	modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go
#	modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
#	modules/mesh-delivery-builder/internal/build/numbers.go
#	modules/mesh-delivery-builder/internal/build/run.go
#	modules/mesh-delivery-builder/internal/build/stuck.go
#	modules/mesh-delivery-builder/internal/versions/publish.go
#	modules/mesh-delivery-builder/internal/versions/reserve.go
#	modules/mesh-delivery-builder/manifest_test.go
#	modules/mesh-delivery-builder/module.json
This commit is contained in:
jschoubben
2026-11-12 00:13:58 +01:00
parent 4f056485f2 a2dfbec30d
commit bf610ec08a
90 changed files with 3464 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/buildrun/asks.go
+4 -1
View File
@@ -35,6 +35,8 @@ type AskWaiter interface {
type Asker struct {
Publisher AskPublisher
Waiter AskWaiter
// Seats finds the build module of an ask's language (design 47 G A-82).
Seats *BuildSeats
// Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase,
// before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault
// (ADR 0525 §6, §9), and the bound starts again when another node takes the ask.
@@ -142,10 +144,11 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error {
// putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node
// that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx.
func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, error) {
module, err := buildstep.BuildModuleOf(a.Language)
bm, err := k.Seats.OfAskLanguage(a.Language)
if err != nil {
return buildstep.Step{}, err
}
module := bm.Module
watch, cancel := context.WithCancel(ctx)
defer cancel()
steps, err := k.Waiter.WatchAsk(watch, module, a.ID)
modules/mesh-delivery-builder/buildrun/asks_integration_test.go
+1 -1
View File
@@ -34,7 +34,7 @@ func TestAnAskIsPutOnceAndItsStepIsRead(t *testing.T) {
t.Fatal(err)
}
bus := JetStreamAsks{JS: js}
k := Asker{Publisher: bus, Waiter: bus, Bound: 5 * time.Second}
k := Asker{Seats: testSeats(), Publisher: bus, Waiter: bus, Bound: 5 * time.Second}
a := goAsk("mvl0frv18d1960")
for i := 0; i < 2; i++ {
if err := k.Put(ctx, a); err != nil {
modules/mesh-delivery-builder/buildrun/asks_test.go
+21 -21
View File
@@ -31,7 +31,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[]}`)}
}}
subjects := &subjectsOf{inner: f}
k := Asker{Publisher: subjects, Waiter: f}
k := Asker{Seats: testSeats(), Publisher: subjects, Waiter: f}
var seen, also []buildstep.Step // issue 1890: a waiter that asks to see each value of the step sees it
// Issue 1921: a second seen on the same context (a check's begin bound inside a try) is added, never in place of the first.
both := WithStepSeen(WithStepSeen(context.Background(), func(s buildstep.Step) { seen = append(seen, s) }),
@@ -68,7 +68,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
if c.transport {
restore = busretry.Waits(20*time.Millisecond, time.Millisecond, time.Millisecond) // a bound this row passes
}
err := (Asker{Publisher: p, Waiter: p.inner}).Put(context.Background(), goAsk("mvl0frv18d1960"))
err := (Asker{Seats: testSeats(), Publisher: p, Waiter: p.inner}).Put(context.Background(), goAsk("mvl0frv18d1960"))
restore()
switch {
case c.transport:
@@ -94,7 +94,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
for _, st := range order {
h.Write(st)
}
if got, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1971")); err != nil || got.State != buildstep.StepPassed {
if got, err := (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1971")); err != nil || got.State != buildstep.StepPassed {
t.Errorf("%s: %+v, %v", name, got, err)
}
}
@@ -106,12 +106,12 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
}
h.Write(st)
}
if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "transport fault: name lookup, node-build-go shanks, lookup proxy: i/o timeout") {
if _, err := (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "transport fault: name lookup, node-build-go shanks, lookup proxy: i/o timeout") {
t.Errorf("three runs then failed: %v", err)
}
h = &FakeAsks{}
h.Write(buildstep.Step{ID: "mvl0frv18d1973", Node: "ace", State: buildstep.StepAgain, Run: 1, AgainAt: time.Now(), Started: time.Now()})
if _, err := (Asker{Publisher: h, Waiter: h, Bound: 20 * time.Millisecond}).Wait(context.Background(), goAsk("mvl0frv18d1973")); err == nil || !strings.Contains(err.Error(), "given again after a transport fault") {
if _, err := (Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: 20 * time.Millisecond}).Wait(context.Background(), goAsk("mvl0frv18d1973")); err == nil || !strings.Contains(err.Error(), "given again after a transport fault") {
t.Errorf("an ask never given again: %v", err)
}
// ADR 0664 §5: that ask is dropped; a wait ended by a named cause is dropped too, an ask no node took included (it waits in
@@ -119,19 +119,19 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
drops := &seatDrops{}
h = &FakeAsks{}
h.Write(buildstep.Step{ID: "mvl0frv18d1974", Node: "ace", State: buildstep.StepAgain, Run: 1, AgainAt: time.Now(), Started: time.Now()})
(Asker{Publisher: h, Waiter: h, Bound: 20 * time.Millisecond, Dropper: drops}).Wait(context.Background(), goAsk("mvl0frv18d1974"))
(Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: 20 * time.Millisecond, Dropper: drops}).Wait(context.Background(), goAsk("mvl0frv18d1974"))
named, stop := context.WithCancelCause(context.Background())
h.Write(buildstep.Step{ID: "mvl0frv18d1975", Node: "g14", State: buildstep.StepRunning, Started: time.Now()})
seenStep := WithStepSeen(named, func(buildstep.Step) { stop(Withdrawn{Why: "superseded by a newer fingerprint"}) })
(Asker{Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(seenStep, goAsk("mvl0frv18d1975"))
(Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(seenStep, goAsk("mvl0frv18d1975"))
shut, off := context.WithCancel(context.Background())
off()
(Asker{Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(shut, goAsk("mvl0frv18d1977"))
(Asker{Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(named, goAsk("mvl0frv18d1976"))
(Asker{Seats: testSeats(), Publisher: h, Waiter: h, Bound: time.Hour, Dropper: drops}).Wait(shut, goAsk("mvl0frv18d1977"))
(Asker{Seats: testSeats(), Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(named, goAsk("mvl0frv18d1976"))
// Review 1 (finding 1): a cause that is no withdrawal (the builder's check bound passing, which waits on) drops nothing.
bounded, passed := context.WithCancelCause(context.Background())
passed(errors.New("the check bound passed"))
(Asker{Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(bounded, goAsk("mvl0frv18d1978"))
(Asker{Seats: testSeats(), Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: drops}).Wait(bounded, goAsk("mvl0frv18d1978"))
if got := strings.Join(drops.got, "|"); !strings.Contains(got, "mvl0frv18d1974") || !strings.Contains(got, "mvl0frv18d1975 superseded by a newer fingerprint") || !strings.Contains(got, "mvl0frv18d1976 superseded by a newer fingerprint") || len(drops.got) != 3 {
t.Errorf("drops %q, want the ask never given again, the superseded one and the one no node took, and no other", got)
}
@@ -139,20 +139,20 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
// how many runs; a drop that failed is said, not discarded.
late := &FakeAsks{}
late.Write(buildstep.Step{ID: "mvl0frv18d1979", Node: "shanks", State: buildstep.StepFailed, Passing: true, Run: 3, Reason: "lookup proxy: i/o timeout"})
if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.HasPrefix(st.Reason, "transport fault: name lookup, node-build-go shanks") {
if st, err := (Asker{Seats: testSeats(), Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.HasPrefix(st.Reason, "transport fault: name lookup, node-build-go shanks") {
t.Errorf("a failed step of run 3 read alone says %q (%v), want it names 3 runs", st.Reason, err)
}
var said []string
bad, off2 := context.WithCancelCause(context.Background())
off2(Withdrawn{Why: "withdrawn"})
(Asker{Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: failingDrops{}, Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}).Wait(bad, goAsk("mvl0frv18d1980"))
(Asker{Seats: testSeats(), Publisher: &FakeAsks{}, Waiter: &FakeAsks{}, Bound: time.Hour, Dropper: failingDrops{}, Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}).Wait(bad, goAsk("mvl0frv18d1980"))
if len(said) != 1 || !strings.Contains(said[0], "mvl0frv18d1980") || !strings.Contains(said[0], "no holder") {
t.Errorf("a drop that failed said %q, want it named with its cause", said)
}
// review round 2 item 1: an ask that waits in its stream for a free slot is not bounded; once a node writes its step
// running, the step is bounded from its start, and the error names the node.
g := &FakeAsks{}
k = Asker{Publisher: g, Waiter: g, Bound: 20 * time.Millisecond}
k = Asker{Seats: testSeats(), Publisher: g, Waiter: g, Bound: 20 * time.Millisecond}
waiting, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if _, err := k.Wait(waiting, goAsk("mvl0frv18d1959")); err != context.DeadlineExceeded {
@@ -170,7 +170,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
// the phase; one three bounds from its start ends, said as kept moving. The steps' own times stand for the clock, so the
// bound is an hour no loaded machine reaches and no case waits on a pause (the check of 2026-11-02 22:35 tripped one
// of 60 ms).
k = Asker{Publisher: g, Waiter: g, Bound: time.Hour}
k = Asker{Seats: testSeats(), Publisher: g, Waiter: g, Bound: time.Hour}
now := time.Now()
g.Write(buildstep.Step{ID: "mvl0frv18d1980", Node: "ace", State: buildstep.StepRunning, Started: now.Add(-90 * time.Minute), Phase: "pb", PhaseAt: now})
g.Write(buildstep.Step{ID: "mvl0frv18d1980", Node: "ace", State: buildstep.StepPassed})
@@ -190,7 +190,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
// first is answered as the node wrote it.
dead := &DeadLetters{}
lw := LostAskWaiter{Inner: g, Dead: dead}
k = Asker{Publisher: g, Waiter: lw, Bound: time.Minute}
k = Asker{Seats: testSeats(), Publisher: g, Waiter: lw, Bound: time.Minute}
g.Write(buildstep.Step{ID: "mvl0frv18d1970", Node: "ace", State: buildstep.StepRunning, Started: time.Now()})
given, ok := GivenUpOf(map[string][]string{"Mesh-Dead-Msg-Id": {"mvl0frv18d1970"}, "Mesh-Dead-Deliveries": {"5"},
"Mesh-Dead-Stream": {"SEAT_NODE_BUILD_GO"}, "Mesh-Dead-Gave-Up": {"2026-10-26T12:26:40Z"}})
@@ -225,7 +225,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
if why := hearing.Deaf(); !strings.Contains(why, "SEAT_NODE_BUILD_GO: permissions violation") {
t.Fatalf("a refused hearing says %q", why)
}
k = Asker{Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute}
k = Asker{Seats: testSeats(), Publisher: g, Waiter: LostAskWaiter{Inner: g, Dead: dead, Hearing: hearing}, Bound: time.Minute}
g.Write(buildstep.Step{ID: "mvl0frv18d1973", Node: "g14", State: buildstep.StepPassed})
if _, err := k.Wait(context.Background(), goAsk("mvl0frv18d1973")); err != nil {
t.Fatal(err)
@@ -245,25 +245,25 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
// ended) and answered; a nudge of another build module reads nothing; a watch that closes is opened again.
d := &FakeAsks{}
heard := &HeardEnds{Inner: deafWatch{}, Read: d}
k = Asker{Publisher: d, Waiter: heard, Bound: time.Minute}
k = Asker{Seats: testSeats(), Publisher: d, Waiter: heard, Bound: time.Minute}
d.Write(buildstep.Step{ID: "mvl0frv18d1980", Node: "shanks", State: buildstep.StepPassed})
got := make(chan buildstep.Step, 1)
go func() { s, _ := k.Wait(context.Background(), goAsk("mvl0frv18d1980")); got <- s }()
for heard.Waiting() == 0 {
time.Sleep(time.Millisecond)
}
heard.Nudge(NodeBuildModule)
heard.Nudge(testNode)
select {
case s := <-got:
t.Fatalf("a nudge of another build module answered %+v", s)
case <-time.After(20 * time.Millisecond):
}
heard.Nudge(GoBuildModule)
heard.Nudge(testGo)
if s := <-got; s.State != buildstep.StepPassed || s.Node != "shanks" {
t.Fatalf("a step its watch never handed over: %+v", s)
}
closing := &HeardEnds{Inner: &closingWatch{inner: d}, Read: d}
if s, err := (Asker{Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != buildstep.StepPassed {
if s, err := (Asker{Seats: testSeats(), Publisher: d, Waiter: closing}).Wait(context.Background(), goAsk("mvl0frv18d1980")); err != nil || s.State != buildstep.StepPassed {
t.Fatalf("a watch that closed once: %+v, %v", s, err)
}
for _, row := range []struct {
@@ -287,7 +287,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
// bound is reached only by a wait that kept the first run's start.
func TestARedeliveryToTheSameNodeRestartsTheBound(t *testing.T) {
f := &FakeAsks{}
k := Asker{Publisher: f, Waiter: f, Bound: time.Hour}
k := Asker{Seats: testSeats(), Publisher: f, Waiter: f, Bound: time.Hour}
const id = "mvl0frv18d1962"
now := time.Now()
f.Write(buildstep.Step{ID: id, Node: "ace", State: buildstep.StepRunning, Started: now})
modules/mesh-delivery-builder/buildrun/buildseats.go
+166
View File
@@ -0,0 +1,166 @@
package buildrun
import (
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// **The builder names no build module** (design 47 G §7, §428, §704; A-82): a build module is the holder of the build seat
// whose acting rule `build` has an `on` matching an artifact's language or `kind:image`, found on the seat list (the seat
// definitions) and the catalog half (the manifest that claims the seat), as the indexer's seed finds it. Every task's module and
// every lane's key is that holder's name as the seat list and the catalog half give it.
// BuildModule is one build seat and the module that holds it.
type BuildModule struct {
Seat string // the seat whose acting rule builds
Module string // the module holding the seat, empty while the catalog half names none
// Toolchain is the language of the toolchain the rule's `on` names, empty for a rule on `kind:image` (no toolchain).
Toolchain string
// ByKind is whether the rule is on `kind:image`: its artifacts have no language and its nodes come from the node-engine's have.
ByKind bool
}
// AskLanguage is the language of the seat's asks (sdkstep.AskSubject): the seat is named node-build-<language>.
func (m BuildModule) AskLanguage() string { return strings.TrimPrefix(m.Seat, "node-build-") }
// BuildSeats finds the build modules from the seat definitions and the holders of seats, both read at each call.
type BuildSeats struct {
// Defs are the seat definitions of the seat list; Holder is a seat's holder on the catalog half.
defs func() map[string]seatdef.Definition
holder func(seat string) (string, bool)
}
// NewBuildSeats is the finder over defs (the seat list's definitions, nil before it is read) and holder.
func NewBuildSeats(defs func() map[string]seatdef.Definition, holder func(seat string) (string, bool)) *BuildSeats {
return &BuildSeats{defs: defs, holder: holder}
}
func (s *BuildSeats) all() []BuildModule {
if s == nil || s.defs == nil {
return nil
}
defs := s.defs()
var out []BuildModule
for _, seat := range slices.Sorted(maps.Keys(defs)) {
for _, r := range defs[seat].Acts {
if r.Step != seatdef.StepBuild || r.Acting() == seatdef.OnEvery {
continue
}
m := BuildModule{Seat: seat, ByKind: r.On == seatdef.OnKindImage}
if !m.ByKind {
m.Toolchain = r.Acting()
}
if s.holder != nil {
m.Module, _ = s.holder(seat)
}
out = append(out, m)
break
}
}
return out
}
// All are the build seats with the module holding each, by seat name; a seat no catalog entry claims has no module.
func (s *BuildSeats) All() []BuildModule { return s.all() }
// Modules are the holders of the build seats, sorted, none that is unnamed.
func (s *BuildSeats) Modules() []string {
var out []string
for _, m := range s.all() {
if m.Module != "" {
out = append(out, m.Module)
}
}
slices.Sort(out)
return slices.Compact(out)
}
// OfModule is the build seat held by module.
func (s *BuildSeats) OfModule(module string) (BuildModule, bool) {
for _, m := range s.all() {
if m.Module == module && module != "" {
return m, true
}
}
return BuildModule{}, false
}
// OfSeat is the build seat named seat.
func (s *BuildSeats) OfSeat(seat string) (BuildModule, bool) {
for _, m := range s.all() {
if m.Seat == seat {
return m, true
}
}
return BuildModule{}, false
}
// For is the build module of an artifact: the build seat whose rule's `on` is language, or `kind:image` when the artifact's
// kind is image. None, or more than one, or a seat no entry claims, is the error.
func (s *BuildSeats) For(language, kind string) (BuildModule, error) {
var found []BuildModule
for _, m := range s.all() {
if (kind == "image" && m.ByKind) || (!m.ByKind && language != "" && strings.EqualFold(m.Toolchain, strings.TrimSpace(language))) {
found = append(found, m)
}
}
switch len(found) {
case 0:
return BuildModule{}, fmt.Errorf("no seat builds an artifact of language %q and kind %q", language, kind)
case 1:
if found[0].Module == "" {
return BuildModule{}, fmt.Errorf("no module claims %s, which builds an artifact of language %q and kind %q", found[0].Seat, language, kind)
}
return found[0], nil
}
return BuildModule{}, fmt.Errorf("the seats %v each build an artifact of language %q and kind %q", seatsOf(found), language, kind)
}
func seatsOf(ms []BuildModule) []string {
var out []string
for _, m := range ms {
out = append(out, m.Seat)
}
return out
}
// LanguageOf is the language of the toolchain the build module builds with, false for a module that is no build seat's holder
// or builds with none.
func (s *BuildSeats) LanguageOf(module string) (string, bool) {
m, ok := s.OfModule(module)
return m.Toolchain, ok && m.Toolchain != ""
}
// Languages are the languages of the toolchains the build seats' rules name, sorted.
func (s *BuildSeats) Languages() []string {
var out []string
for _, m := range s.all() {
if m.Toolchain != "" {
out = append(out, m.Toolchain)
}
}
slices.Sort(out)
return slices.Compact(out)
}
// OfAskLanguage is the build module whose seat takes the asks of an ask language (go, node or image): the seat the sdk names
// for it (buildstep.BuildSeat), held by the module the catalog half names.
func (s *BuildSeats) OfAskLanguage(language string) (BuildModule, error) {
seat, err := buildstep.BuildSeat(language)
if err != nil {
return BuildModule{}, err
}
m, ok := s.OfSeat(seat)
if !ok {
return BuildModule{}, fmt.Errorf("the seat list holds no build seat %s", seat)
}
if m.Module == "" {
return BuildModule{}, fmt.Errorf("no module claims %s", seat)
}
return m, nil
}
modules/mesh-delivery-builder/buildrun/buildseats_test.go
+60
View File
@@ -0,0 +1,60 @@
package buildrun
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The build module of an artifact is the holder of the build seat whose rule `on` names its language, or kind:image, as the seat
// list and the catalog half name it (design 47 G A-82): no module name is in the code.
func TestTheBuildModuleIsTheHolderOfTheBuildSeat(t *testing.T) {
rule := func(step, on string) seatdef.Definition {
return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: step, On: on}}}
}
defs := map[string]seatdef.Definition{
"node-build-go": rule(seatdef.StepBuild, "go"), "node-build-node": rule(seatdef.StepBuild, "typescript"),
"node-build-image": rule(seatdef.StepBuild, seatdef.OnKindImage), "node-build-rust": rule(seatdef.StepBuild, "go"),
"mesh-checks": rule(seatdef.StepCheck, "go"), "node-builder-every": rule(seatdef.StepBuild, "every"),
}
holders := map[string]string{"node-build-go": "acme-go", "node-build-node": "acme-ts", "node-build-image": "acme-img"}
seats := NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(s string) (string, bool) { h, ok := holders[s]; return h, ok })
for _, c := range []struct{ language, kind, module, fail string }{
{"typescript", "bundle", "acme-ts", ""},
{"", "image", "acme-img", ""},
{"python", "bundle", "", "no seat builds"},
{"go", "bundle", "", "each build"}, // node-build-go and node-build-rust both build go
} {
got, err := seats.For(c.language, c.kind)
switch {
case c.fail != "" && (err == nil || !strings.Contains(err.Error(), c.fail)):
t.Errorf("For(%q, %q) = %+v, %v; want the error %q", c.language, c.kind, got, err, c.fail)
case c.fail == "" && (err != nil || got.Module != c.module):
t.Errorf("For(%q, %q) = %+v, %v; want %s", c.language, c.kind, got, err, c.module)
}
}
delete(defs, "node-build-rust")
if m, err := seats.For("go", "bundle"); err != nil || m.Module != "acme-go" || m.Toolchain != "go" || m.AskLanguage() != "go" {
t.Errorf("For(go) = %+v, %v", m, err)
}
if l, ok := seats.LanguageOf("acme-ts"); !ok || l != "typescript" {
t.Errorf("LanguageOf(acme-ts) = %q, %v", l, ok)
}
if _, ok := seats.LanguageOf("acme-img"); ok {
t.Error("an image build module has a toolchain language")
}
if got := strings.Join(seats.Modules(), ","); got != "acme-go,acme-img,acme-ts" {
t.Errorf("Modules = %s", got)
}
delete(holders, "node-build-go") // no catalog entry claims the seat: no build module, said by name
if _, err := seats.For("go", "bundle"); err == nil || !strings.Contains(err.Error(), "no module claims node-build-go") {
t.Errorf("a seat with no holder: %v", err)
}
if m, err := seats.OfAskLanguage("image"); err != nil || m.Module != "acme-img" {
t.Errorf("OfAskLanguage(image) = %+v, %v", m, err)
}
if got := NewBuildSeats(func() map[string]seatdef.Definition { return nil }, nil).Modules(); len(got) != 0 {
t.Errorf("before the seat list is read: %v", got)
}
}
modules/mesh-delivery-builder/buildrun/heardends.go
+24 -6
View File
@@ -36,6 +36,9 @@ type HeardEnds struct {
mu sync.Mutex
waiting map[string]*waitedAsk // by ask id
watched map[string]bool // the modules whose steps are watched
asks JetStreamAsks
ctx context.Context
}
type waitedAsk struct {
@@ -180,16 +183,31 @@ func HearEnds(ctx context.Context, asks JetStreamAsks, inner AskWaiter, modules
if logf == nil {
logf = func(string, ...any) {}
}
h := &HeardEnds{Inner: inner, Read: asks, Logf: logf}
h := &HeardEnds{Inner: inner, Read: asks, Logf: logf, asks: asks, ctx: ctx, watched: map[string]bool{}}
h.Watch(modules)
return h
}
// Watch starts the watch of the steps of each module not watched yet: the build modules are the holders of the build seats the
// seat list and the catalog half name (design 47 G A-82), known only once both are read, and again at each change of them.
func (h *HeardEnds) Watch(modules []string) {
for _, module := range modules {
kv, err := asks.JS.KeyValue(ctx, StepsBucket(module))
if err != nil {
logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err)
h.mu.Lock()
done := h.watched[module]
h.mu.Unlock()
if done {
continue
}
go h.follow(ctx, kv, module)
kv, err := h.asks.JS.KeyValue(h.ctx, StepsBucket(module))
if err != nil {
h.logf("the steps of %s cannot be read (%v): an ask whose own watch goes deaf is read only at a reconnect", module, err)
continue
}
h.mu.Lock()
h.watched[module] = true
h.mu.Unlock()
go h.follow(h.ctx, kv, module)
}
return h
}
func (h *HeardEnds) follow(ctx context.Context, kv jetstream.KeyValue, module string) {
modules/mesh-delivery-builder/buildrun/modules.go
-7
View File
@@ -1,7 +0,0 @@
package buildrun
// The build modules the mesh runs, by the language each builds (ADR 0469).
const (
GoBuildModule = "mesh-build-go"
NodeBuildModule = "mesh-build-node"
)
modules/mesh-delivery-builder/buildrun/seats_test.go
+22
View File
@@ -0,0 +1,22 @@
package buildrun
import (
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The build modules of the fixture seat list: the holders of the three build seats (design 47 G A-82).
const (
testGo = "mesh-build-go"
testNode = "mesh-build-node"
testImage = "mesh-build-image"
)
// testSeats is the finder over a seat list whose build seats act on go, typescript and kind:image, each held by its module.
func testSeats() *BuildSeats {
rule := func(on string) seatdef.Definition {
return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}}
}
defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript"), "node-build-image": rule(seatdef.OnKindImage)}
holders := map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage}
return NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok })
}
modules/mesh-delivery-builder/buildrun/standing_test.go
+3 -3
View File
@@ -41,7 +41,7 @@ func TestPlacedNodes(t *testing.T) {
t.Run("the assignments names the counted nodes once it is settled", func(t *testing.T) {
gate := NewPlacedNodes(func(string, ...any) {})
for _, node := range three {
k, v := placing(t, node, NodeBuildModule)
k, v := placing(t, node, testNode)
if gate.Assignments(k, v, false, old) {
t.Fatal("an assignments before it is settled changed the counted nodes")
}
@@ -50,7 +50,7 @@ func TestPlacedNodes(t *testing.T) {
if got := strings.Join(gate.Nodes(), ","); got != "g14,novox,shanks" {
t.Fatalf("nodes = %q", got)
}
k, v := placing(t, "ace", GoBuildModule)
k, v := placing(t, "ace", testGo)
if !gate.Assignments(k, v, false, old) || len(gate.Nodes()) != 4 {
t.Fatalf("ace placed: nodes %v", gate.Nodes())
}
@@ -65,7 +65,7 @@ func TestPlacedNodes(t *testing.T) {
at := old.Add(time.Duration(n) * time.Hour)
r := meshwants.AssignmentRevision{Revision: n, Kind: "trunk", At: &at}
for _, node := range nodes {
r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: NodeBuildModule, Digest: "d-" + node})
r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: testNode, Digest: "d-" + node})
}
return r
}
modules/mesh-delivery-builder/buildrun/toolchaingate_test.go
+32 -32
View File
@@ -42,16 +42,16 @@ func TestToolchainGate(t *testing.T) {
placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again
three := []string{"g14", "novox", "shanks"}
start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) {
set := &ToolchainSet{}
set := &ToolchainSet{Seats: testSeats()}
said := &[]string{}
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
gate := NewToolchainGate([]string{testGo, testNode}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
for _, node := range three {
if seeded {
key, value := placing(t, node, NodeBuildModule)
key, value := placing(t, node, testNode)
gate.Assignments(key, value, false, old.Add(-time.Hour))
}
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
gate.Toolchain(NodeBuildModule, key, value, false)
gate.Toolchain(testNode, key, value, false)
}
return gate, set, said
}
@@ -59,11 +59,11 @@ func TestToolchainGate(t *testing.T) {
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
gate, set, said := start(t, true)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(NodeBuildModule, key, value, false)
gate.Toolchain(testNode, key, value, false)
if a := buildstep.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
t.Fatalf("before the assignments is settled every report counts; got %+v", a)
}
k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter
k, v := placing(t, "ace", testGo) // another build module on ace: no matter
gate.Assignments(k, v, false, old)
if !gate.Settled() {
t.Fatal("settling left ace's report out, so what was passed on changed")
@@ -74,11 +74,11 @@ func TestToolchainGate(t *testing.T) {
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
t.Fatalf("ace's report is said left out once; said %q", *said)
}
if gate.Toolchain(NodeBuildModule, key, value, false) {
if gate.Toolchain(testNode, key, value, false) {
t.Fatal("a report of a node not placed changed what was passed on")
}
// Placed on ace with a report written since: it counts, and the builders really disagree.
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
k, v = placing(t, "ace", testGo, testNode)
gate.Assignments(k, v, false, old.Add(-time.Minute))
if a := buildstep.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
@@ -92,24 +92,24 @@ func TestToolchainGate(t *testing.T) {
}
// A report deleted from the bucket goes whether its node is placed or not.
key, _ = gateReport(t, "g14", "v26.12.1", placed)
if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 {
if !gate.Toolchain(testNode, key, nil, true) || len(set.Entries("typescript")) != 2 {
t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript")))
}
})
t.Run("build modules given later pass their reports on when the assignments moves", func(t *testing.T) {
set := &ToolchainSet{}
set := &ToolchainSet{Seats: testSeats()}
gate := NewToolchainGate(nil, set.Put, func(string, ...any) {})
gate.Settled()
k, v := placing(t, "novox", GoBuildModule)
k, v := placing(t, "novox", testGo)
gate.Assignments(k, v, false, old)
key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
gate.Toolchain(NodeBuildModule, key, value, false) // ace is not placed: left out
k, v = placing(t, "ace", NodeBuildModule)
gate.Toolchain(testNode, key, value, false) // ace is not placed: left out
k, v = placing(t, "ace", testNode)
if gate.Assignments(k, v, false, placed) || len(set.Entries("typescript")) != 0 {
t.Fatalf("a module the gate was not given passed on at the assignments: %v", set.Entries("typescript"))
}
if !gate.UseModules([]string{NodeBuildModule}) || len(set.Entries("typescript")) != 1 {
if !gate.UseModules([]string{testNode}) || len(set.Entries("typescript")) != 1 {
t.Fatalf("given mesh-build-node, ace's report placed since is passed on; entries %v", set.Entries("typescript"))
}
})
@@ -127,16 +127,16 @@ func TestToolchainGate(t *testing.T) {
k, v := placing(t, "ace") // ace's key, first heard placing nothing
gate.Assignments(k, v, false, old.Add(-time.Hour))
gate.Settled()
k, v = placing(t, "ace", NodeBuildModule)
k, v = placing(t, "ace", testNode)
if gate.Assignments(k, v, false, placed) {
t.Fatal("assignments with no report passed something on")
}
key, value := gateReport(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts {
if got := gate.Toolchain(testNode, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// ace's assignments written again with another module beside: its standing does not move, nothing is said again.
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
k, v = placing(t, "ace", testGo, testNode)
if gate.Assignments(k, v, false, placed.Add(time.Minute)) {
t.Fatal("the same assignments of mesh-build-node changed what was passed on")
}
@@ -150,7 +150,7 @@ func TestToolchainGate(t *testing.T) {
t.Fatalf("said %q, want once why ace's report is left out", *said)
}
key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
if !gate.Toolchain(testNode, key, value, false) || len(set.Entries("typescript")) != 4 {
t.Fatal("ace's new report was not passed on")
}
})
@@ -159,12 +159,12 @@ func TestToolchainGate(t *testing.T) {
t.Run("no assignments held: every report counts", func(t *testing.T) {
gate, set, _ := start(t, false)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(NodeBuildModule, key, value, false)
gate.Toolchain(testNode, key, value, false)
gate.Settled()
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("before the first assignments file every report counts; got %d", n)
}
k, v := placing(t, "g14", NodeBuildModule)
k, v := placing(t, "g14", testNode)
if !gate.Assignments(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 {
t.Fatalf("the first assignments leaves every node it does not place out; got %d", len(set.Entries("typescript")))
}
@@ -177,20 +177,20 @@ func TestToolchainGate(t *testing.T) {
// leaves out a node no revision places the build module on, and counts a node placed in a later revision only by a
// report written since that revision's time.
t.Run("fed by the assignment half", func(t *testing.T) {
set := &ToolchainSet{}
set := &ToolchainSet{Seats: testSeats()}
said := &[]string{}
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
gate := NewToolchainGate([]string{testGo, testNode}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
for _, node := range append([]string{"ace"}, three...) {
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
if node == "ace" {
key, value = gateReport(t, node, "v26.12.0", old)
}
gate.Toolchain(NodeBuildModule, key, value, false)
gate.Toolchain(testNode, key, value, false)
}
revision := func(n int64, at time.Time, nodes ...string) meshwants.AssignmentRevision {
r := meshwants.AssignmentRevision{Revision: n, Kind: "trunk", At: &at}
for _, node := range nodes {
r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: NodeBuildModule, Digest: "d-" + node})
r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: testNode, Digest: "d-" + node})
}
return r
}
@@ -207,7 +207,7 @@ func TestToolchainGate(t *testing.T) {
t.Fatalf("said %q, want why ace's old report is left out", *said)
}
key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
if !gate.Toolchain(testNode, key, value, false) || len(set.Entries("typescript")) != 4 {
t.Fatal("ace's report written since revision 8 is not passed on")
}
if !gate.Revision(revision(9, placed.Add(time.Hour), "ace", "novox", "shanks")) || len(set.Entries("typescript")) != 3 || len(gate.Nodes()) != 3 {
@@ -218,27 +218,27 @@ func TestToolchainGate(t *testing.T) {
t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) {
p := meshmoduleassignerclient.NewPlaced()
// ace's node key is the first revision heard of it: its module stands since before.
k, v := placing(t, "ace", NodeBuildModule)
k, v := placing(t, "ace", testNode)
p.Put(k, v, false, old)
if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 {
if _, changed, _ := p.Put("ace."+testNode, []byte("{}"), false, placed); len(changed) != 0 {
t.Errorf("a module key says which modules stand: %v", changed)
}
k2, v2 := placing(t, "g14")
p.Put(k2, v2, false, old)
k2, v2 = placing(t, "g14", NodeBuildModule)
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule {
k2, v2 = placing(t, "g14", testNode)
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != testNode {
t.Errorf("a module placed says changed %v", changed)
}
if _, _, err := p.Put("novox.node", []byte("not json"), false, old); err == nil {
t.Error("a value that does not read was not said")
}
if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() {
if s, ok := p.Since("ace", testNode); !ok || !s.IsZero() {
t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok)
}
if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) {
if s, ok := p.Since("g14", testNode); !ok || !s.Equal(placed) {
t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok)
}
if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
if got := p.PlacedNodes(testNode); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode())
}
})
modules/mesh-delivery-builder/buildrun/toolchains.go
+5 -6
View File
@@ -7,9 +7,6 @@ import (
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
// LanguageOf is the language each build module's toolchain reports hold.
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
// ToolchainsHeardState is the builder's old state section of the toolchains it heard (design 47 G §604, §546): one key
// <node>.<language>, the toolchain as the build seat said it.
const (
@@ -20,8 +17,10 @@ const (
// ToolchainSet holds each node's toolchain of a language, as the builder heard it (design 47 G §604). Safe for use from
// the consumer and from a build at once.
type ToolchainSet struct {
mu sync.Mutex
by map[string]map[string]buildstep.Toolchain // language -> node -> entry
// Seats finds the build module of a language: a report of a module that holds no build seat is left out (design 47 G A-82).
Seats *BuildSeats
mu sync.Mutex
by map[string]map[string]buildstep.Toolchain // language -> node -> entry
}
// Set takes a node's toolchain in and answers whether the language's entries changed (a node's version, recipe or
@@ -96,7 +95,7 @@ const (
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
// A key or a value that is not an entry of the module's language is left out.
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
language, ok := LanguageOf[module]
language, ok := s.Seats.LanguageOf(module)
if !ok {
return false
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go
+71 -9
View File
@@ -15,10 +15,15 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
@@ -35,8 +40,11 @@ type queueAsker func(key string, value []byte) (json.RawMessage, error)
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
//
// builder_versions, asked by a name, answers also what the verb reserved answers for it (reserve; design 47 G §572): the
// bundle calls the seat mesh-builds through its invokes, as any caller does. Nil answers none.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time, vs versions.Reader,
fw, of versions.Reader) []stdio.Tool {
fw, of versions.Reader, reserve func(name, version string) (meshbuilds.ReservedAnswer, error)) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
// handAct writes a reasks value of a versions hand act and answers the builder's answer, or its refusal as an error.
handAct := func(key string, value []byte, err error) (any, error) {
@@ -83,7 +91,16 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
"version": map[string]any{"type": "string", "description": "one version, to ask whether it is recorded"},
"digest": map[string]any{"type": "string", "description": "with version: the digest the holder received"}}},
Run: func(args map[string]any) (any, error) {
return versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest"))
out, err := versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest"))
if err != nil || reserve == nil {
return out, err
}
ra, err := reserve(out.Name, str(args, "version"))
if err != nil {
return nil, fmt.Errorf("the seat mesh-builds' reserved did not answer for %s: %w", out.Name, err)
}
out.Reserved = &ra
return out, nil
}},
{Name: build.StuckTool,
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
@@ -307,13 +324,15 @@ func written(s stdio.KeptState, key string, revision uint64) time.Time {
return time.Time{}
}
// The seat verbs the bundle calls (the manifest's invokes).
const (
catalogVersionsVerb = "seat:mesh-catalog.versions"
reservedVerb = "seat:mesh-builds.reserved"
)
func main() {
steps := map[string]build.StateRead{}
for _, bm := range build.StuckBuildModules {
steps[bm] = read(stdio.State(bm + ".steps"))
}
reasks := stdio.State(build.ReAsksState)
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), StepsOf: func(seat string) build.StateRead { return read(stdio.State(seat + ".steps")) },
Hearing: read(stdio.State(buildrun.HearingState))}
put := func(key string, value []byte) error {
_, err := reasks.Put(key, json.RawMessage(value))
@@ -363,10 +382,52 @@ func main() {
}
return e.Value, true, nil
}}
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr)
// The catalog half's view of the packages the bundle follows itself (design 47 G §572, A-70), as other bundles follow views.
pk := &views.Reader{Module: builds.BuilderModule, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) },
Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}}
branches, err := meshcatalogclient.FollowBranches(pk, stdio.Ask)
if err != nil {
fmt.Fprintln(os.Stderr, "mesh-delivery-builder-tools:", err)
os.Exit(1)
}
pk.Start(context.Background())
// The build modules are no names (design 47 G A-82): the build seats on the seat list, each held by the module the catalog
// half's manifests say; the seat list and its two tools wired once by mesh-sdk (§204).
gate := seatlist.NewPart("builder", []string{stdio.AnyPerson, "node-tools", "mesh-healers"})
go func() {
<-stdio.Served()
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) })
}()
r.Seats = buildrun.NewBuildSeats(func() map[string]seatdef.Definition {
if rd := gate.Reader(); rd != nil {
return rd.Definitions()
}
return nil
}, versions.HolderOf(branches))
carriers := versions.CarriersFrom(versions.EntriesOf(branches))
released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) {
raw, err := stdio.Ask(catalogVersionsVerb, meshcatalog.VersionsInput{Module: module})
if err != nil {
return nil, err
}
var a meshcatalog.VersionsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the seat mesh-catalog's versions answered what does not read: %w", err)
}
return a.Versions, nil
}
reserve := func(name, version string) (meshbuilds.ReservedAnswer, error) {
raw, err := stdio.Ask(reservedVerb, meshbuilds.ReservedInput{Name: name, Version: version})
if err != nil {
return meshbuilds.ReservedAnswer{}, err
}
var a meshbuilds.ReservedAnswer
return a, json.Unmarshal(raw, &a)
}
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve)
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
Newest: newestVia(stdio.Ask), Steps: r}))
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask}, carriers, released)...)
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
ws := stdio.State(build.WaitsState)
@@ -392,6 +453,7 @@ func main() {
// builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's
// states stand, and asking one again by hand (ADR 0615, DLV-M1).
tools = append(tools, views.StatusTools(builds.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
tools = append(tools, gate.Tools()...)
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go
+11 -3
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"errors"
"fmt"
"strings"
@@ -16,7 +17,10 @@ import (
// reservedTools are builder_reserved, builder_reserved_drop and the seat mesh-builds' verb reserved (design 47 G section
// 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to
// reasks for the builder, which owns the reservations, to answer.
func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
//
// The verb reserved given a name finds the module from the catalog half's view the bundle follows (carriers) and, for a version
// no reservation holds, asks the seat mesh-catalog's verb versions (released); the builder's process serves no call (A-70).
func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder, carriers func(context.Context, string) ([]versions.Built, error), released versions.CatalogVersions) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: versions.ReservedTool, Description: versions.ReservedDescription,
@@ -25,10 +29,14 @@ func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, act
Run: func(args map[string]any) (any, error) { return versions.AnswerReserved(rr, str(args, "module")) }},
{Name: versions.ReservedVerb, Description: "The seat mesh-builds' verb reserved: a module's reserved version with each artifact's digest once built " +
"(design 47 G section 369), for a registry holder deciding whether to publish. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module"},
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module; give this or name"},
"name": map[string]any{"type": "string", "description": "a package's or an image's name in its ecosystem, in place of module"},
"version": map[string]any{"type": "string", "description": "the version, when asked for a particular one"}}},
Run: func(args map[string]any) (any, error) {
if name := str(args, "name"); name != "" {
return versions.AnswerReservedByName(context.Background(), rr, carriers, released, name, str(args, "version"))
}
return versions.AnswerReservedVerb(rr, str(args, "module"), str(args, "version"))
}},
{Name: versions.ReservedDropTool, Description: versions.ReservedDropDescription,
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reservedname_test.go
+58
View File
@@ -0,0 +1,58 @@
package main
import (
"context"
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// Design 47 G §572, §582, A-70: the bundle's verb reserved takes a name and answers from the view it follows and the catalog's
// versions, and builder_versions, asked by a name, calls reserved and answers what it answers.
func TestReservedByNameIsTheBundlesAndBuilderVersionsCallsIt(t *testing.T) {
none := versions.Reader{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, bool, error) { return nil, false, nil }}
carriers := func(_ context.Context, name string) ([]versions.Built, error) {
if name == "@novox/mesh-sdk" {
return []versions.Built{{Module: "mesh-sdk", Artifact: "ts"}}, nil
}
return nil, nil
}
released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) {
return []meshcatalog.VersionEntry{{Module: module, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mesh-sdk:ts", Digest: "sha256:dd"}}}}, nil
}
acts := meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }}
byName := map[string]func(map[string]any) (any, error){}
for _, tl := range reservedTools(none, nil, time.Now, acts, carriers, released) {
byName[tl.Name] = tl.Run
}
got, err := byName[versions.ReservedVerb](map[string]any{"name": "@novox/mesh-sdk", "version": "0.1.2"})
want := meshbuilds.ReservedAnswer{Module: "mesh-sdk", Version: "0.1.2", Released: true, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{{Artifact: "mesh-sdk:ts", Digest: "sha256:dd"}}}
if err != nil || got.(meshbuilds.ReservedAnswer).Released != want.Released || got.(meshbuilds.ReservedAnswer).Artifacts[0] != want.Artifacts[0] {
t.Fatalf("reserved by name = %+v (%v), want %+v", got, err, want)
}
var called []string
reserve := func(name, version string) (meshbuilds.ReservedAnswer, error) {
called = append(called, name+"@"+version)
return want, nil
}
tools := Tools(build.StuckReader{}, nil, nil, nil, nil, time.Now, none, none, none, reserve)
for _, tl := range tools {
if tl.Name == versions.VersionsTool {
out, err := tl.Run(map[string]any{"name": "@novox/mesh-sdk", "version": "0.1.2"})
ans, ok := out.(versions.VersionsAnswer)
if err != nil || !ok || ans.Reserved == nil || ans.Reserved.Module != "mesh-sdk" || !ans.Reserved.Released {
t.Fatalf("builder_versions by name = %+v (%v), want reserved's answer in it", out, err)
}
}
}
if len(called) != 1 || called[0] != "@novox/mesh-sdk@0.1.2" {
t.Errorf("reserved was called %v, want once, by the name and version", called)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go
-24
View File
@@ -3,13 +3,9 @@ package main
import (
"context"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-builder/internal/build"
)
@@ -106,23 +102,3 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key
}
}
}
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
type manifestReader struct{ kv jetstream.KeyValue }
func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) {
key, err := meshcatalogclient.ManifestsKey(digest)
if err != nil {
return catalogue.Manifest{}, err
}
e, err := m.kv.Get(ctx, key)
if err != nil {
return catalogue.Manifest{}, fmt.Errorf("manifests.%s: %w", key, err)
}
kept, err := meshcatalogclient.DecodeManifest(e.Value())
if err != nil {
return catalogue.Manifest{}, err
}
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
return catalogue.Decode([]byte(kept.Manifest))
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog.go
+202
View File
@@ -0,0 +1,202 @@
package main
import (
"context"
"encoding/json"
"fmt"
"slices"
"sync"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-builder/internal/build"
)
// **The builder reads the catalog half per watched branch** (design 47 G §348, §479, §614; A-5, A-15): the shared branch follower
// of meshcatalogclient, adopted on the builder's views reader, gives each watched branch's entries; the manifests, the packages
// and each package's name in its ecosystem are read from the entries, never the indexer's states.
// entryManifests keeps the manifest of each entry heard, by the entry's manifest digest (the entry holds the manifest whole).
type entryManifests struct {
mu sync.Mutex
byFp map[string]catalogue.Manifest
}
// keep takes the manifest of e in; an entry with no manifest or digest keeps none.
func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error {
if e.ManifestDigest == "" || len(e.Manifest) == 0 {
return nil
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return err
}
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
mf, err := catalogue.Decode(raw)
if err != nil {
return fmt.Errorf("the manifest of the entry of %s does not decode: %w", e.Module, err)
}
m.mu.Lock()
defer m.mu.Unlock()
if m.byFp == nil {
m.byFp = map[string]catalogue.Manifest{}
}
m.byFp[e.ManifestDigest] = mf
return nil
}
// Manifest is the manifest the entry heard with that digest holds.
func (m *entryManifests) Manifest(_ context.Context, digest string) (catalogue.Manifest, error) {
m.mu.Lock()
defer m.mu.Unlock()
mf, ok := m.byFp[digest]
if !ok {
return catalogue.Manifest{}, fmt.Errorf("no entry of the catalog half holds the manifest %s", digest)
}
return mf, nil
}
// catalogHalf is the builder's part over the branch follower: what changes in a branch's half goes to the builder, and the
// entries of every followed half are what the package names are read from.
type catalogHalf struct {
branches *meshcatalogclient.Branches
b *build.Builder
manifests *entryManifests
logf func(string, ...any)
// named is, by module, the package names of its first entry last heard, so a name newly heard looks at its module again.
mu sync.Mutex
named map[string]string
// ready is told when every followed branch's first view is in, once; halves are the views adopted (sweepWiring).
ready func()
halves map[string]*views.Follower
once sync.Once
ctx context.Context
// holders is told each entry change: a claim of a build seat may have moved (design 47 G A-82).
holders func()
}
// first says whether branch is the first watched branch of module's registration (design 47 G §371); a module with no
// registration heard is taken as watched on this branch, the first one.
func (c *catalogHalf) first(branch, module string) bool {
reg, ok := c.branches.Registrations.Registration(module)
return !ok || len(reg.Branches) == 0 || reg.Branches[0] == branch
}
// Changed is the follower's Changed: one module's entry of a branch came, or went.
func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, entry json.RawMessage, deleted bool, at time.Time) {
return func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) {
if c.holders != nil {
defer c.holders()
}
first := c.first(branch, module)
if deleted {
c.b.EntryGone(branch, first, module)
return
}
origin := build.Origin{Branch: branch, First: first}
if cat, ok := c.branches.Catalog(branch); ok {
if half, ok := cat.View(); ok {
origin.Source = half.Revision.Source
}
}
var e meshcatalog.CatalogEntry
if err := json.Unmarshal(entry, &e); err != nil || e.Module != module {
c.logf("the catalog half of the branch %q holds what is not the entry of %s, left: %v", branch, module, err)
return
}
if err := c.manifests.keep(e); err != nil {
c.logf("%v", err)
}
c.b.Entry(ctx, origin, e)
c.mu.Lock()
was := c.named[module]
now := namesOf(e)
if c.named == nil {
c.named = map[string]string{}
}
c.named[module] = now
c.mu.Unlock()
if first && now != "" && now != was {
c.logf("the catalog half's entry names a package of %s: its build is looked at again", module)
c.b.PackageNamed(ctx, module)
}
}
}
// namesOf are the package names an entry's artifacts carry, in one line.
func namesOf(e meshcatalog.CatalogEntry) string {
var names []string
for _, a := range e.Artifacts {
if a.Package != nil && a.Package.Name != "" {
names = append(names, a.Ref+"="+a.Package.Name)
}
}
slices.Sort(names)
return fmt.Sprint(names)
}
// sweepWiring is the follower's wiring on the builder's reader that learns each catalog half view adopted, so ready is told once
// when the first view of every branch followed is in (a branch with no revision yet is ready and empty at once).
type sweepWiring struct {
meshcatalogclient.Wiring
half *catalogHalf
}
func (w sweepWiring) Adopt(f *views.Follower, seatName, event string, hear func(context.Context, []byte) error) error {
if err := w.Wiring.Adopt(f, seatName, event, hear); err != nil {
return err
}
if seatName == meshcatalog.Seat && event == meshcatalog.EventCatalogRevision {
w.half.adopted(f)
}
return nil
}
func (w sweepWiring) Drop(owner, state string) error {
err := w.Wiring.Drop(owner, state)
w.half.dropped(state)
return err
}
func (c *catalogHalf) adopted(f *views.Follower) {
c.mu.Lock()
if c.halves == nil {
c.halves = map[string]*views.Follower{}
}
c.halves[f.State] = f
c.mu.Unlock()
go func() {
select {
case <-c.ctx.Done():
case <-f.Ready():
c.checkReady()
}
}()
}
func (c *catalogHalf) dropped(state string) {
c.mu.Lock()
delete(c.halves, state)
c.mu.Unlock()
c.checkReady()
}
// checkReady calls ready once every adopted half's first view is in.
func (c *catalogHalf) checkReady() {
c.mu.Lock()
all := len(c.halves) > 0
for _, f := range c.halves {
select {
case <-f.Ready():
default:
all = false
}
}
c.mu.Unlock()
if all && c.ready != nil {
c.once.Do(c.ready)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalog_test.go
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"context"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest
// digest (design 47 G §348), and an entry with no manifest keeps none.
func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) {
m := &entryManifests{}
e := meshcatalog.CatalogEntry{Module: "netcheck", ManifestDigest: "sha256:aa", Manifest: map[string]any{"module": "netcheck", "version": "1"}}
if err := m.keep(e); err != nil {
t.Fatal(err)
}
if mf, err := m.Manifest(context.Background(), "sha256:aa"); err != nil || mf.Module != "netcheck" {
t.Errorf("Manifest(sha256:aa) = %+v, %v", mf, err)
}
if _, err := m.Manifest(context.Background(), "sha256:bb"); err == nil {
t.Error("Manifest of a digest no entry holds: want an error")
}
if err := m.keep(meshcatalog.CatalogEntry{Module: "bare"}); err != nil || len(m.byFp) != 1 {
t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/catalogholder_integration_test.go
+153
View File
@@ -0,0 +1,153 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
)
// catalogHolder is the holder of the seat mesh-catalog as the bed needs it (design 47 G §348, §614): it answers `revision` for the
// branch main with the entries put so far, `registrations` with none (so the follower follows the branches the gate's `source`
// answers) and `entry`, and says `catalog-revision` at each put, as the follower and the builder hear it.
type catalogHolder struct {
ctx context.Context
js jetstream.JetStream
mu sync.Mutex
entries map[string]meshcatalog.CatalogEntry
revision int64
}
func newCatalogHolder(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder {
t.Helper()
h := &catalogHolder{ctx: ctx, js: js, entries: map[string]meshcatalog.CatalogEntry{}}
answer := func(subject string, with func(in []byte) any) {
s, err := nc.Subscribe(subject, func(m *nats.Msg) {
reply, _ := json.Marshal(map[string]any{"result": with(m.Data), "node": "ctl"})
_ = m.Respond(reply)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Unsubscribe() })
}
answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbRevision+".>", func([]byte) any { return h.whole() })
answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbRegistrations+".>", func([]byte) any {
return meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}}
})
answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbEntry+".>", func(in []byte) any {
var q meshcatalog.EntryInput
_ = json.Unmarshal(in, &q)
h.mu.Lock()
defer h.mu.Unlock()
for _, e := range h.entries {
if e.Fp == q.Fingerprint {
return meshcatalog.EntryAnswer{Entry: e}
}
}
return map[string]any{"error": "no entry of that fingerprint"}
})
answer("mesh.seat."+meshdelivery.Seat+".call."+meshdelivery.VerbSource+".>", func([]byte) any {
return meshdelivery.SourceAnswer{Branches: []string{"main"}}
})
return h
}
// whole is the newest trunk revision of main, every entry in one page; the branch has none yet before the first put.
func (h *catalogHolder) whole() meshcatalog.RevisionAnswer {
h.mu.Lock()
defer h.mu.Unlock()
if h.revision == 0 {
return meshcatalog.RevisionAnswer{NoneYet: true}
}
var refs []meshcatalog.EntryRef
var all []meshcatalog.CatalogEntry
for m, e := range h.entries {
refs = append(refs, meshcatalog.EntryRef{Module: m, Digest: "sha256:" + e.Fp})
all = append(all, e)
}
at := time.Now()
return meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: h.revision, Kind: "trunk", Branch: "main", Entries: refs, Source: 1, At: &at}, Entries: all}
}
// put sets one module's entry in a new trunk revision of main and says catalog-revision.
func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { h.putAll(t, e) }
// putAll sets several modules' entries in one new trunk revision of main.
func (h *catalogHolder) putAll(t *testing.T, es ...meshcatalog.CatalogEntry) {
t.Helper()
h.mu.Lock()
h.revision++
var changed []string
for _, e := range es {
h.entries[e.Module] = e
changed = append(changed, e.Module)
}
ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: changed, V: 1})
h.mu.Unlock()
if _, err := h.js.Publish(h.ctx, meshcatalog.SubjectCatalogRevision("main"), ev); err != nil {
t.Fatal(err)
}
}
// buildSeats is the seat list and the catalog half as the builder finds its build modules from them (design 47 G A-82): the seat
// mesh-catalog's verb seat-definitions answers the three build seats' definitions, each acting on its language (the image seat on
// kind:image), and an entry of each holder's manifest claims its seat.
func (h *catalogHolder) buildSeats(t *testing.T, nc *nats.Conn, holders map[string]string) {
t.Helper()
defs := map[string]seatdef.Definition{}
for _, d := range seatlisttest.Definitions() {
if _, ok := holders[d.Seat]; ok {
defs[d.Seat] = d
}
}
for seat, on := range map[string]string{"node-build-go": "go", "node-build-node": "typescript", "node-build-image": seatdef.OnKindImage} {
d := defs[seat]
d.Acts = []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}
defs[seat] = d
}
s, err := nc.Subscribe("mesh.seat."+meshcatalog.Seat+".call."+seatlist.Verb+".>", func(m *nats.Msg) {
var in struct {
Seat string `json:"seat"`
}
_ = json.Unmarshal(m.Data, &in)
var result any
if in.Seat == "" {
l := seatlist.Listing{Revision: 1}
for seat := range defs {
l.Seats = append(l.Seats, seatlist.Entry{Seat: seat, Source: seatlist.SourceTrunk, Digest: "sha256:" + seat})
}
result = l
} else {
raw, _ := json.Marshal(defs[in.Seat])
result = seatlist.Answer{Seat: in.Seat, Found: true, Revision: 1, Entry: &seatlist.Entry{Seat: in.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:" + in.Seat, Definition: raw}}
}
reply, _ := json.Marshal(map[string]any{"result": result, "node": "ctl"})
_ = m.Respond(reply)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = s.Unsubscribe() })
var entries []meshcatalog.CatalogEntry
for seat, module := range holders {
entries = append(entries, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:" + fmt.Sprintf("%064x", len(module)*7919+int(module[len(module)-1])),
Manifest: map[string]any{"module": module, "claims": []any{map[string]any{"name": seat}}}})
}
h.putAll(t, entries...)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go
+8 -3
View File
@@ -19,7 +19,10 @@ import (
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
// be answered (issue 1945). Never polled.
type heldOnTheBus struct{ js jetstream.JetStream }
type heldOnTheBus struct {
js jetstream.JetStream
seats *buildrun.BuildSeats // the build modules (design 47 G A-82)
}
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
// the ask's subject whose Nats-Msg-Id is the ask's id.
@@ -51,10 +54,11 @@ const (
// the ask is on its stream.
func (h heldOnTheBus) seen(ctx context.Context, a buildstep.StepAsk, later bool) (build.AskSeen, error) {
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
module, err := buildstep.BuildModuleOf(a.Language)
bm, err := h.seats.OfAskLanguage(a.Language)
if err != nil {
return seen, err
}
module := bm.Module
steps, err := h.js.KeyValue(ctx, buildrun.StepsBucket(module))
if err != nil {
return seen, fmt.Errorf("the bucket %s: %w", buildrun.StepsBucket(module), err)
@@ -259,10 +263,11 @@ func (h heldOnTheBus) Drop(ctx context.Context, a buildstep.StepAsk) error {
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
q := build.SeatQueue{Language: language, Note: build.QueueNote}
module, err := buildstep.BuildModuleOf(language)
bm, err := h.seats.OfAskLanguage(language)
if err != nil {
return q, err
}
module := bm.Module
stream, name, _, err := h.stream(ctx, language, buildstep.AskDelivery)
q.Stream = name
if errors.Is(err, jetstream.ErrStreamNotFound) {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+73 -165
View File
@@ -28,15 +28,12 @@ import (
runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
@@ -201,38 +198,22 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
}
}
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
// trunkHead is a trunk head of novox/mesh-catalog.
const trunkHead = "3333333333333333333333333333333333333333"
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
func (b *bed) trunk(t *testing.T, head string) uint64 {
t.Helper()
data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{V: keys.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
if err != nil {
t.Fatal(err)
}
rev, err := b.source.Put(b.ctx, meshdeliveryclient.SourceKey("main"), data)
if err != nil {
t.Fatal(err)
}
return rev
}
type bed struct {
ctx context.Context
js jetstream.JetStream
modules jetstream.KeyValue
candidates jetstream.KeyValue
source jetstream.KeyValue
builds jetstream.KeyValue
heard jetstream.KeyValue // the state toolchains-heard
instances jetstream.KeyValue
packages jetstream.KeyValue // the indexer's packages, served by its read tool and views family
out *saidLog // the builder's process log, every instance's
stop func() // stops the builder started last
fake *fakeBuildModules
srvURL string
dir string
ctx context.Context
js jetstream.JetStream
catalog *catalogHolder // the seat mesh-catalog, as its holder answers the branch follower
source jetstream.KeyValue
builds jetstream.KeyValue
heard jetstream.KeyValue // the state toolchains-heard
instances jetstream.KeyValue
out *saidLog // the builder's process log, every instance's
stop func() // stops the builder started last
fake *fakeBuildModules
srvURL string
dir string
}
// startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the
@@ -300,17 +281,13 @@ func startBedWith(t *testing.T, closed bool) *bed {
}
return kv
}
b.modules = mk(indexerBucket(modulesState), modulesHistory)
b.candidates = mk(indexerBucket(candidatesState), candidatesHistory)
b.source = mk(sourceBucket, sourceHistory)
mk(indexerBucket(manifestsState), manifestsHistory)
// The indexer and the assigner as the owners of the states the builder follows: their read tools and views families
// (ADR 0615, DLV-M1); n1 and n2 each have their durable consumer.
// The seat mesh-catalog's holder and the assigner as the owners of what the builder follows: the catalog half through the seat's
// verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2
// each have their durable consumer.
b.catalog = newCatalogHolder(t, ctx, js, ctl)
b.catalog.buildSeats(t, ctl, map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage,
"git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"}) // the floor's sources are the holders of git and mesh-catalog
readers := []string{"n1_" + builds.BuilderModule, "n2_" + builds.BuilderModule}
for _, st := range []string{modulesState, manifestsState, candidatesState} {
viewstest.Serve(t, ctx, js, ctl, indexerModule, st, indexerModule+".indexer_read_"+st, readers...)
}
b.packages = viewstest.Serve(t, ctx, js, ctl, indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages", readers...)
viewstest.Serve(t, ctx, js, ctl, assignerModule, "assignments", assignerModule+".assigner_read_assignments", readers...)
b.builds = mk(builds.BuilderBuildsBucket, builds.BuilderBuildsHistory)
b.instances = mk(instancesBucket, 1)
@@ -404,28 +381,34 @@ func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[
t.Fatal(err)
}
digest := meshcatalogclient.ManifestDigest(raw)
manifests, err := b.js.KeyValue(b.ctx, indexerBucket(manifestsState))
if err != nil {
t.Fatal(err)
}
mv, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: name, Manifest: string(raw)})
if err != nil {
t.Fatal(err)
}
if _, err := manifests.Put(b.ctx, digest, mv); err != nil {
t.Fatal(err)
}
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: at, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2,
Manifest: digest, Path: path, Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)})
if err != nil {
t.Fatal(err)
}
if _, err := b.modules.Put(b.ctx, name, data); err != nil {
var manifest map[string]any
if err := json.Unmarshal(raw, &manifest); err != nil {
t.Fatal(err)
}
b.catalog.put(t, b.entry(name, path, at, fp2, digest, manifest, toolchains, src))
return digest
}
// manifestOf is a fixture manifest as the entry holds it.
func (b *bed) manifestOf(t *testing.T, fixture string) map[string]any {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", fixture))
if err != nil {
t.Fatal(err)
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
return m
}
// entry is the catalog graph entry of a module as the holder answers it.
func (b *bed) entry(name, path, at, fp2, digest string, manifest map[string]any, toolchains map[string]builds.ToolchainUsed, src uint64) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: name, Repository: "novox/mesh-catalog", Commit: at, Fp: fp2, Path: path, ManifestDigest: digest, Manifest: manifest,
Inputs: &meshcatalog.EntryInputs{Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)}}
}
// archive writes modules.<name> as the real indexer does for a module of an archive only: its fingerprint names no
// toolchain (mesh-delivery-indexer internal/index inputs.go languagesOf gives an archive no language), so Toolchains is empty.
func (b *bed) archive(t *testing.T, name, path, at string) string {
@@ -459,6 +442,12 @@ type saidLog struct {
buf strings.Builder
}
func (l *saidLog) String() string {
l.mu.Lock()
defer l.mu.Unlock()
return l.buf.String()
}
func (l *saidLog) Write(p []byte) (int, error) {
l.mu.Lock()
defer l.mu.Unlock()
@@ -588,11 +577,12 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
}
}
// Design 32, ADR 0615 §1: a package's name the indexer writes later reaches the builder by its packages' views family
// (never their bucket), and looks at the module again; the done bundles are kept, not numbered twice.
if _, err := b.packages.Put(b.ctx, "netcheck.0.1", []byte(`{"v":1,"name":"netcheck","package":"netcheck/lib","series":"0.1"}`)); err != nil {
t.Fatal(err)
}
// Design 32, design 47 G §348: a package's name the catalog half's entry of the module gives later reaches the builder by the
// branch follower's view (never the indexer's state), and looks at the module again; the done bundles are kept, not numbered twice.
named := b.entry("netcheck", "modules/netcheck", commit, fp, digest, nil, bothToolchains, 0)
named.Manifest = b.manifestOf(t, "catalogue-a67f0196-netcheck.module.json")
named.Artifacts = []meshcatalog.Artifact{{Ref: "netcheck:lib", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "netcheck"}}}
b.catalog.put(t, named)
b.awaitSaid(t, "names a package of netcheck")
// Design 47 G §349: version.numbered is no longer said, for a number or for a late name.
if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 {
@@ -647,7 +637,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
reg.awaitFinding(t, failed, true)
b.fake.hang.Store(nil)
close(release)
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) // the indexer writes it again
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, "89abcdef0123456789abcdef0123456789abcdef", third, 3) // the entry moves to a later commit, same fingerprint
b.awaitFP(t, "netcheck", third, isState(builds.BuildDone))
b.awaitSaid(t, "kept: fingerprint "+third+" is numbered under it")
if !slices.ContainsFunc(b.logged(t, "netcheck"), func(e builds.BuildLogEntry) bool {
@@ -747,7 +737,7 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
map[string]builds.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit)
b.await(t, "netcheck2", isState(builds.BuildDone))
if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil {
t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2")
t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2; the log: " + b.out.String())
}
if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 {
t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n)
@@ -786,77 +776,14 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
}
}
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
// fingerprint first appeared at).
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
// TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes: the catalog half's entry of imaged (one image
// artifact) is built on n1, where mesh-build-image runs: its key is done with the digest and the reference, its build call carrying
// publish; the entry's next fingerprint is built again with publish and ends with a reference of its own (ADR 0487 clauses 5 and
// 12; a run's candidate is no input of the builder: design 47 G A-84).
func TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes(t *testing.T) { // 1705
b := startBed(t)
other := "src2:" + strings.Repeat("e", 64)
digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: other,
Manifest: digest, Path: "modules/netcheck", Toolchains: build.CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}})})
if err != nil {
t.Fatal(err)
}
if _, err := b.modules.Put(b.ctx, "netcheck", data); err != nil {
t.Fatal(err)
}
const head = "1111111111111111111111111111111111111111"
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"netcheck"},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"netcheck": {Fp2: fp, Manifest: digest}}}}
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
ckey, err := keys.EscapeName("feat/x"), error(nil)
if err != nil {
t.Fatal(err)
}
if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil {
t.Fatal(err)
}
b.start(t, "n1", nil)
v := b.await(t, "netcheck", isState(builds.BuildDone))
if len(v.Artifacts) != 2 {
t.Fatalf("build = %+v", v)
}
b.fake.mu.Lock()
defer b.fake.mu.Unlock()
found := false
for _, f := range b.fake.fetches {
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
found = true
}
}
if !found {
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
}
}
// TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes: modules.imaged (one image artifact) is built on n1, where
// mesh-build-image runs: a trunk record's key is done with the digest and the reference, its build call carrying publish; a
// candidate's key for the same module is built first with no publish and no reference; when the pull request is merged and the
// trunk's record names the candidate's fingerprint, the key is built again with publish and ends with a reference
// (ADR 0487 clauses 5 and 12).
func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 1705
b := startBed(t)
other := "src2:" + strings.Repeat("e", 64)
digest := b.moduleCommit(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit)
// The trunk carries `other`; its key is built and published.
trunk := func(fp2 string) {
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2,
Manifest: digest, Path: "modules/imaged"})
if err != nil {
t.Fatal(err)
}
if _, err := b.modules.Put(b.ctx, "imaged", data); err != nil {
t.Fatal(err)
}
}
trunk(other)
digest := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, other, 0)
b.start(t, "n1", nil)
okey, _ := builds.BuilderBuildKey("imaged", other)
awaitKey := func(key string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild {
@@ -868,42 +795,23 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
}
}
}
t.Fatalf("builds.%s = %+v within 20 s", key, last)
t.Fatalf("builds.%s = %+v within 20 s; the log: %s", key, last, b.out.String())
return last
}
v := awaitKey(okey, isState(builds.BuildDone))
if len(v.Artifacts) != 1 || v.Artifacts[0].Digest != digestOf("server") || !strings.HasSuffix(v.Artifacts[0].Reference, "@"+digestOf("server")) {
t.Fatalf("trunk build = %+v, want the digest and the reference", v)
t.Fatalf("build = %+v, want the digest and the reference", v)
}
// The candidate carries fp: built, not published.
const head = "1111111111111111111111111111111111111111"
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"imaged"},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"imaged": {Fp2: fp, Manifest: digest}}}}
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
// The module's next fingerprint: built once more, published.
if got := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, fp, 0); got != digest {
t.Fatalf("manifest digest = %s, want %s", got, digest)
}
ckey := keys.EscapeName("feat/x")
if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil {
t.Fatal(err)
}
v = b.await(t, "imaged", isState(builds.BuildDone))
if len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" {
t.Fatalf("candidate build = %+v, want done with no reference", v)
}
b.fake.mu.Lock()
if len(b.fake.builds) != 2 || b.fake.builds[0]["publish"] != true || b.fake.builds[1]["publish"] != nil {
t.Errorf("builds = %v, want the trunk's with publish, then the candidate's with none", b.fake.builds)
}
b.fake.mu.Unlock()
// The pull request is merged: the trunk's record names fp, and its key ends published, built once more with publish.
trunk(fp)
v = b.await(t, "imaged", func(v builds.BuilderBuild) bool {
fkey, _ := builds.BuilderBuildKey("imaged", fp)
v = awaitKey(fkey, func(v builds.BuilderBuild) bool {
return v.State == builds.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Reference != ""
})
if v.Artifacts[0].Digest != digestOf("server") {
t.Errorf("merged build = %+v", v)
t.Errorf("second build = %+v", v)
}
b.fake.mu.Lock()
defer b.fake.mu.Unlock()
@@ -913,8 +821,8 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
published++
}
}
if len(b.fake.builds) != 3 || published != 2 {
t.Errorf("builds = %v, want three, two of them with publish (the trunk's two fingerprints)", b.fake.builds)
if len(b.fake.builds) != 2 || published != 2 {
t.Errorf("builds = %v, want two, both with publish", b.fake.builds)
}
}
@@ -1085,7 +993,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
}
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
held := heldOnTheBus{b.js}
held := heldOnTheBus{js: b.js, seats: testSeats()}
goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery}
stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery}
subject, _ := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskDelivery, sdkstep.AskerBuilder)
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+81 -161
View File
@@ -12,13 +12,10 @@ import (
"io"
"io/fs"
"log"
"maps"
"os"
"os/signal"
"slices"
"strconv"
"strings"
"sync"
"syscall"
"time"
@@ -34,14 +31,13 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-builder/internal/build"
@@ -82,10 +78,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
// The indexer's modules, manifests, candidates and packages and the assigner's assignments are their owners' states, read by the
// first view of their read tools and then their views families, never their buckets (ADR 0615 §1, DLV-M1). An owner that
// is down never holds the start: each is read on a goroutine once its first view is answered, and until then the
// unanswered first view is the builder's finding.
// The assigner's assignments are its state, read by the first view of its read tool and then its views family, never its
// bucket (ADR 0615 §1, DLV-M1); the catalog half is read through the seat mesh-catalog. An owner that is down never holds the
// start: each is read on a goroutine once its first view is answered, and until then the unanswered first view is the
// builder's finding.
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
@@ -109,24 +105,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
})
reader.Findings = func() views.WireWriter { return fw }
modules, err := reader.Follow(indexerModule, modulesState, indexerModule+".indexer_read_modules")
if err != nil {
return err
}
manifests, err := reader.Follow(indexerModule, manifestsState, indexerModule+".indexer_read_manifests")
if err != nil {
return err
}
// The candidates' fingerprints are built too (issue 1684).
candidates, err := reader.Follow(indexerModule, candidatesState, indexerModule+".indexer_read_candidates")
if err != nil {
return err
}
// The indexer's packages, whose names a package is numbered under (design 32).
packages, err := reader.Follow(indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages")
// The catalog half per watched branch (design 47 G §348, §479, §614; A-5, A-15): the shared branch follower of
// meshcatalogclient on the builder's reader, so the modules, their manifests, their packages and each package's name in its
// ecosystem are read from the entries, never the indexer's states.
half := &catalogHalf{manifests: &entryManifests{}, logf: logger.Printf, ctx: ctx}
branches, err := meshcatalogclient.NewBranches(sweepWiring{meshcatalogclient.ProcessWiring(reader), half}, calls.Call)
if err != nil {
return err
}
branches.Logf = logger.Printf
half.branches = branches
// The build modules are no names (design 47 G A-82): the holders of the build seats, the seat list's definitions and the
// catalog half's claims, found at each use.
seatGate := newSeatGate()
seats := buildrun.NewBuildSeats(seatGate.definitions, versions.HolderOf(branches))
assignments, err := reader.Follow(assignerModule, "assignments", assignerModule+".assigner_read_assignments")
if err != nil {
return err
@@ -137,13 +129,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
}
}
tools := &buildrun.ToolchainSet{}
tools := &buildrun.ToolchainSet{Seats: seats}
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
// The read of an ask's steps is a kept call (design 47 G §568): a fault ends no key, and past five minutes unanswered it is a
// finding; its bound is a key of the state bounds, bound once the bus holds it (set below, with the findings).
stepReads := &stepsRead{bounds: func() jetstream.KeyValue { return nil }, bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf}
asks := buildrun.JetStreamAsks{JS: js, Unanswered: stepReads.unanswered, Answered: stepReads.answered}
// The steps of each build module are watched once the seat list and the catalog half name it (seatsChanged).
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
@@ -157,13 +150,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer hearing.Stop()
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
var buildModules []string
for _, l := range buildstep.AskLanguages {
if m, err := buildstep.BuildModuleOf(l); err == nil {
buildModules = append(buildModules, m)
}
}
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
heard := buildrun.HearEnds(ctx, asks, asks, seats.Modules(), logger.Printf)
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
// (the seat verb held, novox/hq issue 1937) before it leaves the key done, on its calls connection.
@@ -171,31 +158,39 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// the bus holds it; the floor's three sources asked as its own account.
versionsKV := process.AwaitBucket(ctx, js, conn, versions.Bucket, "no trunk build is numbered: each is built unnumbered and published by its digest", logger.Printf)
found := &findings{w: fw, logf: logger.Printf, now: time.Now}
floorWaits := &versions.FloorWaits{Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))}
floorWaits := &versions.FloorWaits{Holder: versions.HolderOf(branches), Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))}
// The names waiting on their floor are kept in the state floor-waits (DLV-B4): a restarted builder takes them in, so
// their silent sources' events still wake them; until the state is bound they are kept here only.
process.AwaitBucket(ctx, js, conn, versions.FloorWaitsBucket, "the names waiting on their floor are kept in memory only, and builder_floor_waits shows none", logger.Printf).
Ready(func(kv jetstream.KeyValue) {
floorWaits.Keep(kvKeeper{ctx: ctx, kv: kv}, heldFloorWaits(ctx, kv, logger.Printf))
})
// A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32).
numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits,
Names: versions.NamesIn(indexerPackages{packages})}
// A package is numbered under its name in its ecosystem, read from the catalog half's entries (design 32, design 47 G §348).
numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call, versions.HolderOf(branches)), Waits: floorWaits,
Names: versions.NamesFrom(versions.EntriesOf(branches))}
// A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder
// answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read.
pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf)
stepReads.bounds, stepReads.findings = func() jetstream.KeyValue { return pubBoundsKV.KV() }, found
pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf}
numberer.Publishing = pubs.arm
b := &build.Builder{
var b *build.Builder
seatsChanged := func(ctx context.Context) {
heard.Watch(seats.Modules())
if b != nil {
b.Reconsider(ctx)
}
}
b = &build.Builder{
Claimer: build.Claimer{Store: kvStore{buildsKV}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls}},
Held: heldOnTheBus{js},
Seats: seats,
Asks: buildrun.Asker{Seats: seats, Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls}},
Held: heldOnTheBus{js: js, seats: seats},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js},
Gone: heldOnTheBus{js: js, seats: seats},
CheckAfter: bound,
Toolchains: tools,
Manifests: manifestReader{manifests},
Manifests: half.manifests,
Logf: logger.Printf,
Stored: storedVia(calls.Call),
Node: cred.Node,
@@ -221,6 +216,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// and version-released heard; nothing is reserved before the fill.
reservedKV := process.AwaitBucket(ctx, js, conn, versions.ReservedBucket, "no version is reserved: builds are made unreserved", logger.Printf)
reserved := newReservations(awaitedReserved{awaitedVersions{b: reservedKV, bucket: versions.ReservedBucket}}, calls.Call, pubBoundsKV.KV, found, b.Reconsider, logger.Printf)
// Each artifact the holder says published at a reserved version is said as the seat mesh-builds' package-published (design 47 G §52).
reserved.say = func(ctx context.Context, p meshbuilds.PackagePublished) error {
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
}
b.Reserved = reserved
// A state bound later numbers the builds looked at without it.
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
@@ -251,18 +250,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}}
queueKV.Ready(again.bound(ctx, js, logger.Printf))
actedKV.Ready(again.bound(ctx, js, logger.Printf))
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
errs := make(chan error, 10)
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: func(ctx context.Context) {
// A source revision heard is the forge's merge: it wakes a floor whose bridge_tags was silent (design 51 *The floor*).
(&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "")
b.TrunkMoved(ctx)
}, errs: errs, logf: logger.Printf}
trunks.start()
b.Trunk = trunks.head
b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf}
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465)
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
b.Log = func(e builds.BuildLogEntry) {
@@ -290,7 +280,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
case kv == nil:
default:
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
says := newSayer(ctx, kv, cred.Node, seats.Modules, logger.Printf)
b.Answered = says.Answered
defer func() {
stopping, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
@@ -393,7 +383,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// condition keeper's condition-cleared.
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
events := func(m jetstream.Msg) {
if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them
return
@@ -404,17 +394,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568)
return
}
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seatHeard.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
return
}
heardGone(ctx, b, m, logger.Printf)
}
// The seats' events (artifacts-gone, the registry seats' words, manifest-accepted and manifest-reads-changed and the
// seat mesh-conditions' condition-cleared, which this pattern covers: a consumer takes no two overlapping filters); the
// owners' views events are the views reader's.
if err := hub.Handle("mesh.seat.*.event.>", events); err != nil {
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, manifest-accepted and
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
// one taker, so no pattern here covers them).
// The seat list over the connection and heard on the hub: a change of it, as of the catalog half's holders, names the build
// modules again (design 47 G A-82, §204).
// The runner's build step ask, the builder's one entry point for a candidate (design 47 G A-97).
b.StepEntry = newEntryWaits(branches, half.manifests).entry
if err := serveSteps(ctx, conn, js, cred, b, logger.Printf); err != nil {
return err
}
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf)
half.holders = func() { seatsChanged(ctx) }
for _, pattern := range seatEventPatterns {
if err := hub.Handle(pattern, events); err != nil {
return err
}
}
hub.OnBound(reads.atBind) // what a node's engine said before the consumer was bound (ADR 0668 §4)
if stopped != nil {
return stopped
@@ -426,10 +428,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return hub.Start(ctx, js, cred)
})
go func() { <-ctx.Done(); hub.Stop() }()
// The modules, once their first views and the manifests' are answered: what a module builds is read from the manifests.
go func() { errs <- buildFromModules(ctx, b, modules, manifests, candidates) }()
// A package's name heard in the indexer's packages looks at its module again, so a done build is numbered then.
go func() { errs <- followPackages(ctx, packages, b, logger.Printf) }()
// The catalog half's entries as they come: a module's entry of a branch goes to the builder, and a key left building for a
// fingerprint no branch carries is closed (issue 1660) once the first view of every followed branch is in (never at the start:
// a holder of the seat that is down holds no other part of it).
half.b = b
half.ready = func() {
if err := b.Sweep(ctx); err != nil {
errs <- err
}
}
branches.Changed = half.Changed(ctx)
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
// every other input. The builder asks again by itself without it.
// A state not on the bus yet at this start is watched once it appears (issue 1790).
@@ -454,6 +462,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
stepReads.expired(ctx, e)
return
}
if key, ok := versions.MemberBoundOf(e.Key()); ok {
if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member
b.MemberBoundEnded(ctx, key)
}
return
}
if node, ok := enginereads.BoundOf(e.Key()); ok {
if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended
reads.expiredCall(node)
@@ -473,7 +487,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
// builder alone may read the stream, so it answers there.
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
eachRead := func(e jetstream.KeyValueEntry) {
queueRead(ctx, heldOnTheBus{js: js, seats: seats}, kv, e, logger.Printf)
}
if qw, err := settle(ctx, kv, eachRead); err != nil {
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
} else {
@@ -509,102 +525,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// keeper raises a first view unanswered past its bound (ADR 0615 §6).
var firstViewsBucket = builds.BuilderModule + "_" + views.StatusState
// The indexer's states the builder follows (indexerBucket(modulesState), ManifestsBucket, CandidatesBucket), and the owners'
// views families as their manifests' `streams` name them.
const (
modulesState = "modules"
manifestsState = "manifests"
candidatesState = "candidates"
)
// buildFromModules builds from the indexer's modules and candidates once the first views of the modules, the manifests and
// the candidates are answered (never at the start: an indexer that is down holds no other part of it), and follows them.
func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests, candidates *views.Picture) error {
for _, p := range []*views.Picture{modules, manifests, candidates} {
select {
case <-ctx.Done():
return nil
case <-p.Follower().Ready():
}
}
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
b.Module(ctx, e.Key(), e.Value())
} else {
b.Forget(e.Key())
}
}
w, err := settle(ctx, modules, each)
if err != nil {
return err
}
// A key left building for a fingerprint its module no longer has is closed (issue 1660); the modules are all known now.
if err := b.Sweep(ctx); err != nil {
return err
}
// The fingerprints the indexer gives the candidates' modules are built too (issue 1684), once the trunk's modules are known.
eachCandidate := func(e jetstream.KeyValueEntry) {
if opOf(e) {
b.Candidate(ctx, e.Key(), e.Value())
} else {
b.ForgetCandidate(e.Key())
}
}
cw, err := settle(ctx, candidates, eachCandidate)
if err != nil {
return err
}
go func() { _ = follow(ctx, cw, eachCandidate) }()
return follow(ctx, w, each)
}
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
// sourceHistory of them: what a candidate's base revision names.
type sourceHeads struct {
mu sync.Mutex
byRev map[uint64]map[string]string
}
// put takes one revision in and answers whether it is new.
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
if !opOf(e) {
return false
}
src, err := meshdeliveryclient.DecodeSource(e.Value())
if err != nil {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if s.byRev == nil {
s.byRev = map[uint64]map[string]string{}
}
if _, had := s.byRev[e.Revision()]; had {
return false
}
s.byRev[e.Revision()] = src.Heads
for len(s.byRev) > sourceHistory {
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
}
return true
}
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
heads, ok := s.byRev[rev]
if !ok {
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
}
for r, head := range heads {
if trunk.SameRepository(r, repository) {
return head, nil
}
}
return "", nil
}
// toolchainWaitOf reads the toolchain wait's setting, toolchain-wait-minutes, from the file the manifest declares with the
// content ${config:toolchain-wait-minutes} (design 47 G §672, A-71), at the builder's start: no file, or an empty one, is
// the default; a content that is no number of minutes is an error naming the file.
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/memberbounds.go
+42
View File
@@ -0,0 +1,42 @@
package main
import (
"context"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/versions"
)
// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of
// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded).
// No timer.
type memberBounds struct {
bounds func() jetstream.KeyValue
now func() time.Time
logf func(string, ...any)
}
// Arm is build.MemberBounds' Arm.
func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error {
kv := m.bounds()
if kv == nil {
return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket)
}
if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
return err
}
return nil
}
// End is build.MemberBounds' End.
func (m *memberBounds) End(ctx context.Context, key string) {
if kv := m.bounds(); kv != nil {
if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
m.logf("the member bound of %s is not deleted: %v", key, err)
}
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names.go
+17 -13
View File
@@ -18,20 +18,24 @@ const (
indexerModule = "mesh-delivery-indexer"
// assignerModule is the holder of the state assignments.
assignerModule = buildrun.AssignerModule
// sourceBucket is the gate's state source (reads: mesh-delivery-gate.source), the heads a candidate's base names.
sourceBucket = gateModule + "_source"
// packagesBucket is the indexer's state packages.
packagesBucket = indexerModule + "_" + meshcatalog.StatePackages
// The revisions each keeps per key (the beds of the integration test create them alike).
sourceHistory = 64
candidatesHistory = 10
modulesHistory = 20
manifestsHistory = 1
)
func indexerBucket(state string) string { return indexerModule + "_" + state }
// namedTreePrefix opens the tree input of a module whose every artifact names its paths.
const namedTreePrefix = "named:"
// seatEventPatterns are the seats' events the builder's own handler takes off its durable consumer (its manifest's `consumes` of
// them): the registry seats' words, artifacts-gone, the build seats' toolchain-reported, condition-cleared, the node-engine
// seat's events and, of the seat mesh-catalog, manifest-accepted, version-released and versions-filled. The seat mesh-catalog's
// catalog-revision and registration-changed and the seat mesh-delivery's source-moved are the views reader's (the branch
// follower's), so no pattern here covers them: a message has one taker.
var seatEventPatterns = func() []string {
var out []string
for _, seat := range []string{"container-image-registry", "go-package-registry", "npm-package-registry", "oci-artifact-registry",
"mesh-artifact-sweep", "mesh-conditions", "node-engine", "node-build-go", "node-build-image", "node-build-node"} {
out = append(out, "mesh.seat."+seat+".event.>")
}
for _, event := range []string{meshcatalog.EventManifestAccepted, meshcatalog.EventVersionReleased, meshcatalog.EventVersionsFilled} {
out = append(out, "mesh.seat."+meshcatalog.Seat+".event."+event, "mesh.seat."+meshcatalog.Seat+".event."+event+".>")
}
return out
}()
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/names_test.go
+11 -6
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"os"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
@@ -25,22 +26,26 @@ func TestTheNamesAreTheManifests(t *testing.T) {
t.Fatal(err)
}
for _, read := range []string{
gateModule + ".source",
"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps",
"node-build-go.steps", "node-build-image.steps", "node-build-node.steps",
} {
if !slices.Contains(m.Reads, read) {
t.Errorf("the builder reads %s, its manifest's reads are %v", read, m.Reads)
}
}
for _, view := range []string{indexerModule + ".views.modules", indexerModule + ".views.manifests", indexerModule + ".views.candidates",
indexerModule + ".views.packages", assignerModule + ".views.assignments"} {
// The catalog half (design 47 G §348, §614) is followed through the seat, and no state of the indexer is.
for _, c := range m.Consumes {
if strings.HasPrefix(c, indexerModule+".") {
t.Errorf("the builder follows %s, an indexer state, its manifest consumes it", c)
}
}
for _, view := range []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", assignerModule + ".views.assignments"} {
if !slices.Contains(m.Consumes, view) {
t.Errorf("the builder follows %s, its manifest consumes %v", view, m.Consumes)
}
}
for want, got := range map[string]string{
"mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"),
buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard", "mesh-delivery-indexer_packages": packagesBucket,
"mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"),
buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard",
"mesh-delivery-assigner_assignments": buildrun.AssignmentsBucket,
} {
if want != got {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/optional.go
-68
View File
@@ -1,76 +1,8 @@
package main
import (
"context"
"fmt"
"sync"
"github.com/nats-io/nats.go/jetstream"
)
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
// and opened again at the next event that needs it where there is one.
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
// and a candidate's preview waits, saying why.
type trunkSource struct {
ctx context.Context
js bucketOpener
heads *sourceHeads
moved func(context.Context)
errs chan<- error
logf func(string, ...any)
mu sync.Mutex
opened bool
}
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
func (t *trunkSource) ensure() error {
t.mu.Lock()
defer t.mu.Unlock()
if t.opened {
return nil
}
kv, err := t.js.KeyValue(t.ctx, sourceBucket)
if err != nil {
return fmt.Errorf("the bucket %s cannot be read (%v)", sourceBucket, err)
}
w, err := kv.WatchAll(t.ctx, jetstream.IncludeHistory()) // every watched branch's key: a revision names its head whichever branch it is of (design 47 G §15)
if err != nil {
return fmt.Errorf("the bucket %s cannot be watched (%v)", sourceBucket, err)
}
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
if err != nil {
return fmt.Errorf("the bucket %s: %w", sourceBucket, err)
}
t.opened = true
go func() {
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
if t.heads.put(e) && t.moved != nil {
t.moved(t.ctx)
}
})
}()
return nil
}
// start opens the source at start; one that cannot be is said, and the builder goes on.
func (t *trunkSource) start() {
if err := t.ensure(); err != nil {
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
}
}
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
if err := t.ensure(); err != nil {
return "", err
}
return t.heads.head(ctx, rev, repository)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go
+2 -13
View File
@@ -43,23 +43,11 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard)
t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding)
t.Run("a branch's end is acknowledged once its tagged versions are removed (design 51 Tagged versions)", aBranchsEndIsHeard)
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), sourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads)
}
@@ -258,7 +246,8 @@ func (m *wakeMsg) Headers() nats.Header {
func aFloorsWakeIsHeard(t *testing.T) {
var logs bytes.Buffer
waits := &versions.FloorWaits{}
holders := map[string]string{"git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"}
waits := &versions.FloorWaits{Holder: func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }}
waits.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceRequirements}}, nil, time.Unix(0, 0))
waits.Wait("m/img", versions.SeatImages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
f := &floorWake{b: &build.Builder{}, waits: waits, logf: log.New(&logs, "", 0).Printf}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go
+16 -3
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"strings"
"sync/atomic"
"time"
@@ -48,6 +49,8 @@ type reservations struct {
logf func(string, ...any)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
call toolCaller
// say says package-published on the seat mesh-builds; nil says none.
say func(context.Context, meshbuilds.PackagePublished) error
}
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
@@ -82,9 +85,9 @@ func (rv *reservations) Reserve(ctx context.Context, module, series string, arti
// Built records the digests; once every artifact has one, the publish's bound starts (a key of the state bounds with its
// own time to live): its end with the reservation standing is version-unreleased.
func (rv *reservations) Built(ctx context.Context, module, version string, digests map[string]string) error {
func (rv *reservations) Built(ctx context.Context, module, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
for ref, d := range digests {
if _, err := rv.r.Built(ctx, module, version, ref, d); err != nil {
if _, err := rv.r.Built(ctx, module, version, ref, d, deps[ref]); err != nil {
return err
}
}
@@ -203,9 +206,19 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
// published marks the artifacts of the reservation a registry seat's published names (its version and digest).
func (rv *reservations) published(ctx context.Context, o versions.Outcome) {
if _, err := rv.r.Published(ctx, o.Version, o.Digest); err != nil {
marked, err := rv.r.Published(ctx, o.Version, o.Digest)
if err != nil {
rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err)
}
// Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334).
for _, m := range marked {
if rv.say == nil {
continue
}
if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil {
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
}
}
}
// dropped is builder_reserved_drop: the reservation is deleted and its bound ended.
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go
+11 -2
View File
@@ -11,6 +11,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"mesh-delivery-builder/internal/versions"
@@ -93,11 +94,13 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
if !rv.heard(ctx, filled) || !filled.acked || !rv.Filled() || woke != 1 {
t.Fatalf("versions-filled: acked %v, filled %v, woke %d", filled.acked, rv.Filled(), woke)
}
res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{})
var said []meshbuilds.PackagePublished
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil }
res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{Branch: "main", Revision: 41})
if err != nil || res.Version != "1.4.3" {
t.Fatalf("reserved %q (%v), want 1.4.3 after the catalog's 1.4.2", res.Version, err)
}
if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}); err != nil {
if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil {
t.Fatal(err)
}
boundKey := versions.ReservedBoundKey("m", "1.4")
@@ -105,6 +108,12 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
t.Fatalf("the publish's bound is not armed: %v", bounds.keys)
}
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"})
// Design 47 G §52, A-57: the holder's published is said once, per artifact, as the seat mesh-builds' package-published with the
// branch and source revision of the reservation; a second published of the same artifact says nothing.
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"})
if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != "1.4.3" || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 {
t.Fatalf("package-published said %+v, want one of m:a at 1.4.3 for main at source 41", said)
}
if !rv.ended(ctx, purged{key: boundKey}) {
t.Fatal("a reservation's bound was not taken in")
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go
+3 -3
View File
@@ -31,7 +31,7 @@ type sayer struct {
ctx context.Context
store sayStore
node string
need []string
need func() []string
logf func(string, ...any)
mu sync.Mutex
@@ -39,7 +39,7 @@ type sayer struct {
said bool
}
func newSayer(ctx context.Context, store sayStore, node string, need []string, logf func(string, ...any)) *sayer {
func newSayer(ctx context.Context, store sayStore, node string, need func() []string, logf func(string, ...any)) *sayer {
return &sayer{ctx: ctx, store: store, node: node, need: need, logf: logf, answered: map[string]bool{}}
}
@@ -51,7 +51,7 @@ func (s *sayer) Answered(module string) {
if s.said {
return
}
for _, m := range s.need {
for _, m := range s.need() {
if !s.answered[m] {
return
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says_test.go
+1 -1
View File
@@ -24,7 +24,7 @@ func (s saysOf) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt
// one; stopped, it says nothing any more.
func TestTheBuilderSaysItWritesReferencesOnceEveryBuildModuleAnsweredOne(t *testing.T) {
s := saysOf{}
say := newSayer(t.Context(), s, "shanks", []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"}, t.Logf)
say := newSayer(t.Context(), s, "shanks", func() []string { return []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"} }, t.Logf)
say.Answered("mesh-build-go")
say.Answered("mesh-build-node")
if v, ok := s["shanks"]; ok {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seatlist.go
+26
View File
@@ -0,0 +1,26 @@
package main
import (
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The builder reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
// definitions name the build seats (the seats whose acting rule is the step build), its tools `builder_seat_list` and
// `builder_seat_list_again`. A call through the mesh MCP server is proved only as its machine's tool runner (issue 627).
var seatListAgainCallers = []string{stdio.AnyPerson, "node-tools", "mesh-healers"}
// seatGate is the builder's wiring of the seat list.
type seatGate struct{ *seatlist.Part }
func newSeatGate() *seatGate { return &seatGate{seatlist.NewPart("builder", seatListAgainCallers)} }
// definitions are the seat definitions the list holds, none before it is read.
func (g *seatGate) definitions() map[string]seatdef.Definition {
if r := g.Reader(); r != nil {
return r.Definitions()
}
return nil
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats_test.go
+23
View File
@@ -0,0 +1,23 @@
package main
import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The build modules of the fixture seat list: the holders of the three build seats (design 47 G A-82).
const (
testGo = "mesh-build-go"
testNode = "mesh-build-node"
testImage = "mesh-build-image"
)
// testSeats is the finder over a seat list whose build seats act on go, typescript and kind:image, each held by its module.
func testSeats() *buildrun.BuildSeats {
rule := func(on string) seatdef.Definition {
return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}}
}
defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript"), "node-build-image": rule(seatdef.OnKindImage)}
holders := map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage}
return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok })
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step.go
+101
View File
@@ -0,0 +1,101 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
)
// **The runner's build step ask is the builder's one entry point for a candidate** (design 47 G A-97, A-84, A-99): the seat
// mesh-builds' accepted kind `step`, heard on its subject, names the entry of one module by fingerprint; the builder reads it through
// the seat mesh-catalog's `entry`, builds it (build.Builder.BuildStep) and says the end as the seat's event step-ended.<node>, the
// ask's id as the event's id, which the runner holds the end to.
// entryWaits answers an entry by fingerprint from the catalog half or the seat's `entry`, waiting for the answer the entries
// keeper is told of (meshcatalogclient.Entries.Changed), never a timer.
type entryWaits struct {
branches *meshcatalogclient.Branches
manifest *entryManifests
mu sync.Mutex
waiting map[string][]chan meshcatalogclient.EntryRow
}
func newEntryWaits(branches *meshcatalogclient.Branches, manifests *entryManifests) *entryWaits {
w := &entryWaits{branches: branches, manifest: manifests, waiting: map[string][]chan meshcatalogclient.EntryRow{}}
prior := branches.Entries.Changed
branches.Entries.Changed = func(row meshcatalogclient.EntryRow) {
w.mu.Lock()
chans := w.waiting[row.Fingerprint]
delete(w.waiting, row.Fingerprint)
w.mu.Unlock()
for _, ch := range chans {
ch <- row
}
if prior != nil {
prior(row)
}
}
return w
}
// entry is build.Builder.StepEntry.
func (w *entryWaits) entry(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
ch := make(chan meshcatalogclient.EntryRow, 1)
w.mu.Lock()
w.waiting[fingerprint] = append(w.waiting[fingerprint], ch)
w.mu.Unlock()
e, state := w.branches.Resolve(ctx, fingerprint)
if state == meshcatalogclient.EntryAsked {
select {
case row := <-ch:
if row.Entry == nil {
return meshcatalog.CatalogEntry{}, fmt.Errorf("the seat mesh-catalog holds no entry of it (%s)", row.State)
}
e, state = *row.Entry, row.State
case <-ctx.Done():
return meshcatalog.CatalogEntry{}, ctx.Err()
}
}
if state != meshcatalogclient.EntryAnswered {
return meshcatalog.CatalogEntry{}, fmt.Errorf("the seat mesh-catalog's entry of it is %s", state)
}
return e, w.manifest.keep(e) // the manifest the build is planned from
}
// serveSteps hears the step asks on the seat's accept subject and answers each one's end, in its own goroutine.
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, cred process.Account, b *build.Builder, logf func(string, ...any)) error {
sub, err := conn.Subscribe(meshbuilds.AcceptSubjectStep, func(m *nats.Msg) {
id := m.Header.Get("x-event-id")
if id == "" {
id = m.Header.Get(nats.MsgIdHdr)
}
var run meshbuilds.StepRun
if err := json.Unmarshal(m.Data, &run); err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
logf("a step ask on %s that is no build step with an id is left: %v", m.Subject, err)
return
}
go func() {
end := b.BuildStep(ctx, run)
end.ID = id
if err := emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectStepEnded(cred.Node), id, end); err != nil {
logf("saying the end of the build step %s: %v", id, err)
}
}()
})
if err != nil {
return err
}
go func() { <-ctx.Done(); _ = sub.Drain() }()
return nil
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/step_integration_test.go
+52
View File
@@ -0,0 +1,52 @@
//go:build integration
package main
import (
"encoding/json"
"testing"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// The runner's build step ask on the seat mesh-builds' accept subject, naming an entry by its fingerprint, is built through the
// module's one claim and answered as the seat's step-ended.<node> under the ask's id (design 47 G A-97, A-99).
func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
b := startBed(t)
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
defer sub.Close()
ended := make(chan *nats.Msg, 4)
s, err := sub.ChanSubscribe(meshbuilds.SubjectStepEndedAll, ended)
if err != nil {
t.Fatal(err)
}
defer s.Unsubscribe()
b.start(t, "n1", nil)
b.await(t, "netcheck", isState(builds.BuildDone))
run := meshbuilds.StepRun{Pipeline: "main.1.netcheck", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 1,
Entries: []meshbuilds.BuildEntry{{Module: "netcheck", Fingerprint: fp}}}
data, _ := json.Marshal(run)
msg := nats.NewMsg(meshbuilds.AcceptSubjectStep)
msg.Data = data
msg.Header.Set("x-event-id", "ask-1")
if err := sub.PublishMsg(msg); err != nil {
t.Fatal(err)
}
select {
case m := <-ended:
var end meshbuilds.StepEnded
if err := json.Unmarshal(m.Data, &end); err != nil || end.ID != "ask-1" || end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Pipeline != "main.1.netcheck" || m.Subject != meshbuilds.SubjectStepEnded("n1") {
t.Fatalf("step-ended on %s = %+v, %v; want the ask ask-1 passed on n1", m.Subject, end, err)
}
case <-time.After(20 * time.Second):
t.Fatal("no step-ended within 20 s")
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go
+3 -3
View File
@@ -104,7 +104,7 @@ func (h *toolchainsHeard) bind(ctx context.Context, state heardState) {
h.mu.Lock()
h.kept = state
h.mu.Unlock()
for _, language := range buildrun.LanguageOf {
for _, language := range h.set.Seats.Languages() {
for _, e := range h.set.Entries(language) {
h.keep(e.Node, e.Toolchain)
}
@@ -192,7 +192,7 @@ func (h *toolchainsHeard) running(ctx context.Context, data []byte) {
h.runAt[id] = r.At
kept := h.kept
h.mu.Unlock()
language, build := buildrun.LanguageOf[r.Module]
language, build := h.set.Seats.LanguageOf(r.Module)
if !build || r.Fingerprint != "" {
return
}
@@ -224,7 +224,7 @@ func (h *toolchainsHeard) Dropped(ctx context.Context, key string) {
// agree says toolchain-agreed for each language whose agreed version is not the one last said (design 47 G §472: said
// only when the agreed version changes, ADR 0584 §1).
func (h *toolchainsHeard) agree(ctx context.Context) {
for _, language := range buildrun.LanguageOf {
for _, language := range h.set.Seats.Languages() {
entries := h.set.Entries(language)
tc, none := buildstep.AgreedToolchain(entries)
if none != "" {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains_test.go
+5 -5
View File
@@ -75,7 +75,7 @@ type heardRig struct {
func newHeardRig(t *testing.T, state *memState) *heardRig {
t.Helper()
r := &heardRig{state: state}
r.h = newToolchainsHeard(&buildrun.ToolchainSet{}, func(context.Context) { r.changed++ },
r.h = newToolchainsHeard(&buildrun.ToolchainSet{Seats: testSeats()}, func(context.Context) { r.changed++ },
func(_ context.Context, e meshbuilds.ToolchainAgreed) error { r.said = append(r.said, e); return nil }, log.New(io.Discard, "", 0))
r.h.bind(context.Background(), state)
return r
@@ -130,18 +130,18 @@ func TestTheBuilderKeepsTheToolchainsItHeardAndForgetsThemOnStopped(t *testing.T
t.Fatalf("a changed agreed version was not said: %+v", again.said)
}
at := time.Now()
again.running(t, "ace", buildrun.GoBuildModule, "src2:x", at) // running: nothing is forgotten
again.running(t, "ace", testGo, "src2:x", at) // running: nothing is forgotten
if k, _ := state.Keys(); len(k) != 3 {
t.Fatalf("a running forgot a key: %v", k)
}
again.running(t, "ace", buildrun.GoBuildModule, "", at.Add(time.Second)) // stopped: no fingerprint
again.running(t, "ace", testGo, "", at.Add(time.Second)) // stopped: no fingerprint
if k, _ := state.Keys(); len(k) != 2 || k[0] != "g14.go" {
t.Fatalf("stopped on ace left %v, want g14.go and novox.go", k)
}
// An older running read after (the start's read of the stream kept last per subject) changes nothing.
before := again.changed
again.running(t, "g14", buildrun.GoBuildModule, "", at)
again.running(t, "g14", buildrun.GoBuildModule, "src2:y", at.Add(-time.Hour))
again.running(t, "g14", testGo, "", at)
again.running(t, "g14", testGo, "src2:y", at.Add(-time.Hour))
if k, _ := state.Keys(); len(k) != 1 || k[0] != "novox.go" || again.changed != before+1 {
t.Fatalf("after g14 stopped and an older running: %v", k)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go
+21 -54
View File
@@ -21,8 +21,8 @@ import (
// The floor's two module tools (novox/hq design 51 *The floor*); the seat's is `seat:<registry>.versions`.
const (
bridgeTagsTool = "mesh-delivery-forge-bridge.bridge_tags"
requirementsTool = "mesh-delivery-indexer.indexer_requirements"
bridgeTagsTool = "bridge_tags"
requirementsTool = "indexer_requirements"
)
// awaitedVersions is the state versions as versions.Store: bound once the bus holds it (the grants part makes it, issue
@@ -60,55 +60,6 @@ func (s awaitedVersions) Get(ctx context.Context, key string) ([]byte, uint64, b
return e.Value(), e.Revision(), true, nil
}
// indexerPackages is the indexer's packages state as the builder reads a package's name in its ecosystem from it (design
// 32): the picture of its read tool's first view and its views family (ADR 0615 §1), never its bucket. Until the first
// view is answered a package is built unnumbered and said (versions.Numberer.Names), and numbered once its name is heard
// (followPackages).
type indexerPackages struct{ kv jetstream.KeyValue }
func (s indexerPackages) Keys(ctx context.Context) ([]string, error) {
keys, err := s.kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
func (s indexerPackages) Get(ctx context.Context, key string) ([]byte, bool, error) {
e, err := s.kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
return e.Value(), true, nil
}
// followPackages watches the picture of the indexer's packages state: each series key heard that names a package's name
// (versions.NamedModule), its first view's included, looks at its module again (build.Builder.PackageNamed), so a done
// build that left the package unnumbered is numbered then with its digests. Woken by the views family, never a timer; a
// key heard again naming the name and module it named (the indexer writes a series at each publish) looks at nothing.
func followPackages(ctx context.Context, kv jetstream.KeyValue, b *build.Builder, logf func(string, ...any)) error {
w, err := kv.WatchAll(ctx)
if err != nil {
return err
}
heard := map[string]string{}
return follow(ctx, w, func(e jetstream.KeyValueEntry) {
if e.Operation() != jetstream.KeyValuePut {
delete(heard, e.Key())
return
}
module, name, named := versions.NamedModule(e.Key(), e.Value())
if named && heard[e.Key()] != module+" "+name {
heard[e.Key()] = module + " " + name
logf("%s.%s names a package of %s: its build is looked at again", packagesBucket, e.Key(), module)
b.PackageNamed(ctx, module)
}
})
}
func versionsConflict(err error) error {
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return versions.ErrConflict
@@ -154,7 +105,15 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
// versions, the forge bridge's bridge_tags and the indexer's indexer_requirements. A call that fails is that source
// silent.
func floorVia(call toolCaller) versions.FloorSources {
func floorVia(call toolCaller, holder func(seat string) (string, bool)) versions.FloorSources {
sourceTool := func(source, tool string) (string, error) {
seat := versions.SourceSeats[source]
module, ok := holder(seat)
if !ok {
return "", fmt.Errorf("no module is known to hold the seat %s, the source of %s", seat, tool)
}
return module + "." + tool, nil
}
return versions.FloorSources{
Seat: func(_ context.Context, seat, name string) ([]string, error) {
raw, err := call("seat:"+seat+".versions", map[string]any{"name": name})
@@ -178,7 +137,11 @@ func floorVia(call toolCaller) versions.FloorSources {
return out, nil
},
Tags: func(_ context.Context, repository, prefix string) ([]string, error) {
raw, err := call(bridgeTagsTool, map[string]any{"repository": repository, "prefix": prefix})
tool, err := sourceTool(versions.SourceTags, bridgeTagsTool)
if err != nil {
return nil, err
}
raw, err := call(tool, map[string]any{"repository": repository, "prefix": prefix})
if err != nil {
return nil, err
}
@@ -191,7 +154,11 @@ func floorVia(call toolCaller) versions.FloorSources {
return a.Tags, nil
},
Requirements: func(_ context.Context, name string) ([]string, error) {
raw, err := call(requirementsTool, map[string]any{"package": name})
tool, err := sourceTool(versions.SourceRequirements, requirementsTool)
if err != nil {
return nil, err
}
raw, err := call(tool, map[string]any{"package": name})
if err != nil {
return nil, err
}
modules/mesh-delivery-builder/internal/build/candidates.go
+202 -4
View File
@@ -3,10 +3,13 @@ package build
import (
"context"
"fmt"
"maps"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
@@ -20,7 +23,15 @@ import (
// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk
// commit the head is merged onto.
type via struct{ Ref, Base string }
type via struct {
Ref, Base string
// Members are the group's package members the module is built against by their builds (design 47 G §373, §465).
Members []member
}
// member is one package member of a group's run: the package the module's fingerprint was computed against at the member's
// head, the member's module and artifact that build it, and the member's fingerprint in the same candidate.
type member struct{ Package, Module, Artifact, Fp string }
// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it
// carries. A value that is not a candidate is said and left.
@@ -46,6 +57,14 @@ func (b *Builder) ForgetCandidate(key string) {
}
func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.CandidatesByGroupValue) {
group := map[string]string{} // the fingerprint of each module the group's pull requests change
for _, p := range c.Pulls {
for name, cm := range p.Fps {
if cm.Fp2 != "" && cm.Manifest != "" {
group[name] = cm.Fp2
}
}
}
for _, p := range c.Pulls {
names := make([]string, 0, len(p.Fps))
for name := range p.Fps {
@@ -83,9 +102,14 @@ func (b *Builder) considerCandidate(ctx context.Context, c meshcatalog.Candidate
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
continue
}
members, err := b.membersOf(ctx, name, cm, group)
if err != nil {
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.Fp2, err)
continue
}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base, Members: members}
}
b.mu.Unlock()
b.consider(ctx, name, m)
@@ -106,7 +130,7 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]bui
if !plan.NeedsToolchain(bm) {
continue
}
language := buildrun.LanguageOf[bm]
language, _ := b.Seats.LanguageOf(bm)
if tc, none := buildstep.AgreedToolchain(b.Toolchains.Entries(language)); none == "" {
out[language] = builds.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
}
@@ -131,3 +155,177 @@ func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string)
}
return head, nil
}
// membersOf are the package members of a group's run that module name is built against (design 47 G §373, §465): each package
// its fingerprint names at a full commit, the head of a group member, whose module and artifact the catalog half's view of the
// packages names (Builder.Members) and which the group changes. A package at a version, or one no module of the group builds,
// is no member: it is fetched as before.
func (b *Builder) membersOf(ctx context.Context, name string, cm meshcatalog.CandidatesByGroupValuePullsItemFpsValue, group map[string]string) ([]member, error) {
if b.Members == nil {
return nil, nil
}
var out []member
for _, pkg := range slices.Sorted(maps.Keys(cm.Packages)) {
if len(cm.Packages[pkg]) != 40 {
continue
}
module, artifact, found, err := b.Members(ctx, pkg)
if err != nil {
return nil, fmt.Errorf("the module and artifact of the package %s cannot be read: %w", pkg, err)
}
if !found || module == name || group[module] == "" {
continue
}
out = append(out, member{Package: pkg, Module: module, Artifact: artifact, Fp: group[module]})
}
return out, nil
}
// MemberBound is how long a dependent of a group's run waits for a package member's build (design 47 G §373: building's
// 60-minute bound).
const MemberBound = 60 * time.Minute
// MemberBounds keeps the bound of a member wait, a key of the builder's state with its own time to live (design 47 G §373, A-77):
// Arm writes it for key, the dependent's build key; End deletes it.
type MemberBounds interface {
Arm(ctx context.Context, key, why string, bound time.Duration) error
End(ctx context.Context, key string)
}
// MemberBoundEnded is told that key's bound ended (the bus's marker of the expiry): the dependent's wait ends then, and the
// candidates are looked at again, which fails its step naming the member still not built.
func (b *Builder) MemberBoundEnded(ctx context.Context, key string) {
b.init()
b.mu.Lock()
waiting := b.memberWaits[key]
if waiting {
b.memberExpired[key] = true
}
b.mu.Unlock()
if waiting {
b.TrunkMoved(ctx)
}
}
// endMemberWait forgets key's wait and deletes its bound.
func (b *Builder) endMemberWait(ctx context.Context, key string) {
b.mu.Lock()
had := b.memberWaits[key]
delete(b.memberWaits, key)
delete(b.memberExpired, key)
b.mu.Unlock()
if had && b.MemberBounds != nil {
b.MemberBounds.End(ctx, key)
}
}
// memberBuilds reads the builds of the package members key's module is built against, from the builder's own keys: the
// build each is fetched by, why the step fails (a member's build failed, holds no digest, or is missing past the bound), or
// that it waits (a member not built yet), saying wait-held naming the member once (design 47 G §373). The bound is read at each look: the
// builder holds no timer.
func (b *Builder) memberBuilds(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, key string) (got []buildstep.FetchBuild, fail string, waiting bool) {
b.mu.Lock()
v := b.via[key]
b.mu.Unlock()
if len(v.Members) == 0 {
return nil, "", false
}
defer func() {
if !waiting {
b.endMemberWait(ctx, key) // a wait that ends, by a build or a failure, deletes its bound
}
}()
var pending []string
for _, mem := range v.Members {
ref := mem.Module + "/" + mem.Artifact
mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp)
if err != nil {
return nil, fmt.Sprintf("the package member %s (%s) has no build key: %v", mem.Package, ref, err), false
}
raw, _, found, err := b.Claimer.Store.Get(ctx, mk)
if err != nil {
return nil, fmt.Sprintf("the build of the package member %s (%s) cannot be read: %v", mem.Package, ref, err), false
}
if !found {
pending = append(pending, ref)
continue
}
cur, err := builds.DecodeBuilderBuild(raw)
if err != nil {
return nil, fmt.Sprintf("the build of the package member %s (%s) is not read: %v", mem.Package, ref, err), false
}
switch cur.State {
case builds.BuildDone:
digest := ""
for _, a := range cur.Artifacts {
if a.Name == mem.Artifact {
digest = a.Digest
}
}
if digest == "" {
return nil, fmt.Sprintf("the build of the package member %s (%s) names no digest of it", mem.Package, ref), false
}
got = append(got, buildstep.FetchBuild{Package: mem.Package, Module: mem.Module, Artifact: mem.Artifact, Digest: digest})
case builds.BuildFailed:
return nil, fmt.Sprintf("the build of the package member %s (%s) failed: %s", mem.Package, ref, cur.Error), false
default:
pending = append(pending, ref)
}
}
if len(pending) == 0 {
b.mu.Lock()
b.fetched[key] = got
b.mu.Unlock()
return got, "", false
}
b.mu.Lock()
said := b.memberWaits[key]
expired := b.memberExpired[key]
b.mu.Unlock()
if expired {
return nil, fmt.Sprintf("the package member %s was not built within %s", strings.Join(pending, ", "), MemberBound), false
}
if !said {
line := module + " at " + m.Fp2 + " is not built yet: wait-held, the package member " + strings.Join(pending, ", ") + " of the group is not built, within " + MemberBound.String()
if b.MemberBounds != nil {
// The bound is a key with its own time to live, whose expiry is heard (MemberBoundEnded); one that cannot be written
// would never end, so the step fails at once, saying so.
if err := b.MemberBounds.Arm(ctx, key, line, MemberBound); err != nil {
return nil, fmt.Sprintf("the package member %s is not built, and the bound of the wait cannot be written: %v", strings.Join(pending, ", "), err), false
}
}
b.mu.Lock()
b.memberWaits[key] = true
b.mu.Unlock()
b.logf("%s", line)
if b.Log != nil {
b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, At: b.now().UTC(), Line: line, Why: "wait-held"})
}
}
return nil, "", true
}
// memberEnded looks at the candidates again once the build of key ended, when it is a package member of a dependent of a group's
// run: a dependent that read it as not built just before this end is not waiting yet, so the look cannot rest on the waits.
func (b *Builder) memberEnded(ctx context.Context, key string) {
b.mu.Lock()
member := false
for _, v := range b.via {
for _, mem := range v.Members {
if mk, err := builds.BuilderBuildKey(mem.Module, mem.Fp); err == nil && mk == key {
member = true
}
}
}
b.mu.Unlock()
if member {
b.TrunkMoved(ctx)
}
}
// fetchedBuilds are the builds key's module is fetched against (memberBuilds), nil for none.
func (b *Builder) fetchedBuilds(key string) []buildstep.FetchBuild {
b.mu.Lock()
defer b.mu.Unlock()
return b.fetched[key]
}
modules/mesh-delivery-builder/internal/build/claim.go
+17 -9
View File
@@ -60,6 +60,8 @@ type ClaimOptions struct {
// ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken
// again only then. Nil means none does.
ImageBuilder func() bool
// Seats finds the build modules of the artifacts a done key holds unpublished (design 47 G A-82).
Seats *buildrun.BuildSeats
// Trunk is whether the trunk's record names this fingerprint. A done key whose image or package has no reference was
// built as a candidate's, which never pushes or publishes (ADR 0487 clause 12, ADR 0498 clauses 5 and 6); on the trunk it
// is taken again to publish them, an image only when an image builder runs (ImageBuilder).
@@ -96,11 +98,13 @@ func GoneStep(gone []string) string {
// Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has
// no reference (a candidate's, ADR 0487 clause 12, ADR 0498 clauses 5 and 6).
func Unpublished(b builds.BuilderBuild) bool { return len(UnpublishedModules(b)) > 0 }
func Unpublished(b builds.BuilderBuild, seats *buildrun.BuildSeats) bool {
return len(UnpublishedModules(b, seats)) > 0
}
// UnpublishedModules are the build modules that publish what a done build holds unpublished: the image build module for
// an image, the Node build module for a package; sorted, none when it holds nothing unpublished.
func UnpublishedModules(b builds.BuilderBuild) []string {
func UnpublishedModules(b builds.BuilderBuild, seats *buildrun.BuildSeats) []string {
if b.State != builds.BuildDone {
return nil
}
@@ -111,10 +115,14 @@ func UnpublishedModules(b builds.BuilderBuild) []string {
}
var out []string
if image {
out = append(out, ImageBuildModule)
if bm, err := seats.For("", "image"); err == nil {
out = append(out, bm.Module)
}
}
if pkg {
out = append(out, buildrun.NodeBuildModule)
if bm, err := seats.For("typescript", catalogue.ArtifactPackage); err == nil {
out = append(out, bm.Module)
}
}
sort.Strings(out)
return out
@@ -122,10 +130,10 @@ func UnpublishedModules(b builds.BuilderBuild) []string {
// publishable are the build modules of UnpublishedModules that can publish now: the image build module only while a node
// runs it (imageBuilder).
func publishable(b builds.BuilderBuild, imageBuilder func() bool) []string {
func publishable(b builds.BuilderBuild, seats *buildrun.BuildSeats, imageBuilder func() bool) []string {
var out []string
for _, bm := range UnpublishedModules(b) {
if bm != ImageBuildModule || (imageBuilder != nil && imageBuilder()) {
for _, bm := range UnpublishedModules(b, seats) {
if seat, _ := seats.OfModule(bm); !seat.ByKind || (imageBuilder != nil && imageBuilder()) {
out = append(out, bm)
}
}
@@ -194,7 +202,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
if err != nil {
err = fmt.Errorf("%s: whether the artifact store still holds %s cannot be asked, so the key is "+
"left done (the controller's take-in refuses it if one is gone): %w", key, strings.Join(refs, ", "), err)
if len(publishable(cur, opt.ImageBuilder)) > 0 {
if len(publishable(cur, opt.Seats, opt.ImageBuilder)) > 0 {
return &Claim{Key: key, Rev: rev, Build: cur, Unasked: err}, nil
}
return nil, err
@@ -210,7 +218,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
}
}
}
if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 {
if opt.Trunk && len(publishable(cur, opt.Seats, opt.ImageBuilder)) > 0 {
return &Claim{Key: key, Rev: rev, Build: cur}, nil
}
return nil, nil
modules/mesh-delivery-builder/internal/build/entry_test.go
+28
View File
@@ -0,0 +1,28 @@
package build
import (
"reflect"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy: a catalog graph entry gives the builder its fingerprint, manifest digest,
// source, the toolchains its fingerprint was computed with, its bases, packages, needs and the named paths of its artifacts by path
// (design 47 G §348, §614).
func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
e := meshcatalog.CatalogEntry{Module: "netcheck", Repository: "novox/mesh-catalog", Path: "modules/netcheck", Commit: "abc", Fp: "src2:f",
ManifestDigest: "sha256:m", Bases: []string{"base"}, Packages: map[string]string{"p": "1"}, Needs: []string{"n"},
Inputs: &meshcatalog.EntryInputs{Src: 7, Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}},
Paths: map[string][]string{"lib": {"a/b", "c"}}}}
got := ModuleOf(e)
if got.Fp2 != "src2:f" || got.Manifest != "sha256:m" || got.Path != "modules/netcheck" || got.Commit != "abc" || got.Src != 7 ||
got.Toolchains["go"].Version != "go1" || got.Packages["p"] != "1" || !reflect.DeepEqual(got.Needs, []string{"n"}) ||
got.Inputs2 == nil || !reflect.DeepEqual(got.Inputs2.Bases, []string{"base"}) ||
!reflect.DeepEqual(got.Inputs2.Paths, map[string]string{"a/b": "lib", "c": "lib"}) {
t.Errorf("ModuleOf = %+v", got)
}
if bare := ModuleOf(meshcatalog.CatalogEntry{Module: "m"}); bare.Inputs2 != nil || bare.Fp2 != "" {
t.Errorf("ModuleOf of an entry with none = %+v, want nothing set", bare)
}
}
modules/mesh-delivery-builder/internal/build/members_test.go
+166
View File
@@ -0,0 +1,166 @@
package build
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
const memberPackage = "example.org/p"
var fpP = "src2:" + hex64('2')
// groupValue is a group's candidate: the module m, built against the head of the package member p, and p.
func groupValue(t *testing.T) []byte {
t.Helper()
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: headX, Modules: []string{"m", "p"},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{
"m": {Fp2: fp1, Manifest: mdigest('a'), Packages: map[string]string{memberPackage: headY, "example.org/q": "v1.2.3"}},
"p": {Fp2: fpP, Manifest: mdigest('b')}}}}
hash, err := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: 7, Head: headX}})
if err != nil {
t.Fatal(err)
}
data, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
return data
}
// memberRig is a rig whose group changes m and its package member p (artifact lib); p has a trunk record when known.
func memberRig(t *testing.T, known bool) *rig {
t.Helper()
r := newRig(t, goBundle)
lib := catalogue.Artifact{Name: "lib", Kind: "bundle", Language: "go"}
r.mf.byDigest[mdigest('b')] = catalogue.Manifest{Module: "p", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{lib}}}
r.b.Members = func(_ context.Context, pkg string) (string, string, bool, error) {
if pkg == memberPackage {
return "p", "lib", true, nil
}
return "", "", false, nil
}
trunkModule(t, r)
if known {
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: trunkHead, Fp2: "src2:" + hex64('d'),
Manifest: mdigest('b'), Path: "modules/p", Toolchains: CatalogToolchains(agreedToolchains)})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "p", data)
r.b.Wait()
}
return r
}
func TestAGroupRunBuildsADependentAgainstItsPackageMembersBuildByDigest(t *testing.T) { // design 47 G §373, §465; test 103
t.Run("the member is fetched by its build's digest, and no package of it is fetched by version", func(t *testing.T) {
r := memberRig(t, true)
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
t.Fatalf("m = %+v, want built", v)
}
var found bool
for _, c := range r.bm.callsOf("build_go_fetch") {
f := c.body.(map[string]any)
bs, _ := f["builds"].([]any)
if len(bs) == 0 {
continue
}
found = true
want := map[string]any{"package": memberPackage, "module": "p", "artifact": "lib", "digest": digest}
if len(bs) != 1 || !jsonEqual(bs[0], want) {
t.Errorf("builds = %v, want %v", bs, want)
}
pk, _ := f["packages"].(map[string]any)
if _, named := pk[memberPackage]; named || pk["example.org/q"] != "v1.2.3" {
t.Errorf("packages = %v, want the member out of it and the other package kept", pk)
}
}
if !found {
t.Error("no fetch named the member's build")
}
})
t.Run("a member not built holds the dependent with wait-held naming it, and past the bound fails it", func(t *testing.T) {
r := memberRig(t, false)
var mu sync.Mutex
bounds := &fakeMemberBounds{}
r.b.MemberBounds = bounds
var said []builds.BuildLogEntry
r.b.Log = func(e builds.BuildLogEntry) { mu.Lock(); said = append(said, e); mu.Unlock() }
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
r.b.Wait()
if _, written := r.st.vals[r.key]; written || len(r.bm.callsOf("build_go_fetch")) != 0 {
t.Fatal("the dependent was claimed or built while its package member is not built")
}
mu.Lock()
if len(said) != 1 || said[0].Why != "wait-held" || !strings.Contains(said[0].Line, "p/lib") {
t.Errorf("said = %+v, want one wait-held naming p/lib", said)
}
mu.Unlock()
if got := bounds.armed(); len(got) != 1 || got[0] != r.key+" "+MemberBound.String() {
t.Fatalf("bounds armed = %v, want one of 60 minutes on the dependent's key", got)
}
r.b.MemberBoundEnded(context.Background(), r.key) // the bus's marker of the bound's expiry
r.b.Wait()
if got := bounds.ended(); len(got) != 1 || got[0] != r.key {
t.Errorf("bounds ended = %v, want the wait's bound deleted", got)
}
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || len(r.bm.callsOf("build_go_fetch")) != 0 {
t.Errorf("m = %+v, want failed naming the member and nothing built", v)
}
})
t.Run("a member whose build failed fails the dependent's step naming it", func(t *testing.T) {
r := memberRig(t, false)
mk, _ := builds.BuilderBuildKey("p", fpP)
data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, Error: "boom"})
if err != nil {
t.Fatal(err)
}
r.st.put(mk, data)
r.b.Candidate(context.Background(), "feat%2Fx", groupValue(t))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "p/lib") || !strings.Contains(v.Error, "boom") || len(r.bm.callsOf("build_go_fetch")) != 0 {
t.Errorf("m = %+v, want failed naming the member and its error", v)
}
})
}
// fakeMemberBounds keeps the bounds a builder arms and ends.
type fakeMemberBounds struct {
mu sync.Mutex
arms, ends_ []string
}
func (f *fakeMemberBounds) Arm(_ context.Context, key, _ string, bound time.Duration) error {
f.mu.Lock()
defer f.mu.Unlock()
f.arms = append(f.arms, key+" "+bound.String())
return nil
}
func (f *fakeMemberBounds) End(_ context.Context, key string) {
f.mu.Lock()
defer f.mu.Unlock()
f.ends_ = append(f.ends_, key)
}
func (f *fakeMemberBounds) armed() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.arms }
func (f *fakeMemberBounds) ended() []string { f.mu.Lock(); defer f.mu.Unlock(); return f.ends_ }
func jsonEqual(a, b any) bool {
x, _ := json.Marshal(a)
y, _ := json.Marshal(b)
return string(x) == string(y)
}
modules/mesh-delivery-builder/internal/build/numbers.go
+6 -4
View File
@@ -3,6 +3,7 @@ package build
import (
"context"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"strconv"
"strings"
@@ -135,8 +136,9 @@ type Reservations interface {
Filled() bool
// Reserve reserves module's next version in series for the artifacts (their references, <module>:<artifact>).
Reserve(ctx context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error)
// Built records the digests of the artifacts built under the reserved version, by reference.
Built(ctx context.Context, module, version string, digests map[string]string) error
// Built records the digests of the artifacts built under the reserved version, by reference, and what each resolved and was
// built with (design 47 G §466, §673).
Built(ctx context.Context, module, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error
}
// reserves says whether a look at a trunk build is one that reserves: the build has a series of the form MAJOR.MINOR
@@ -172,12 +174,12 @@ func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bo
// reserve reserves the version of the build that starts; a reservation that cannot be made is said and the build goes on
// unreserved, as an unnumbered build does.
func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan) (versions.ReservedVersion, bool) {
func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, o Origin) (versions.ReservedVersion, bool) {
refs := make([]string, 0, len(plan.Tasks))
for _, t := range plan.Tasks {
refs = append(refs, module+":"+t.Artifact)
}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source}
for _, t := range plan.Tasks {
src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go")
}
modules/mesh-delivery-builder/internal/build/plan.go
+34 -14
View File
@@ -13,11 +13,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// ImageBuildModule is the build module that builds an image artifact (novox/hq ADR 0487): buildah, rootless, on the nodes that
// run it. It has no toolchain language; the builder finds its nodes from the node-engine's have.
const ImageBuildModule = "mesh-build-image"
// NoImageBuilderPrefix opens the error of a build key failed because no node runs ImageBuildModule (ADR 0487 clause 5): such a
// NoImageBuilderPrefix opens the error of a build key failed because no node runs the image build module (ADR 0487 clause 5): such a
// key is not built again as a new revision, only when a node reports the module (ClaimOptions.ImageBuilder) or the fingerprint
// changes.
const NoImageBuilderPrefix = "no image builder:"
@@ -82,34 +78,55 @@ func (n *NotPinned) Error() string {
return fmt.Sprintf("%s%s: %q names no digest (@sha256:), so what would be copied is whatever its tag means today", builds.NotPinned, n.Artifact, n.From)
}
// NoBuildModule is the error of an artifact whose build seat or its holder the seat list and the catalog half do not name (yet):
// the plan is not kept, so it is made again when they are read (design 47 G A-82).
type NoBuildModule struct {
Artifact string
Err error
}
func (n *NoBuildModule) Error() string { return fmt.Sprintf("artifact %s: %v", n.Artifact, n.Err) }
func (n *NoBuildModule) Unwrap() error { return n.Err }
// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2, ADR 0498 clauses 4 and 5): a Go bundle and an archive are
// built by the Go build module, an image by the image build module (ADR 0487), which also copies an upstream image pinned
// by its digest, and a TypeScript bundle or package by the Node one, each in the order of the artifacts' names. A manifest
// with an upstream artifact pinned by no digest, or a bundle or a package of another language, or a kind no build module
// builds, is refused as a whole, naming the first such artifact by name. A manifest with no artifact has an empty plan.
func PlanOf(m catalogue.Manifest) (Plan, error) {
func PlanOf(m catalogue.Manifest, seats *buildrun.BuildSeats) (Plan, error) {
if m.Build == nil {
return Plan{}, nil
}
artifacts := append([]catalogue.Artifact(nil), m.Build.Artifacts...)
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
p := Plan{Series: m.Version}
task := func(a catalogue.Artifact, language, kind string) error {
bm, err := seats.For(language, kind)
if err != nil {
return &NoBuildModule{Artifact: a.Name, Err: err}
}
t := Task{Artifact: a.Name, Kind: a.Kind, Module: bm.Module, Language: bm.Toolchain, From: a.From}
if a.Kind != catalogue.ArtifactUpstream {
t.From = ""
}
p.Tasks = append(p.Tasks, t)
return nil
}
for _, a := range artifacts {
var err error
switch {
case a.Kind == catalogue.ArtifactArchive:
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"})
case a.Kind == catalogue.ArtifactBundle && a.Language == "go":
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"})
case a.Kind == catalogue.ArtifactBundle && a.Language == "typescript":
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"})
err = task(a, "go", a.Kind) // an archive is packed by the Go build seat's holder, which the indexer gives no language
case a.Kind == catalogue.ArtifactBundle && (a.Language == "go" || a.Language == "typescript"):
err = task(a, a.Language, a.Kind)
case a.Kind == catalogue.ArtifactImage:
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule})
err = task(a, "", "image")
case a.Kind == catalogue.ArtifactUpstream && !strings.Contains(a.From, "@sha256:"):
return Plan{}, &NotPinned{Artifact: a.Name, From: a.From}
case a.Kind == catalogue.ArtifactUpstream:
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: ImageBuildModule, From: a.From})
err = task(a, "", "image")
case a.Kind == catalogue.ArtifactPackage && a.Language == "typescript":
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"})
err = task(a, a.Language, a.Kind)
case a.Kind == catalogue.ArtifactBundle:
return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)}
case a.Kind == catalogue.ArtifactPackage:
@@ -117,6 +134,9 @@ func PlanOf(m catalogue.Manifest) (Plan, error) {
default:
return Plan{}, &Refused{Artifact: a.Name, What: a.Kind}
}
if err != nil {
return Plan{}, err
}
}
return p, nil
}
modules/mesh-delivery-builder/internal/build/plan_test.go
+40 -17
View File
@@ -10,6 +10,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// manifestOf is a manifest holding these artifacts: the plan reads only their name, kind and language.
@@ -22,14 +23,14 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2
catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"},
catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"},
catalogue.Artifact{Name: "files", Kind: "archive"})
p, err := PlanOf(m)
p, err := PlanOf(m, testSeats())
if err != nil {
t.Fatal(err)
}
want := []Task{
{Artifact: "files", Kind: "archive", Module: buildrun.GoBuildModule, Language: "go"},
{Artifact: "tool", Kind: "bundle", Module: buildrun.GoBuildModule, Language: "go"},
{Artifact: "web", Kind: "bundle", Module: buildrun.NodeBuildModule, Language: "typescript"},
{Artifact: "files", Kind: "archive", Module: testGo, Language: "go"},
{Artifact: "tool", Kind: "bundle", Module: testGo, Language: "go"},
{Artifact: "web", Kind: "bundle", Module: testNode, Language: "typescript"},
}
if len(p.Tasks) != len(want) {
t.Fatalf("tasks = %+v, want %+v", p.Tasks, want)
@@ -39,7 +40,7 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2
t.Errorf("task %d = %+v, want %+v", i, p.Tasks[i], want[i])
}
}
if modules := p.Modules(); len(modules) != 2 || modules[0] != buildrun.GoBuildModule || modules[1] != buildrun.NodeBuildModule {
if modules := p.Modules(); len(modules) != 2 || modules[0] != testGo || modules[1] != testNode {
t.Errorf("modules = %v", modules)
}
}
@@ -47,24 +48,24 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2
const pinned = "nats:2.11.17-alpine@sha256:3333333333333333333333333333333333333333333333333333333333333333"
func TestPlanSendsAPinnedUpstreamToTheImageBuildModulesCopyAndRefusesAnUnpinnedOne(t *testing.T) { // B3, ADR 0498 clause 4
p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "up", Kind: "upstream", From: pinned}))
p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "up", Kind: "upstream", From: pinned}), testSeats())
if err != nil || len(p.Tasks) != 2 {
t.Fatalf("plan %+v, error %v", p, err)
}
if up := p.Tasks[1]; up.Artifact != "up" || up.Kind != "upstream" || up.Module != ImageBuildModule || up.From != pinned || up.Language != "" {
t.Errorf("task = %+v, want the upstream sent to %s with its pinned reference and no toolchain", up, ImageBuildModule)
if up := p.Tasks[1]; up.Artifact != "up" || up.Kind != "upstream" || up.Module != testImage || up.From != pinned || up.Language != "" {
t.Errorf("task = %+v, want the upstream sent to %s with its pinned reference and no toolchain", up, testImage)
}
if p.NeedsToolchain(ImageBuildModule) {
if p.NeedsToolchain(testImage) {
t.Error("a copy needs no toolchain")
}
_, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}))
_, err = PlanOf(manifestOf(catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}), testSeats())
if err == nil || !strings.HasPrefix(err.Error(), builds.NotPinned+"up") || !builds.FailedUntilChanged(err.Error()) {
t.Errorf("an upstream with no digest: error = %v, want it to open %q naming the artifact", err, builds.NotPinned)
}
}
func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498 clause 5
_, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"}))
_, err := PlanOf(manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, catalogue.Artifact{Name: "pkg", Kind: "package", Language: "python"}), testSeats())
var r *Refused
if err == nil || !asRefused(err, &r) || !strings.HasPrefix(err.Error(), builds.RefusedKind) || !strings.Contains(err.Error(), "python") || !strings.Contains(err.Error(), "pkg") {
t.Errorf("error = %v, want it to open %q naming the artifact and its language", err, builds.RefusedKind)
@@ -72,10 +73,10 @@ func TestPlanRefusesAPackageOfAnotherLanguageByName(t *testing.T) { // ADR 0498
}
func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.T) {
if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), builds.RefusedKind) {
if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"}), testSeats()); err == nil || !strings.HasPrefix(err.Error(), builds.RefusedKind) {
t.Errorf("a python bundle: error = %v, want a refusal", err)
}
p, err := PlanOf(manifestOf())
p, err := PlanOf(manifestOf(), testSeats())
if err != nil || len(p.Tasks) != 0 {
t.Errorf("no artifact: plan %+v error %v, want an empty plan", p, err)
}
@@ -84,14 +85,36 @@ func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.
func asRefused(err error, target **Refused) bool { return errors.As(err, target) }
func TestPlanSendsAnImageToMeshBuildImageWithNoToolchain(t *testing.T) { // 1705a, ADR 0487 clause 5
p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "img", Kind: "image"}, catalogue.Artifact{Name: "files", Kind: "archive"}))
p, err := PlanOf(manifestOf(catalogue.Artifact{Name: "img", Kind: "image"}, catalogue.Artifact{Name: "files", Kind: "archive"}), testSeats())
if err != nil || len(p.Tasks) != 2 {
t.Fatalf("plan %+v, error %v", p, err)
}
if p.Tasks[1].Artifact != "img" || p.Tasks[1].Module != ImageBuildModule {
t.Errorf("task = %+v, want the image on %s", p.Tasks[1], ImageBuildModule)
if p.Tasks[1].Artifact != "img" || p.Tasks[1].Module != testImage {
t.Errorf("task = %+v, want the image on %s", p.Tasks[1], testImage)
}
if p.NeedsToolchain(ImageBuildModule) {
if p.NeedsToolchain(testImage) {
t.Error("an image needs no toolchain")
}
}
// A plan's task names the holder of the build seat the seat list and the catalog half give, and a plan made before they name one
// is no refusal and is not kept (design 47 G A-82).
func TestAPlanNamesTheHoldersOfTheBuildSeats(t *testing.T) {
m := manifestOf(catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}, catalogue.Artifact{Name: "img", Kind: "image"})
defs := map[string]seatdef.Definition{
"node-build-go": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "go"}}},
"node-build-image": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: seatdef.OnKindImage}}},
}
holders := map[string]string{}
seats := buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(s string) (string, bool) { h, ok := holders[s]; return h, ok })
_, err := PlanOf(m, seats)
var none *NoBuildModule
if !errors.As(err, &none) {
t.Fatalf("with no holder: %v, want NoBuildModule", err)
}
holders["node-build-go"], holders["node-build-image"] = "acme-go", "acme-img"
p, err := PlanOf(m, seats)
if err != nil || len(p.Tasks) != 2 || p.Tasks[0].Module != "acme-img" || p.Tasks[1].Module != "acme-go" || p.Tasks[1].Language != "go" || p.Tasks[0].Language != "" {
t.Fatalf("plan = %+v, %v", p, err)
}
}
modules/mesh-delivery-builder/internal/build/reserve_test.go
+80 -3
View File
@@ -2,12 +2,16 @@ package build
import (
"context"
"encoding/json"
"fmt"
"reflect"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -21,16 +25,19 @@ type fakeReservations struct {
reserved [][]string
series []string
built map[string]string
sources []versions.Source
deps map[string][]meshbuilds.Dependency
}
func (f *fakeReservations) Filled() bool { return f.filled }
func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string, _ versions.Source) (versions.ReservedVersion, error) {
f.reserved, f.series = append(f.reserved, artifacts), append(f.series, series)
func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) {
f.reserved, f.series, f.sources = append(f.reserved, artifacts), append(f.series, series), append(f.sources, src)
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
}
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string) error {
func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
f.deps = deps
f.built = map[string]string{version: ""}
for ref, d := range digests {
f.built[ref] = d
@@ -127,3 +134,73 @@ func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) {
}
}
}
// TestOnlyABuildOfTheFirstWatchedBranchReservesAndItCarriesItsCatalogRevision: design 47 G §371, A-57 — a build of the registration's
// first watched branch reserves a version and the reservation carries the branch and the source revision of the catalog revision its
// entry came from; a build of another watched branch reserves none and is built and done.
func TestOnlyABuildOfTheFirstWatchedBranchReservesAndItCarriesItsCatalogRevision(t *testing.T) {
for _, c := range []struct {
name string
origin Origin
want []versions.Source
}{
{"the first branch", Origin{Branch: "main", Source: 41, First: true},
[]versions.Source{{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Branch: "main", Revision: 41}}},
{"another branch", Origin{Branch: "beta", Source: 42}, nil},
} {
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true}
r.b.Reserved = res
r.b.Entry(context.Background(), c.origin, meshcatalog.CatalogEntry{Module: "m", Repository: "novox/mesh-catalog", Path: "modules/m",
Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: fp1, ManifestDigest: mdigest('a'), Inputs: &meshcatalog.EntryInputs{Toolchains: CatalogToolchains(agreedToolchains)}})
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone {
t.Fatalf("build = %+v, want done", v)
}
if !reflect.DeepEqual(res.sources, c.want) {
t.Errorf("reserved from %+v, want %+v", res.sources, c.want)
}
})
}
}
// TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith: design 47 G §466, §673, A-57 — the `resolved` of each artifact and the
// toolchain digest of the step's result are kept on the reservation as dependencies of kind 1 and 3, their parts the artifact
// references the entry's edges name; a package no edge names is left out and said.
func TestAReservedBuildKeepsWhatItsStepResolvedAndWasBuiltWith(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true}
r.b.Reserved = res
var journal []string
r.b.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
st := r.answer(a)
var out buildstep.BuildAnswer
if err := json.Unmarshal(st.Result, &out); err != nil || len(out.Artifacts) == 0 {
return st
}
out.Artifacts[0].Resolved = []buildstep.AskResolved{{Package: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.14", Digest: "sha256:11"},
{Package: "unnamed/pkg", Version: "1.0.0", Digest: "sha256:22"}}
out.Toolchain = &buildstep.ToolchainUsed{Language: "go", Version: "go1.27.1", Recipe: "1", Digest: "sha256:tc"}
st.Result, _ = json.Marshal(out)
return st
}
r.b.Entry(context.Background(), Origin{Branch: "main", Source: 41, First: true}, meshcatalog.CatalogEntry{Module: "m", Repository: "novox/mesh-catalog",
Path: "modules/m", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: fp1, ManifestDigest: mdigest('a'),
Inputs: &meshcatalog.EntryInputs{Toolchains: CatalogToolchains(agreedToolchains)},
Edges: []meshcatalog.Edge{{Kind: 1, From: "mesh-sdk:go", To: "m:tool", Why: "git.novox.be/novox/mesh-sdk/go"}, {Kind: 3, From: "go-toolchain:program", To: "m:tool"}}})
r.b.Wait()
want := map[string][]meshbuilds.Dependency{"m:tool": {{Kind: 1, Part: "mesh-sdk:go", Version: "0.1.14", Digest: "sha256:11"},
{Kind: 3, Part: "go-toolchain:program", Version: "go1.27.1", Digest: "sha256:tc"}}}
if !reflect.DeepEqual(res.deps, want) {
t.Errorf("dependencies = %+v, want %+v", res.deps, want)
}
if !slices.ContainsFunc(journal, func(l string) bool {
return strings.Contains(l, "unnamed/pkg") && strings.Contains(l, "names no artifact")
}) {
t.Errorf("journal %q, want the package no edge names said", journal)
}
}
modules/mesh-delivery-builder/internal/build/round4_test.go
+5 -5
View File
@@ -42,12 +42,12 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes
}
func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1
p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}})
if err != nil || !p.NeedsToolchain(buildrun.GoBuildModule) {
t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(buildrun.GoBuildModule), err)
p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}}, testSeats())
if err != nil || !p.NeedsToolchain(testGo) {
t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(testGo), err)
}
p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}})
if p.NeedsToolchain(buildrun.GoBuildModule) {
p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}}, testSeats())
if p.NeedsToolchain(testGo) {
t.Error("a plan of an archive needs a toolchain")
}
}
modules/mesh-delivery-builder/internal/build/run.go
+242 -31
View File
@@ -1,6 +1,7 @@
package build
import (
"cmp"
"context"
"encoding/json"
"errors"
@@ -15,6 +16,7 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
@@ -39,6 +41,10 @@ type ToolchainEntries interface {
// change of the toolchains.
type Builder struct {
Claimer Claimer
// StepEntry reads the entry of a fingerprint through the seat mesh-catalog's entry (design 47 G A-84, A-97).
StepEntry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
// Seats finds the build modules: the holders of the build seats on the seat list and the catalog half (design 47 G A-82).
Seats *buildrun.BuildSeats
// Asks puts each build module's part of a build as one ask on its build seat's stream and waits for its step (novox/hq
// ADR 0525 §1, §3, §6; issue 1887 A4): the builder chooses no node, whichever build node pulls the ask builds it.
Asks interface {
@@ -74,6 +80,11 @@ type Builder struct {
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
// Members answers the module and artifact that build a package, from the catalog half's view of the packages (design 47 G
// §373, §465): a package named at a group member's head is fetched by that member's build. Nil takes no package as a
// member's. MemberBounds keeps the bound of a dependent's wait for a member's build (A-77); nil keeps none.
Members func(ctx context.Context, pkg string) (module, artifact string, found bool, err error)
MemberBounds MemberBounds
// Gone says why an ask with no ended step can no longer be answered, or "" when it can (issue 1945): asked once the
// ask has waited CheckAfter, and again each CheckAfter it goes on waiting. An ask that cannot be answered is asked
// again under a new id, at most Askings times. Nil, or CheckAfter 0, never asks it.
@@ -112,10 +123,15 @@ type Builder struct {
mu sync.Mutex
// fillWaits are the builds that said wait-held for the fill (module.fingerprint), so it is said once.
fillWaits sync.Map
known map[string]meshcatalog.ModulesByModuleValue // by modules key
known map[string]meshcatalog.ModulesByModuleValue // by modules key: the first watched branch's
heard map[string]meshcatalog.ModulesByModuleValue // by branch and module (take): what supersedes what
origin map[string]Origin // by build key: the catalog revision its entry came from (take)
edges map[string][]meshcatalog.Edge // by build key: the edges of its entry (take), whose parts a package-published names
deps map[string]map[string][]meshbuilds.Dependency // by build key: what each artifact resolved and was built with (keepDependencies)
inflight map[string]bool // by build key
again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built
plans map[string]planned // by manifest digest
stepWaits map[string][]chan builds.BuildLogEntry // by build key: the step asks waiting for its end (step.go)
cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go)
via map[string]via // by build key: a candidate's fetch (candidates.go)
// outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them.
@@ -125,6 +141,10 @@ type Builder struct {
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
// waiting for one is built once it is here (ADR 0500 clause 6).
published map[string]bool
// memberWaits are the build keys of the dependents of a group's run that wait for a package member's build, and memberExpired
// those whose bound ended; fetched are the member builds a key's module is fetched against (candidates.go).
memberWaits, memberExpired map[string]bool
fetched map[string][]sdkstep.FetchBuild
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
held map[string]bool
@@ -162,9 +182,12 @@ func (b *Builder) init() {
}
b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
b.heard, b.known = map[string]meshcatalog.ModulesByModuleValue{}, map[string]meshcatalog.ModulesByModuleValue{}
b.origin, b.edges = map[string]Origin{}, map[string][]meshcatalog.Edge{}
b.inflight, b.again, b.plans, b.tried = map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
b.memberWaits, b.memberExpired, b.fetched = map[string]bool{}, map[string]bool{}, map[string][]sdkstep.FetchBuild{}
b.builds = map[string]context.CancelCauseFunc{}
})
}
@@ -178,8 +201,9 @@ func (b *Builder) logf(format string, args ...any) {
// Wait returns when every build started so far has ended.
func (b *Builder) Wait() { b.wg.Wait() }
// Module takes one value of the indexer's modules bucket in (the key is the module's name) and builds it if it is
// to be built. A value that is not a module is said and left.
// Module takes one value of a modules bucket in (the key is the module's name) and builds it if it is to be built. A value that
// is not a module is said and left. The builder takes its modules from the catalog half (Entry); this is the same take of one
// module's value, with no branch, the first one's.
func (b *Builder) Module(ctx context.Context, key string, value []byte) {
b.init()
m, err := meshcatalogclient.DecodeModule(value)
@@ -187,9 +211,49 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
b.logf("modules.%s holds what is not a module, left: %v", key, err)
return
}
b.take(ctx, Origin{First: true}, key, m, nil)
}
// Origin is the catalog revision an entry came from (design 47 G §371, A-57): the watched branch it is of, as named, the source
// revision the revision was taken in from, and whether the branch is the first of the module's registration, the only one whose
// builds reserve a version and whose entry is the module's value for what the builder reads by name (a version, a build again).
type Origin struct {
Branch string
Source int64
First bool
}
// Entry takes the catalog graph entry of one module on one watched branch in (design 47 G §348, §614) and builds its
// fingerprint if it is to be built. An entry of a branch other than the first builds its fingerprint, reserves none, and
// supersedes only that branch's older one. An entry naming no module is said and left.
func (b *Builder) Entry(ctx context.Context, o Origin, e meshcatalog.CatalogEntry) {
b.init()
if e.Module == "" {
b.logf("the catalog half of the branch %q holds an entry that names no module, left", o.Branch)
return
}
b.take(ctx, o, e.Module, ModuleOf(e), e.Edges)
}
// EntryGone takes a module's entry out of a branch's half (the module was dropped from it, or the branch is followed no more).
func (b *Builder) EntryGone(branch string, first bool, module string) {
b.init()
b.mu.Lock()
old, had := b.known[key]
b.known[key] = m
delete(b.heard, branch+"\x00"+module)
b.mu.Unlock()
if first {
b.Forget(module)
}
}
func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue, e []meshcatalog.Edge) {
b.mu.Lock()
hk := o.Branch + "\x00" + key
old, had := b.heard[hk]
b.heard[hk] = m
if o.First {
b.known[key] = m
}
var superseded context.CancelCauseFunc
if had && old.Fp2 != "" && old.Fp2 != m.Fp2 {
if ok, err := builds.BuilderBuildKey(key, old.Fp2); err == nil {
@@ -197,8 +261,10 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
}
}
if bk, err := builds.BuilderBuildKey(key, m.Fp2); err == nil && m.Fp2 != "" {
// A fingerprint the trunk carries is built from the trunk's commit, whoever asked for it first.
// A fingerprint a watched branch carries is built from that branch's commit, whoever asked for it first.
b.trunkFP[bk] = true
b.origin[bk] = o
b.edges[bk] = e
delete(b.via, bk)
}
b.mu.Unlock()
@@ -210,6 +276,43 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
b.consider(ctx, key, m)
}
// ModuleOf is a catalog entry in the form the builder builds by: its fingerprint, manifest digest, source (repository,
// directory, commit, the source revision it was taken in at), the toolchains its fingerprint was computed with, the bases it is
// built against, its packages and needs, and the named paths of its artifacts.
func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None,
Manifest: e.ManifestDigest, Packages: e.Packages, Needs: e.Needs}
if len(e.Bases) > 0 {
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases}
}
if in := e.Inputs; in != nil {
m.Src, m.Toolchains, m.Problems = in.Src, in.Toolchains, in.Problems
if len(in.Paths) > 0 {
if m.Inputs2 == nil {
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{}
}
m.Inputs2.Paths = map[string]string{}
for artifact, paths := range in.Paths {
for _, p := range paths {
m.Inputs2.Paths[p] = artifact
}
}
}
}
return m
}
// originOf is the catalog revision the entry of a build key came from; a key no entry named is of the first branch, as a build
// of the trunk's value was before branches were watched.
func (b *Builder) originOf(key string) Origin {
b.mu.Lock()
defer b.mu.Unlock()
if o, ok := b.origin[key]; ok {
return o
}
return Origin{First: true}
}
// NodesChanged is Reconsider after a toolchain moved, so a build that waited for the nodes to agree on one may go on.
func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) }
@@ -226,10 +329,14 @@ func (b *Builder) Sweep(ctx context.Context) error {
}
b.mu.Lock()
current := map[string]bool{}
for name, m := range b.known {
if m.Fp2 != "" {
if k, err := builds.BuilderBuildKey(name, m.Fp2); err == nil {
current[k] = true
// What every followed branch carries now is current, not only the first branch's (design 47 G §348).
for _, held := range []map[string]meshcatalog.ModulesByModuleValue{b.known, b.heard} {
for hk, m := range held {
name := hk[strings.LastIndex(hk, "\x00")+1:]
if m.Fp2 != "" {
if k, err := builds.BuilderBuildKey(name, m.Fp2); err == nil {
current[k] = true
}
}
}
}
@@ -399,7 +506,11 @@ func (b *Builder) planOf(ctx context.Context, digest string) (Plan, error, bool)
if err != nil {
return Plan{}, fmt.Errorf("its manifest %s cannot be read: %w", digest, err), false
}
plan, err := PlanOf(m)
plan, err := PlanOf(m, b.Seats)
var none *NoBuildModule
if errors.As(err, &none) {
return plan, err, false // the seat list or the catalog half names no build module yet: made again when they do
}
b.mu.Lock()
b.plans[digest] = planned{plan, err}
b.mu.Unlock()
@@ -475,6 +586,13 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
defer b.handEnded(key)
defer b.unrelease(key)
plan, planErr, read := b.planOf(ctx, m.Manifest)
var noModule *NoBuildModule
if errors.As(planErr, &noModule) {
// The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again
// when they do (Reconsider, at each change of either; design 47 G A-82).
b.logf("%s at %s is not built yet: %v", module, m.Fp2, planErr)
return
}
nothing := read && planErr == nil && len(plan.Tasks) == 0
if planErr == nil && !nothing && !b.isReleased(key) {
if why := b.stale(m, plan); why != "" {
@@ -505,13 +623,21 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
return
}
}
reserving := trunk && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
memberFail := ""
if !trunk && planErr == nil && !nothing {
var waiting bool
if _, memberFail, waiting = b.memberBuilds(ctx, module, m, key); waiting {
return // a package member of the group is not built (design 47 G §373): its end looks at this key again
}
}
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
if reserving && b.waitFill(module, m) {
return
}
// A key built again by hand is taken over as it stands too: one left building by a look that ended without its write
// (issue 1966) is the hand tool's to finish.
cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished,
Stored: b.storedOnce(key)})
if err != nil {
b.logf("%s: %v", module, err)
@@ -537,7 +663,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
if cl.Build.State == builds.BuildDone {
prior, only = &cl.Build, map[string]bool{}
if trunk {
for _, bm := range publishable(cl.Build, imageSeat) {
for _, bm := range publishable(cl.Build, b.Seats, imageSeat) {
only[bm] = true
publishing = true
}
@@ -546,13 +672,15 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
var reserved versions.ReservedVersion
hasReserved := false
if reserving && cl.Build.State != builds.BuildDone {
reserved, hasReserved = b.reserve(ctx, module, m, plan)
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key))
}
var result builds.BuilderBuild
if numbering.Refused != "" && cl.Build.State != builds.BuildDone {
// Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key
// fails with why, and is taken again at the next look.
result = failed("%s", numbering.Refused)
} else if memberFail != "" && cl.Build.State != builds.BuildDone {
result = failed("%s", memberFail) // a group's run: a package member's build failed or is missing (design 47 G §373)
} else if nothing {
// A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
@@ -599,6 +727,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
if b.Ended != nil {
b.Ended(ctx, cl.Key)
}
b.memberEnded(ctx, cl.Key)
ended := LogEnded(module, m, b.refOf(module, m.Fp2), b.Node, b.now(), result)
ended.OutOfStep = b.takeOutOfStep(key) // the generated files the build module's compare found out of step (design 26)
b.logBuild(ended)
@@ -612,7 +741,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
for _, a := range result.Artifacts {
digests[module+":"+a.Name] = a.Digest
}
if err := b.Reserved.Built(ctx, module, reserved.Version, digests); err != nil {
if err := b.Reserved.Built(ctx, module, reserved.Version, digests, b.takeDependencies(key)); err != nil {
b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err)
}
}
@@ -635,7 +764,7 @@ func (b *Builder) stale(m meshcatalog.ModulesByModuleValue, plan Plan) string {
if !plan.NeedsToolchain(bm) {
continue // an archive is packed with no toolchain
}
language := buildrun.LanguageOf[bm]
language, _ := b.Seats.LanguageOf(bm)
ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language))
if ag.Wait != "" {
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, ag.Wait)
@@ -659,11 +788,14 @@ type lane struct {
version string // the version the build reserved, MAJOR.MINOR.PATCH, empty for a build that reserves none (design 47 G §573)
tasks []Task
toolchain *builds.ToolchainUsed
digests map[string]string // artifact -> digest
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
imports []string // the mesh packages the lane's programs import, as its build module answered them
kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty
outOfStep []string // the generated files its build module's compare found out of step, passed or failed
digests map[string]string // artifact -> digest
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
imports []string // the mesh packages the lane's programs import, as its build module answered them
resolved map[string][]sdkstep.AskResolved // artifact -> the mesh packages its build resolved (design 47 G §466)
tcDigest string // the digest of the toolchain program the building node ran, kind 3's (design 47 G §466, §472)
tcVersion string // and the version it ran
kept string // the digest of the artifact the build node kept instead of storing it (BuildAnswer.Kept), else empty
outOfStep []string // the generated files its build module's compare found out of step, passed or failed
err error
}
@@ -686,9 +818,9 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
continue
}
var toolchain *builds.ToolchainUsed
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
if plan.NeedsToolchain(bm) {
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
language := buildrun.LanguageOf[bm]
language, _ := b.Seats.LanguageOf(bm)
ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language))
if ag.Wait != "" {
return failed("the %s toolchain is not agreed: %s", language, ag.Wait)
@@ -712,7 +844,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
}
}
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
lanes = append(lanes, &lane{module: bm, version: version, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
lanes = append(lanes, &lane{module: bm, version: version, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}, resolved: map[string][]sdkstep.AskResolved{}})
}
if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
b.handLanes(key, len(lanes))
@@ -725,6 +857,7 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
wg.Wait()
if key, err := builds.BuilderBuildKey(module, m.Fp2); err == nil {
b.keepOutOfStep(key, lanes)
b.keepDependencies(key, module, lanes)
}
var errs []string
for _, l := range lanes {
@@ -797,8 +930,9 @@ func (b *Builder) build(ctx context.Context, module string, m meshcatalog.Module
}
for _, bm := range kept {
steps = append(steps, bm+" kept")
kept, _ := b.Seats.LanguageOf(bm)
for lang, u := range prior.Toolchains {
if _, ok := used[lang]; !ok && buildrun.LanguageOf[bm] == lang {
if _, ok := used[lang]; !ok && kept == lang {
used[lang] = u
}
}
@@ -883,9 +1017,6 @@ func keyError(errs []string) string {
// want of an image builder before R4g is taken again (ADR 0525 §1).
func imageSeat() bool { return true }
// askLanguage is the build seat language of each build module (sdkstep.AskSubject).
var askLanguage = map[string]string{buildrun.GoBuildModule: sdkstep.AskGo, buildrun.NodeBuildModule: sdkstep.AskNode, ImageBuildModule: sdkstep.AskImage}
// askNotes writes a claimed key's asks into it, building, before each is put (ADR 0525 §1; issue 1887 A4b), and holds
// the asks a key taken over at start already names, so they are waited for and never put again.
type askNotes struct {
@@ -942,6 +1073,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
b.mu.Unlock()
if candidate {
req.Ref, req.Base = v.Ref, v.Base
req.Builds = b.fetchedBuilds(key) // a group's run: each package member by its build's digest (design 47 G §465)
}
}
if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
@@ -954,7 +1086,8 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
sort.Strings(req.Paths)
}
candidate := req.Base != ""
language := askLanguage[l.module]
seat, _ := b.Seats.OfModule(l.module)
language := seat.AskLanguage()
kind := sdkstep.AskDelivery
if candidate {
kind = sdkstep.AskCheck
@@ -978,6 +1111,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
// them into the workspace, whatever the repository's go.mod names.
ask.Packages = maps.Clone(m.Packages)
for _, fb := range req.Builds {
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
}
}
if id := asked.resume(language); id != "" {
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
@@ -1039,6 +1175,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
}
l.imports = answer.Imports
l.kept = answer.Kept
if answer.Toolchain != nil {
l.tcDigest, l.tcVersion = answer.Toolchain.Digest, answer.Toolchain.Version
}
built := map[string]sdkstep.AskBuilt{}
for _, a := range answer.Artifacts {
built[a.Artifact] = a
@@ -1050,6 +1189,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
return
}
l.digests[t.Artifact] = a.Digest
l.resolved[t.Artifact] = a.Resolved
if a.Reference != "" && (stored(t.Kind) || !candidate) {
l.references[t.Artifact] = a.Reference
}
@@ -1078,6 +1218,7 @@ func (b *Builder) now() time.Time {
// logBuild tells the log an entry, said in the journal when it cannot be one.
func (b *Builder) logBuild(e builds.BuildLogEntry) {
b.stepEnded(e)
if b.Log == nil {
return
}
@@ -1184,3 +1325,73 @@ func SittingOutWords(language string, a sdkstep.Agreement) []string {
}
return out
}
// keepDependencies keeps, for a build key, what its step results say each artifact resolved (kind 1) and was built with (kind 3) as
// the dependencies `package-published` carries (design 47 G §466, §673, A-57): the part of a kind-1 item is the artifact reference of
// the package, and of the kind-3 item the toolchain's part, each as the edges of the entry name it. A part no edge names is said and
// its item left out.
func (b *Builder) keepDependencies(key, module string, lanes []*lane) {
b.mu.Lock()
edges := b.edges[key]
b.mu.Unlock()
out := map[string][]meshbuilds.Dependency{}
for _, l := range lanes {
for _, t := range l.tasks {
ref := module + ":" + t.Artifact
var deps []meshbuilds.Dependency
for _, r := range l.resolved[t.Artifact] {
part := ""
for _, e := range edges {
if e.Kind == 1 && e.To == ref && e.Why == r.Package {
part = cmp.Or(e.Part, e.From)
}
}
if part == "" && b.Members != nil {
if mod, art, ok, err := b.Members(context.Background(), r.Package); ok && err == nil {
part = mod + ":" + art
}
}
if part == "" {
b.logf("%s: the package %s the build of %s resolved names no artifact: its dependency is not said", key, r.Package, ref)
continue
}
deps = append(deps, meshbuilds.Dependency{Kind: 1, Part: part, Version: r.Version, Digest: r.Digest})
}
if l.tcDigest != "" {
part := ""
for _, e := range edges {
if e.Kind == 3 && e.To == ref {
part = cmp.Or(e.Part, e.From)
}
}
if part == "" {
b.logf("%s: no edge of kind 3 names the toolchain part the build of %s used: its dependency is not said", key, ref)
} else {
deps = append(deps, meshbuilds.Dependency{Kind: 3, Part: part, Version: l.tcVersion, Digest: l.tcDigest})
}
}
if len(deps) > 0 {
out[ref] = deps
}
}
}
b.mu.Lock()
defer b.mu.Unlock()
if b.deps == nil {
b.deps = map[string]map[string][]meshbuilds.Dependency{}
}
if len(out) == 0 {
delete(b.deps, key)
return
}
b.deps[key] = out
}
// takeDependencies answers the dependencies held for a build key and forgets them.
func (b *Builder) takeDependencies(key string) map[string][]meshbuilds.Dependency {
b.mu.Lock()
defer b.mu.Unlock()
d := b.deps[key]
delete(b.deps, key)
return d
}
modules/mesh-delivery-builder/internal/build/run_test.go
+9 -9
View File
@@ -145,7 +145,7 @@ type rig struct {
}
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"}
var toolPrefix = map[string]string{testGo: "build_go_", testNode: "build_node_", testImage: "build_image_"}
// answer runs an ask as the build module that pulls it does (runAsk), with the fake build modules' tools on the
// rig's pulling node, and answers the step that node writes.
@@ -178,14 +178,14 @@ func (r *rig) answer(a buildstep.StepAsk) buildstep.Step {
func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
t.Helper()
r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{Seats: testSeats()}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
for _, n := range []string{"ace", "novox"} {
r.tools.Set(n, tcOf("go", "go1.27.1"))
r.tools.Set(n, tcOf("typescript", "tsc5"))
}
r.asks = &buildrun.FakeAsks{Answer: r.answer}
r.b = &Builder{
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
Seats: testSeats(), Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
}
@@ -667,7 +667,7 @@ func aNewerFingerprintWithdrawsTheBuildOfTheOlderOne(t *testing.T) {
}
return r.answer(a)
}
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second, Dropper: drops}
r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second, Dropper: drops}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
<-running
fp2 := "src2:" + hex64('2')
@@ -915,7 +915,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
t.Run("waits in its stream: never asked again", func(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
gone := &goneAsks{why: func(buildstep.StepAsk) string { return "" }}
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
@@ -1122,12 +1122,12 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
gone := &goneAsks{}
dead := &buildrun.DeadLetters{}
if c.dead {
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
}
var heard *buildrun.HeardEnds
if c.deaf {
heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks}
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
r.b.Asks = buildrun.Asker{Seats: testSeats(), Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
}
if !c.byHand && !c.dead && !c.deaf {
gone.why = func(a buildstep.StepAsk) string {
@@ -1158,7 +1158,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
t.Fatal("no ask is waited on")
}
}
heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended
heard.Nudge(testGo) // a step of mesh-build-go ended
}
if c.done && !c.deaf {
asks := waitForAsks(t, r, c.puts)
@@ -1269,7 +1269,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
running, _ := json.Marshal(buildstep.Step{V: keys.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: buildstep.StepRunning, Started: at})
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Seats: testSeats(), Steps: map[string]StateRead{"node-build-go": state(map[string]string{}), "node-build-image": state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
Now: func() time.Time { return at.Add(time.Hour) }}
out, err := r.Stuck("")
modules/mesh-delivery-builder/internal/build/seats_test.go
+23
View File
@@ -0,0 +1,23 @@
package build
import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The build modules of the fixture seat list: the holders of the three build seats (design 47 G A-82).
const (
testGo = "mesh-build-go"
testNode = "mesh-build-node"
testImage = "mesh-build-image"
)
// testSeats is the finder over a seat list whose build seats act on go, typescript and kind:image, each held by its module.
func testSeats() *buildrun.BuildSeats {
rule := func(on string) seatdef.Definition {
return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}}
}
defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript"), "node-build-image": rule(seatdef.OnKindImage)}
holders := map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage}
return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok })
}
modules/mesh-delivery-builder/internal/build/step.go
+156
View File
@@ -0,0 +1,156 @@
package build
import (
"context"
"fmt"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// **A candidate's build is the runner's step ask** (design 47 G A-97, A-84, A-99): the runner's asking half asks the seat
// mesh-builds a `build` step naming each entry by fingerprint, and the builder, its one entry point for a candidate, reads each
// entry through the seat mesh-catalog's `entry` (the repository and the commit are the member's head), builds it as it builds any
// fingerprint (the existing claim, plan and lanes) and answers the step's end. A build is per fingerprint, so no pull request
// number is needed, and the builder follows no candidate.
// BuildStep builds the entries of one step ask and answers its end: passed when every entry's build is done, failed when one
// failed (said), could not run when an entry could not be read, nothing to run for an ask naming none. The ask's id and the node
// are the caller's to set.
func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbuilds.StepEnded {
b.init()
started := b.now()
end := meshbuilds.StepEnded{V: 1, Pipeline: run.Pipeline, Step: meshbuilds.StepEndedStepBuild, Rerun: run.Rerun, By: run.By, Entries: run.Entries, Node: b.Node, Started: &started}
finish := func(outcome, said string) meshbuilds.StepEnded {
at := b.now()
end.Ended, end.Outcome, end.Said = &at, outcome, said
return end
}
if len(run.Entries) == 0 {
return finish(meshbuilds.StepEndedOutcomeNothingToRun, "the ask names no entry")
}
if b.StepEntry == nil {
return finish(meshbuilds.StepEndedOutcomeCouldNotRun, "the builder reads no entry of the seat mesh-catalog")
}
var failed []string
for _, e := range run.Entries {
outcome, said := b.buildEntry(ctx, e)
switch outcome {
case meshbuilds.StepEndedOutcomeCouldNotRun:
return finish(outcome, said)
case meshbuilds.StepEndedOutcomeFailed:
failed = append(failed, said)
}
}
if len(failed) > 0 {
return finish(meshbuilds.StepEndedOutcomeFailed, strings.Join(failed, "; "))
}
return finish(meshbuilds.StepEndedOutcomePassed, "")
}
// buildEntry builds one entry at its fingerprint and waits for its end.
func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outcome, said string) {
entry, err := b.StepEntry(ctx, e.Fingerprint)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry %s at %s cannot be read: %v", e.Module, e.Fingerprint, err)
}
if entry.Module != e.Module || (e.Commit != "" && entry.Commit != e.Commit) {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("the entry of %s at %s is of %s at %s", e.Module, e.Fingerprint, entry.Module, entry.Commit)
}
key, err := builds.BuilderBuildKey(entry.Module, entry.Fp)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error()
}
ended := b.awaitEnd(key)
defer b.dropEnd(key, ended)
if done, failedWords, found := b.keyEnd(ctx, key); found {
if done {
return meshbuilds.StepEndedOutcomePassed, ""
}
return meshbuilds.StepEndedOutcomeFailed, failedWords
}
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path,
Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{}
}
b.mu.Unlock()
b.consider(ctx, entry.Module, m)
select {
case log := <-ended:
if log.Outcome == builds.BuildDone {
return meshbuilds.StepEndedOutcomePassed, ""
}
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error)
case <-ctx.Done():
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the builder stopped before its build ended", entry.Module, entry.Fp)
}
}
// keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended.
func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) {
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
if err != nil || !ok {
return false, "", false
}
v, err := builds.DecodeBuilderBuild(raw)
if err != nil {
return false, "", false
}
switch {
case v.State == builds.BuildDone && !Unpublished(v, b.Seats):
return true, "", true
case v.State == builds.BuildFailed && builds.FailedUntilChanged(v.Error):
return false, key + ": " + v.Error, true
}
return false, "", false
}
// awaitEnd registers a wait for the end of the build of key, heard at its log entry (logBuild).
func (b *Builder) awaitEnd(key string) chan builds.BuildLogEntry {
ch := make(chan builds.BuildLogEntry, 1)
b.mu.Lock()
if b.stepWaits == nil {
b.stepWaits = map[string][]chan builds.BuildLogEntry{}
}
b.stepWaits[key] = append(b.stepWaits[key], ch)
b.mu.Unlock()
return ch
}
func (b *Builder) dropEnd(key string, ch chan builds.BuildLogEntry) {
b.mu.Lock()
defer b.mu.Unlock()
for i, c := range b.stepWaits[key] {
if c == ch {
b.stepWaits[key] = append(b.stepWaits[key][:i], b.stepWaits[key][i+1:]...)
break
}
}
if len(b.stepWaits[key]) == 0 {
delete(b.stepWaits, key)
}
}
// stepEnded tells the step asks waiting for a build that its end was logged.
func (b *Builder) stepEnded(e builds.BuildLogEntry) {
if e.Event != builds.BuildEndedEvent {
return
}
key, err := builds.BuilderBuildKey(e.Module, e.FP)
if err != nil {
return
}
b.mu.Lock()
chans := append([]chan builds.BuildLogEntry(nil), b.stepWaits[key]...)
b.mu.Unlock()
for _, ch := range chans {
select {
case ch <- e:
default:
}
}
}
modules/mesh-delivery-builder/internal/build/step_test.go
+76
View File
@@ -0,0 +1,76 @@
package build
import (
"context"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The runner's build step ask is the builder's one entry point for a candidate (design 47 G A-97): each entry is read by its
// fingerprint, built through the one claim, and the step's end says passed only for a build that is done.
func TestABuildStepAnswersItsEnd(t *testing.T) {
r := newRig(t, catalogue.Artifact{Name: "files", Kind: "archive"})
entryOf := func(fp, commit string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: commit, Fp: fp, ManifestDigest: mdigest('a'), Path: "."}
}
r.b.StepEntry = func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
if fp == fp1 {
return entryOf(fp1, "c1"), nil
}
return meshcatalog.CatalogEntry{}, errors.New("not found")
}
ask := func(entries ...meshbuilds.BuildEntry) meshbuilds.StepEnded {
return r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries})
}
if end := ask(); end.Outcome != meshbuilds.StepEndedOutcomeNothingToRun {
t.Errorf("an ask naming no entry: %+v", end)
}
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: "src2:" + strings.Repeat("2", 64)}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "cannot be read") {
t.Errorf("an entry the seat does not hold: %+v", end)
}
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: fp1, Commit: "other"}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun {
t.Errorf("an entry at another commit: %+v", end)
}
// A build already done is passed at once, naming each entry as it was asked.
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a')})
if _, err := r.st.Create(t.Context(), r.key, done); err != nil {
t.Fatal(err)
}
entries := []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1, Commit: "c1"}}
if end := ask(entries...); end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Step != meshbuilds.StepEndedStepBuild || len(end.Entries) != 1 || end.Entries[0].Commit != "c1" || end.Pipeline != "main.1.m" {
t.Errorf("a done build: %+v", end)
}
}
// A step ask waits for the end of its build without a timer: the log entry of that end wakes it (design 47 G A-97).
func TestAStepAskIsWokenByItsBuildsEnd(t *testing.T) {
r := newRig(t)
key := r.key
ch := r.b.awaitEnd(key)
r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildStartedEvent, Module: "m", FP: fp1}) // a start wakes nothing
select {
case <-ch:
t.Fatal("a build's start woke the wait")
default:
}
r.b.stepEnded(builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: fp1, Outcome: builds.BuildDone})
select {
case e := <-ch:
if e.Outcome != builds.BuildDone {
t.Errorf("end = %+v", e)
}
case <-time.After(time.Second):
t.Fatal("the end of the build did not wake the wait")
}
r.b.dropEnd(key, ch)
if len(r.b.stepWaits) != 0 {
t.Errorf("waits left: %v", r.b.stepWaits)
}
}
modules/mesh-delivery-builder/internal/build/stuck.go
+19 -6
View File
@@ -236,8 +236,13 @@ type Stuck struct {
type StuckReader struct {
Builds StateRead
ReAsks StateRead
// Steps are the build modules' steps states, by build module.
// Steps are the build seats' steps states (`<seat>.steps`, design 47 G A-82), by build seat.
Steps map[string]StateRead
// StepsOf, when set, reads the steps state of a build seat the seat list names (`<seat>.steps`), instead of Steps: the seats
// are known only once the list is read.
StepsOf func(seat string) StateRead
// Seats finds the build module of a build seat, whose name the key's step words give.
Seats *buildrun.BuildSeats
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
Hearing StateRead
Now func() time.Time
@@ -365,13 +370,14 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
s.Age = now().Sub(at).Round(time.Second).String()
}
for language, id := range v.Asks {
bm, _ := buildstep.BuildModuleOf(language)
a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
seat, _ := buildstep.BuildSeat(language)
bm, _ := r.Seats.OfSeat(seat)
a := StuckAsk{Language: language, BuildModule: bm.Module, ID: id, Asking: askingOf(v.Step, bm.Module), Step: "none"}
if t, ok := idTime(id); ok {
a.Asked = t
}
if st, ok := r.Steps[bm]; ok {
if k, ok := stepKeys[bm][id]; ok {
if st, ok := steps[seat]; ok {
if k, ok := stepKeys[seat][id]; ok {
reads++
if raw, _, err := st.Get(k); err == nil && raw != nil {
if step, err := buildstep.DecodeStep(raw); err == nil {
@@ -416,7 +422,14 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
// until an ask names it.
func (r StuckReader) stepIndex() (map[string]map[string]string, error) {
stepKeys := map[string]map[string]string{}
for bm, s := range r.Steps {
steps := r.Steps
if r.StepsOf != nil {
steps = map[string]StateRead{}
for _, m := range r.Seats.All() {
steps[m.Seat] = r.StepsOf(m.Seat)
}
}
for bm, s := range steps {
ks, err := s.Keys()
if err != nil {
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
modules/mesh-delivery-builder/internal/build/waits.go
+1 -2
View File
@@ -9,7 +9,6 @@ import (
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
@@ -57,7 +56,7 @@ func (b *Builder) toolchainWait(module string, m meshcatalog.ModulesByModuleValu
if !plan.NeedsToolchain(bm) {
continue
}
language := buildrun.LanguageOf[bm]
language, _ := b.Seats.LanguageOf(bm)
ag := sdkstep.ToolchainAgreement(b.Toolchains.Entries(language))
if want, ok := m.Toolchains[language]; ag.Wait == "" && ok && want.Version != ag.Toolchain.Version {
return ToolchainWait{Module: module, Fingerprint: m.Fp2, Language: language, Version: want.Version, Since: b.now()}, true
modules/mesh-delivery-builder/internal/versions/entrynames.go
+137
View File
@@ -0,0 +1,137 @@
package versions
import (
"context"
"encoding/json"
"slices"
"sort"
"strings"
"sync"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Entries are the catalog graph entries of the halves the builder follows, one per module and watched branch (design 47 G §348,
// §614): where a package's name in its ecosystem is read from, each artifact's `package` (§676), never the indexer's state.
type Entries func() []meshcatalog.CatalogEntry
// artifactOf is the artifact name of an artifact reference `<module>:<artifact>` of module.
func artifactOf(module, ref string) (string, bool) {
m, a, ok := strings.Cut(ref, ":")
return a, ok && m == module && a != ""
}
// NamesFrom answers a package's name in its ecosystem from the entries (design 47 G §348): the name the entry of module gives the
// artifact; known is false when no entry names one for it.
func NamesFrom(all Entries) PackageNames {
return func(_ context.Context, module, artifact string) (string, bool, error) {
for _, e := range all() {
if e.Module != module {
continue
}
for _, a := range e.Artifacts {
if name, ok := artifactOf(module, a.Ref); ok && name == artifact && a.Package != nil && a.Package.Name != "" {
return a.Package.Name, true, nil
}
}
}
return "", false, nil
}
}
// CarriersFrom are the module artifacts a package's name in its ecosystem is the name of, as the entries hold them (design 47 G
// §572, §478), each distinct module and artifact once, sorted; a name no entry carries answers none.
func CarriersFrom(all Entries) func(ctx context.Context, name string) ([]Built, error) {
return func(_ context.Context, name string) ([]Built, error) {
seen := map[Built]bool{}
var out []Built
for _, e := range all() {
for _, a := range e.Artifacts {
artifact, ok := artifactOf(e.Module, a.Ref)
if !ok || a.Package == nil || a.Package.Name != name || seen[Built{e.Module, artifact}] {
continue
}
seen[Built{e.Module, artifact}] = true
out = append(out, Built{e.Module, artifact})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Module+"/"+out[i].Artifact < out[j].Module+"/"+out[j].Artifact })
return out, nil
}
}
// ModuleFrom is the module and artifact a package's name is built from, from the entries (CarriersFrom); more than one entry
// carrying it is an error naming them.
func ModuleFrom(all Entries) func(ctx context.Context, name string) (module, artifact string, found bool, err error) {
return moduleOfCarriers(CarriersFrom(all))
}
// EntriesOf are the entries of every half the branch follower holds, the followed branches in the order they came.
func EntriesOf(branches *meshcatalogclient.Branches) Entries {
return func() []meshcatalog.CatalogEntry {
var out []meshcatalog.CatalogEntry
for _, br := range branches.Followed() {
if cat, ok := branches.Catalog(br); ok {
if half, ok := cat.View(); ok {
for _, e := range half.Entries {
out = append(out, e)
}
}
}
}
return out
}
}
// HolderOf finds the module that holds a seat: the one whose entry in the first followed branch's half (the trunk's) has a
// manifest that claims it (design 47 G A-82; the seat list answers seat definitions alone, §527). More than one claimant, or
// none, is no holder.
func HolderOf(branches *meshcatalogclient.Branches) func(seat string) (string, bool) {
var mu sync.Mutex
claims := map[string][]string{} // manifest digest -> the seats it claims
claimed := func(e meshcatalog.CatalogEntry) []string {
mu.Lock()
defer mu.Unlock()
if seats, ok := claims[e.ManifestDigest]; ok && e.ManifestDigest != "" {
return seats
}
var seats []string
if raw, err := json.Marshal(e.Manifest); err == nil {
if mf, err := catalogue.Decode(raw); err == nil {
for _, c := range mf.Claims {
seats = append(seats, c.Name)
}
}
}
if e.ManifestDigest != "" {
claims[e.ManifestDigest] = seats
}
return seats
}
return func(seat string) (string, bool) {
followed := branches.Followed()
if len(followed) == 0 {
return "", false
}
cat, ok := branches.Catalog(followed[0])
if !ok {
return "", false
}
half, ok := cat.View()
if !ok {
return "", false
}
var holders []string
for _, e := range half.Entries {
if slices.Contains(claimed(e), seat) {
holders = append(holders, e.Module)
}
}
if len(holders) != 1 {
return "", false
}
return holders[0], true
}
}
modules/mesh-delivery-builder/internal/versions/entrynames_test.go
+55
View File
@@ -0,0 +1,55 @@
package versions
import (
"context"
"reflect"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
func named(module, artifact, name string) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: module, Artifacts: []meshcatalog.Artifact{
{Ref: module + ":" + artifact, Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: name}}}}
}
// TestPackageNamesAreReadFromTheCatalogHalfsEntries: a package's name in its ecosystem is the name the entry of its module gives the
// artifact (design 47 G §348, §676), the module and artifact a name is built from are the entries carrying it, and a name two
// entries carry is an error naming both (§491); no state of the indexer is read.
func TestPackageNamesAreReadFromTheCatalogHalfsEntries(t *testing.T) {
entries := func() []meshcatalog.CatalogEntry {
return []meshcatalog.CatalogEntry{named("mesh-sdk", "go", "git.novox.be/novox/mesh-sdk/go"), named("netcheck", "lib", "netcheck"),
named("twin-a", "lib", "twin"), named("twin-b", "lib", "twin"), {Module: "bare"}}
}
ctx := context.Background()
for _, c := range []struct {
module, artifact string
name string
known bool
}{
{"mesh-sdk", "go", "git.novox.be/novox/mesh-sdk/go", true},
{"netcheck", "lib", "netcheck", true},
{"netcheck", "other", "", false},
{"bare", "lib", "", false},
{"unknown", "lib", "", false},
} {
got, known, err := NamesFrom(entries)(ctx, c.module, c.artifact)
if err != nil || got != c.name || known != c.known {
t.Errorf("NamesFrom(%s/%s) = %q, %v, %v; want %q, %v", c.module, c.artifact, got, known, err, c.name, c.known)
}
}
module, artifact, found, err := ModuleFrom(entries)(ctx, "netcheck")
if module != "netcheck" || artifact != "lib" || !found || err != nil {
t.Errorf("ModuleFrom(netcheck) = %q, %q, %v, %v", module, artifact, found, err)
}
if _, _, found, err := ModuleFrom(entries)(ctx, "nobody"); found || err != nil {
t.Errorf("ModuleFrom(nobody) = %v, %v; want none", found, err)
}
if _, _, _, err := ModuleFrom(entries)(ctx, "twin"); err == nil {
t.Error("ModuleFrom(twin): two entries carry it, want an error")
}
got, _ := CarriersFrom(entries)(ctx, "twin")
if want := []Built{{"twin-a", "lib"}, {"twin-b", "lib"}}; !reflect.DeepEqual(got, want) {
t.Errorf("CarriersFrom(twin) = %v, want %v", got, want)
}
}
modules/mesh-delivery-builder/internal/versions/floor.go
+15 -6
View File
@@ -77,11 +77,9 @@ const (
// sourceOfWake is the floor source each event says something of; a clear names the source by its module.
var sourceOfWake = map[string]string{WakePublished: SourceSeat, WakeForge: SourceTags, WakeAccepted: SourceRequirements}
// ModuleOfSource is the module whose health condition's clear wakes a silent source.
var ModuleOfSource = map[string]string{
SourceTags: "mesh-delivery-forge-bridge",
SourceRequirements: "mesh-delivery-indexer",
}
// SourceSeats are the seats whose holders are the floor's tag and requirement sources (design 47 G A-82): the seat git (tags)
// and the seat mesh-catalog (requirements), the holders read from the seat list and the catalog half.
var SourceSeats = map[string]string{SourceTags: "git", SourceRequirements: "mesh-catalog"}
// FloorWait is one name waiting on its floor: the sources still silent and why, since when; the finding the builder
// writes names them, cleared once all three answer.
@@ -135,6 +133,8 @@ type FloorWaits struct {
keeper FloorKeeper
// Logf says a wait that could not be kept; nil says nothing.
Logf func(string, ...any)
// Holder finds the module holding a seat (the catalog half's claim); a source's module is the holder of its SourceSeats seat.
Holder func(seat string) (string, bool)
// Said is told each wait recorded (raised) and each answered (not raised): the builder's finding floor-silent.
Said func(w FloorWait, raised bool)
}
@@ -290,7 +290,7 @@ func (fw *FloorWaits) Wake(kind, seat, module, name string) []string {
}
case WakeCleared:
for _, s := range w.Silent {
if ModuleOfSource[s] == module || (s == SourceSeat && w.Holder != "" && w.Holder == module) {
if fw.holds(SourceSeats[s], module) || (s == SourceSeat && w.Holder != "" && w.Holder == module) {
out = append(out, n)
break
}
@@ -317,3 +317,12 @@ func (fw *FloorWaits) Waiting() []FloorWait {
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out
}
// holds is whether module holds seat on the catalog half.
func (fw *FloorWaits) holds(seat, module string) bool {
if seat == "" || module == "" || fw.Holder == nil {
return false
}
holder, ok := fw.Holder(seat)
return ok && holder == module
}
modules/mesh-delivery-builder/internal/versions/names.go
+72
View File
@@ -2,6 +2,8 @@ package versions
import (
"context"
"errors"
"fmt"
"sort"
"strings"
@@ -70,3 +72,73 @@ func isSeriesKey(key string) bool {
_, _, ok := meshcatalog.ParsePackagesByPackageSeriesKey(key)
return ok
}
// Built is a module's artifact, which a package's name is the name of.
type Built struct{ Module, Artifact string }
// Carriers are the module artifacts a package's name in its ecosystem is the name of, as the catalog half's packages view
// holds them (the inverse of NamesIn; design 47 G §572, §478): each distinct module and artifact of a series key whose value
// names that name, sorted. A name no entry carries answers none.
func Carriers(s PackagesState) func(ctx context.Context, name string) ([]Built, error) {
return func(ctx context.Context, name string) ([]Built, error) {
keys, err := s.Keys(ctx)
if err != nil {
return nil, err
}
sort.Strings(keys)
seen := map[Built]bool{}
var out []Built
for _, key := range keys {
if !isSeriesKey(key) {
continue
}
value, found, err := s.Get(ctx, key)
if err != nil {
return nil, err
}
if !found {
continue
}
v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value)
if err != nil || v.Name != name {
continue
}
if module, artifact, cut := strings.Cut(v.Package, "/"); cut && module != "" && artifact != "" && !seen[Built{module, artifact}] {
seen[Built{module, artifact}] = true
out = append(out, Built{module, artifact})
}
}
return out, nil
}
}
// ModuleOf is the module and artifact a package's name is built from (Carriers), found false for a name no entry carries; more
// than one entry carrying it is an error naming them.
func ModuleOf(s PackagesState) func(ctx context.Context, name string) (module, artifact string, found bool, err error) {
return moduleOfCarriers(Carriers(s))
}
// moduleOfCarriers is ModuleOf over one way of finding the carriers.
func moduleOfCarriers(carriers func(ctx context.Context, name string) ([]Built, error)) func(ctx context.Context, name string) (module, artifact string, found bool, err error) {
return func(ctx context.Context, name string) (string, string, bool, error) {
got, err := carriers(ctx, name)
switch {
case err != nil:
return "", "", false, err
case len(got) == 0:
return "", "", false, nil
case len(got) > 1:
return "", "", false, errors.New(MoreThanOne(name, got))
}
return got[0].Module, got[0].Artifact, true, nil
}
}
// MoreThanOne is the words of a name that more than one entry carries.
func MoreThanOne(name string, got []Built) string {
var refs []string
for _, b := range got {
refs = append(refs, b.Module+"/"+b.Artifact)
}
return fmt.Sprintf("the name %s is carried by more than one entry: %s", name, strings.Join(refs, ", "))
}
modules/mesh-delivery-builder/internal/versions/publish.go
+10 -1
View File
@@ -38,8 +38,16 @@ const (
oldFormBoundPrefix = "old-form."
unputBoundPrefix = "unput."
stepsBoundPrefix = "steps."
memberBoundPrefix = "member."
)
// MemberBoundKey is the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77):
// `member.<build key>`. Its end while the member is not built fails the dependent's step naming the member.
func MemberBoundKey(buildKey string) string { return memberBoundPrefix + buildKey }
// MemberBoundOf is the build key a member wait's bound names, and whether key is one.
func MemberBoundOf(key string) (string, bool) { return strings.CutPrefix(key, memberBoundPrefix) }
// UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966,
// novox/hq ADR 0668 §3): `unput.<build key>`. Its end while the key still stands so, with no look holding it, is the
// finding call-unanswered about the key.
@@ -84,7 +92,8 @@ const BoundsTool = "builder_bounds"
const BoundsDescription = "The builder's expiring keys, its state " + BoundsState + " (design 51 *Health and conditions*): each key with what it " +
"bounds (a version's publish, whose end unanswered is the finding " + PublishUnanswered + "; or a node's seven days holding the old " +
"form's window, whose end is the finding " + OldFormHeld + "; or a build key left building because its end could not be written, " +
"whose end is the finding call-unanswered), when it ends, and how long is left. The bus ends each key by its own time to live."
"whose end is the finding call-unanswered; or a dependent's wait for a package member of a group's run, whose end fails its " +
"step), when it ends, and how long is left. The bus ends each key by its own time to live."
// Bound is a key's value in BoundsState: what it bounds, in plain words, and when it ends.
type Bound struct {
modules/mesh-delivery-builder/internal/versions/reserve.go
+91 -6
View File
@@ -11,6 +11,7 @@ import (
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// **The reservation is one key per module and MAJOR.MINOR** (design 47 G §369, §52): the builder keeps in its old state
@@ -60,6 +61,13 @@ type ReservedVersion struct {
// Unstored are the artifacts a holder refused as not stored, with its words: said once, and not published again on
// events (design 47 G §588); builder_build_again builds the key again, which reserves anew.
Unstored map[string]string `json:"unstored,omitempty"`
// Branch and Source are the watched branch and the source revision of the catalog revision whose entry the build came from
// (design 47 G §371, A-57): what `package-published` says of the publication.
Branch string `json:"branch,omitempty"`
Source int64 `json:"source,omitempty"`
// Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with
// (kind 3), kept for `package-published` (design 47 G §466, §673, A-57).
Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"`
}
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
@@ -69,6 +77,10 @@ type Source struct {
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
Fingerprint string
GoPackage bool
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
Branch string
Revision int64
}
// ReservedKey is the key of module's reservation of series (MAJOR.MINOR).
@@ -143,7 +155,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
}
}
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(),
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint}
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision}
if src.GoPackage {
out.Tag = GoTagOf(out.Version)
}
@@ -180,7 +192,7 @@ func (r *Reserver) get(ctx context.Context, key string) (ReservedVersion, uint64
// Built records artifact's digest under the reserved version of module's series; once every artifact has one the build is
// ended (BuiltAt). A reservation moved on to another version (a newer build took the next) is left alone and answers false.
func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string) (bool, error) {
func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string, deps []meshbuilds.Dependency) (bool, error) {
series, _, err := SplitVersion(version)
if err != nil {
return false, err
@@ -192,6 +204,12 @@ func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest
return false, err
}
cur.Artifacts[artifact] = digest
if len(deps) > 0 {
if cur.Dependencies == nil {
cur.Dependencies = map[string][]meshbuilds.Dependency{}
}
cur.Dependencies[artifact] = deps
}
if cur.BuiltAt.IsZero() && allSet(cur.Artifacts) {
cur.BuiltAt = r.now()
}
@@ -312,9 +330,24 @@ func (r *Reserver) Drop(ctx context.Context, module, series string) (bool, error
return true, r.Store.Delete(ctx, key)
}
// Marked is an artifact of a reservation a registry seat's published newly marked: the reservation as it stands then.
type Marked struct {
Reserved ReservedVersion
Artifact string
}
// PackagePublished is the seat mesh-builds' package-published of one artifact the holder said published at the reservation's
// version (design 47 G §52, §334, §673, A-57): the artifact reference, the version and the digest published, the watched branch and
// the source revision the build was made from, and the dependencies its step result named.
func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.PackagePublished {
at = at.UTC()
return meshbuilds.PackagePublished{V: 1, Artifact: m.Artifact, At: &at, Branch: m.Reserved.Branch, Source: m.Reserved.Source, Module: m.Reserved.Module,
Version: m.Reserved.Version, Digest: digest, Dependencies: m.Reserved.Dependencies[m.Artifact]}
}
// Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form)
// whose built digest it names are marked published. It answers the keys it marked.
func (r *Reserver) Published(ctx context.Context, version, digest string) ([]string, error) {
// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked.
func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Marked, error) {
version = strings.TrimPrefix(version, "v")
if version == "" || digest == "" {
return nil, nil
@@ -324,7 +357,7 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]str
return nil, fmt.Errorf("the reservations cannot be listed: %w", err)
}
sort.Strings(keys)
var marked []string
var marked []Marked
for _, k := range keys {
again:
cur, rev, found, err := r.get(ctx, k)
@@ -335,14 +368,17 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]str
continue
}
changed := false
var arts []string
for a, d := range cur.Artifacts {
if d == digest && !cur.Published[a] {
if cur.Published == nil {
cur.Published = map[string]bool{}
}
cur.Published[a], changed = true, true
arts = append(arts, a)
}
}
sort.Strings(arts)
if !changed {
continue
}
@@ -352,7 +388,9 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]str
} else if err != nil {
return marked, fmt.Errorf("%s cannot be written: %w", k, err)
}
marked = append(marked, k)
for _, a := range arts {
marked = append(marked, Marked{Reserved: cur, Artifact: a})
}
}
return marked, nil
}
@@ -444,6 +482,53 @@ func AnswerReservedVerb(r Reader, module, version string) (meshbuilds.ReservedAn
return out, nil
}
// CatalogVersions answers a module's released versions, the seat mesh-catalog's verb versions (design 47 G §582).
type CatalogVersions func(ctx context.Context, module string) ([]meshcatalog.VersionEntry, error)
// AnswerReservedByName is the verb reserved given a package's name (design 47 G §572, §582, A-70): the module is found from the
// name in the catalog half's view (carriers); that module's reserved version answers as AnswerReservedVerb does. A version
// given with no reservation of it (a reservation goes once its version is released) is asked of the catalog, and answers
// released with the artifacts and digests the release recorded. A name no entry carries, more than one entry carries, or a
// version neither reserved nor released is answered error, naming which, set alone.
func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.Context, string) ([]Built, error), released CatalogVersions, name, version string) (meshbuilds.ReservedAnswer, error) {
name, version = strings.TrimSpace(name), strings.TrimSpace(version)
got, err := carriers(ctx, name)
if err != nil {
return meshbuilds.ReservedAnswer{}, fmt.Errorf("the entries of %s cannot be read: %w", name, err)
}
switch {
case len(got) == 0:
return meshbuilds.ReservedAnswer{Error: "no entry carries the name " + name}, nil
case len(got) > 1:
return meshbuilds.ReservedAnswer{Error: MoreThanOne(name, got)}, nil
}
module := got[0].Module
rs := &Reserver{Store: readOnlyReserve{readStore{r}}}
if _, found, err := rs.Of(ctx, module, version); err != nil {
return meshbuilds.ReservedAnswer{}, err
} else if found {
return AnswerReservedVerb(r, module, version)
}
if version != "" {
entries, err := released(ctx, module)
if err != nil {
return meshbuilds.ReservedAnswer{}, fmt.Errorf("the released versions of %s cannot be read: %w", module, err)
}
for _, e := range entries {
if e.Version != version {
continue
}
out := meshbuilds.ReservedAnswer{Module: module, Version: version, Released: true, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{}}
for _, a := range e.Artifacts {
out.Artifacts = append(out.Artifacts, meshbuilds.ReservedAnswerArtifactsItem{Artifact: a.Artifact, Digest: a.Digest})
}
return out, nil
}
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no version %s reserved or released", name, module, version)}, nil
}
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil
}
// The descriptions of the reservation's tools.
const (
ReservedDescription = "The versions the builder reserved (its state reserved, design 47 G section 369): one per module and MAJOR.MINOR, " +
modules/mesh-delivery-builder/internal/versions/reserve_test.go
+15 -5
View File
@@ -78,11 +78,11 @@ func TestReservation(t *testing.T) {
// The artifacts' digests, then the bound.
for _, a := range arts {
if ok, err := r.Built(ctx, "m", "0.1.23", a, "sha256:"+a); err != nil || !ok {
if ok, err := r.Built(ctx, "m", "0.1.23", a, "sha256:"+a, nil); err != nil || !ok {
t.Fatalf("built %s: %v %v", a, ok, err)
}
}
if ok, _ := r.Built(ctx, "m", "0.1.22", "m:a", "x"); ok {
if ok, _ := r.Built(ctx, "m", "0.1.22", "m:a", "x", nil); ok {
t.Error("a digest was written under a version a newer build superseded")
}
published := func(_, _, a string) bool { return a == "m:a" }
@@ -118,11 +118,12 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
ctx := context.Background()
store := &reserveMem{mem: newMem()}
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }}
v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{})
v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, Source{Branch: "main", Revision: 41})
if err != nil || v.Version != "1.4.3" {
t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err)
}
if _, err := r.Built(ctx, "m", "1.4.3", "m:a", "sha256:aa"); err != nil {
deps := []meshbuilds.Dependency{{Kind: 1, Part: "mesh-sdk:go", Version: "0.1.14", Digest: "sha256:dd"}, {Kind: 3, Part: "go-toolchain:program", Version: "go1.27", Digest: "sha256:tt"}}
if _, err := r.Built(ctx, "m", "1.4.3", "m:a", "sha256:aa", deps); err != nil {
t.Fatal(err)
}
rd := Reader{
@@ -142,9 +143,18 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
if got, err := AnswerReserved(rd, "other"); err != nil || len(got["reserved"].([]ReservedVersion)) != 0 {
t.Errorf("builder_reserved of another module = %v (%v), want none", got, err)
}
if marked, err := r.Published(ctx, "v1.4.3", "sha256:aa"); err != nil || len(marked) != 1 {
marked, err := r.Published(ctx, "v1.4.3", "sha256:aa")
if err != nil || len(marked) != 1 {
t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err)
}
// Design 47 G §52, §673, A-57: the package-published of that artifact carries the digest published, the branch and the source
// revision the reservation was made for, and the dependencies its step result named.
at := time.Unix(100, 0)
said := marked[0].PackagePublished("sha256:aa", at)
atUTC := at.UTC()
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
t.Errorf("package-published = %+v, want %+v", said, wantSaid)
}
if marked, _ := r.Published(ctx, "1.4.3", "sha256:other"); len(marked) != 0 {
t.Errorf("a published with another digest marked %v", marked)
}
modules/mesh-delivery-builder/internal/versions/reservedname_test.go
+76
View File
@@ -0,0 +1,76 @@
package versions
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// TestReservedByNameAnswersFromTheViewAndTheCatalog: design 47 G §572, §582, A-70 — a package's name (an npm name or a Go
// module path) is mapped to its module from the catalog half's view; the reserved version answers as for the module, a version
// no reservation holds is asked of the catalog's versions and answers released with the release's digests, and a name no entry
// carries, one more than one carries, or a version neither reserved nor released answers error, set alone.
func TestReservedByNameAnswersFromTheViewAndTheCatalog(t *testing.T) {
ctx := context.Background()
view := packagesMem{
"git=2enovox=2ebe/novox/mesh-sdk/go.0.1": []byte(`{"v":1,"name":"git.novox.be/novox/mesh-sdk/go","package":"mesh-sdk/go","series":"0.1"}`),
"git=2enovox=2ebe/novox/mesh-sdk/go.0.2": []byte(`{"v":1,"name":"git.novox.be/novox/mesh-sdk/go","package":"mesh-sdk/go","series":"0.2"}`),
"@novox/mesh-sdk.0.1": []byte(`{"v":1,"name":"@novox/mesh-sdk","package":"mesh-sdk/ts","series":"0.1"}`),
"twice.0.1": []byte(`{"v":1,"name":"twice","package":"a/x","series":"0.1"}`),
"twice-b.0.1": []byte(`{"v":1,"name":"twice","package":"b/y","series":"0.1"}`),
}
store := &reserveMem{mem: newMem()}
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"0.1.3"}, nil }}
if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{}); err != nil {
t.Fatal(err)
}
rd := Reader{
Keys: func() ([]string, error) { return store.KeysOf(ctx, AllFilter) },
Get: func(key string) (json.RawMessage, bool, error) {
raw, _, ok, err := store.Get(ctx, key)
return raw, ok, err
}}
var asked []string
released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) {
asked = append(asked, module)
return []meshcatalog.VersionEntry{{Module: module, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mesh-sdk:go", Digest: "sha256:dd"}}}}, nil
}
carriers := Carriers(view)
ask := func(name, version string) meshbuilds.ReservedAnswer {
t.Helper()
got, err := AnswerReservedByName(ctx, rd, carriers, released, name, version)
if err != nil {
t.Fatalf("%s %s: %v", name, version, err)
}
return got
}
if got := ask("@novox/mesh-sdk", ""); got.Module != "mesh-sdk" || got.Version != "0.1.4" || got.Released || got.Error != "" {
t.Errorf("an npm name = %+v, want the module's reserved version", got)
}
if got := ask("git.novox.be/novox/mesh-sdk/go", "0.1.2"); !got.Released || got.Module != "mesh-sdk" || len(got.Artifacts) != 1 || got.Artifacts[0].Digest != "sha256:dd" {
t.Errorf("a Go module path = %+v, want the release's artifacts and digests (two series of one entry are one entry)", got)
}
if len(asked) != 1 || asked[0] != "mesh-sdk" {
t.Errorf("the catalog was asked for %v, want the module's versions once, for the version no reservation holds", asked)
}
for name, c := range map[string]struct{ name, version, want string }{
"unknown name": {"nothing", "", "no entry carries"},
"two entries": {"twice", "", "more than one entry"},
"neither released": {"git.novox.be/novox/mesh-sdk/go", "9.9.9", "reserved or released"},
} {
if got := ask(c.name, c.version); !strings.Contains(got.Error, c.want) || got.Module != "" || len(got.Artifacts) != 0 {
t.Errorf("%s = %+v, want error naming %q alone", name, got, c.want)
}
}
if _, err := AnswerReservedByName(ctx, rd, Carriers(packagesMem(nil)), released, "x", ""); err == nil {
t.Error("an unreadable view was answered as if no entry carried the name")
}
if _, err := AnswerReservedByName(ctx, rd, carriers, func(context.Context, string) ([]meshcatalog.VersionEntry, error) { return nil, errors.New("down") }, "git.novox.be/novox/mesh-sdk/go", "0.1.2"); err == nil {
t.Error("an unreadable catalog was answered as if the version were not released")
}
}
modules/mesh-delivery-builder/internal/versions/tools.go
+5
View File
@@ -6,6 +6,8 @@ import (
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
)
// The builder's tools of its versions (design 51 *Events of the builder*, *Tools of the builder*).
@@ -64,6 +66,9 @@ type VersionsAnswer struct {
Asked string `json:"asked,omitempty"`
Recorded *bool `json:"recorded,omitempty"`
Why string `json:"why,omitempty"`
// Reserved is what the seat mesh-builds' verb reserved answers for the name (and version): the reserved or released version
// with each artifact's digest, or its error (design 47 G §572, §582).
Reserved *meshbuilds.ReservedAnswer `json:"reserved,omitempty"`
}
// AnswerVersions is builder_versions: a name's versions in order, in one call; with a version (and a digest), whether
modules/mesh-delivery-builder/internal/versions/versions_test.go
+2 -1
View File
@@ -258,7 +258,8 @@ func TestTheBuildersVersions(t *testing.T) {
if f.Known() || !reflect.DeepEqual(f.Silent, []string{SourceTags}) || why[SourceTags] != "no answer" {
t.Fatalf("Ask = %+v, %v; want bridge_tags silent", f, why)
}
var fw FloorWaits
holders := map[string]string{SourceSeats[SourceTags]: "mesh-delivery-forge-bridge", SourceSeats[SourceRequirements]: "mesh-delivery-indexer"}
fw := FloorWaits{Holder: func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }}
w := fw.Wait("m/program", "oci-artifact-registry", "distribution", f, why, time.Unix(0, 0))
if !strings.Contains(w.Finding(), "bridge_tags (no answer)") {
t.Errorf("finding = %q", w.Finding())
modules/mesh-delivery-builder/manifest_test.go
+16 -14
View File
@@ -119,8 +119,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). Whether a
// node's machine is heard it asks the watchdogs seat's presence (issue 1966, ADR 0596 §2, ADR 0668 §3).
if want := []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps",
"mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
if want := []string{"mesh-delivery-assigner.assignments",
"node-build-go.steps", "node-build-image.steps", "node-build-node.steps"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"node-build-go", "node-build-image", "node-build-node", "operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) {
@@ -129,9 +129,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
// other tool; it holds no capacity.
// 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955).
// DLV-M1 (ADR 0615 §1): the indexer's modules, manifests and candidates and the assigner's assignments are asked by their
// read tools and followed by their views families, never read; so are the indexer's packages, whose names a package is
// numbered under (design 32).
// DLV-M1 (ADR 0615 §1): the assigner's assignments are asked by their read tool and followed by their views family, never
// read; the catalog half (the modules, their manifests, their packages and the package names a package is numbered under) is
// read through the seat mesh-catalog's branch follower (design 47 G §348, §614), and the indexer's states not at all.
for _, r := range m.Reads {
// The toolchain gate follows the assigner's assignments through meshmoduleassignerclient (buildrun.AssignmentsBucket), so
// it declares that one read, as the console and route-proxy do; everything else of theirs is asked by tools.
@@ -141,11 +141,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
}
// Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements.
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests",
"mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_requirements", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
"seat:git.tag",
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no
@@ -156,12 +155,11 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
// DLV-M1: the indexer's and the assigner's views families of the states it follows.
if want := []string{"container-image-registry.kept", "container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed",
"go-package-registry.kept", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed",
"mesh-artifact-sweep.artifacts-gone", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled",
"mesh-artifact-sweep.artifacts-gone", "mesh-catalog.manifest-accepted", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-catalog.version-released", "mesh-catalog.versions-filled",
"mesh-conditions.condition-cleared", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", "mesh-delivery-forge-bridge.forge.**",
"mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules",
"mesh-delivery-indexer.views.packages", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running",
"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running",
"npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed",
"oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
"oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), sorted(want)) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -172,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) {
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
@@ -278,6 +276,10 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR
return err
}
for _, f := range forbidden {
// The one subscription is the seat mesh-builds' accepted kind `step`, the runner's build ask (design 47 G A-97).
if f == "Subscribe(" && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/step.go" {
continue
}
if strings.Contains(string(src), f) {
t.Errorf("%s holds %s: the builder serves no request subject and holds no timer", path, f)
}
modules/mesh-delivery-builder/module.json
+18 -17
View File
@@ -13,6 +13,8 @@
],
"tools": [
"builder_announce_again",
"builder_seat_list",
"builder_seat_list_again",
"builder_ask_again",
"builder_bounds",
"builder_build",
@@ -53,6 +55,8 @@
"go-package-registry.removed",
"mesh-artifact-sweep.artifacts-gone",
"mesh-catalog.manifest-accepted",
"mesh-catalog.seat-definition-changed",
"mesh-catalog.seat-definition-removed",
"mesh-catalog.version-released",
"mesh-catalog.versions-filled",
"mesh-conditions.condition-cleared",
@@ -71,10 +75,9 @@
"oci-artifact-registry.refused",
"oci-artifact-registry.removed",
"mesh-delivery-assigner.views.assignments",
"mesh-delivery-indexer.views.candidates",
"mesh-delivery-indexer.views.manifests",
"mesh-delivery-indexer.views.modules",
"mesh-delivery-indexer.views.packages"
"mesh-catalog.catalog-revision",
"mesh-catalog.registration-changed",
"mesh-delivery.source-moved"
],
"invokes": [
"mesh-delivery-forge-bridge.bridge_tags",
@@ -85,9 +88,7 @@
"seat:go-package-registry.remove",
"seat:go-package-registry.versions",
"seat:mesh-artifact-sweep.held",
"seat:mesh-catalog.entry",
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.revision",
"seat:mesh-builds.reserved",
"seat:mesh-catalog.versions",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-module-assigner.assignments",
@@ -101,14 +102,15 @@
"seat:oci-artifact-registry.remove",
"seat:oci-artifact-registry.versions",
"mesh-delivery-assigner.assigner_read_assignments",
"mesh-delivery-indexer.indexer_read_candidates",
"mesh-delivery-indexer.indexer_read_manifests",
"mesh-delivery-indexer.indexer_read_modules",
"mesh-delivery-indexer.indexer_read_packages",
"seat:container-image-registry.publish",
"seat:go-package-registry.publish",
"seat:npm-package-registry.publish",
"seat:oci-artifact-registry.publish"
"seat:oci-artifact-registry.publish",
"seat:mesh-catalog.entry",
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.seat-definitions",
"seat:mesh-catalog.revision",
"seat:mesh-delivery.source"
],
"state": [
{ "name": "builds", "history": 5 },
@@ -157,11 +159,10 @@
{ "name": "build-log", "max-age-days": 30, "max-mib": 3072 }
],
"reads": [
"mesh-build-go.steps",
"mesh-build-image.steps",
"mesh-build-node.steps",
"mesh-delivery-assigner.assignments",
"mesh-delivery-gate.source"
"node-build-go.steps",
"node-build-image.steps",
"node-build-node.steps",
"mesh-delivery-assigner.assignments"
],
"uses": [
"node-build-go",
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/buildmodules.go
+23
View File
@@ -0,0 +1,23 @@
package main
import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// The checker's names of the build modules, kept until its part finds them on the seat list as the builder does (design 47 G
// A-82 is the builder's change; the builder's package holds no name of them any more): the holders of the three build seats.
const (
goBuildModule = "mesh-build-go"
nodeBuildModule = "mesh-build-node"
)
// checkerBuildSeats is the finder over the build seats those names hold.
var checkerBuildSeats = func() *buildrun.BuildSeats {
rule := func(on string) seatdef.Definition {
return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}}
}
defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript")}
holders := map[string]string{"node-build-go": goBuildModule, "node-build-node": nodeBuildModule}
return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok })
}()
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go
+3 -3
View File
@@ -404,7 +404,7 @@ func (r *repoModules) PackagePaths(repository string) []string {
// is judged by both, each finding its own directories in the tree (go.mod files; package.json files), so a repository
// with tests in either language and no manifest is judged whole (issue 1887 A12).
func (r *repoModules) BuildModules(repository string) []string {
out := []string{buildrun.GoBuildModule}
out := []string{goBuildModule}
r.mu.Lock()
defer r.mu.Unlock()
known := false
@@ -414,12 +414,12 @@ func (r *repoModules) BuildModules(repository string) []string {
}
known = true
if r.langs[r.typed[module]]["typescript"] {
out = append(out, buildrun.NodeBuildModule)
out = append(out, nodeBuildModule)
break
}
}
if !known {
out = append(out, buildrun.NodeBuildModule)
out = append(out, nodeBuildModule)
}
sort.Strings(out)
return out
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go
+3 -4
View File
@@ -14,7 +14,6 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -70,10 +69,10 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test
r := &repoModules{manifests: store}
putModule(t, r, "mesh-delivery-checker", "novox/mesh-delivery", goManifest)
putModule(t, r, "mesh-ui", "novox/mesh-catalog", tsManifest)
if got := r.BuildModules("novox/mesh-delivery"); strings.Join(got, ",") != buildrun.GoBuildModule {
if got := r.BuildModules("novox/mesh-delivery"); strings.Join(got, ",") != goBuildModule {
t.Errorf("mesh-delivery: %v, want the Go build module alone", got)
}
if got := r.BuildModules("novox/mesh-catalog"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule {
if got := r.BuildModules("novox/mesh-catalog"); strings.Join(got, ",") != goBuildModule+","+nodeBuildModule {
t.Errorf("mesh-catalog: %v, want both", got)
}
if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" {
@@ -81,7 +80,7 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test
}
// 1887/A12: a repository with no module the mesh knows (mesh-lab: nested go.mod files and a root package.json) is
// judged by both build modules.
if got := r.BuildModules("novox/mesh-lab"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule {
if got := r.BuildModules("novox/mesh-lab"); strings.Join(got, ",") != goBuildModule+","+nodeBuildModule {
t.Errorf("a repository with no module: %v, want both build modules", got)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go
+6 -6
View File
@@ -210,7 +210,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if link == "" {
link = defaultLink
}
tools := &buildrun.ToolchainSet{}
tools := &buildrun.ToolchainSet{Seats: checkerBuildSeats}
presence := &buildrun.PresenceSet{}
trunks := &check.Trunks{}
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
@@ -228,7 +228,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer hearing.Stop()
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
heard := buildrun.HearEnds(ctx, asks, asks, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, logger.Printf)
heard := buildrun.HearEnds(ctx, asks, asks, []string{goBuildModule, nodeBuildModule}, logger.Printf)
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}
asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf)
@@ -285,7 +285,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Asker: engineAsks{call: caller.CallOn},
Placed: placedNodes{kv: func(context.Context) (jetstream.KeyValue, error) { return assignments, nil }}.Placed,
Bound: answerBound,
Sandbox: seatSandbox{asker: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}},
Sandbox: seatSandbox{asker: buildrun.Asker{Seats: checkerBuildSeats, Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}},
Verdicts: verdicts.all,
// The node-engine's trunk head: a group with no node-engine member composes with the composer built there (ADR 0656 §5).
Composer: func() string { return trunks.Heads()[check.NodeEngineRepository] },
@@ -314,7 +314,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Says = verdictSays{js: js, node: cred.Node, now: time.Now} // a pull request's verdict event, on the stream asks (design 26 *Out of step*)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}, tools: tools, trunks: trunks,
asks: buildrun.Asker{Seats: checkerBuildSeats, Publisher: asks, Waiter: waiter, Bound: stepBound, Dropper: buildrun.SeatDrops{Caller: caller}}, tools: tools, trunks: trunks,
kept: kvStore{buckets[check.CheckerKeptBucket]}, previews: kvStore{buckets[check.PreviewsBucket]}, candidates: tryCandidates{candidates},
repos: repos, published: published, builds: buildStates, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions, inputs: runInputs{awaitedKV{inputsKV}}, beginBound: beginBound,
@@ -331,7 +331,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
// assignments is settled first, so the toolchains settle through the gate already knowing it.
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
gate := buildrun.NewToolchainGate([]string{goBuildModule, nodeBuildModule}, tools.Put, logger.Printf)
watch.put = gate.Toolchain
// Until the assignments's first view is answered the gate passes every report on, as before the assignments is settled.
assignmentsErrs := make(chan error, 1)
@@ -354,7 +354,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A build module says each time it wrote its toolchains key: its stream, and the checker's durable consumer on it, are
// made by the controller. Where the stream is not there yet the checker says so and is started again with the build
// module.
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
for _, module := range []string{goBuildModule, nodeBuildModule} {
stream := toolchainStreams[module]
cons, err := cred.EventConsumer(ctx, js, stream)
if err != nil {
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/refused_test.go
+1 -1
View File
@@ -20,7 +20,7 @@ import (
// its grant. The gate over the seat's assignments, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
// issue 774).
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
set := &buildrun.ToolchainSet{}
set := &buildrun.ToolchainSet{Seats: checkerBuildSeats}
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go
+4 -4
View File
@@ -17,7 +17,7 @@ import (
// toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it
// (declared by the build modules, heard by this module through consumes).
var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO", buildrun.NodeBuildModule: "TOOLCHAINS_NODE"}
var toolchainStreams = map[string]string{goBuildModule: "TOOLCHAINS_GO", nodeBuildModule: "TOOLCHAINS_NODE"}
// toolchainWatch keeps the build modules' toolchains in a buildrun.ToolchainSet and tells the builder when one changed. A
// build module's bucket is made when it is assigned, after the builder started, and the builder holds no timer (ADR 0437
@@ -37,12 +37,12 @@ type toolchainWatch struct {
}
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(checkerBuildSeats.Modules())), watched: map[string]bool{}}
}
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
func (w *toolchainWatch) start() error {
for module := range buildrun.LanguageOf {
for _, module := range checkerBuildSeats.Modules() {
if err := w.ensure(module); err != nil {
return err
}
@@ -96,7 +96,7 @@ type delivered interface {
// terminated; a failure stays unacknowledged, and the server delivers it again.
func (w *toolchainWatch) Reported(m delivered) {
module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.")
if _, known := buildrun.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") {
if _, known := checkerBuildSeats.LanguageOf(module); !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") {
w.logger.Printf("%s: not a build module's event, dropped", m.Subject())
_ = m.Term()
return
modules/mesh-delivery-checker/internal/check/build_change.go
+2 -3
View File
@@ -7,7 +7,6 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/sdk/graph"
@@ -75,13 +74,13 @@ func (st *state) buildsOfTheCheck(ctx context.Context) (map[string]string, []str
if !readable {
return wait, words
}
for _, bm := range slices.Sorted(maps.Keys(buildrun.LanguageOf)) {
for _, bm := range slices.Sorted(maps.Keys(languageOf)) {
if _, alters := changed[bm]; !alters {
continue
}
var built []string
for module, m := range changed {
if _, isBuildModule := buildrun.LanguageOf[module]; !isBuildModule && buildsWith(m, buildrun.LanguageOf[bm]) {
if _, isBuildModule := languageOf[module]; !isBuildModule && buildsWith(m, languageOf[bm]) {
built = append(built, module)
delete(wait, module)
}
modules/mesh-delivery-checker/internal/check/buildmodules.go
+11
View File
@@ -0,0 +1,11 @@
package check
// The checker's names of the build modules, kept until its part finds them on the seat list as the builder does (design 47 G
// A-82 is the builder's change; the builder's package holds no name of them any more).
const (
goBuildModule = "mesh-build-go"
nodeBuildModule = "mesh-build-node"
)
// languageOf is the language each build module's toolchain reports hold.
var languageOf = map[string]string{goBuildModule: "go", nodeBuildModule: "typescript"}
modules/mesh-delivery-checker/internal/check/buildseats_test.go
+16
View File
@@ -0,0 +1,16 @@
package check
import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// testSeats is the finder over the build seats the checker's names hold (buildmodules.go).
func testSeats() *buildrun.BuildSeats {
rule := func(on string) seatdef.Definition {
return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}}
}
defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript")}
holders := map[string]string{"node-build-go": goBuildModule, "node-build-node": nodeBuildModule}
return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok })
}
modules/mesh-delivery-checker/internal/check/checker_test.go
+1 -1
View File
@@ -450,7 +450,7 @@ func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testi
done := make(chan struct{})
go func() {
defer close(done)
(buildrun.Asker{Publisher: fake, Waiter: fake, Bound: time.Hour, Dropper: drops}).Wait(run, buildstep.StepAsk{ID: "mvl0frv18d2000", Language: buildstep.AskGo, Kind: buildstep.AskCheck})
(buildrun.Asker{Seats: testSeats(), Publisher: fake, Waiter: fake, Bound: time.Hour, Dropper: drops}).Wait(run, buildstep.StepAsk{ID: "mvl0frv18d2000", Language: buildstep.AskGo, Kind: buildstep.AskCheck})
}()
ch.Forget("g")
<-done
modules/mesh-delivery-checker/internal/check/examiner_test.go
+9 -9
View File
@@ -106,7 +106,7 @@ func (l langs) BuildModules(repository string) []string {
if m, ok := l[repository]; ok {
return m
}
return []string{buildrun.GoBuildModule}
return []string{goBuildModule}
}
func goEntries(nodes ...string) entries {
@@ -207,7 +207,7 @@ func refsOf(v any) []BesideRef {
}
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"}
var toolPrefix = map[string]string{goBuildModule: "build_go_", nodeBuildModule: "build_node_"}
// answer runs a check ask as the build module that pulls it does (gobuild.RunAsk), with the fake tools on shanks and the
// bench's kept verdicts, and answers the step it writes.
@@ -228,7 +228,7 @@ func (b *bench) answer(a buildstep.StepAsk) buildstep.Step {
k, found, err := (Kept{Store: b.kept}).Lookup(ctx, level, digest)
return int(k.Tests), found, err
}}
if module == buildrun.GoBuildModule {
if module == goBuildModule {
steps.Digest = tool("digest")
}
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
@@ -278,7 +278,7 @@ func newBench(t testing.TB) *bench {
}
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
b.asks = &buildrun.FakeAsks{Answer: b.answer}
b.ex = &Examiner{Asks: buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev,
b.ex = &Examiner{Asks: buildrun.Asker{Seats: testSeats(), Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev,
Modules: langs{}, Trunks: &Trunks{heads: map[string]string{DeliveryRepository: trunkD}}, Inputs: memInputs{b.inputs}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }}
return b
}
@@ -559,7 +559,7 @@ func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1292/AC1
b := newBench(t)
b.ex.Modules = langs{repoA: {buildrun.NodeBuildModule}}
b.ex.Modules = langs{repoA: {nodeBuildModule}}
b.fetch = captured[Fetched](t, "node-fetch-clean")
b.test = func(c stepCall) (any, error) {
return Tested{Level: c.args["level"].(string), Module: "modules/m", Passed: true, Elapsed: 0.4, Bound: 5}, nil
@@ -991,7 +991,7 @@ func (p pkgPaths) PackagePaths(repository string) []string { return p[repository
// groupPackageBench is a group of a TypeScript member (mesh-delivery) and the mesh-sdk member, whose merge changes sdkChanged.
func groupPackageBench(t *testing.T, sdkChanged []string) *bench {
b := newBench(t)
b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}, repoSDK: {buildrun.GoBuildModule, buildrun.NodeBuildModule}}
b.ex.Modules = langs{repoA: {goBuildModule, nodeBuildModule}, repoSDK: {goBuildModule, nodeBuildModule}}
b.ex.Packages = pkgPaths{repoSDK: {"src", "test", "package.json", "package-lock.json", "tsconfig.json"}}
fetch := func(c stepCall) (any, error) {
f := b.fetch
@@ -1006,7 +1006,7 @@ func groupPackageBench(t *testing.T, sdkChanged []string) *bench {
func nodeFetchOf(b *bench, repo string) stepCall {
for _, c := range b.steps.tool("fetch") {
if c.module == buildrun.NodeBuildModule && c.args["repository"] == repo {
if c.module == nodeBuildModule && c.args["repository"] == repo {
return c
}
}
@@ -1065,7 +1065,7 @@ func TestAnAskWhoseStepNeverEndsEndsTheCheckLoudly(t *testing.T) {
b.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
return buildstep.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: buildstep.StepRunning, Started: time.Now()}
}
b.ex.Asks = buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond}
b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if f := out.Verdict.Failed; out.Verdict.State != meshdeliverycheckerclient.VerdictFail || !strings.Contains(f.Words, "on shanks did not end within") {
t.Fatalf("verdict = %+v", out.Verdict)
@@ -1105,7 +1105,7 @@ func TestACheckAskWaitingInItsStreamIsNotBounded(t *testing.T) {
b := newBench(t)
answer := b.asks.Answer
b.asks.Answer = nil
b.ex.Asks = buildrun.Asker{Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond}
b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(nil, nil), Waiter: b.asks, Bound: 20 * time.Millisecond}
done := make(chan Outcome, 1)
go func() { done <- b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) }()
for deadline := time.Now().Add(10 * time.Second); len(b.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
modules/mesh-delivery-checker/internal/check/judge.go
+8 -8
View File
@@ -19,7 +19,7 @@ import (
)
// goModule is the build module the Go repositories are judged by.
const goModule = buildrun.GoBuildModule
const goModule = goBuildModule
// RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build
// module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut
@@ -69,7 +69,7 @@ type BuildModules interface {
}
// askLanguage is the build seat language of each build module the checker asks.
var askLanguage = map[string]string{buildrun.GoBuildModule: buildstep.AskGo, buildrun.NodeBuildModule: buildstep.AskNode}
var askLanguage = map[string]string{goBuildModule: buildstep.AskGo, nodeBuildModule: buildstep.AskNode}
// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
@@ -545,17 +545,17 @@ func (st *state) judgeAll(ctx context.Context) {
modules := st.j.Modules.BuildModules(p.Repository)
for _, bm := range modules {
t := &target{pull: p, module: bm}
if tc, none := buildstep.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
if tc, none := buildstep.AgreedToolchain(st.j.Toolchains.Entries(languageOf[bm])); none == "" {
t.toolchain = tc.Version
t.agreed = &buildstep.ToolchainUsed{Language: buildrun.LanguageOf[bm], Version: tc.Version, Recipe: tc.Recipe}
t.agreed = &buildstep.ToolchainUsed{Language: languageOf[bm], Version: tc.Version, Recipe: tc.Recipe}
}
args := buildstep.Fetch{Repository: p.Repository, Commit: p.Head, Ref: refOf(p.Number), Base: p.Trunk}
if bm == buildrun.NodeBuildModule {
if bm == nodeBuildModule {
if groupPackage != nil && p.Repository != SDKRepository {
args.Package = groupPackage
}
}
if bm == buildrun.GoBuildModule {
if bm == goBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
st.fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
@@ -797,7 +797,7 @@ func unmetOf(levels []buildstep.LevelAnswer) string {
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
// running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la buildstep.LevelAnswer) float64 {
if t.runner != "" && t.module == buildrun.GoBuildModule && la.Digest != nil {
if t.runner != "" && t.module == goBuildModule && la.Digest != nil {
t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
}
if t.runner != "" && la.Judged != "" {
@@ -806,7 +806,7 @@ func (st *state) level(ctx context.Context, t *target, la buildstep.LevelAnswer)
_, elapsed, results := st.levels.Level(t.pull.Repository, t.pull.Number, t.module, t.node, la)
st.facts = append(st.facts, rules.Step{Repository: t.pull.Repository, Number: t.pull.Number, Module: t.module, Node: t.node, Level: rules.LevelFact(la)})
var digests map[string]string
if t.module == buildrun.GoBuildModule && la.Digest != nil {
if t.module == goBuildModule && la.Digest != nil {
// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
digests = la.Digest.Map()
}
modules/mesh-delivery-checker/internal/check/lostask_test.go
+8 -8
View File
@@ -63,7 +63,7 @@ func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []
func losing(b *bench, lose int) *losingAsks {
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
b.ex.Asks = buildrun.Asker{Publisher: b.asker(l, l.FakeAsks.Write), Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(l, l.FakeAsks.Write), Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
return l
}
@@ -288,7 +288,7 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
tr = &troubled{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, restarts: row.restarts, deaf: row.deaf,
deafIDs: map[string]bool{}, steps: map[string]buildstep.Step{}}
heard = &buildrun.HeardEnds{Inner: tr, Read: tr}
b.ex.Asks = buildrun.Asker{Publisher: b.asker(tr, tr.Write), Waiter: heard, Bound: 5 * time.Second}
b.ex.Asks = buildrun.Asker{Seats: testSeats(), Publisher: b.asker(tr, tr.Write), Waiter: heard, Bound: 5 * time.Second}
asked = func() int { return len(tr.Asks()) }
}
r := newRig(t, b.ex)
@@ -307,23 +307,23 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
}
switch row.how {
case "nudge":
heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read
heard.Nudge(buildrun.GoBuildModule)
heard.Nudge(nodeBuildModule) // another build module's step: not this one's to read
heard.Nudge(goBuildModule)
case "hand":
cands := newMemKV()
ckey, _ := meshcatalogclient.CandidatesKey("feat/x")
_, _ = cands.Create(ctx, ckey, cand)
none := StateRead{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, error) { return nil, nil }}
rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept),
StepStates: map[string]StateRead{buildrun.GoBuildModule: none}} // no node took its ask yet
StepStates: map[string]StateRead{goBuildModule: none}} // no node took its ask yet
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "waits in its build seat's stream") {
t.Errorf("asked again while its ask waits in the stream (issue 1791's kind): %v", err)
}
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(true)
rd.StepStates[goBuildModule] = tr.Steps(true)
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "a step of it runs") {
t.Errorf("asked again while its step runs: %v", err)
}
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(false)
rd.StepStates[goBuildModule] = tr.Steps(false)
if st, _ := rd.Status(); !strings.Contains(st.Lost["feat/x"], AskAgainTool) {
t.Errorf("status lost = %v, want feat/x named with %s", st.Lost, AskAgainTool)
}
@@ -368,7 +368,7 @@ func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
id := fmt.Sprint("mvl0frv18d19", 80+i)
vals["ace."+id], _ = buildstep.EncodeStep(buildstep.Step{ID: id, Node: "ace", Tool: "check ask", State: buildstep.StepFailed, Passing: p, Ended: time.Now(), What: "check " + repoA + "@" + short12(head1)})
}
rd := Reader{CandidateState: kvRead(cands2(t, cand)), VerdictState: kvRead(failedKV), AskState: kvRead(newMemKV()), StepStates: map[string]StateRead{buildrun.GoBuildModule: {
rd := Reader{CandidateState: kvRead(cands2(t, cand)), VerdictState: kvRead(failedKV), AskState: kvRead(newMemKV()), StepStates: map[string]StateRead{goBuildModule: {
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(vals)), nil }, Get: func(k string) (json.RawMessage, error) { return vals[k], nil }}}}
if got, err := rd.AskAgain(asVerdicts{failedKV}, "feat/x", "the network"); (err == nil) != c.ok || (c.ok && !strings.HasPrefix(got.Said, AskedAgainWords)) {
t.Errorf("%s: asked again = %+v, %v, want asked again %v", c.name, got, err, c.ok)
modules/mesh-delivery-checker/internal/check/pins_test.go
+1 -2
View File
@@ -9,7 +9,6 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -119,7 +118,7 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes
b = newBench(t)
b.fetch.Modules = []string{"modules/m"}
b.ex.Read = fixedManifests{digest: sdk}
b.ex.Modules = langs{repoA: {buildrun.GoBuildModule, buildrun.NodeBuildModule}}
b.ex.Modules = langs{repoA: {goBuildModule, nodeBuildModule}}
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), sdkPull)).Verdict; v.State != meshdeliverycheckerclient.VerdictPass {
t.Errorf("a group's verdict = %+v", v)
}
modules/mesh-delivery-checker/rules/buildmodules.go
+8
View File
@@ -0,0 +1,8 @@
package rules
// The checker's names of the build modules, kept until its part finds them on the seat list as the builder does (design 47 G
// A-82 is the builder's change; the builder's package holds no name of them any more).
const (
goBuildModule = "mesh-build-go"
nodeBuildModule = "mesh-build-node"
)
modules/mesh-delivery-checker/rules/facts.go
+1 -2
View File
@@ -15,7 +15,6 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
@@ -159,7 +158,7 @@ func (t Tally) Level(repository string, number int, module, node string, la buil
}
results = append(results, res)
elapsed += res.Elapsed + res.Setup + res.Lint
if module == buildrun.NodeBuildModule {
if module == nodeBuildModule {
if !res.NoTests {
a.Run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
}
modules/mesh-delivery-checker/rules/graph.go
+1 -3
View File
@@ -4,8 +4,6 @@ import (
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
)
// The module graph's cycle rule (novox/hq ADR 0603 §2, ADR 0614 §3, §6), judged by the checking program (ADR 0656 §5):
@@ -60,7 +58,7 @@ func JudgeGraph(g Graph) (*FailedFirst, []string) {
return nil, []string{"the module graph has the cycle " + words + ", which the trunk already has among the same modules (ADR 0603 §2)"}
}
}
return &FailedFirst{Machine: buildrun.GoBuildModule, Words: fmt.Sprintf("the change's module graph has the cycle %s: these modules would "+
return &FailedFirst{Machine: goBuildModule, Words: fmt.Sprintf("the change's module graph has the cycle %s: these modules would "+
"each wait for the other for good, through their bases, the provisions and bus state they await, or the verbs they invoke (ADR 0603 §2, ADR 0614 §6)", words)}, nil
}
modules/mesh-delivery-checker/rules/judged.go
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/sdk/composer"
)
@@ -21,7 +20,7 @@ import (
// (milestone 47/B0, D1b): the running checker asks and records; these rules give the verdict.
// manifestsMachine is the machine a manifest's failure names: the build seat that ran the manifest validator.
const manifestsMachine = buildrun.GoBuildModule
const manifestsMachine = goBuildModule
// CatalogSeat is the seat the indexer holds: the module whose manifest claims it is the indexer, found by the claim and
// never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4).
modules/mesh-delivery-checker/rules/words.go
+1 -2
View File
@@ -6,7 +6,6 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
@@ -183,7 +182,7 @@ func LevelFailure(repository, module, level string, results ...Tested) *meshdeli
}
for _, f := range res.Failed {
name := FailedName(f)
if module == buildrun.NodeBuildModule && f.Package == "" {
if module == nodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, FailedFirst{Test: level + ": " + name, Words: FailureWordsAfter(f, HidWords(module, res.By, f))})