novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/47-G-builder6' into feat/47-G-join8

Browse Source
This commit is contained in:
jschoubben
2026-11-12 00:12:54 +01:00
parent 3524c880ea dc024ae552
commit 4f056485f2
49 changed files with 3246 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/buildrun/asks.go
+93 -29
View File
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"log"
"strings"
"sync"
"time"
@@ -102,17 +101,18 @@ func (k Asker) dropped(a buildstep.StepAsk, why string) {
// PutTry is how long one publish of an ask waits for its stream's answer before it is made again (Put).
var PutTry = 10 * time.Second
// Put puts a on its stream: the seat's accept subject of its language and kind. The put is a kept call (novox/hq ADR 0668
// Put puts a on its stream: the seat's accept subject of its language and kind, under the builder's name as asker (design 47 G
// §481, §600: `accept.<queue>.builder`). The put is a kept call (novox/hq ADR 0668
// §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it
// is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by
// PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it
// PutTry, until ctx ends. A put that still meets one is an error opening `transport fault:` (TransportFault), so the key it
// fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does.
func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error {
raw, err := buildstep.EncodeStepAsk(a)
if err != nil {
return err
}
subject, err := buildstep.AskSubject(a.Language, a.Kind)
subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, buildstep.AskerBuilder)
if err != nil {
return err
}
@@ -132,7 +132,8 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error {
case err == nil:
return nil
case busretry.TimedOut(err) || busretry.Undelivered(err):
return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err)
seat, _ := buildstep.BuildSeat(a.Language)
return errors.New(TransportFault(FaultClass(err.Error()), seat, "no node", fmt.Sprintf("the put of the %s ask %s: %v", a.Kind, a.ID, err)))
}
return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
}
@@ -211,7 +212,8 @@ func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, e
if step.Passing && step.State == buildstep.StepFailed && (len(agains) > 0 || step.Run > 1) {
// The step's own count of runs (ADR 0664 §2), not only the ones this wait saw: a builder that started again
// reads the key's last value and none of the runs before it.
step.Reason = fmt.Sprintf(buildstep.TransportFaultWords+"%d runs (%s): %s", max(step.Run, len(agains)+1), strings.Join(append(agains, step.Node), ", "), step.Reason)
seat, _ := buildstep.BuildSeat(a.Language)
step.Reason = TransportFault(FaultClass(step.Reason), seat, step.Node, step.Reason)
}
return step, nil
}
@@ -297,6 +299,14 @@ func (k Asker) Passed(step buildstep.Step, err error) (buildstep.Step, error) {
// of `*.<ask id>` in the build module's steps bucket.
type JetStreamAsks struct {
JS jetstream.JetStream
// Unanswered is told each time the read of an ask's steps meets a transport fault it keeps waiting out (design 47 G §568, ADR
// 0668 §3): the builder arms the call's bound, a key with its own time to live (A-77), and raises call-unanswered when it
// ends unanswered. Answered is told when the read is answered again. Both may be nil.
Unanswered func(module, askID string, fault error)
Answered func(module, askID string)
// OpenWatch opens the watch of keys filtered by pattern in the build module's steps state, from the revision after `after`
// when after is not 0; nil is the bus's.
OpenWatch func(ctx context.Context, module, pattern string, after uint64) (jetstream.KeyWatcher, error)
}
// PublishAsk is AskPublisher's.
@@ -307,46 +317,100 @@ func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data
return err
}
// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
// once, and every change after.
func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) {
pattern, err := buildstep.AskStepWatch(askID)
if err != nil {
return nil, err
// openWatch opens the watch of pattern in the steps state of module, from the revision after `after` when it is not 0.
func (s JetStreamAsks) openWatch(ctx context.Context, module, pattern string, after uint64) (jetstream.KeyWatcher, error) {
if s.OpenWatch != nil {
return s.OpenWatch(ctx, module, pattern, after)
}
kv, err := s.JS.KeyValue(ctx, StepsBucket(module))
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", StepsBucket(module), err)
}
w, err := kv.Watch(ctx, pattern)
if after > 0 {
return kv.Watch(ctx, pattern, jetstream.ResumeFromRevision(after))
}
return kv.Watch(ctx, pattern)
}
// transportFault says err is a transport fault of ADR 0664 §1's classes or the read's own deadline: nothing was answered.
func transportFault(err error) bool {
return busretry.Undelivered(err) || busretry.TimedOut(err)
}
// openKept is a kept call: the watch is opened, and opened again while the bus answers nothing, until it is or ctx ends
// (design 47 G §568: a transport fault of the builder's own read ends no build key). Each fault past the call's own asking again
// is told to Unanswered; the answer, to Answered. An error that is no transport fault is the caller's.
func (s JetStreamAsks) openKept(ctx context.Context, module, askID, pattern string, after uint64) (jetstream.KeyWatcher, error) {
for {
w, err := busretry.DoRepeatable("opening the watch of the ask "+askID+"'s step on "+module, func() (jetstream.KeyWatcher, error) {
return s.openWatch(ctx, module, pattern, after)
}, nil)
if err == nil {
if s.Answered != nil {
s.Answered(module, askID)
}
return w, nil
}
if ctx.Err() != nil {
return nil, ctx.Err()
}
if !transportFault(err) {
return nil, err
}
if s.Unanswered != nil {
s.Unanswered(module, askID, err)
}
}
}
// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
// once, and every change after. A watch that cannot be opened for a transport fault is opened again once the bus answers, and one
// that closes is taken up again from the revision after the last it handed over (design 47 G §568), so a fault fails no key.
func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) {
pattern, err := buildstep.AskStepWatch(askID)
if err != nil {
return nil, err
}
w, err := s.openKept(ctx, module, askID, pattern, 0)
if err != nil {
return nil, err
}
out := make(chan buildstep.Step)
go func() {
defer close(out)
defer w.Stop()
var last uint64
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
step, err := buildstep.DecodeStep(e.Value())
if err != nil {
continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
}
closed := false
for !closed {
select {
case out <- step:
case <-ctx.Done():
_ = w.Stop()
return
case e, ok := <-w.Updates():
if !ok {
closed = true
break
}
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
last = max(last, e.Revision())
step, err := buildstep.DecodeStep(e.Value())
if err != nil {
continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
}
select {
case out <- step:
case <-ctx.Done():
_ = w.Stop()
return
}
}
}
_ = w.Stop()
if w, err = s.openKept(ctx, module, askID, pattern, last+1); err != nil {
return
}
}
}()
return out, nil
modules/mesh-delivery-builder/buildrun/asks_test.go
+21 -5
View File
@@ -30,7 +30,8 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
f := &FakeAsks{Answer: func(a buildstep.StepAsk) buildstep.Step {
return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[]}`)}
}}
k := Asker{Publisher: f, Waiter: f}
subjects := &subjectsOf{inner: f}
k := Asker{Publisher: subjects, Waiter: f}
var seen, also []buildstep.Step // issue 1890: a waiter that asks to see each value of the step sees it
// Issue 1921: a second seen on the same context (a check's begin bound inside a try) is added, never in place of the first.
both := WithStepSeen(WithStepSeen(context.Background(), func(s buildstep.Step) { seen = append(seen, s) }),
@@ -39,6 +40,10 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" || len(also) != 1 {
t.Fatalf("step %+v, %v, asks %d, seen %+v and %+v", step, err, len(f.Asks()), seen, also)
}
// Design 47 G §481, §600: the ask is put under the builder's name, accept.<queue>.builder.
if len(subjects.seen) != 1 || subjects.seen[0] != "mesh.seat.node-build-go.accept."+goAsk("x").Kind+".builder" {
t.Fatalf("the ask was put on %v", subjects.seen)
}
f.Answer = func(a buildstep.StepAsk) buildstep.Step {
return buildstep.Step{ID: a.ID, Node: "shanks", State: buildstep.StepRefused, Reason: "go 1.27 installed"}
}
@@ -67,11 +72,11 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
restore()
switch {
case c.transport:
if err == nil || !strings.HasPrefix(err.Error(), buildstep.TransportFaultWords) {
if err == nil || !strings.HasPrefix(err.Error(), TransportFaultPrefix+" deadline, node-build-go no node, ") {
t.Errorf("%s: %v, want it opens with the transport fault's words", name, err)
}
case c.tries == 1:
if err == nil || p.tries != 1 || strings.Contains(err.Error(), buildstep.TransportFaultWords) {
if err == nil || p.tries != 1 || strings.HasPrefix(err.Error(), TransportFaultPrefix) {
t.Errorf("%s: %v after %d tries, want one try and no transport fault", name, err, p.tries)
}
case err != nil || p.tries != c.tries || len(p.inner.Asks()) != 1 || len(p.ids) != c.tries || p.ids[0] != p.ids[len(p.ids)-1]:
@@ -101,7 +106,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
}
h.Write(st)
}
if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "a transport fault on 3 runs (ace, g14, shanks): lookup proxy: i/o timeout") {
if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "transport fault: name lookup, node-build-go shanks, lookup proxy: i/o timeout") {
t.Errorf("three runs then failed: %v", err)
}
h = &FakeAsks{}
@@ -134,7 +139,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
// how many runs; a drop that failed is said, not discarded.
late := &FakeAsks{}
late.Write(buildstep.Step{ID: "mvl0frv18d1979", Node: "shanks", State: buildstep.StepFailed, Passing: true, Run: 3, Reason: "lookup proxy: i/o timeout"})
if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.Contains(st.Reason, "a transport fault on 3 runs") {
if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.HasPrefix(st.Reason, "transport fault: name lookup, node-build-go shanks") {
t.Errorf("a failed step of run 3 read alone says %q (%v), want it names 3 runs", st.Reason, err)
}
var said []string
@@ -381,3 +386,14 @@ func (p *faultyPuts) PublishAsk(ctx context.Context, subject, id string, data []
}
return p.inner.PublishAsk(ctx, subject, id, data)
}
// subjectsOf records the subject each ask is published on.
type subjectsOf struct {
inner AskPublisher
seen []string
}
func (p *subjectsOf) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
p.seen = append(p.seen, subject)
return p.inner.PublishAsk(ctx, subject, id, data)
}
modules/mesh-delivery-builder/buildrun/faults.go
+53
View File
@@ -0,0 +1,53 @@
package buildrun
import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
// The words of a key that failed on a transport fault (design 47 G §568, A-76): `transport fault: <class>, <seat> <node>,
// <cause>`, the class one of ADR 0664 §1's four, the seat and node the step's, the cause the last run's in the build
// module's words. No other failure opens `transport fault:`.
// TransportFaultPrefix opens the error of a key that failed on a transport fault.
const TransportFaultPrefix = builds.TransportFaultPrefix
// The classes of a transport fault.
const (
ClassNoResponders = "no responders"
ClassNameLookup = "name lookup"
ClassConnection = "connection"
ClassDeadline = "deadline"
)
// FaultClass is the class of a call's error or a step's reason: the words the builder's retake reads a transport fault by
// (buildstep.PassingCause), read as no responders, a name lookup, a deadline or, for every other refused, reset or cut
// connection, a connection.
func FaultClass(cause string) string {
c := strings.ToLower(cause)
has := func(words ...string) bool {
for _, w := range words {
if strings.Contains(c, w) {
return true
}
}
return false
}
switch {
case has("no responders", strings.ToLower(buildstep.HolderSilentWords), "jetstream not enabled"):
return ClassNoResponders
case has("no such host", "name resolution", "lookup ", "could not resolve", "enotfound", "eai_again"):
return ClassNameLookup
case has("deadline", "timeout", "timed out", "etimedout"):
return ClassDeadline
}
return ClassConnection
}
// TransportFault is the error of a key that failed on a transport fault of the seat's step on node, with the cause of its
// last run.
func TransportFault(class, seat, node, cause string) string {
return TransportFaultPrefix + " " + class + ", " + seat + " " + node + ", " + cause
}
modules/mesh-delivery-builder/buildrun/faults_test.go
+23
View File
@@ -0,0 +1,23 @@
package buildrun
import "testing"
// Design 47 G §568, A-76: the class of a transport fault is read from the call's error, and its words are one form.
func TestFaultClassAndWords(t *testing.T) {
for cause, class := range map[string]string{
"nats: no responders available for request": ClassNoResponders,
"the holder is silent: seat x answered": ClassNoResponders,
"lookup proxy: no such host": ClassNameLookup,
"Get https://r: context deadline exceeded": ClassDeadline,
"read tcp: i/o timeout": ClassDeadline,
"dial tcp: connection refused": ClassConnection,
"ssh: kex_exchange_identification: read: reset": ClassConnection,
} {
if got := FaultClass(cause); got != class {
t.Errorf("FaultClass(%q) = %q, want %q", cause, got, class)
}
}
if got, want := TransportFault(ClassDeadline, "node-build-go", "ace", "i/o timeout"), "transport fault: deadline, node-build-go ace, i/o timeout"; got != want {
t.Errorf("TransportFault = %q, want %q", got, want)
}
}
modules/mesh-delivery-builder/buildrun/steps_resume_test.go
+88
View File
@@ -0,0 +1,88 @@
package buildrun
import (
"context"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/busretry"
)
type fakeEntry struct {
jetstream.KeyValueEntry
rev uint64
value []byte
}
func (e fakeEntry) Revision() uint64 { return e.rev }
func (e fakeEntry) Value() []byte { return e.value }
func (e fakeEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
type fakeWatcher struct {
jetstream.KeyWatcher
updates chan jetstream.KeyValueEntry
}
func (w fakeWatcher) Updates() <-chan jetstream.KeyValueEntry { return w.updates }
func (w fakeWatcher) Stop() error { return nil }
// Design 47 G §568: the read of an ask's steps is a kept call. A transport fault opening the watch ends no key (the wait is
// not failed): it is opened again once the bus answers, said unanswered meanwhile and answered after; and a watch that closes
// is taken up again from the revision after the last it handed over, so the step that ended meanwhile is heard.
func TestTheWatchOfAStepResumesAfterAFault(t *testing.T) {
t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 2*time.Millisecond))
passed, _ := buildstep.EncodeStep(buildstep.Step{ID: "mvl0frv18d1958", Tool: "build", Node: "ace", State: buildstep.StepPassed})
running, _ := buildstep.EncodeStep(buildstep.Step{ID: "mvl0frv18d1958", Tool: "build", Node: "ace", State: buildstep.StepRunning})
var mu sync.Mutex
var opens []uint64
var unanswered, answered int
asks := JetStreamAsks{
Unanswered: func(string, string, error) { mu.Lock(); unanswered++; mu.Unlock() },
Answered: func(string, string) { mu.Lock(); answered++; mu.Unlock() },
OpenWatch: func(_ context.Context, _, _ string, after uint64) (jetstream.KeyWatcher, error) {
mu.Lock()
defer mu.Unlock()
opens = append(opens, after)
ch := make(chan jetstream.KeyValueEntry, 2)
// The bus answers nothing past the kept call's own asking again (its bound), so the fault is told and the watch tried again.
switch n := len(opens); {
case n <= 40:
return nil, nats.ErrNoResponders
case n == 41:
ch <- fakeEntry{rev: 5, value: running}
close(ch) // the watch closes under the wait
default:
ch <- fakeEntry{rev: 6, value: passed}
}
return fakeWatcher{updates: ch}, nil
}}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
steps, err := asks.WatchAsk(ctx, "mesh-build-go", "mvl0frv18d1958")
if err != nil {
t.Fatalf("a transport fault opening the watch failed the wait: %v", err)
}
var states []string
for s := range steps {
states = append(states, s.State)
if buildstep.Ended(s) {
break
}
}
mu.Lock()
defer mu.Unlock()
if len(states) != 2 || states[0] != buildstep.StepRunning || states[1] != buildstep.StepPassed {
t.Fatalf("steps heard = %v, want running then passed", states)
}
if len(opens) != 42 || opens[41] != 6 {
t.Errorf("opens after revisions %v, want the last taken up again after the last revision heard (6)", opens)
}
if unanswered == 0 || answered != 2 {
t.Errorf("unanswered %d, answered %d, want the fault said and each answer said", unanswered, answered)
}
}
modules/mesh-delivery-builder/builds/builds.go
+28 -4
View File
@@ -194,11 +194,15 @@ func FailedUntilChanged(err string) bool {
return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind)
}
// FailedOnTransportFault says whether a failed key's error is a step's end after its third run met a transport fault
// (buildstep.TransportFaultWords): a fault of the network, not of the source, so the builder does not retake it by itself
// (ADR 0664 §3, which changes ADR 0429 clause 18 for this case); a person's hand act or a builder that starts again does.
// TransportFaultPrefix opens the error of a key that failed on a transport fault: `transport fault: <class>, <seat> <node>,
// <cause>` (design 47 G §568, A-76). No other failure opens it.
const TransportFaultPrefix = "transport fault:"
// FailedOnTransportFault says whether a failed key's error opens TransportFaultPrefix: a fault of the network, not of the
// source, so the builder does not retake it on its look (ADR 0664 §3, which ADR 0673 changes): a person's hand act, a builder
// that starts again, or an event that says its cause passed (design 47 G §568) builds it again.
func FailedOnTransportFault(err string) bool {
return strings.Contains(err, buildstep.TransportFaultWords)
return strings.HasPrefix(err, TransportFaultPrefix)
}
// ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and
@@ -241,6 +245,11 @@ type BuilderBuild struct {
// oci-artifact-registry (design 47 G §671): every artifact of the build is kept there, each with its digest in
// Artifacts. A build not kept names none; the builder carries it onto the build's end (BuildLogEntry.KeptOn).
KeptOn string `json:"kept-on,omitempty"`
// EventBuilds counts, in the key's newest revision, the revisions an event started since the last one a person started
// (design 47 G §568, ADR 0673): the builder builds a key that failed on a transport fault again on an event for the third
// time at most, then asks the operator. A person's build (builder_build_again, the operator's answer) starts the count
// again. A reader that does not know the field reads none.
EventBuilds int `json:"event-builds,omitempty"`
}
func (b BuilderBuild) check() error {
@@ -361,3 +370,18 @@ func DecodeBuilderBuild(data []byte) (BuilderBuild, error) {
// SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ.
func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) }
// TransportFaultOf reads a key's error `transport fault: <class>, <seat> <node>, <cause>` (design 47 G §568, A-76): ok is false
// for an error that does not open so. The node of a fault no node ran (the put of an ask) is `no node`.
func TransportFaultOf(err string) (class, seat, node, cause string, ok bool) {
rest, found := strings.CutPrefix(err, TransportFaultPrefix+" ")
if !found {
return "", "", "", "", false
}
parts := strings.SplitN(rest, ", ", 3)
if len(parts) != 3 {
return "", "", "", "", false
}
seat, node, _ = strings.Cut(parts[1], " ")
return parts[0], seat, node, parts[2], seat != "" && node != ""
}
modules/mesh-delivery-builder/builds/reference_test.go
+2 -2
View File
@@ -103,13 +103,13 @@ func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) {
{VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false},
{RefusedKind + "a package in go", false},
{"go test: 2 failed", true},
{"a transport fault on 3 runs (ace, g14, novox)", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's
{"transport fault: deadline, node-build-go ace, i/o timeout", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's
} {
if got := FailedUntilChanged(c.err); got == c.again {
t.Errorf("FailedUntilChanged(%q) = %v", c.err, got)
}
}
if !FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace): i/o timeout") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") {
if !FailedOnTransportFault("transport fault: connection, node-build-go ace, connection refused") || FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace)") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") {
t.Error("FailedOnTransportFault must name the third run's end alone: the builder's own retake holds it (ADR 0664 §3)")
}
if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go
+79
View File
@@ -0,0 +1,79 @@
package main
import (
"context"
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-builder/internal/build"
)
// catalogWithin is how long a call waits for the catalog half's first view of one branch.
var catalogWithin = 30 * time.Second
// buildTool is builder_build (design 47 G §589): a module's build keys whole.
func buildTool(r *build.BuildReader) stdio.Tool {
return stdio.Tool{Name: build.BuildTool,
Description: "A module's build keys, whole, newest first (at most 20): per key its fingerprint (whole, as the build-input fingerprint the catalog graph " +
"entry carries it), state (queued, building, done or failed), revision and since when, the step and its node, the error whole, event-builds " +
"(the revisions events started since the last a person started; after three the operator is asked) and the open ask; and newest, the watched " +
"branches whose newest catalog graph entry of the module names that fingerprint, read from the catalog half. A module none of whose keys the newest " +
"entry names says so, naming the entry's fingerprint. With fingerprint, that key alone. Which node wants which fingerprint is the assigner's " +
"to answer (the seat mesh-wants). Read-only.",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module"},
"fingerprint": map[string]any{"type": "string", "description": "one key: its fingerprint, src2:<hex>"}}},
Run: func(args map[string]any) (any, error) {
module, _ := args["module"].(string)
fp, _ := args["fingerprint"].(string)
return r.Build(module, fp)
}}
}
// newestVia reads the fingerprint of a module's newest catalog graph entry on each of its watched branches, from the catalog
// half through the seat mesh-catalog (the verb registrations for the branches, the verb revision for each branch's view), as
// the delivery view's tools read it: once per call (design 47 G §532, §589, A-87).
func newestVia(ask meshcatalogclient.Call) func(module string) (map[string]string, error) {
return func(module string) (map[string]string, error) {
raw, err := ask(meshcatalogclient.RegistrationsVerb, meshcatalog.RegistrationsInput{Module: module})
if err != nil {
return nil, fmt.Errorf("the registrations of %s: %w", module, err)
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RegistrationsVerb, said)
}
var regs meshcatalog.RegistrationsAnswer
if err := json.Unmarshal(answer, &regs); err != nil {
return nil, fmt.Errorf("%s answered no registrations: %w", meshcatalogclient.RegistrationsVerb, err)
}
out := map[string]string{}
for _, reg := range regs.Registrations {
if reg.Module != module {
continue
}
for _, branch := range reg.Branches {
c, err := meshcatalogclient.NewCatalog(ask, branch)
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), catalogWithin)
err = c.Start(ctx)
cancel()
if err != nil {
return nil, fmt.Errorf("the catalog half of %s has no first view: %w", branch, err)
}
if e, ok := c.Entry(module); ok && e.Fp != "" {
out[branch] = e.Fp
}
}
}
return out, nil
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go
+32
View File
@@ -0,0 +1,32 @@
package main
import (
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// Design 47 G §589, A-87: the tools bundle reads each watched branch's newest catalog graph entry of the module from the catalog
// half itself: the verb registrations for the branches, the verb revision for each branch's view.
func TestNewestIsReadFromTheCatalogHalfOfEachWatchedBranch(t *testing.T) {
fp := map[string]string{"main": "src2:" + strings.Repeat("1", 64), "feature": "src2:" + strings.Repeat("2", 64)}
ask := func(key string, body any) (json.RawMessage, error) {
switch key {
case meshcatalogclient.RegistrationsVerb:
return json.Marshal(meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{{Module: "m", Branches: []string{"main", "feature"}}}})
case meshcatalogclient.RevisionVerb:
in := body.(meshcatalog.RevisionInput)
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: 3, Kind: meshcatalogclient.KindTrunk},
Entries: []meshcatalog.CatalogEntry{{Module: "m", Fp: fp[in.Branch]}, {Module: "z", Fp: "src2:" + strings.Repeat("9", 64)}}})
}
t.Fatalf("asked %s", key)
return nil, nil
}
got, err := newestVia(ask)("m")
if err != nil || len(got) != 2 || got["main"] != fp["main"] || got["feature"] != fp["feature"] {
t.Fatalf("newest = %v, %v, want each branch's entry fingerprint whole", got, err)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go
+20 -12
View File
@@ -54,17 +54,9 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
if a.Refused {
return nil, errors.New(a.Answer)
}
return map[string]any{"key": key, "name": a.Name, "do": a.Do, "version": a.Version, "answer": a.Answer}, nil
return map[string]any{"key": key, "name": a.Name, "answer": a.Answer}, nil
}
nameFP := map[string]any{"type": "object", "required": []string{"name", "fp", "why"}, "properties": map[string]any{
"name": map[string]any{"type": "string", "description": "the numbered name, <module>/<artifact>, as builder_versions lists it"},
"fp": map[string]any{"type": "string", "description": "the fingerprint, src2:<hex>"},
"why": map[string]any{"type": "string", "description": "why, in plain words"}}}
return []stdio.Tool{
{Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, Input: nameFP,
Run: func(args map[string]any) (any, error) {
return handAct(build.NewPublishAgain(str(args, "name"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()))
}},
{Name: versions.FloorAgainTool, Description: versions.FloorAgainDescription,
Input: map[string]any{"type": "object", "required": []string{"name", "why"}, "properties": map[string]any{
"name": map[string]any{"type": "string", "description": "the numbered name, <module>/<artifact>"},
@@ -164,15 +156,16 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference; a failed " +
"build carries its error and the step it was at. more says the " +
"log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.",
"log held more than one answer carries: ask again from the last build's time. With module, that module's builds only. What the operator's command nox builds reads. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"},
"module": map[string]any{"type": "string", "description": "only this module's builds"}}},
Run: func(args map[string]any) (any, error) {
at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
if err != nil {
return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
}
return build.BuildsSince(since, at)
return build.BuildsSince(since, at, str(args, "module"))
}},
{Name: build.AnnounceAgainTool,
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
@@ -371,7 +364,11 @@ func main() {
return e.Value, true, nil
}}
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr)
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
Newest: newestVia(stdio.Ask), Steps: r}))
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
ws := stdio.State(build.WaitsState)
wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
e, err := ws.Get(key)
@@ -400,3 +397,14 @@ func main() {
os.Exit(1)
}
}
// entryOf reads a key with its revision and when it was written.
func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) {
return func(key string) (json.RawMessage, uint64, time.Time, error) {
e, err := s.Get(key)
if err != nil || e == nil {
return nil, 0, time.Time{}, err
}
return e.Value, e.Revision, written(s, key, e.Revision), nil
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go
+65
View File
@@ -0,0 +1,65 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// readerOf is a kept state as the versions package's reader.
func readerOf(s stdio.KeptState) versions.Reader {
return versions.Reader{Keys: s.Keys, Get: func(key string) (json.RawMessage, bool, error) {
e, err := s.Get(key)
if err != nil || e == nil {
return nil, false, err
}
return e.Value, true, nil
}}
}
// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act:
// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer
// with the versions it queued.
func republishTools(queue, acted versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription,
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's reserved versions"},
"version": map[string]any{"type": "string", "description": "with module: only this reserved version, MAJOR.MINOR.PATCH"},
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewRepublish(str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("published reserved versions again by hand (module %q, version %q)", str(args, "module"), str(args, "version")),
Why: str(args, "why"), Cause: versions.PublishAgainTool, Verb: "publish-again"}); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was queued: %w", err)
}
raw, err := ask(key, value)
if err != nil {
return nil, err
}
a, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if a.Refused {
return nil, errors.New(a.Answer)
}
return map[string]any{"key": key, "queued": a.Queued, "answer": a.Answer}, nil
}},
{Name: versions.PublishQueueTool, Description: versions.PublishQueueDescription,
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted) }},
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go
+2 -2
View File
@@ -115,7 +115,7 @@ func (h heldOnTheBus) seen(ctx context.Context, a buildstep.StepAsk, later bool)
// stream is the build seat's stream of language, its name and the subject of kind on it.
func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) {
subject, err := buildstep.AskSubject(language, kind)
subject, err := buildstep.AskSubjectBy(language, kind, buildstep.AskerBuilder)
if err != nil {
return nil, "", "", err
}
@@ -286,7 +286,7 @@ func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (bu
return q, err
}
for _, kind := range buildstep.AskKinds {
subject, _ := buildstep.AskSubject(language, kind)
subject, _ := buildstep.AskSubjectBy(language, kind, buildstep.AskerBuilder)
var ahead []build.Queued
if err := onSubject(ctx, stream, subject, func(m build.Queued) bool {
ahead = append(ahead, m)
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+13 -9
View File
@@ -165,6 +165,18 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
t.Fatal(err)
}
// Design 47 G §600: each queue's worker takes an ask in both forms, main's and the builder's (accept.<queue>.builder); the
// build modules' helper above filters main's alone, so the worker is declared again with both.
stream, _ := sdkstep.AskStream(language)
for _, kind := range sdkstep.AskKinds {
worker, _ := sdkstep.AskWorker(language, kind)
main, _ := sdkstep.AskSubject(language, kind)
byBuilder, _ := sdkstep.AskSubjectBy(language, kind, sdkstep.AskerBuilder)
if _, err := f.js.CreateOrUpdateConsumer(f.ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{main, byBuilder},
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: 5}); err != nil {
t.Fatal(err)
}
}
put := func(key string, value any) (uint64, error) {
raw, _ := value.(json.RawMessage)
return kv.Put(f.ctx, key, raw)
@@ -586,18 +598,10 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 {
t.Errorf("version.numbered of netcheck/tools-go said %v, want none", got)
}
// DLV-B10: builder_publish_again of a published, held build is refused: nothing says a version numbered (§349).
for _, a := range []string{"tools-go", "tools-typescript"} {
reg.published(t, "netcheck/"+a, "0.1.5", digestOf(a))
}
reg.awaitOf(t, "netcheck/tools-go", func(es []versions.Entry) bool { return len(es) == 1 && es[0].State == versions.Published })
key, again, err := build.NewPublishAgain("netcheck/tools-go", fp, "the holder lost it", "me@shanks", time.Now())
if err != nil {
t.Fatal(err)
}
if a := reg.answered(t, key, again); !a.Refused {
t.Errorf("builder_publish_again = %+v, want a refusal", a)
}
// A numbered version a holder will refuse below, written to the state as the builder numbers one.
refusable, _ := json.Marshal(versions.Entry{Name: "netcheck/tools-go", Kind: versions.KindBundle, Series: "0.1", Version: "0.1.6", Module: "netcheck",
Fingerprint: "src2:" + strings.Repeat("6", 64), Seat: ociartifactregistry.Seat, State: versions.Numbered, At: time.Now().UTC()})
@@ -1084,7 +1088,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
held := heldOnTheBus{b.js}
goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery}
stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery}
subject, _ := sdkstep.AskSubject(sdkstep.AskGo, sdkstep.AskDelivery)
subject, _ := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskDelivery, sdkstep.AskerBuilder)
if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil {
t.Fatal(err)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+76 -19
View File
@@ -7,8 +7,10 @@ package main
import (
"cmp"
"context"
"errors"
"fmt"
"io"
"io/fs"
"log"
"maps"
"os"
@@ -22,6 +24,8 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
@@ -136,7 +140,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
tools := &buildrun.ToolchainSet{}
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
asks := buildrun.JetStreamAsks{JS: js}
// The read of an ask's steps is a kept call (design 47 G §568): a fault ends no key, and past five minutes unanswered it is a
// finding; its bound is a key of the state bounds, bound once the bus holds it (set below, with the findings).
stepReads := &stepsRead{bounds: func() jetstream.KeyValue { return nil }, bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf}
asks := buildrun.JetStreamAsks{JS: js, Unanswered: stepReads.unanswered, Answered: stepReads.answered}
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
@@ -177,6 +184,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder
// answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read.
pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf)
stepReads.bounds, stepReads.findings = func() jetstream.KeyValue { return pubBoundsKV.KV() }, found
pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf}
numberer.Publishing = pubs.arm
b := &build.Builder{
@@ -200,13 +208,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
b.Numbers = build.VersionNumbers{N: numberer}
// The builder's toolchain waits (design 47 G §672): its state waits, and wait-held and wait-ended on its stream waits; its
// bound is its setting toolchain-wait-minutes.
waitBound := defaultToolchainWait
if v := strings.TrimSpace(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES")); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return fmt.Errorf("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES %q is not a number of minutes", v)
}
waitBound = time.Duration(n) * time.Minute
waitBound, err := toolchainWaitOf(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_FILE"))
if err != nil {
return err
}
waitsBucket := process.AwaitBucket(ctx, js, conn, build.WaitsBucket, "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf)
b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error {
@@ -221,17 +225,32 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A state bound later numbers the builds looked at without it.
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Done: doneVia(b), Reconsider: b.Reconsider,
BuildAgain: func(ctx context.Context, module, fp, why, by string) (string, bool) {
r := b.BuildAgain(ctx, module, fp, why, by, buildAgainBound)
if r.Refused != "" {
return r.Refused, true
}
if r.Failed != "" {
return "built again, and it failed before its asks were put: " + r.Failed, false
}
return fmt.Sprintf("built again as revision %d", r.Revision), false
}}
acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider}
// A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G
// §588): its states publish-queue and acted, bound once the bus holds them.
republishMax, err := republishMaxOf(getenv("MESH_BUILDER_REPUBLISH_MAX_FILE"))
if err != nil {
return err
}
queueKV := process.AwaitBucket(ctx, js, conn, versions.PublishQueueBucket, "no version is published again on an event or by builder_publish_again", logger.Printf)
actedKV := process.AwaitBucket(ctx, js, conn, versions.ActedBucket, "no version is published again on an event or by builder_publish_again", logger.Printf)
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: versions.PublishQueueBucket}}},
Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}}, Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf}
acts.Republish = republish.Republish
again := &republishing{rp: republish}
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
decided := &decisions{subject: oc.SubjectDecided(cred.Module)}
retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}}
operator := &build.OperatorAsk{Builder: b, Acted: retakeActed, Ctx: ctx, Logf: logger.Printf,
Client: &occ.Client{Module: cred.Module, State: calls.State("asked"), Publish: publishVia(ctx, js),
Record: recordVia(calls.Call, calls.Ask), Subscribe: decided.hear(logger.Printf)}}
if err := operator.Listen(); err != nil {
return fmt.Errorf("hearing the operator's answers: %w", err)
}
retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}}
queueKV.Ready(again.bound(ctx, js, logger.Printf))
actedKV.Ready(again.bound(ctx, js, logger.Printf))
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
@@ -376,6 +395,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
events := func(m jetstream.Msg) {
if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them
return
}
if decided.heard(m) { // the operator's answer to an ask about a key (§568)
return
}
if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568)
return
}
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
return
}
@@ -422,6 +450,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
unput.expired(ctx, e)
return
}
if _, _, ok := versions.StepsBoundOf(e.Key()); ok {
stepReads.expired(ctx, e)
return
}
if node, ok := enginereads.BoundOf(e.Key()); ok {
if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended
reads.expiredCall(node)
@@ -572,3 +604,28 @@ func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (st
}
return "", nil
}
// toolchainWaitOf reads the toolchain wait's setting, toolchain-wait-minutes, from the file the manifest declares with the
// content ${config:toolchain-wait-minutes} (design 47 G §672, A-71), at the builder's start: no file, or an empty one, is
// the default; a content that is no number of minutes is an error naming the file.
func toolchainWaitOf(path string) (time.Duration, error) {
if path == "" {
return defaultToolchainWait, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return defaultToolchainWait, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return defaultToolchainWait, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a number of minutes", path, v)
}
return time.Duration(n) * time.Minute, nil
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go
+10 -5
View File
@@ -58,7 +58,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream
released(ctx, b, kv, e, r, logf)
return
}
if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop {
if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop {
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
go versionActed(ctx, acts, kv, e, r, logf)
return
@@ -192,10 +192,15 @@ func emitBuildEnded(ctx context.Context, js jetstream.JetStream, cred process.Ac
// versionActed answers a reasks value of builder_publish_again, builder_floor_again or builder_old_form's drop, a hand act said in the log.
func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
var err error
if r.Mode == build.ModePublishAgain {
var ag versions.Again
ag, r.Answer, err = acts.PublishAgain(ctx, r.Name, r.FP, r.Why, r.By, r.At)
r.Do, r.Version = ag.Do, ag.Entry.Version
if r.Mode == build.ModeRepublish {
if acts.Republish == nil {
err = errors.New("this builder publishes nothing again")
} else {
r.Queued, err = acts.Republish(ctx, r.Module, r.Version)
if err == nil {
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
}
}
} else if r.Mode == build.ModeReservedDrop {
if acts.DropReserved == nil {
err = errors.New("this builder keeps no reservations")
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go
+146
View File
@@ -0,0 +1,146 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"sync/atomic"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"mesh-delivery-builder/internal/versions"
)
// The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus.
// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's.
type awaitedQueue struct{ awaitedReserved }
func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) {
kv, err := s.kv()
if err != nil {
return 0, err
}
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
return st.Bytes(), nil
}
// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry.
func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
return func(_ context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
raw, err := call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint})
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return meshcatalog.CatalogEntry{}, fmt.Errorf("the catalog answered what is no entry: %w", err)
}
return a.Entry, nil
}
}
// sendVia sends a publish to the registry seat with the digest and no parts (A-73): the holder takes the bytes from the
// artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes).
func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error {
return func(_ context.Context, p versions.Publish) error {
raw, err := call("seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
if err != nil {
return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err)
}
var a gopackageregistry.PublishAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return fmt.Errorf("%s answered what is no publish answer: %w", p.Seat, err)
}
if len(a.Lacks) > 0 {
return fmt.Errorf("%s%s holds %d of the parts of %s %s", versions.NotStoredWords, p.Seat, len(a.Lacks), p.Name, p.Version)
}
return nil
}
}
// publishBody is the seat's publish input for p: the digest, its kind and name and version, no parts.
func publishBody(p versions.Publish) gopackageregistry.PublishInput {
return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version}
}
// republishMaxOf reads the setting republish-max-versions from the file the manifest declares (design 47 G §633, A-71's
// form): no file or an empty one is the default; a content that is no positive number is an error naming the file.
func republishMaxOf(path string) (int, error) {
if path == "" {
return versions.DefaultRepublishMax, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return versions.DefaultRepublishMax, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return versions.DefaultRepublishMax, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a number of versions", path, v)
}
return n, nil
}
// republishing starts the publish again once both its states are bound: the queue is taken up again and the last running
// of every node's module is read (§556), acted on where its sequence is newer than the kept one.
type republishing struct {
rp *versions.Republisher
ready atomic.Int32
}
func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) {
return func(jetstream.KeyValue) {
if g.ready.Add(1) != 2 {
return
}
if err := g.rp.Resume(ctx); err != nil {
logf("the queued publishes are not taken up again: %v", err)
}
err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(subject string, data []byte, seq uint64) {
if _, err := g.rp.Heard(ctx, subject, data, seq); err != nil {
logf("the running %s is not acted on at the start: %v", subject, err)
}
})
if err != nil {
logf("the running of the node-engine seat is not read for the publish again at the start: %v", err)
}
}
}
// heard hands a message of the events consumer to the publish again: one of its subjects whose state cannot take it is left
// for the bus to give again (Nak); every message goes on to the other parts.
func (g *republishing) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) {
meta, err := m.Metadata()
if err != nil {
return false
}
handled, err := g.rp.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream)
if handled && err != nil {
logf("%v; heard again", err)
if nakErr := m.Nak(); nakErr != nil {
logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
}
return true
}
return false
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go
+79
View File
@@ -0,0 +1,79 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// TestRepublishSend: design 47 G §588, A-73: a publish again is the registry seat's publish with the digest and no parts; the
// four seats take one wire form, a call's fault or an answer that lacks parts is the holder's refusal in its words, and its
// setting is read from the file the manifest declares.
func TestRepublishSend(t *testing.T) {
p := versions.Publish{Seat: versions.SeatArtifacts, Kind: "bundle", Name: "m/prog", Version: "0.1.3", Digest: "sha256:aa", Fingerprint: "src2:m", Commit: "c", Transfer: "t"}
want, _ := json.Marshal(publishBody(p))
if strings.Contains(string(want), "parts") {
t.Errorf("the publish names parts: %s", want)
}
for name, v := range map[string]any{"container image": containerimageregistry.PublishInput{Digest: "sha256:aa"}, "oci": ociartifactregistry.PublishInput{Digest: "sha256:aa"},
"go": gopackageregistry.PublishInput{Digest: "sha256:aa"}, "npm": npmpackageregistry.PublishInput{Digest: "sha256:aa"}} {
var ref, got map[string]any
a, _ := json.Marshal(gopackageregistry.PublishInput{Digest: "sha256:aa"})
b, _ := json.Marshal(v)
_ = json.Unmarshal(a, &ref)
_ = json.Unmarshal(b, &got)
if len(ref) != len(got) {
t.Errorf("the %s seat's publish input is %s, the go seat's %s: one wire form is sent to all four", name, b, a)
}
}
var asked string
for name, c := range map[string]struct {
answer string
err error
want string
}{
"held": {`{"held":true,"transfer":"t"}`, nil, ""},
"lacks": {`{"lacks":["sha256:bb"],"transfer":"t"}`, nil, versions.NotStoredWords},
"a fault": {"", errors.New("no responders"), "no responders"},
"no sense": {`[`, nil, "no publish answer"},
} {
send := sendVia(func(key string, body any) (json.RawMessage, error) {
asked = key
return json.RawMessage(c.answer), c.err
})
err := send(context.Background(), p)
if asked != "seat:oci-artifact-registry.publish" || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) {
t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want)
}
}
dir := t.TempDir()
for content, want := range map[string]int{"": 50, "7": 7} {
path := filepath.Join(dir, "max")
_ = os.WriteFile(path, []byte(content), 0o600)
if got, err := republishMaxOf(path); err != nil || got != want {
t.Errorf("republishMaxOf(%q) = %d, %v, want %d", content, got, err, want)
}
}
_ = os.WriteFile(filepath.Join(dir, "bad"), []byte("0"), 0o600)
if _, err := republishMaxOf(filepath.Join(dir, "bad")); err == nil {
t.Error("a setting of 0 is accepted")
}
if _, _, err := build.NewRepublish("", "0.1.3", "why", "me", versionsNow()); err == nil {
t.Error("a version with no module is accepted")
}
}
func versionsNow() time.Time { return time.Unix(1, 0) }
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go
+5
View File
@@ -125,6 +125,11 @@ func (rv *reservations) unreleased(ctx context.Context, v versions.ReservedVersi
needs: "builder_reserved shows the reservation; builder_publish_again publishes the missing artifacts under it; builder_reserved_drop drops it."})
}
// unreleasedWords raises version-unreleased about a version with the words of a publish again that ended badly.
func (rv *reservations) unreleasedWords(ctx context.Context, v versions.ReservedVersion, why string) {
rv.unreleased(ctx, v, why)
}
// ended is told each change of the state bounds: a reservation's bound whose time ended raises version-unreleased while
// the reservation stands built.
func (rv *reservations) ended(ctx context.Context, e jetstream.KeyValueEntry) bool {
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go
+124
View File
@@ -0,0 +1,124 @@
package main
import (
"context"
"encoding/json"
"fmt"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
stdio "git.novox.be/novox/mesh-sdk/go"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
)
// The builder's retake of a key failed on a transport fault, on the node-engine seat's events (design 47 G §568, ADR 0673): the
// wiring of build.Retaker to the events consumer.
type retaking struct{ rt *build.Retaker }
// heard hands a message of the events consumer to the retake: one of its subjects whose state cannot take it is left for the bus to
// give again (Nak); every message goes on to the other parts.
func (g *retaking) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) {
meta, err := m.Metadata()
if err != nil {
return false
}
handled, err := g.rt.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream)
if handled && err != nil {
logf("%v; heard again", err)
if nakErr := m.Nak(); nakErr != nil {
logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
}
return true
}
return false
}
// The builder's three bus adapters for the operator-channel client (design 47 G §568), as the teller has them: its own ask
// published, the router's record read through the seat, its own `decided` heard through the module's hub.
// publishVia publishes an accept of the seat, de-duplicated by id: the same id twice is one message.
func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string, body any, id string) error {
return func(subject string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
var opts []jetstream.PublishOpt
if id != "" {
opts = append(opts, jetstream.WithMsgID(id))
}
_, err = js.Publish(ctx, subject, data, opts...)
return err
}
}
// routerBucket is the bucket of the router's asks: the state `asks` of the seat operator-channel (the contract's StateAsks) in
// the bucket of the seat's holder on the seat list, which the seat mesh-module-assigner's verb assignments answers with seats
// (design 47 G A-74); no module's name is in the program.
func routerBucket(call func(key string, body any) (json.RawMessage, error)) (string, error) {
raw, err := call("seat:"+mma.Seat+"."+mma.VerbAssignments, mma.AssignmentsInput{Seats: mma.AssignmentsInputSeatsTrue})
if err != nil {
return "", fmt.Errorf("who holds the seat %s is not asked of the seat %s: %w", oc.Seat, mma.Seat, err)
}
var held struct {
Seats []mma.SeatHolders `json:"seats"`
}
if err := json.Unmarshal(raw, &held); err != nil {
return "", fmt.Errorf("the answer of %s.%s is not its seat's: %w", mma.Seat, mma.VerbAssignments, err)
}
for _, s := range held.Seats {
if s.Seat == oc.Seat && len(s.Holders) > 0 {
return s.Holders[0].Module + "_" + oc.StateAsks, nil
}
}
return "", fmt.Errorf("no module holds the seat %s on the seat list, so the router's record of an ask is not read", oc.Seat)
}
// recordVia reads the router's record of one of this module's asks: a get of `<module>.<id>` in the bucket the seat's holder
// keeps (record is process.Conn.Ask).
func recordVia(call func(key string, body any) (json.RawMessage, error), record func(bucket, id string) (*stdio.StateEntry, error)) func(id string) (*stdio.StateEntry, error) {
return func(id string) (*stdio.StateEntry, error) {
bucket, err := routerBucket(call)
if err != nil {
return nil, err
}
return record(bucket, id)
}
}
// decisions holds the handler of the router's `decided` event for this module: the module's one hub has the pattern
// `mesh.seat.*.event.>`, which covers it, so the events handler hands the message here (a hub refuses two patterns that overlap).
type decisions struct {
subject string
handle func(jetstream.Msg)
}
// hear is the client's Subscribe: it keeps the handler of the warrants the router says to this module.
func (d *decisions) hear(logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error {
return func(pattern string, handle func(stdio.Envelope) error) error {
d.handle = func(m jetstream.Msg) {
if err := handle(stdio.Envelope{Key: pattern, Body: m.Data()}); err != nil {
logf("a warrant could not be handled and is given again: %v", err)
if nakErr := m.Nak(); nakErr != nil {
logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
}
return
}
if err := m.Ack(); err != nil {
logf("a warrant was handled but could not be acknowledged: %v", err)
}
}
return nil
}
}
// heard takes the message when it is this module's `decided`.
func (d *decisions) heard(m jetstream.Msg) bool {
if d.handle == nil || m.Subject() != d.subject {
return false
}
d.handle(m)
return true
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go
+95
View File
@@ -0,0 +1,95 @@
package main
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"mesh-delivery-builder/internal/versions"
)
// **The builder's read of an ask's steps is a kept call, unanswered past five minutes a finding** (design 47 G §568, A-77, ADR
// 0668 §3): a transport fault opening or taking up the watch of `steps` ends no build key; the read is made again once the bus
// answers. The first fault arms a key of the state bounds (StepsBoundKey) with its own time to live; the bus's marker of its end,
// heard on the watch of the state, raises call-unanswered about the ask while the read is still unanswered; the answer deletes
// the key and clears the finding. No timer.
type stepsRead struct {
bounds func() jetstream.KeyValue
findings *findings
bound time.Duration
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
open map[string]string // the asks whose read is unanswered, by ask id: the last fault
}
// unanswered is told each fault of the read of the ask's steps on module: the first arms the bound.
func (s *stepsRead) unanswered(module, askID string, fault error) {
s.mu.Lock()
if s.open == nil {
s.open = map[string]string{}
}
_, armed := s.open[askID]
s.open[askID] = fault.Error()
s.mu.Unlock()
if armed {
return
}
ctx := context.Background()
var err error
if kv := s.bounds(); kv == nil {
err = fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket)
} else if _, err = kv.Create(ctx, versions.StepsBoundKey(module, askID), versions.EncodeBound("the read of the steps of the ask "+askID+" on "+module, s.now().Add(s.bound)), jetstream.KeyTTL(s.bound)); errors.Is(err, jetstream.ErrKeyExists) {
err = nil
}
if err != nil {
s.logf("the bound of the read of the steps of the ask %s is not written, so it is raised now: %v", askID, err)
s.raise(ctx, module, askID, fault.Error(), s.now())
}
}
// answered ends the read's bound and clears its finding.
func (s *stepsRead) answered(module, askID string) {
s.mu.Lock()
_, was := s.open[askID]
delete(s.open, askID)
s.mu.Unlock()
if !was {
return
}
ctx := context.Background()
if kv := s.bounds(); kv != nil {
if err := kv.Delete(ctx, versions.StepsBoundKey(module, askID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
s.logf("the bound of the read of the steps of the ask %s is not deleted: %v", askID, err)
}
}
s.findings.clear(ctx, meshconditionsclient.KindCallUnanswered, meshconditionsclient.ScopeCall, askID)
}
// expired is told each change of the state bounds: the end of a steps bound raises call-unanswered while its read is unanswered.
func (s *stepsRead) expired(ctx context.Context, e jetstream.KeyValueEntry) {
module, askID, ok := versions.StepsBoundOf(e.Key())
if !ok || e.Operation() != jetstream.KeyValuePurge {
return
}
s.mu.Lock()
fault, open := s.open[askID]
s.mu.Unlock()
if !open {
return
}
s.raise(ctx, module, askID, fault, s.now().Add(-s.bound))
}
func (s *stepsRead) raise(ctx context.Context, module, askID, fault string, since time.Time) {
o := callUnanswered(askID, "the read of the steps of "+module, "", fault, "builder."+meshconditionsclient.KindCallUnanswered, since)
o.needs = "the bus answers the read again by itself; builder_stuck shows the key waiting on the ask."
s.findings.raise(ctx, o)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"context"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"mesh-delivery-builder/internal/versions"
)
// Design 47 G §568, A-77: the first fault of the read of an ask's steps arms a bound key with its own time to live; its end
// while the read is unanswered raises call-unanswered about the ask; the answer deletes the bound and clears the finding.
func TestTheStepsReadIsCallUnansweredAfterItsBound(t *testing.T) {
st := &memFindings{vals: map[string][]byte{}}
f := &findings{w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
marks := &memBounds{keys: map[string]bool{}}
s := &stepsRead{bounds: func() jetstream.KeyValue { return marks }, findings: f, bound: meshconditionsclient.CallUnansweredAfter,
now: func() time.Time { return time.Unix(600, 0) }, logf: t.Logf}
ask := "mvl0frv18d1958"
callKey := meshconditionsclient.Key(meshconditionsclient.ScopeCall, ask, meshconditionsclient.KindCallUnanswered)
s.unanswered("mesh-build-go", ask, errors.New("nats: no responders available for request"))
if !marks.keys[versions.StepsBoundKey("mesh-build-go", ask)] || st.vals[callKey] != nil {
t.Fatalf("a first fault: bound %v, finding %v; want the bound armed, no finding yet", marks.keys, st.vals[callKey] != nil)
}
s.expired(context.Background(), purged{key: versions.StepsBoundKey("mesh-build-go", ask)})
if got, err := decodeFinding(st.vals[callKey]); err != nil || got.Kind != meshconditionsclient.KindCallUnanswered || !strings.Contains(got.Summary, "the steps of mesh-build-go") || !strings.Contains(got.Summary, ask) {
t.Fatalf("the bound's end: the finding = %+v, %v", got, err)
}
s.answered("mesh-build-go", ask)
if marks.keys[versions.StepsBoundKey("mesh-build-go", ask)] || st.vals[callKey] != nil {
t.Errorf("the read answered: bound %v, finding left %v", marks.keys, st.vals[callKey] != nil)
}
s.expired(context.Background(), purged{key: versions.StepsBoundKey("mesh-build-go", ask)})
if st.vals[callKey] != nil {
t.Error("a bound's end after the answer raised call-unanswered")
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go
+8 -4
View File
@@ -254,8 +254,8 @@ func (h *toolchainsHeard) agree(ctx context.Context) {
}
// lastPerSubject reads the last message of every subject of a stream kept last per subject, through an ordered consumer
// made for that read (design 47 G §556), and hands each to each; it answers when the last of them was handed.
func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter string, each func(subject string, data []byte)) error {
// made for that read (design 47 G §556), and hands each to each with its stream sequence; it answers when the last of them was handed.
func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter string, each func(subject string, data []byte, sequence uint64)) error {
cons, err := js.OrderedConsumer(ctx, stream, jetstream.OrderedConsumerConfig{FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
if err != nil {
return fmt.Errorf("reading %s: %w", stream, err)
@@ -278,14 +278,18 @@ func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter
if err != nil {
return fmt.Errorf("reading %s: %w", stream, err)
}
each(m.Subject(), m.Data())
seq := uint64(0)
if meta, err := m.Metadata(); err == nil {
seq = meta.Sequence.Stream
}
each(m.Subject(), m.Data(), seq)
}
return nil
}
// readRunning is the start's read of every node's module's last running (§556), before the consumer's.
func (h *toolchainsHeard) readRunning(ctx context.Context, js jetstream.JetStream) {
if err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(_ string, data []byte) { h.running(ctx, data) }); err != nil {
if err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(_ string, data []byte, _ uint64) { h.running(ctx, data) }); err != nil {
h.logf("the running of the node-engine seat is not read at the start, so a build module stopped while this builder was away is forgotten at its next running: %v", err)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go
+39
View File
@@ -0,0 +1,39 @@
package main
import (
"os"
"path/filepath"
"testing"
"time"
)
// A-71: the toolchain wait's setting is read at the start from the file the manifest declares, never from the environment.
func TestToolchainWaitIsReadFromItsFile(t *testing.T) {
dir := t.TempDir()
n := 0
file := func(content string) string {
n++
p := filepath.Join(dir, "w"+string(rune('a'+n)))
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return p
}
for _, c := range []struct {
name, path string
want time.Duration
bad bool
}{
{"no file named", "", defaultToolchainWait, false},
{"file absent", filepath.Join(dir, "absent"), defaultToolchainWait, false},
{"empty", file(" \n"), defaultToolchainWait, false},
{"minutes", file("15\n"), 15 * time.Minute, false},
{"not a number", file("soon"), 0, true},
{"zero", file("0"), 0, true},
} {
got, err := toolchainWaitOf(c.path)
if (err != nil) != c.bad || got != c.want {
t.Errorf("%s: %v, %v; want %v, error %v", c.name, got, err, c.want, c.bad)
}
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go
-20
View File
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
@@ -306,22 +305,3 @@ func (f *floorWake) wake(ctx context.Context, kind, seat, module string) {
func conditionModules(id string) []string {
return strings.FieldsFunc(id, func(r rune) bool { return r == '/' || r == '.' || r == ':' })
}
// doneVia says whether a build key is done, read from the builder's own builds state: the build's file is held.
func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (bool, error) {
return func(ctx context.Context, module, fp string) (bool, error) {
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return false, err
}
raw, _, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil || !found {
return false, err
}
cur, err := builds.DecodeBuilderBuild(raw)
if err != nil {
return false, err
}
return cur.State == builds.BuildDone, nil
}
}
modules/mesh-delivery-builder/go.mod
+2
View File
@@ -9,6 +9,8 @@ require (
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000
modules/mesh-delivery-builder/internal/build/buildread.go
+222
View File
@@ -0,0 +1,222 @@
package build
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// **builder_build answers a module's build keys whole** (design 47 G §589, A-87): with `fingerprint` that key whole, with
// `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and
// since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch
// whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle.
// BuildTool is the builder's read tool of a module's build keys.
const BuildTool = "builder_build"
// MostBuilds is how many keys of a module builder_build answers by module alone.
const MostBuilds = 20
// BuildRow is one build key as builder_build answers it.
type BuildRow struct {
Key string `json:"key"`
Fingerprint string `json:"fingerprint"`
State string `json:"state"`
Revision uint64 `json:"revision,omitempty"`
Since time.Time `json:"since,omitzero"`
Step string `json:"step,omitempty"`
Node string `json:"node,omitempty"`
Error string `json:"error,omitempty"`
EventBuilds int `json:"event-builds"`
// Ask is the open ask of the operator about the key (§568).
Ask *OpenAsk `json:"ask,omitempty"`
// Newest names each watched branch whose newest catalog graph entry of the module names this fingerprint.
Newest []string `json:"newest,omitempty"`
}
// OpenAsk is the ask the operator has not answered.
type OpenAsk struct {
ID string `json:"id"`
Asked string `json:"asked"`
Expires string `json:"expires"`
}
// BuildAnswer is what builder_build answers.
type BuildAnswer struct {
Module string `json:"module"`
Builds []BuildRow `json:"builds"`
// Newest is the fingerprint of the module's newest catalog graph entry on each watched branch.
Newest map[string]string `json:"newest-entries,omitempty"`
// Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint.
Note string `json:"note,omitempty"`
}
// BuildReader reads what builder_build answers.
type BuildReader struct {
Builds StateRead
// Entry is a key's value with its revision and when it was written; nil leaves both out.
Entry func(key string) (json.RawMessage, uint64, time.Time, error)
// Asked is the builder's state asked, which holds the asks the operator was asked.
Asked StateRead
// Newest is the fingerprint of the module's newest catalog graph entry on each watched branch, read from the catalog half.
Newest func(module string) (map[string]string, error)
Steps StuckReader
index map[string]map[string]string
}
// Build answers for module, and for one fingerprint when fingerprint is not empty.
func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) {
out := BuildAnswer{Module: module, Builds: []BuildRow{}}
if module == "" {
return out, errors.New("name the module (module)")
}
all, err := r.Builds.Keys()
if err != nil {
return out, err
}
var rows []BuildRow
for _, key := range all {
mod, fp, err := builds.ParseBuilderBuildKey(key)
if err != nil || mod != module || (fingerprint != "" && fp != fingerprint) {
continue
}
raw, rev, at, err := r.read(key)
if err != nil {
return out, fmt.Errorf("%s cannot be read: %w", key, err)
}
if raw == nil {
continue
}
v, err := builds.DecodeBuilderBuild(raw)
if err != nil {
return out, fmt.Errorf("%s holds what is not a build: %w", key, err)
}
row := BuildRow{Key: key, Fingerprint: fp, State: v.State, Revision: rev, Since: at, Step: v.Step, Error: v.Error, EventBuilds: v.EventBuilds}
if _, _, node, _, ok := builds.TransportFaultOf(v.Error); ok && node != "no node" {
row.Node = node
}
if row.Node == "" && v.State == builds.BuildBuilding {
row.Node = r.stepNode(v)
}
rows = append(rows, row)
}
if fingerprint != "" && len(rows) == 0 {
return out, fmt.Errorf("the builder holds no build key of %s at %s", module, fingerprint)
}
sort.Slice(rows, func(i, j int) bool {
if !rows[i].Since.Equal(rows[j].Since) {
return rows[i].Since.After(rows[j].Since)
}
return rows[i].Key > rows[j].Key
})
if len(rows) > MostBuilds {
rows = rows[:MostBuilds]
}
asks := r.openAsks()
for i := range rows {
rows[i].Ask = asks[rows[i].Key]
}
if r.Newest != nil {
newest, err := r.Newest(module)
if err != nil {
return out, fmt.Errorf("the catalog half's newest entry of %s cannot be read: %w", module, err)
}
out.Newest = newest
named := false
for i := range rows {
for branch, fp := range newest {
if fp == rows[i].Fingerprint {
rows[i].Newest, named = append(rows[i].Newest, branch), true
}
}
slices.Sort(rows[i].Newest)
}
if !named && len(newest) > 0 {
var says []string
for branch, fp := range newest {
says = append(says, branch+" names "+fp)
}
slices.Sort(says)
out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ")
}
}
out.Builds = append(out.Builds, rows...)
return out, nil
}
// stepNode is the node a building key's step runs on, read from its asks' steps in the build modules' steps; empty when none runs.
func (r *BuildReader) stepNode(v builds.BuilderBuild) string {
if len(v.Asks) == 0 || len(r.Steps.Steps) == 0 {
return ""
}
if r.index == nil {
idx, err := r.Steps.stepIndex()
if err != nil {
return ""
}
r.index = idx
}
languages := make([]string, 0, len(v.Asks))
for l := range v.Asks {
languages = append(languages, l)
}
slices.Sort(languages)
for _, l := range languages {
bm, _ := buildstep.BuildModuleOf(l)
st, ok := r.Steps.Steps[bm]
k, found := r.index[bm][v.Asks[l]]
if !ok || !found {
continue
}
if raw, _, err := st.Get(k); err == nil && raw != nil {
if step, err := buildstep.DecodeStep(raw); err == nil && step.Node != "" {
return step.Node
}
}
}
return ""
}
func (r *BuildReader) read(key string) (json.RawMessage, uint64, time.Time, error) {
if r.Entry != nil {
return r.Entry(key)
}
raw, at, err := r.Builds.Get(key)
return raw, 0, at, err
}
// openAsks are the asks kept in the state asked that nobody answered yet, by the key each is about.
func (r *BuildReader) openAsks() map[string]*OpenAsk {
out := map[string]*OpenAsk{}
if r.Asked.Keys == nil {
return out
}
keys, err := r.Asked.Keys()
if err != nil {
return out
}
for _, k := range keys {
raw, _, err := r.Asked.Get(k)
if err != nil || raw == nil {
continue
}
var kept occ.Kept
if json.Unmarshal(raw, &kept) != nil || kept.Ended != "" || kept.Acted || kept.Ask.About == "" {
continue
}
if t, err := occ.Time(kept.Ask.Expires); err == nil && !t.IsZero() && time.Now().After(t) {
continue
}
out[kept.Ask.About] = &OpenAsk{ID: kept.Ask.ID, Asked: kept.Asked.UTC().Format(time.RFC3339), Expires: kept.Ask.Expires}
}
return out
}
modules/mesh-delivery-builder/internal/build/buildread_test.go
+83
View File
@@ -0,0 +1,83 @@
package build
import (
"encoding/json"
"sort"
"strings"
"testing"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
const fp2 = "src2:2222222222222222222222222222222222222222222222222222222222222222"
func stateOf(vals map[string][]byte) StateRead {
return StateRead{
Keys: func() ([]string, error) {
var ks []string
for k := range vals {
ks = append(ks, k)
}
sort.Strings(ks)
return ks, nil
},
Get: func(k string) (json.RawMessage, time.Time, error) { return vals[k], time.Time{}, nil }}
}
// Design 47 G §589: builder_build by module answers its keys newest first with whole fingerprints, event-builds and the open
// ask, and names the watched branches whose newest entry names a key's fingerprint; a module none of whose keys the newest
// entry names says so; with a fingerprint, that key alone.
func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) {
failed, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, EventBuilds: 3, Manifest: mdigest('a'),
Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"})
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a'), Artifacts: []builds.BuiltArtifact{{Name: "x", Kind: "file", Digest: "sha256:" + strings.Repeat("a", 64)}}})
k1, _ := builds.BuilderBuildKey("m", fp1)
k2, _ := builds.BuilderBuildKey("m", fp2)
other, _ := builds.BuilderBuildKey("other", fp1)
vals := map[string][]byte{k1: failed, k2: done, other: done}
ask := occ.Kept{Ask: oc.Ask{ID: "retake-1", About: k1, Expires: occ.FormatTime(time.Now().Add(time.Hour))}, Asked: time.Now()}
askRaw, _ := json.Marshal(ask)
r := &BuildReader{Builds: stateOf(vals), Asked: stateOf(map[string][]byte{"ask.retake-1": askRaw}),
Newest: func(string) (map[string]string, error) { return map[string]string{"main": fp1, "feature": fp2}, nil }}
a, err := r.Build("m", "")
if err != nil || len(a.Builds) != 2 {
t.Fatalf("builds = %+v, %v, want the module's two keys", a, err)
}
var row BuildRow
for _, b := range a.Builds {
if b.Fingerprint == fp1 {
row = b
}
}
if row.Fingerprint != fp1 || len(row.Fingerprint) != len("src2:")+64 || row.State != builds.BuildFailed || row.EventBuilds != 3 || row.Node != "ace" ||
!strings.HasPrefix(row.Error, "transport fault: connection, node-build-go ace, dial tcp") || row.Ask == nil || row.Ask.ID != "retake-1" || len(row.Newest) != 1 || row.Newest[0] != "main" {
t.Errorf("the failed key = %+v, want its fingerprint whole, its error whole, event-builds 3, the open ask and newest main", row)
}
if one, err := r.Build("m", fp2); err != nil || len(one.Builds) != 1 || one.Builds[0].Fingerprint != fp2 || one.Builds[0].Newest[0] != "feature" {
t.Errorf("one key = %+v, %v", one, err)
}
r.Newest = func(string) (map[string]string, error) {
return map[string]string{"main": "src2:" + strings.Repeat("c", 64)}, nil
}
if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) {
t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err)
}
}
// Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it.
func TestBuilderStuckAnswersTheFingerprintWhole(t *testing.T) {
building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, Step: "asked", Manifest: mdigest('a')})
k, _ := builds.BuilderBuildKey("m", fp1)
out, err := StuckReader{Builds: stateOf(map[string][]byte{k: building})}.Stuck("m")
if err != nil {
t.Fatal(err)
}
rows := out["stuck"].([]Stuck)
if len(rows) != 1 || rows[0].FP != fp1 {
t.Fatalf("stuck = %+v, want the fingerprint %s whole", rows, fp1)
}
}
modules/mesh-delivery-builder/internal/build/claim.go
+11 -1
View File
@@ -54,6 +54,9 @@ type ClaimOptions struct {
// ByHand is a person's hand act (BuildAgain): a key failed on a transport fault is taken again only by it, or by a builder
// that has just started (TakeOver), never by the builder's own retake (ADR 0664 §3).
ByHand bool
// ByEvent is the builder's retake on an event that says a transport fault's cause passed (design 47 G §568): it takes a key
// failed on a transport fault, as a new revision that counts one more of EventBuilds.
ByEvent bool
// ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken
// again only then. Nil means none does.
ImageBuilder func() bool
@@ -213,11 +216,18 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
return nil, nil
case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
return nil, nil
case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver:
case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent:
return nil, nil
case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
return nil, nil
case cur.State == builds.BuildFailed:
building.EventBuilds = cur.EventBuilds // a person's build starts the count again, an event's adds one (design 47 G §568)
switch {
case opt.ByHand:
building.EventBuilds = 0
case opt.ByEvent:
building.EventBuilds++
}
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
case cur.State == builds.BuildQueued && cur.Waits != "":
if opt.Published == nil || !opt.Published(cur.Waits) {
modules/mesh-delivery-builder/internal/build/claim_test.go
+1 -1
View File
@@ -166,7 +166,7 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) {
take bool
}{{ClaimOptions{}, false}, {ClaimOptions{ByHand: true}, true}, {ClaimOptions{TakeOver: true}, true}} {
st = newMem()
seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "mesh-build-go on ace: a transport fault on 3 runs (ace, g14, novox): lookup proxy: i/o timeout"})
seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "transport fault: name lookup, node-build-go ace, lookup proxy: i/o timeout"})
if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", o.opt); err != nil || (got != nil) != o.take {
t.Errorf("a transport fault on 3 runs with %+v: claim = %v, %v, want taken %v", o.opt, got, err, o.take)
}
modules/mesh-delivery-builder/internal/build/keyerror_test.go
+17
View File
@@ -0,0 +1,17 @@
package build
import (
"strings"
"testing"
)
// Design 47 G §568: a key's error opens `transport fault:` only when every lane failed on one.
func TestKeyErrorOpensTransportFaultOnlyWhenAllLanesDid(t *testing.T) {
tf := "transport fault: deadline, node-build-go ace, i/o timeout"
if got := keyError([]string{tf, "mesh-build-node on g14: the step failed: tsc: 2 errors"}); strings.HasPrefix(got, "transport fault:") {
t.Errorf("beside a real failure the key opens %q", got)
}
if got := keyError([]string{tf, tf}); !strings.HasPrefix(got, "transport fault:") {
t.Errorf("two transport faults open %q", got)
}
}
modules/mesh-delivery-builder/internal/build/numbers.go
+1 -1
View File
@@ -177,7 +177,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
for _, t := range plan.Tasks {
refs = append(refs, module+":"+t.Artifact)
}
src := versions.Source{Repository: m.Repository, Commit: m.Commit}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2}
for _, t := range plan.Tasks {
src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go")
}
modules/mesh-delivery-builder/internal/build/retake.go
+226
View File
@@ -0,0 +1,226 @@
package build
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-builder/internal/versions"
)
// **A key that failed on a transport fault is built again on an event that says its cause passed** (design 47 G §568, ADR 0673
// §1 to §4, which changes ADR 0664 §3): the node-engine seat's `running` of a module whose catalog graph entry claims the build
// seat the failed step ran on, on any node, and its `manifest-reads-changed` of the node the step ran on with `why` `returned`.
// Each message is acted on once, by its stream sequence (state `acted`). A key is built again once per event and only while its
// newest revision is that failure, as a new revision counting one more of EventBuilds; after MostEventBuilds the next event
// starts nothing and asks the operator once.
// MostEventBuilds is how many revisions events start, since the last a person started, before the operator is asked.
const MostEventBuilds = 3
// Retake builds the failed key again on an event: it starts the look of the key's module at its fingerprint, which claims the
// key as an event's (ClaimOptions.ByEvent). started is false when the key is not failed on a transport fault, when the module
// has another fingerprint now, or when a look holds the key; ask is true when MostEventBuilds are spent, with the key as it
// stands and its revision.
func (b *Builder) Retake(ctx context.Context, key string) (started, ask bool, cur builds.BuilderBuild, rev uint64, err error) {
b.init()
module, fp, err := builds.ParseBuilderBuildKey(key)
if err != nil {
return false, false, cur, 0, err
}
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil || !found {
return false, false, cur, 0, err
}
if cur, err = builds.DecodeBuilderBuild(value); err != nil {
return false, false, cur, rev, fmt.Errorf("%s holds what is not a build: %w", key, err)
}
if cur.State != builds.BuildFailed || !builds.FailedOnTransportFault(cur.Error) {
return false, false, cur, rev, nil
}
if cur.EventBuilds >= MostEventBuilds {
return false, true, cur, rev, nil
}
b.mu.Lock()
m, known := b.known[module]
if !known || m.Fp2 != fp || m.Manifest == "" || b.inflight[key] || b.hands[key] != nil {
b.mu.Unlock()
return false, false, cur, rev, nil
}
b.inflight[key], b.events[key] = true, true
b.mu.Unlock()
b.logf("%s: failed (%s); built again on an event, build %d of %d since a person's", key, cur.Error, cur.EventBuilds+1, MostEventBuilds)
b.startOnce(ctx, key, module, m)
return true, false, cur, rev, nil
}
// Retaker hears the node-engine seat's events and builds again what they say can pass (Builder.Retake).
type Retaker struct {
Builder *Builder
// Acted is the state acted: the last stream sequence acted on for each subject, and the revision the operator was asked at.
Acted versions.ReserveStore
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
// Ask asks the operator about a key whose event builds are spent (the seat operator-channel's ask).
Ask func(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error
Logf func(string, ...any)
}
func (r *Retaker) logf(f string, a ...any) {
if r.Logf != nil {
r.Logf(f, a...)
}
}
// retakeKey is the acted key of a subject's sequence, apart from the publish again's.
func retakeKey(subject string) string { return versions.ActedKey("retake:" + subject) }
// askedKey is the acted key of the revision of a build key the operator was asked at.
func askedKey(key string) string { return versions.ActedKey("retake-asked:" + key) }
// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq, and acts on
// it once. A subject that is neither is not its; an error leaves it not acted on, so it is read again.
func (r *Retaker) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) {
running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*"))
changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".")
if !running && !changed {
return false, nil
}
key := retakeKey(subject)
raw, rev, found, err := r.Acted.Get(ctx, key)
if err != nil {
return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err)
}
if found {
var a versions.Acted
if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq {
return true, nil
}
}
var want func(seat, node string) bool
if running {
want, err = r.runningWant(ctx, data)
} else {
want, err = r.returnedWant(data)
}
if err == nil && want != nil {
err = r.retakeAll(ctx, want)
}
if err != nil {
return true, err
}
val, _ := json.Marshal(versions.Acted{Sequence: seq})
if found {
_, err = r.Acted.Update(ctx, key, val, rev)
} else {
_, err = r.Acted.Create(ctx, key, val)
}
if err != nil && !errors.Is(err, versions.ErrConflict) {
return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err)
}
return true, nil
}
// runningWant is what a running says passed: the build seats the running module's catalog graph entry claims, on any node.
func (r *Retaker) runningWant(ctx context.Context, data []byte) (func(seat, node string) bool, error) {
var ev nodeengine.Running
if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" {
r.logf("a running that does not read is not acted on: %v", err)
return nil, nil
}
if ev.Stopped || ev.Fingerprint == "" {
return nil, nil
}
e, err := r.Entry(ctx, ev.Fingerprint)
if err != nil {
return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err)
}
var seats []string
for _, s := range e.Serves {
if seat, _, _ := strings.Cut(s, "."); strings.HasPrefix(seat, "node-build-") && !slices.Contains(seats, seat) {
seats = append(seats, seat)
}
}
if len(seats) == 0 {
return nil, nil
}
return func(seat, _ string) bool { return slices.Contains(seats, seat) }, nil
}
// returnedWant is what a manifest-reads-changed says passed: only one with why returned, for the node it names.
func (r *Retaker) returnedWant(data []byte) (func(seat, node string) bool, error) {
var ev nodeengine.ManifestReadsChanged
if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" {
r.logf("a manifest-reads-changed that does not read is not acted on: %v", err)
return nil, nil
}
if ev.Why != nodeengine.ManifestReadsChangedWhyReturned {
return nil, nil
}
return func(_, node string) bool { return node == ev.Node }, nil
}
// retakeAll builds again every key failed on a transport fault whose seat and node want says passed.
func (r *Retaker) retakeAll(ctx context.Context, want func(seat, node string) bool) error {
keys, err := r.Builder.Claimer.Store.Keys(ctx)
if err != nil {
return fmt.Errorf("the build keys cannot be listed: %w", err)
}
for _, key := range keys {
raw, _, found, err := r.Builder.Claimer.Store.Get(ctx, key)
if err != nil || !found {
continue
}
v, err := builds.DecodeBuilderBuild(raw)
if err != nil || v.State != builds.BuildFailed {
continue
}
_, seat, node, _, ok := builds.TransportFaultOf(v.Error)
if !ok || !want(seat, node) {
continue
}
started, ask, cur, rev, err := r.Builder.Retake(ctx, key)
if err != nil {
return err
}
if started || !ask || r.Ask == nil {
continue
}
if err := r.askOnce(ctx, key, cur, rev); err != nil {
return err
}
}
return nil
}
// askOnce asks the operator about key once per revision of the key.
func (r *Retaker) askOnce(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error {
ak := askedKey(key)
raw, arev, found, err := r.Acted.Get(ctx, ak)
if err != nil {
return err
}
val := []byte(fmt.Sprint(rev))
if found && string(raw) == string(val) {
return nil
}
if err := r.Ask(ctx, key, cur, rev); err != nil {
return fmt.Errorf("the operator cannot be asked about %s: %w", key, err)
}
if found {
_, err = r.Acted.Update(ctx, ak, val, arev)
} else {
_, err = r.Acted.Create(ctx, ak, val)
}
if err != nil && !errors.Is(err, versions.ErrConflict) {
return err
}
return nil
}
modules/mesh-delivery-builder/internal/build/retake_test.go
+130
View File
@@ -0,0 +1,130 @@
package build
import (
"context"
"encoding/json"
"fmt"
"sync"
"testing"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
type actedStore struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
}
func (s *actedStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.vals[k]
return v, s.revs[k], ok, nil
}
func (s *actedStore) Create(_ context.Context, k string, v []byte) (uint64, error) {
return s.Update(context.Background(), k, v, 0)
}
func (s *actedStore) Update(_ context.Context, k string, v []byte, _ uint64) (uint64, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.vals == nil {
s.vals, s.revs = map[string][]byte{}, map[string]uint64{}
}
s.vals[k], s.revs[k] = v, s.revs[k]+1
return s.revs[k], nil
}
func (s *actedStore) KeysOf(context.Context, string) ([]string, error) { return nil, nil }
func (s *actedStore) Delete(_ context.Context, k string) error { delete(s.vals, k); return nil }
// Design 47 G §568, ADR 0673: a key failed on a transport fault is built again on the node-engine seat's events that say its
// cause passed, once per event, counted in event-builds; after three the next event asks the operator once.
func TestRetakeOnAnEvent(t *testing.T) {
r := newRig(t, goBundle)
// The first look creates the key and its note meets the fault; a retake's claim is an update that succeeds and the note after
// it is the second.
fault := fmt.Errorf("the key: %w", context.DeadlineExceeded)
failOnce := func() { r.st.before = func() { r.st.before = func() { r.st.faults = []error{fault} } } }
r.st.faults = []error{fault}
if v := r.consider(t, "modules/m"); !builds.FailedOnTransportFault(v.Error) {
t.Fatalf("the key = %+v, want failed on a transport fault", v)
}
var asked []string
rt := &Retaker{Builder: r.b, Acted: &actedStore{},
Entry: func(context.Context, string) (meshcatalog.CatalogEntry, error) {
return meshcatalog.CatalogEntry{Serves: []string{"node-build-go.steps"}}, nil
},
Ask: func(_ context.Context, key string, _ builds.BuilderBuild, _ uint64) error {
asked = append(asked, key)
return nil
}}
running, _ := json.Marshal(nodeengine.Running{Module: "mesh-build-go", Node: "g14", Fingerprint: "f1"})
started, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "ace", Why: "started"})
heard := func(seq uint64, subject string, data []byte) {
t.Helper()
if ok, err := rt.Heard(context.Background(), subject, data, seq); !ok || err != nil {
t.Fatalf("heard %s: %v, %v", subject, ok, err)
}
r.b.Wait()
}
runningSubject := "mesh.seat.node-engine.event.running.g14.mesh-build-go"
readsSubject := "mesh.seat.node-engine.event.manifest-reads-changed.ace"
// A `started` builds nothing, and the same message again builds nothing.
heard(1, readsSubject, started)
if v := valueOf(t, r.st, r.key); v.EventBuilds != 0 || v.State != builds.BuildFailed {
t.Fatalf("a started: the key = %+v", v)
}
for i, ev := range []struct {
seq uint64
subject string
data []byte
}{{2, runningSubject, running}, {3, runningSubject + ".2", running}, {4, runningSubject + ".3", running}} {
failOnce()
heard(ev.seq, ev.subject, ev.data)
if v := valueOf(t, r.st, r.key); v.EventBuilds != i+1 || !builds.FailedOnTransportFault(v.Error) {
t.Fatalf("event %d: the key = %+v, want event-builds %d failed on a transport fault", i+1, v, i+1)
}
}
heard(4, runningSubject+".3", running) // a sequence heard is not acted on again
if v := valueOf(t, r.st, r.key); v.EventBuilds != 3 || len(asked) != 0 {
t.Fatalf("a sequence heard again: the key = %+v, asked %v", v, asked)
}
heard(5, runningSubject, running)
heard(6, runningSubject, running)
if v := valueOf(t, r.st, r.key); v.EventBuilds != 3 || len(asked) != 1 || asked[0] != r.key {
t.Fatalf("the fourth event: the key = %+v, asked %v, want nothing built and the operator asked once", v, asked)
}
// A person's build starts the count again.
if res := r.b.BuildAgain(context.Background(), "m", fp1, "Build once more", "operator", 2e9); res.Refused != "" {
t.Fatalf("built again by hand: %+v", res)
}
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.EventBuilds != 0 {
t.Fatalf("after a person's build: the key = %+v, want event-builds 0", v)
}
}
// A manifest-reads-changed with why returned builds again the keys whose step ran on its node, and no other's.
func TestRetakeOnANodeReturned(t *testing.T) {
r := newRig(t, goBundle)
r.consider(t, "modules/m")
seed(t, r.st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.builder", Manifest: mdigest('a'),
Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"})
rt := &Retaker{Builder: r.b, Acted: &actedStore{}}
for seq, c := range []struct {
node, why string
built bool
}{{"g14", "returned", false}, {"ace", "started", false}, {"ace", "returned", true}} {
data, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: c.node, Why: c.why})
if _, err := rt.Heard(context.Background(), "mesh.seat.node-engine.event.manifest-reads-changed."+c.node, data, uint64(seq+1)); err != nil {
t.Fatal(err)
}
r.b.Wait()
if v := valueOf(t, r.st, r.key); (v.State == builds.BuildDone) != c.built {
t.Fatalf("%s %s: the key = %+v, want built %v", c.node, c.why, v, c.built)
}
}
}
modules/mesh-delivery-builder/internal/build/retakeask.go
+162
View File
@@ -0,0 +1,162 @@
package build
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-builder/internal/versions"
)
// **The operator is asked once a key's event builds are spent** (design 47 G §568, ADR 0673 §3): the seat operator-channel's ask,
// level approve, with the options *Build once more* and *Leave it*, `about` the key so a newer ask replaces it (ADR 0666 §12),
// naming the module, the fingerprint and the last error whole. *Build once more* builds one revision as a person's build
// (BuildAgain, which starts the count again); *Leave it*, a decline or the ask's expiry leaves the key failed.
// The ids of the ask's options.
const (
OptionBuildOnceMore = "build-once-more"
OptionLeaveIt = "leave-it"
)
// AskLasts is how long the ask stays open: the longest an ask that approves may (the seat's own bound).
const AskLasts = occ.ApproveLasts
// askBuildBound is how long the builder waits for the claim and the asks of the build *Build once more* starts.
const askBuildBound = 15 * time.Second
// OperatorAsk asks the operator about a key and carries out the answer.
type OperatorAsk struct {
// Client is the builder's asker on the seat operator-channel: its ask published under the builder's own name, its state
// `asked` (the asks it kept and the warrants it acted on), the router's record, the `decided` event.
Client *occ.Client
// Acted keeps which key an ask was made for, beside the revision of the key it was made at (Retaker.Acted).
Acted versions.ReserveStore
Builder *Builder
// Ctx is the builder's own context: the build a person's answer starts lives on it.
Ctx context.Context
Now func() time.Time
Logf func(string, ...any)
}
func (o *OperatorAsk) logf(f string, a ...any) {
if o.Logf != nil {
o.Logf(f, a...)
}
}
func (o *OperatorAsk) now() time.Time {
if o.Now != nil {
return o.Now()
}
return time.Now()
}
// AskID is the id of the ask about one revision of a key: the same revision is the same ask, a newer one a new ask.
func AskID(key string, rev uint64) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s@%d", key, rev)))
return "retake-" + hex.EncodeToString(sum[:16])
}
// askedOf is the acted key of the key an ask was made for.
func askedOf(id string) string { return versions.ActedKey("retake-ask:" + id) }
// BuildOnceMoreAct is what *Build once more* binds: the verb and the key's module and fingerprint.
func BuildOnceMoreAct(module, fingerprint string) occ.Act {
return occ.Act{"verb": "build-again", "module": module, "fingerprint": fingerprint}
}
// Compose is the ask about key as it stands, at revision rev.
func (o *OperatorAsk) Compose(key string, cur builds.BuilderBuild, rev uint64) (oc.Ask, error) {
module, fp, err := builds.ParseBuilderBuildKey(key)
if err != nil {
return oc.Ask{}, err
}
binds, err := occ.ActDigest(BuildOnceMoreAct(module, fp))
if err != nil {
return oc.Ask{}, err
}
headline := "Build " + module + " once more?"
if r := []rune(headline); len(r) > occ.HeadlineLength {
headline = string(r[:occ.HeadlineLength-1]) + "…"
}
last := []rune(cur.Error)
if len(last) > 120 {
last = append(last[:119], '…')
}
return oc.Ask{ID: AskID(key, rev), Headline: headline, Who: occ.Operator, About: key,
Explanation: fmt.Sprintf("The build of %s failed on a transport fault, and events built it again %d times without it passing.\nLast error: %s", module, cur.EventBuilds, string(last)),
Details: fmt.Sprintf("Key %s\nFingerprint %s\nLast error whole: %s", key, fp, cur.Error),
Expires: occ.FormatTime(o.now().Add(AskLasts)),
OnExpiry: "the key stays failed until builder_build_again or a new fingerprint",
Options: []oc.Option{
{ID: OptionBuildOnceMore, Label: "Build once more", Does: "the builder builds this fingerprint once more, and counts the build as a person's", Level: occ.Approve, Binds: binds},
{ID: OptionLeaveIt, Label: "Leave it", Does: "nothing: the key stays failed", Level: occ.Acknowledge},
}}, nil
}
// Ask asks the operator about key at revision rev. The same revision asked twice is one ask.
func (o *OperatorAsk) Ask(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error {
a, err := o.Compose(key, cur, rev)
if err != nil {
return err
}
k := askedOf(a.ID)
if _, _, found, err := o.Acted.Get(ctx, k); err != nil {
return err
} else if !found {
if _, err := o.Acted.Create(ctx, k, []byte(key)); err != nil && !errors.Is(err, versions.ErrConflict) {
return err
}
}
// An ask published and not marked asked (the write after it failed) is read again: it is one ask.
if e, err := o.Client.State.Get("ask." + a.ID); err != nil {
return err
} else if e != nil {
return nil
}
return o.Client.Ask(a)
}
// Listen hears the operator's answers: each warrant of this builder's asks is acted on once.
func (o *OperatorAsk) Listen() error {
return o.Client.Listen(o.act, func(h occ.Handled, w oc.Warrant, err error) {
if err != nil {
o.logf("the answer to the ask %s: %s: %v", w.Ask, h, err)
return
}
o.logf("the answer to the ask %s: %s", w.Ask, h)
})
}
// act carries out the option the operator chose: Build once more builds the key as a person's build, Leave it nothing.
func (o *OperatorAsk) act(opt oc.Option, w oc.Warrant) error {
if opt.ID != OptionBuildOnceMore {
return nil
}
raw, _, found, err := o.Acted.Get(o.Ctx, askedOf(w.Ask))
if err != nil {
return err
}
if !found {
return fmt.Errorf("the builder holds no key the ask %s was made for", w.Ask)
}
module, fp, err := builds.ParseBuilderBuildKey(string(raw))
if err != nil {
return err
}
o.logf("%s: the operator chose %s; built again as a person's build", raw, opt.Label)
go func() {
if res := o.Builder.BuildAgain(o.Ctx, module, fp, "the operator chose "+opt.Label, "operator-channel", askBuildBound); res.Refused != "" {
o.logf("%s: not built again: %s", raw, res.Refused)
}
}()
return nil
}
modules/mesh-delivery-builder/internal/build/retakeask_test.go
+113
View File
@@ -0,0 +1,113 @@
package build
import (
"context"
"encoding/json"
"testing"
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// askedState is the client's state `asked`.
type askedState struct{ vals map[string][]byte }
func (s *askedState) Get(key string) (*stdio.StateEntry, error) {
if v, ok := s.vals[key]; ok {
return &stdio.StateEntry{Key: key, Value: v, Revision: 1}, nil
}
return nil, nil
}
func (s *askedState) Create(key string, value any) (uint64, error) {
if _, ok := s.vals[key]; ok {
return 0, stdio.ErrChanged
}
return s.Update(key, value, 0)
}
func (s *askedState) Update(key string, value any, _ uint64) (uint64, error) {
if s.vals == nil {
s.vals = map[string][]byte{}
}
s.vals[key], _ = json.Marshal(value)
return 1, nil
}
// Design 47 G §568: after three event builds the operator is asked, through the seat operator-channel's ask (level approve,
// *Build once more* and *Leave it*, about the key), and *Build once more* builds the key as a person's build, *Leave it* does not.
func TestTheOperatorIsAskedAndAnswers(t *testing.T) {
for _, option := range []string{OptionLeaveIt, OptionBuildOnceMore} {
t.Run(option, func(t *testing.T) { theOperatorAnswers(t, option) })
}
}
func theOperatorAnswers(t *testing.T, option string) {
r := newRig(t, goBundle)
r.consider(t, "modules/m")
seed(t, r.st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.builder", Manifest: mdigest('a'), EventBuilds: 3,
Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"})
var published []oc.Ask
acted := &actedStore{}
asker := &OperatorAsk{Builder: r.b, Acted: acted, Ctx: context.Background(), Now: func() time.Time { return time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) },
Client: &occ.Client{Module: "mesh-delivery-builder", State: &askedState{},
Publish: func(subject string, body any, id string) error {
if subject != oc.AcceptSubjectAsk("mesh-delivery-builder") {
t.Errorf("published on %s", subject)
}
published = append(published, body.(oc.Ask))
return nil
}}}
rt := &Retaker{Builder: r.b, Acted: acted, Ask: asker.Ask,
Entry: nil}
data, _ := json.Marshal(map[string]string{"node": "ace", "why": "returned"})
for seq := uint64(1); seq <= 2; seq++ { // the same key and revision asked at two events is one ask
if _, err := rt.Heard(context.Background(), "mesh.seat.node-engine.event.manifest-reads-changed.ace", data, seq); err != nil {
t.Fatal(err)
}
r.b.Wait()
}
if len(published) != 1 {
t.Fatalf("asked %d times, want once", len(published))
}
a := published[0]
if a.About != r.key || len(a.Options) != 2 || a.Options[0].Level != occ.Approve || a.Options[0].Label != "Build once more" || a.Options[1].Label != "Leave it" {
t.Fatalf("the ask = %+v, want about the key with the options Build once more (approve) and Leave it", a)
}
if err := occ.Check(a, time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("the ask does not hold to the seat's checks: %v", err)
}
if err := occ.Glance(a.Explanation); err != nil {
t.Fatal(err)
}
warrant := func(option string) oc.Warrant {
return oc.Warrant{Ask: a.ID, AskDigest: occ.Digest(a), Asker: "mesh-delivery-builder", Outcome: occ.OutcomeChosen, Option: option, Level: levelOf(a, option),
At: occ.FormatTime(time.Date(2026, 11, 11, 12, 5, 0, 0, time.UTC)), By: &oc.Person{Who: occ.Operator, Kind: "telegram", Identity: "op", Verified: "user"}}
}
h, err := asker.Client.Take(warrant(option), asker.act)
if err != nil || h != occ.Acted {
t.Fatalf("%s: %v, %v", option, h, err)
}
for i := 0; i < 50 && option == OptionBuildOnceMore; i++ { // the build the answer starts runs on its own goroutine
r.b.Wait()
if valueOf(t, r.st, r.key).State == builds.BuildDone {
break
}
time.Sleep(time.Millisecond)
}
r.b.Wait()
v := valueOf(t, r.st, r.key)
if option == OptionLeaveIt && (v.State != builds.BuildFailed || v.EventBuilds != 3) {
t.Fatalf("after Leave it the key = %+v, want failed as it was", v)
}
if option == OptionBuildOnceMore && (v.State != builds.BuildDone || v.EventBuilds != 0) {
t.Fatalf("after Build once more the key = %+v, want built as a person's build (event-builds 0)", v)
}
}
func levelOf(a oc.Ask, option string) string {
o, _ := occ.OptionOf(a, option)
return o.Level
}
modules/mesh-delivery-builder/internal/build/run.go
+19 -6
View File
@@ -132,6 +132,8 @@ type Builder struct {
waiting map[string]map[string]chan string
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
hands map[string]*handWatch
// events are the keys an event asks the builder to build again (Retake), until their look claims them.
events map[string]bool
// released are the keys builder_release is building now, whatever toolchain their fingerprint names (§276, §672).
released map[string]bool
// builds are the ends that withdraw the asks of a key's build in flight, by build key (ADR 0664 §4): a newer fingerprint of
@@ -162,7 +164,7 @@ func (b *Builder) init() {
b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.hands = map[string]*handWatch{}
b.hands, b.events = map[string]*handWatch{}, map[string]bool{}
b.builds = map[string]context.CancelCauseFunc{}
})
}
@@ -493,6 +495,8 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
first := !b.tried[key]
b.tried[key] = true
byHand := b.hands[key] != nil
byEvent := b.events[key]
delete(b.events, key)
b.mu.Unlock()
var numbering versions.Numbering
if trunk && planErr == nil && !nothing {
@@ -507,7 +511,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
}
// A key built again by hand is taken over as it stands too: one left building by a look that ended without its write
// (issue 1966) is the hand tool's to finish.
cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
Stored: b.storedOnce(key)})
if err != nil {
b.logf("%s: %v", module, err)
@@ -576,7 +580,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
b.logf("%s: left building, the toolchains moved under it", cl.Key)
return
}
result.By, result.Manifest = b.Claimer.By, m.Manifest
result.By, result.Manifest, result.EventBuilds = b.Claimer.By, m.Manifest, cl.Build.EventBuilds
if len(cl.Gone) > 0 && result.State == builds.BuildDone {
result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")"
}
@@ -855,6 +859,12 @@ var keyPrefixes = []string{builds.NotPinned, builds.VersionTaken, builds.Refused
// (`<tool> on <node>: the step <key> failed: not stored: ...`), the key's error opens with that prefix, so a reader of the
// key need not search for it.
func keyError(errs []string) string {
// A transport fault is the key's own words only when every lane failed on one: beside a real failure it is said after it,
// so the key is not built again on an event for a failure the source made (design 47 G §568).
errs = slices.Clone(errs)
slices.SortStableFunc(errs, func(a, b string) int {
return boolInt(strings.HasPrefix(a, builds.TransportFaultPrefix)) - boolInt(strings.HasPrefix(b, builds.TransportFaultPrefix))
})
for _, prefix := range keyPrefixes {
for i, e := range errs {
if strings.HasPrefix(e, prefix) {
@@ -991,11 +1001,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil {
// The note and the put are one act (novox/hq ADR 0668 §1; issue 1748 note 27): a note a transport fault kept from
// the key fails the key with that fault's words, which builder_build_again takes (ADR 0664 §3).
fault := ""
l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err)
if busretry.TimedOut(err) || busretry.Undelivered(err) {
fault = sdkstep.TransportFaultWords + "the note of the ask: "
seat, _ := sdkstep.BuildSeat(language)
l.err = errors.New(buildrun.TransportFault(buildrun.FaultClass(err.Error()), seat, "no node", l.err.Error()))
}
l.err = fmt.Errorf("%sthe ask of %s could not be written to the key: %w", fault, l.module, err)
return
}
if err := b.Asks.Put(ctx, ask); err != nil {
@@ -1017,6 +1027,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
}
if step.State != sdkstep.StepPassed {
l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason)
if strings.HasPrefix(step.Reason, builds.TransportFaultPrefix) {
l.err = errors.New(step.Reason) // a key's error opens with it (design 47 G §568)
}
return
}
var answer sdkstep.BuildAnswer
modules/mesh-delivery-builder/internal/build/run_test.go
+30 -2
View File
@@ -270,12 +270,19 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At})
}
page.Next = 3
since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At)
since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, "")
if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != builds.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More {
t.Errorf("builds since = %+v, %v, want the one end and more", since, err)
}
// With module (design 47 G §589) the builds of that module alone.
if other, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, "another"); err != nil || len(other.Builds) != 0 {
t.Errorf("builds since of another module = %+v, %v, want none", other, err)
}
if mine, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, logged[0].Module); err != nil || len(mine.Builds) != 1 {
t.Errorf("builds since of the module = %+v, %v, want its one end", mine, err)
}
page.Entries[0].Value = []byte(`{"v":0}`)
if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil {
if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, ""); err == nil {
t.Error("an entry that does not read was skipped, want it refused")
}
if v.State != builds.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 {
@@ -1328,6 +1335,27 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
if maxFlight.Load() < 2 {
t.Errorf("the values were read one at a time")
}
// Design 47 G §484: a step key neither form parses is written in the journal naming it.
var said []string
steps := StateRead{Keys: func() ([]string, error) {
return []string{"ace.mvl0frv18d1957", "builder.ace.mvl0frv18d1958", "a.b.c.d"}, nil
}}
if _, err := (StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{"mesh-build-go": steps}, Now: func() time.Time { return at },
Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}).Stuck("mod10"); err != nil {
t.Fatal(err)
}
n := 0
for _, l := range said {
if strings.Contains(l, `"a.b.c.d"`) {
n++
}
if strings.Contains(l, "ace.mvl0frv18d1957") || strings.Contains(l, "builder.ace.mvl0frv18d1958") {
t.Errorf("a key of one of the two forms is said not to parse: %s", l)
}
}
if n != 1 {
t.Errorf("the key of neither form is named %d times in %q", n, said)
}
})
}
modules/mesh-delivery-builder/internal/build/since.go
+6 -2
View File
@@ -2,6 +2,7 @@ package build
import (
"fmt"
"slices"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
@@ -16,9 +17,9 @@ type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error)
// BuildsSince answers builds.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on,
// oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again
// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that
// from the last build's time. With module (design 47 G §589) only that module's builds are answered. An entry the log holds that does not read is refused, naming its sequence: a reader that
// skipped it would register too little.
func BuildsSince(read LogSince, since time.Time) (builds.BuildsSince, error) {
func BuildsSince(read LogSince, since time.Time, module string) (builds.BuildsSince, error) {
out := builds.BuildsSince{Since: since.UTC(), Builds: []builds.BuildLogEntry{}}
page, err := read(since, "", BuildsSinceLimit)
if err != nil {
@@ -38,6 +39,9 @@ func BuildsSince(read LogSince, since time.Time) (builds.BuildsSince, error) {
if ended := builds.EndedSince(entries, since); ended != nil {
out.Builds = ended
}
if module != "" {
out.Builds = slices.DeleteFunc(out.Builds, func(e builds.BuildLogEntry) bool { return e.Module != module })
}
out.More = page.Next != 0
return out, nil
}
modules/mesh-delivery-builder/internal/build/stuck.go
+45 -33
View File
@@ -57,35 +57,32 @@ type ReAsk struct {
Failed string `json:"failed,omitempty"`
Revision uint64 `json:"revision,omitempty"`
Asks map[string]string `json:"asks,omitempty"`
// Name is the numbered name (`<module>/<artifact>`) of a ModePublishAgain or ModeFloorAgain value; Do and Version the
// builder's answer: what it did (versions.Again*) and the version published.
Name string `json:"name,omitempty"`
Do string `json:"do,omitempty"`
Version string `json:"version,omitempty"`
// Name is the numbered name (`<module>/<artifact>`) of a ModeFloorAgain value; Version is a ModeRepublish value's
// request and Queued the builder's answer to it: the versions it queued, `<module> <version>`.
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
Queued []string `json:"queued,omitempty"`
}
// ModePublishAgain is the Mode of a reasks value of builder_publish_again (design 51 *Tools of the builder*).
const ModePublishAgain = "publish-again"
// ModeRepublish is the Mode of a reasks value of builder_publish_again (design 47 G §588): Module and Version, each
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
const ModeRepublish = "republish"
// ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*).
const ModeFloorAgain = "floor-again"
// NewPublishAgain is the value builder_publish_again writes for a name's fingerprint: the key of the name's module and
// fingerprint, and the value.
func NewPublishAgain(name, fp, why, by string, at time.Time) (string, []byte, error) {
module, artifact, ok := strings.Cut(strings.TrimSpace(name), "/")
if !ok || module == "" || artifact == "" {
return "", nil, fmt.Errorf("name %q is not a numbered name, <module>/<artifact>, as builder_versions lists it", name)
}
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
// NewRepublish is the value builder_publish_again writes (design 47 G §588): its key, `republish.<unix nanoseconds>`
// (a call is one value, never a build key), and the value. Module and version are each optional.
func NewRepublish(module, version, why, by string, at time.Time) (string, []byte, error) {
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
if version != "" && module == "" {
return "", nil, errors.New("a version is published again under its module: give module with version")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why it is published again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModePublishAgain, Module: module, FP: fp, Name: strings.TrimSpace(name)})
return key, data, err
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeRepublish, Module: module, Version: version})
return "republish." + strconv.FormatInt(at.UnixNano(), 10), data, err
}
// NewFloorAgain is the value builder_floor_again writes for a name: the key `floor.<name escaped>`, never a build key,
@@ -316,20 +313,9 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
if r.Now != nil {
now = r.Now
}
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
stepKeys := map[string]map[string]string{}
for bm, s := range r.Steps {
ks, err := s.Keys()
if err != nil {
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
}
byID := make(map[string]string, len(ks))
for _, k := range ks {
if _, _, id, err := buildstep.ParseStepKey(k); err == nil {
byID[id] = k
}
}
stepKeys[bm] = byID
stepKeys, err := r.stepIndex()
if err != nil {
return nil, err
}
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
// parallel, at most MaxReads of them.
@@ -426,6 +412,32 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
return answer, nil
}
// stepIndex finds the build modules' steps by the ask's id, which their key ends in: one index per build module, no value read
// until an ask names it.
func (r StuckReader) stepIndex() (map[string]map[string]string, error) {
stepKeys := map[string]map[string]string{}
for bm, s := range r.Steps {
ks, err := s.Keys()
if err != nil {
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
}
byID := make(map[string]string, len(ks))
for _, k := range ks {
_, _, id, err := buildstep.ParseStepKey(k)
if err != nil {
// A key neither form parses is named, never passed over in silence (design 47 G §484).
if r.Logf != nil {
r.Logf("builder_stuck: the key %q of the steps of %s is a step key of neither form, <asker>.<node>.<id> or <node>.<id>: %v", k, bm, err)
}
continue
}
byID[id] = k
}
stepKeys[bm] = byID
}
return stepKeys, nil
}
// StuckBuildModules are the build modules whose steps builder_stuck reads.
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}
modules/mesh-delivery-builder/internal/versions/again.go
+2 -92
View File
@@ -2,63 +2,13 @@ package versions
import (
"context"
"errors"
"fmt"
"strings"
"time"
)
// What builder_publish_again does with one fingerprint of a name (design 51 *Tools of the builder*), after a holder failed.
const (
// AgainPublish is a held build whose recorded version is published again: refused since design 47 G §349 (no version.numbered is said).
AgainPublish = "publish"
// AgainBuild builds the fingerprint again under the same number: the build's file is gone and no published was said
// for the number, so the new digest replaces the recorded one (Numberer.Built).
AgainBuild = "build"
// AgainRenumber takes a new number: a published was said for the number, so other bytes never go under it.
AgainRenumber = "renumber"
)
// Again is what builder_publish_again does, with the entry it acts on and why.
type Again struct {
Do string `json:"do"`
Entry Entry `json:"entry"`
Why string `json:"why"`
}
// PublishAgain decides builder_publish_again for fingerprint among a name's entries; built says whether the builder still
// holds the build's file (its done key). It refuses a fingerprint not numbered, and a version refused for other bytes
// than the ones recorded under it.
func PublishAgain(entries []Entry, fingerprint string, built bool) (Again, error) {
fingerprint = strings.TrimSpace(fingerprint)
if fingerprint == "" {
return Again{}, errors.New("fp is required: the fingerprint whose version is published again")
}
for _, e := range latestFirst(entries) {
if e.Tag != "" || e.Fingerprint != fingerprint {
continue
}
switch {
case e.State == Refused && e.RefusedDigest != "" && e.Digest != "" && e.RefusedDigest != e.Digest:
return Again{}, fmt.Errorf("%s@%s was refused for other bytes (%s, recorded %s): it is not published again; a new build takes a new number",
e.Name, e.Version, e.RefusedDigest, e.Digest)
case e.State == Published || e.State == Removed:
return Again{Do: AgainRenumber, Entry: e, Why: fmt.Sprintf("%s@%s was %s, so the fingerprint takes a new number", e.Name, e.Version, e.State)}, nil
case !built:
return Again{Do: AgainBuild, Entry: e, Why: fmt.Sprintf("the build's file is gone and no published was said for %s@%s: built again under the same number", e.Name, e.Version)}, nil
}
return Again{Do: AgainPublish, Entry: e, Why: fmt.Sprintf("%s@%s (%s) is held", e.Name, e.Version, e.State)}, nil
}
return Again{}, fmt.Errorf("fingerprint %s is not numbered for this name: builder_versions lists what is", fingerprint)
}
// The tools' descriptions.
const (
PublishAgainDescription = "Publish one fingerprint's version again after its holder failed (design 51): with the build's file held, the " +
"nothing says a version numbered since design 47 G §349, so a held build is refused; with the file gone, it builds the fingerprint " +
"again under the same number, the new digest replacing the recorded one, since no published was said for it; after a published (or " +
"a removal) it takes a new number. It refuses a fingerprint not numbered and a version the holder refused for other bytes. A hand " +
"act, recorded with its caller and why in the builder's log."
FloorAgainDescription = "Ask the floor of one name again now, by hand (design 51 *The floor*): the seat's versions, the forge bridge's " +
"bridge_tags and the indexer's indexer_requirements, as the builder asks them by itself when a silent source says something. " +
"Answers which sources were silent and whether the floor is now known. A name the builder's versions already hold needs no floor: " +
@@ -68,56 +18,16 @@ const (
// Acts are the builder's hand acts on its versions, answered by its program (the tools bundle writes them to reasks).
type Acts struct {
Numberer *Numberer
// Done says whether the build key of module and fingerprint is done: the build's file is held.
Done func(ctx context.Context, module, fp string) (bool, error)
// BuildAgain builds a failed key again (builder_build_again's path): its answer, or why it refused.
BuildAgain func(ctx context.Context, module, fp, why, by string) (answer string, refused bool)
// Reconsider looks at every module again, so a claim asks a name's floor.
Reconsider func(ctx context.Context)
// Republish queues the reserved versions builder_publish_again names (design 47 G §588): the versions it queued.
Republish func(ctx context.Context, module, version string) ([]string, error)
// DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not.
DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error)
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
}
// PublishAgain is builder_publish_again of name's fingerprint: what it did, the version, and its answer; an error refuses.
func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at time.Time) (Again, string, error) {
module, _, _ := strings.Cut(name, "/")
entries, err := a.Numberer.Versions.Of(ctx, name)
if err != nil {
return Again{}, "", err
}
done, err := a.Done(ctx, module, fp)
if err != nil {
return Again{}, "", fmt.Errorf("the build key of %s at %s cannot be read: %w", module, fp, err)
}
ag, err := PublishAgain(entries, fp, done)
if err != nil {
return Again{}, "", err
}
if ag.Do == AgainPublish || (ag.Do == AgainRenumber && done) {
return Again{}, "", errors.New("nothing says a version numbered since design 47 G §349: the version is published under its reservation (§369)")
}
e := ag.Entry
if ag.Do == AgainRenumber {
_, ne, err := a.Numberer.Versions.Number(ctx, Claim{Name: e.Name, Kind: e.Kind, Series: e.Series, Module: e.Module, Fingerprint: e.Fingerprint,
Commit: e.Commit, Revision: e.Revision, Seat: e.Seat, Merge: e.Merge, Fresh: true})
if err != nil {
return Again{}, "", err
}
ag.Entry = ne
}
answer := ag.Why
if ag.Do == AgainBuild || (ag.Do == AgainRenumber && !done) {
said, refused := a.BuildAgain(ctx, module, fp, why, by)
if refused {
return Again{}, "", fmt.Errorf("%s; and the build again was refused: %s", ag.Why, said)
}
answer += "; " + said
}
return ag, answer, nil
}
// FloorAgain is builder_floor_again of name: the floor counts only for a name the versions hold nothing of.
func (a Acts) FloorAgain(ctx context.Context, name string) (string, error) {
entries, err := a.Numberer.Versions.Of(ctx, name)
modules/mesh-delivery-builder/internal/versions/publish.go
+16
View File
@@ -37,6 +37,7 @@ const (
publishBoundPrefix = "publish."
oldFormBoundPrefix = "old-form."
unputBoundPrefix = "unput."
stepsBoundPrefix = "steps."
)
// UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966,
@@ -44,6 +45,21 @@ const (
// finding call-unanswered about the key.
func UnputBoundKey(buildKey string) string { return unputBoundPrefix + buildKey }
// StepsBoundKey is the bound of the builder's read of an ask's steps while the bus does not answer it (design 47 G §568, A-77):
// `steps.<build module>.<ask id>`. Its end, the bus's marker of its time to live, while the read is unanswered, is the finding
// call-unanswered about the ask.
func StepsBoundKey(module, askID string) string { return stepsBoundPrefix + module + "." + askID }
// StepsBoundOf is the build module and ask a steps bound names, and whether key is one.
func StepsBoundOf(key string) (module, askID string, ok bool) {
rest, ok := strings.CutPrefix(key, stepsBoundPrefix)
if !ok {
return "", "", false
}
module, askID, ok = strings.Cut(rest, ".")
return module, askID, ok && module != "" && askID != ""
}
// UnputBoundOf is the build key an unput bound names, and whether key is one.
func UnputBoundOf(key string) (string, bool) { return strings.CutPrefix(key, unputBoundPrefix) }
modules/mesh-delivery-builder/internal/versions/republish.go
+646
View File
@@ -0,0 +1,646 @@
package versions
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
// **A reserved version left unpublished is published again by the builder, on an event or by a person's tool** (design 47
// G §588, ADR 0674): on the node-engine seat's `running` of a module whose catalog graph entry claims the registry seat
// of the missing artifacts' kind, and on its `manifest-reads-changed` with `why` `returned` of the node such a holder
// runs on. Each message is acted on once, by its stream sequence (state `acted`); each holder has one queue of versions,
// kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds.
const (
// PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB.
PublishQueueState = "publish-queue"
PublishQueueBucket = "mesh-delivery-builder_" + PublishQueueState
PublishQueueMiB = 4
// ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB.
ActedState = "acted"
ActedBucket = "mesh-delivery-builder_" + ActedState
ActedMiB = 1
// PublishQueueTool is builder_publish_queue.
PublishQueueTool = "builder_publish_queue"
// DefaultRepublishMax is the setting republish-max-versions unless set (§633).
DefaultRepublishMax = 50
// QueueFull is what version-unreleased says of a version the queue's cap refused.
QueueFull = "queue full"
// NotStoredWords opens what a refused artifact is said as.
NotStoredWords = "not stored: "
// VerbPublish is the registry seats' verb a publish again sends.
VerbPublish = "publish"
)
// RegistrySeats are the four registry seats a version is published to.
var RegistrySeats = []string{SeatImages, SeatArtifacts, SeatGoPackages, SeatNPM}
// SeatOfArtifact is the registry seat of one artifact of the catalog graph entry: from its package's ecosystem when the
// entry names one (go, npm, image), else from its kind (SeatOf).
func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) {
if a.Package != nil {
switch a.Package.Ecosystem {
case "go":
return SeatGoPackages, true
case "npm":
return SeatNPM, true
case "image":
return SeatImages, true
}
}
seat, _, ok := SeatOf(a.Kind, a.Language)
return seat, ok
}
// QueuedPublish is the value of a publish-queue key `<holder module>.<module>.<version>` (each token escaped, §73): the
// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its
// publish has begun.
type QueuedPublish struct {
V int `json:"v"`
Holder string `json:"holder"`
Module string `json:"module"`
Version string `json:"version"`
Artifacts map[string]string `json:"artifacts"`
Queued time.Time `json:"queued"`
Begun bool `json:"begun,omitempty"`
}
// QueueKey is the key of holder's queued version of module.
func QueueKey(holder, module, version string) string {
return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version)
}
// Acted is the value of an acted key: the last stream sequence acted on for its subject.
type Acted struct {
Sequence uint64 `json:"sequence"`
}
// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap.
type QueueStore interface {
ReserveStore
Bytes(ctx context.Context) (uint64, error)
}
// Publish is one artifact's publish again: what the registry seat's publish takes, without parts.
type Publish struct {
Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string
}
// Republisher publishes reserved versions again (design 47 G §588).
type Republisher struct {
Reserved *Reserver
Queue QueueStore
Acted ReserveStore
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
// Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words.
Send func(ctx context.Context, p Publish) error
// Unreleased says version-unreleased about a version with the holder's words.
Unreleased func(ctx context.Context, v ReservedVersion, why string)
// Max is the setting republish-max-versions: the most versions one event queues.
Max int
Now func() time.Time
Logf func(string, ...any)
// Go starts the sending of a holder's queue; nil starts a goroutine.
Go func(func())
mu sync.Mutex
runs map[string]map[string][]string // node -> holder module -> the registry seats it claims
busy map[string]bool
}
func (r *Republisher) now() time.Time {
if r.Now != nil {
return r.Now().UTC()
}
return time.Now().UTC()
}
func (r *Republisher) logf(f string, a ...any) {
if r.Logf != nil {
r.Logf(f, a...)
}
}
// ActedKey is the acted key of a subject.
func ActedKey(subject string) string { return EscapeToken(subject) }
// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq: it acts
// on it only when its sequence is newer than the one kept for its subject, and keeps it after. A subject that is neither is
// not its; an error leaves the message not acted on, so it is read again.
func (r *Republisher) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) {
running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*"))
changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".")
if !running && !changed {
return false, nil
}
key := ActedKey(subject)
raw, rev, found, err := r.Acted.Get(ctx, key)
if err != nil {
return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err)
}
if found {
var a Acted
if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq {
return true, nil
}
}
if running {
err = r.running(ctx, data)
} else {
err = r.returned(ctx, data)
}
if err != nil {
return true, err
}
val, _ := json.Marshal(Acted{Sequence: seq})
if found {
_, err = r.Acted.Update(ctx, key, val, rev)
} else {
_, err = r.Acted.Create(ctx, key, val)
}
if err != nil && !errors.Is(err, ErrConflict) {
return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err)
}
return true, nil
}
// registrySeatsOf are the registry seats the module of the catalog entry fingerprint claims, by their publish verb.
func (r *Republisher) registrySeatsOf(ctx context.Context, fingerprint string) ([]string, error) {
e, err := r.Entry(ctx, fingerprint)
if err != nil {
return nil, err
}
var out []string
for _, seat := range RegistrySeats {
if slices.Contains(e.Serves, seat+"."+VerbPublish) {
out = append(out, seat)
}
}
return out, nil
}
// running takes a running in: a module stopped is forgotten; one running whose entry claims a registry seat is a holder on
// its node, and its versions are queued.
func (r *Republisher) running(ctx context.Context, data []byte) error {
var ev nodeengine.Running
if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" {
r.logf("a running that does not read is not acted on: %v", err)
return nil
}
if ev.Stopped || ev.Fingerprint == "" {
r.mu.Lock()
delete(r.runs[ev.Node], ev.Module)
r.mu.Unlock()
return nil
}
seats, err := r.registrySeatsOf(ctx, ev.Fingerprint)
if err != nil {
return fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err)
}
r.mu.Lock()
if r.runs == nil {
r.runs = map[string]map[string][]string{}
}
if r.runs[ev.Node] == nil {
r.runs[ev.Node] = map[string][]string{}
}
if len(seats) == 0 {
delete(r.runs[ev.Node], ev.Module)
} else {
r.runs[ev.Node][ev.Module] = seats
}
r.mu.Unlock()
if len(seats) == 0 {
return nil
}
_, err = r.queueEvent(ctx, map[string][]string{ev.Module: seats})
return err
}
// returned takes a manifest-reads-changed in: only one with why returned acts, for every holder known to run on its node.
func (r *Republisher) returned(ctx context.Context, data []byte) error {
var ev nodeengine.ManifestReadsChanged
if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" {
r.logf("a manifest-reads-changed that does not read is not acted on: %v", err)
return nil
}
if ev.Why != nodeengine.ManifestReadsChangedWhyReturned {
return nil
}
r.mu.Lock()
holders := map[string][]string{}
for m, seats := range r.runs[ev.Node] {
holders[m] = seats
}
r.mu.Unlock()
if len(holders) == 0 {
return nil
}
_, err := r.queueEvent(ctx, holders)
return err
}
// view is a reserved version with the artifacts of it a holder would publish.
type view struct {
v ReservedVersion
arts map[string]meshcatalog.Artifact // reference -> its catalog artifact
}
// queueEvent queues, for each holder (module -> the seats it claims), the unreleased versions whose missing artifacts
// that holder's seats take: newest keys first, at most Max versions in all (one key per module and MAJOR.MINOR is the
// reservation's own).
func (r *Republisher) queueEvent(ctx context.Context, holders map[string][]string) ([]string, error) {
over, err := r.Reserved.Overdue(ctx, nil)
if err != nil {
return nil, err
}
var vs []ReservedVersion
for _, o := range over {
if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok {
vs = append(vs, v)
}
}
return r.queue(ctx, vs, func(seat string) []string { return holdersOf(holders, seat) }, r.Max)
}
func holdersOf(holders map[string][]string, seat string) []string {
var out []string
for m, seats := range holders {
if slices.Contains(seats, seat) {
out = append(out, m)
}
}
sort.Strings(out)
return out
}
// queue queues vs newest first, at most max of them (0 is no limit), each artifact to the first holder holdersOf names for
// its seat. It answers the versions queued, `<module> <version>`.
func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf func(seat string) []string, max int) ([]string, error) {
sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) })
var queued []string
started := map[string]bool{}
for _, v := range vs {
if max > 0 && len(queued) >= max {
break
}
if v.Fingerprint == "" || v.BuiltAt.IsZero() {
continue
}
e, err := r.Entry(ctx, v.Fingerprint)
if err != nil {
return queued, fmt.Errorf("the catalog entry %s of %s %s cannot be read: %w", v.Fingerprint, v.Module, v.Version, err)
}
byHolder := map[string]map[string]string{}
for _, ref := range v.Missing() {
if _, no := v.Unstored[ref]; no {
continue
}
for _, a := range e.Artifacts {
seat, ok := SeatOfArtifact(a)
if a.Ref != ref || !ok {
continue
}
if hs := holdersOf(seat); len(hs) > 0 {
if byHolder[hs[0]] == nil {
byHolder[hs[0]] = map[string]string{}
}
byHolder[hs[0]][ref] = v.Artifacts[ref]
}
}
}
did := false
for holder, arts := range byHolder {
ok, err := r.enqueue(ctx, v, holder, arts)
if err != nil {
return queued, err
}
did = did || ok
started[holder] = true
}
if did {
queued = append(queued, v.Module+" "+v.Version)
}
}
for holder := range started {
r.drainLater(ctx, holder)
}
return queued, nil
}
// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the
// section's cap (said as queue full).
func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) {
key := QueueKey(holder, v.Module, v.Version)
if _, _, found, err := r.Queue.Get(ctx, key); err != nil {
return false, err
} else if found {
return false, nil
}
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.now()})
used, err := r.Queue.Bytes(ctx)
if err != nil {
return false, fmt.Errorf("the size of %s cannot be read: %w", PublishQueueBucket, err)
}
if used+uint64(len(raw)) > PublishQueueMiB<<20 {
if r.Unreleased != nil {
r.Unreleased(ctx, v, QueueFull)
}
return false, nil
}
if _, err := r.Queue.Create(ctx, key, raw); err != nil {
if errors.Is(err, ErrConflict) {
return false, nil
}
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
}
return true, nil
}
// Republish is builder_publish_again: with module and version, that reserved version's missing artifacts; with module
// alone, each reserved version of it whose artifacts are not all published; with neither, every version the finding
// version-unreleased names. It queues as the events do (no limit but the cap), for the holders known to run, and answers
// the versions it queued.
func (r *Republisher) Republish(ctx context.Context, module, version string) ([]string, error) {
var vs []ReservedVersion
if module == "" {
over, err := r.Reserved.Overdue(ctx, nil)
if err != nil {
return nil, err
}
for _, o := range over {
if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok {
vs = append(vs, v)
}
}
} else {
all, err := r.Reserved.All(ctx)
if err != nil {
return nil, err
}
for _, v := range all {
if v.Module == module && (version == "" || v.Version == version) && len(v.Missing()) > 0 {
vs = append(vs, v)
}
}
if len(vs) == 0 {
return nil, fmt.Errorf("%s holds no reserved version%s with an artifact unpublished", module, map[bool]string{true: " " + version, false: ""}[version != ""])
}
}
r.mu.Lock()
known := map[string][]string{}
for _, mods := range r.runs {
for m, seats := range mods {
known[m] = seats
}
}
r.mu.Unlock()
return r.queue(ctx, vs, func(seat string) []string { return holdersOf(known, seat) }, 0)
}
// Queued are the queued versions, per holder in the order they are sent, and the acted keys with their sequences:
// builder_publish_queue.
type Queued struct {
Queue []QueuedPublish `json:"queue"`
Acted map[string]uint64 `json:"acted"`
}
// AnswerQueue is builder_publish_queue over the two sections as the tools bundle reads them.
func AnswerQueue(queue, acted Reader) (Queued, error) {
out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}}
keys, err := queue.Keys()
if err != nil {
return out, fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
}
for _, k := range keys {
raw, ok, err := queue.Get(k)
if err != nil || !ok {
continue
}
var q QueuedPublish
if json.Unmarshal(raw, &q) == nil {
out.Queue = append(out.Queue, q)
}
}
sort.SliceStable(out.Queue, func(i, j int) bool {
if out.Queue[i].Holder != out.Queue[j].Holder {
return out.Queue[i].Holder < out.Queue[j].Holder
}
return out.Queue[i].Queued.Before(out.Queue[j].Queued)
})
akeys, err := acted.Keys()
if err != nil {
return out, fmt.Errorf("the state %s cannot be listed: %w", ActedState, err)
}
for _, k := range akeys {
raw, ok, err := acted.Get(k)
if err != nil || !ok {
continue
}
var a Acted
if json.Unmarshal(raw, &a) == nil {
out.Acted[k] = a.Sequence
}
}
return out, nil
}
// PublishQueueDescription is builder_publish_queue's description.
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588): per registry " +
"seat holder, in the order they are sent, each with its artifacts' digests, when it was queued and whether its publish has begun; and the " +
"keys of the state " + ActedState + ", the last stream sequence the builder acted on for each running and manifest-reads-changed subject. Read-only."
// PublishAgainDescription is builder_publish_again's description.
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
"version's missing artifacts; with module alone, each reserved version of that module whose artifacts are not all published; with neither, every " +
"version the finding version-unreleased names, newest first. It queues them as the builder does on a registry holder's running, and answers at once " +
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
"recorded in the hand-act log."
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588).
func (r *Republisher) Resume(ctx context.Context) error {
keys, err := r.Queue.KeysOf(ctx, AllFilter)
if err != nil {
return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
}
holders := map[string]bool{}
for _, k := range keys {
raw, _, found, err := r.Queue.Get(ctx, k)
if err != nil || !found {
continue
}
var q QueuedPublish
if json.Unmarshal(raw, &q) == nil {
holders[q.Holder] = true
}
}
for h := range holders {
r.drainLater(ctx, h)
}
return nil
}
func (r *Republisher) drainLater(ctx context.Context, holder string) {
r.mu.Lock()
if r.busy == nil {
r.busy = map[string]bool{}
}
if r.busy[holder] {
r.mu.Unlock()
return
}
r.busy[holder] = true
r.mu.Unlock()
run := func() {
defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }()
r.Drain(ctx, holder)
}
if r.Go != nil {
r.Go(run)
return
}
go run()
}
// Drain sends holder's queued versions one at a time, oldest first, each bounded by UnreleasedBound; a key is deleted when
// its outcome is said.
func (r *Republisher) Drain(ctx context.Context, holder string) {
for ctx.Err() == nil {
keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*")
if err != nil {
r.logf("the queue of %s cannot be listed: %v", holder, err)
return
}
var items []QueuedPublish
for _, k := range keys {
if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found {
var q QueuedPublish
if json.Unmarshal(raw, &q) == nil {
items = append(items, q)
}
}
}
if len(items) == 0 {
return
}
sort.SliceStable(items, func(i, j int) bool { return items[i].Queued.Before(items[j].Queued) })
r.send(ctx, items[0])
}
}
// send publishes one queued version's artifacts and says the outcome, then deletes its key.
func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
key := QueueKey(q.Holder, q.Module, q.Version)
defer func() {
if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil {
r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err)
}
}()
if !q.Begun {
q.Begun = true
raw, _ := json.Marshal(q)
if _, rev, found, err := r.Queue.Get(ctx, key); err == nil && found {
_, _ = r.Queue.Update(ctx, key, raw, rev)
}
}
v, found, err := r.Reserved.Of(ctx, q.Module, q.Version)
if err != nil || !found {
return // released meanwhile, or dropped
}
e, err := r.Entry(ctx, v.Fingerprint)
if err != nil {
r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err))
return
}
refs := make([]string, 0, len(q.Artifacts))
for ref := range q.Artifacts {
refs = append(refs, ref)
}
sort.Strings(refs)
for _, ref := range refs {
var art *meshcatalog.Artifact
for i := range e.Artifacts {
if e.Artifacts[i].Ref == ref {
art = &e.Artifacts[i]
}
}
seat, ok := "", false
if art != nil {
seat, ok = SeatOfArtifact(*art)
}
if !ok {
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
return
}
name := NameOf(q.Module, strings.TrimPrefix(ref, q.Module+":"))
if art.Package != nil && art.Package.Name != "" {
name = art.Package.Name
}
kind := art.Kind
bound, cancel := context.WithTimeout(ctx, UnreleasedBound) // a context's bound, no timer (ADR 0437 §8)
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
Commit: v.Commit, Transfer: "republish-" + q.Module + "-" + v.Version + "-" + strings.TrimPrefix(ref, q.Module+":")})
cancel()
if err != nil {
words := err.Error()
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
if ok, merr := r.Reserved.MarkUnstored(ctx, q.Module, q.Version, ref, words); merr != nil {
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
} else if ok {
r.say(ctx, v, ref+" "+NotStoredWords+words)
}
continue
}
r.say(ctx, v, ref+": "+words)
}
}
}
func (r *Republisher) say(ctx context.Context, v ReservedVersion, why string) {
r.logf("%s %s is not published again: %s", v.Module, v.Version, why)
if r.Unreleased != nil {
r.Unreleased(ctx, v, why)
}
}
// MarkUnstored records that a holder refused artifact of module's reserved version as not stored; it answers whether that
// was new (said once).
func (r *Reserver) MarkUnstored(ctx context.Context, module, version, artifact, words string) (bool, error) {
series, _, err := SplitVersion(version)
if err != nil {
return false, err
}
key := ReservedKey(module, series)
for {
cur, rev, found, err := r.get(ctx, key)
if err != nil || !found || cur.Version != version {
return false, err
}
if _, was := cur.Unstored[artifact]; was {
return false, nil
}
if cur.Unstored == nil {
cur.Unstored = map[string]string{}
}
cur.Unstored[artifact] = words
raw, _ := json.Marshal(cur)
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
continue
} else if err != nil {
return false, fmt.Errorf("%s cannot be written: %w", key, err)
}
return true, nil
}
}
modules/mesh-delivery-builder/internal/versions/republish_test.go
+220
View File
@@ -0,0 +1,220 @@
package versions
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
)
type queueMem struct {
*reserveMem
bytes uint64
}
func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil }
// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by
// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record.
type republishBed struct {
*Republisher
reserved *reserveMem
queue *queueMem
acted *reserveMem
now time.Time
sent []Publish
said []string
refuse string // the words Send fails with, empty for none
}
func newRepublishBed(t *testing.T, max int) *republishBed {
t.Helper()
b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, acted: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Acted: b.acted, Max: max, Now: func() time.Time { return b.now }, Go: func(f func()) { f() },
Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
if fp == "src2:holder" {
return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil
}
m := strings.TrimPrefix(fp, "src2:")
return meshcatalog.CatalogEntry{Artifacts: []meshcatalog.Artifact{
{Ref: m + ":prog", Kind: "bundle", Language: "go"},
{Ref: m + ":lib", Kind: "package", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/" + m + "/go"}}}}, nil
},
Send: func(_ context.Context, p Publish) error {
b.sent = append(b.sent, p)
if b.refuse != "" {
return errors.New(b.refuse)
}
return nil
},
Unreleased: func(_ context.Context, v ReservedVersion, why string) {
b.said = append(b.said, v.Module+" "+v.Version+": "+why)
},
Logf: t.Logf}
return b
}
// reserve reserves and builds a version of module m (its patch given by the reservation order), its build ended long ago.
func (b *republishBed) reserve(t *testing.T, module string) ReservedVersion {
t.Helper()
b.now = b.now.Add(time.Minute)
ctx := context.Background()
v, err := b.Reserved.Reserve(ctx, module, "0.1", []string{module + ":prog", module + ":lib"}, Source{Fingerprint: "src2:" + module})
if err != nil {
t.Fatal(err)
}
for ref, d := range map[string]string{module + ":prog": "sha256:aa", module + ":lib": "sha256:bb"} {
if _, err := b.Reserved.Built(ctx, module, v.Version, ref, d); err != nil {
t.Fatal(err)
}
}
b.now = b.now.Add(UnreleasedBound + time.Minute)
return v
}
func running(node, module, fp string, stopped bool) (string, []byte) {
data, _ := json.Marshal(nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Stopped: stopped, At: time.Unix(1, 0)})
return nodeengine.SubjectRunning(node, module), data
}
// TestRepublish: design 47 G §588 (A-73, A-76): a reserved version left unpublished is published again, on a registry
// holder's running and on its node's returned, once per message by its stream sequence, as a publish by digest with no parts,
// at most the setting's number per event, newest first; an artifact no store holds is said once and not tried again.
func TestRepublish(t *testing.T) {
ctx := context.Background()
t.Run("running of a holder, once per sequence", func(t *testing.T) {
b := newRepublishBed(t, 50)
v := b.reserve(t, "m")
if v.Fingerprint != "src2:m" {
t.Fatalf("the reservation keeps the entry's fingerprint: %q", v.Fingerprint)
}
subject, data := running("n1", "oci", "src2:holder", false)
for _, seq := range []uint64{5, 4} { // 4 is older than the acted 5: it starts nothing
if _, err := b.Heard(ctx, subject, data, seq); err != nil || len(b.sent) != 1 {
t.Fatalf("seq %d: sent %d (%v), want 1", seq, len(b.sent), err)
}
}
p := b.sent[0]
if p.Seat != SeatArtifacts || p.Name != "m/prog" || p.Kind != "bundle" || p.Version != v.Version || p.Digest != "sha256:aa" || p.Fingerprint != "src2:m" {
t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p)
}
if _, err := b.Heard(ctx, subject, data, 5); err != nil || len(b.sent) != 1 {
t.Errorf("the same sequence read again sent %d, want still 1", len(b.sent))
}
if _, err := b.Heard(ctx, subject, data, 6); err != nil || len(b.sent) != 2 {
t.Errorf("a newer sequence sent %d, want 2", len(b.sent))
}
if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 {
t.Errorf("a key stays in the queue after its outcome: %v", keys)
}
})
t.Run("returned acts, started does not", func(t *testing.T) {
b := newRepublishBed(t, 50)
b.reserve(t, "m")
subject, data := running("n1", "oci", "src2:holder", true) // stopped: not a holder yet
_, _ = b.Heard(ctx, subject, data, 1)
subject, data = running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 2)
b.sent = nil
for i, why := range []string{nodeengine.ManifestReadsChangedWhyStarted, "", nodeengine.ManifestReadsChangedWhyReturned} {
ev, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "n1", Why: why})
handled, err := b.Heard(ctx, nodeengine.SubjectManifestReadsChanged("n1"), ev, uint64(10+i))
if !handled || err != nil {
t.Fatalf("%q: handled %v, %v", why, handled, err)
}
if want := map[string]int{"started": 0, "": 0, "returned": 1}[why]; len(b.sent) != want {
t.Errorf("why %q: sent %d, want %d", why, len(b.sent), want)
}
}
if handled, _ := b.Heard(ctx, "mesh.seat.node-engine.event.composed.n1", nil, 1); handled {
t.Error("a subject that is neither is taken as its own")
}
})
t.Run("the setting limits an event, newest first", func(t *testing.T) {
b := newRepublishBed(t, 2)
var versions []string
for _, m := range []string{"m1", "m2", "m3"} {
versions = append(versions, b.reserve(t, m).Module)
}
subject, data := running("n1", "oci", "src2:holder", false)
if _, err := b.Heard(ctx, subject, data, 1); err != nil {
t.Fatal(err)
}
if len(b.sent) != 2 || !strings.HasPrefix(b.sent[0].Name, "m3/") || !strings.HasPrefix(b.sent[1].Name, "m2/") {
t.Errorf("sent %+v, want the two newest, m3 then m2", b.sent)
}
})
t.Run("an artifact no store holds is said once and not tried again", func(t *testing.T) {
b := newRepublishBed(t, 50)
b.reserve(t, "m")
b.refuse = "oci-artifact-registry's publish of m/prog: not stored: no blob sha256:aa"
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1)
_, _ = b.Heard(ctx, subject, data, 2)
if len(b.sent) != 1 || len(b.said) != 1 || !strings.Contains(b.said[0], "not stored: ") {
t.Errorf("sent %d, said %v: want one try and one saying of not stored", len(b.sent), b.said)
}
})
t.Run("a refusal in other words is said each time", func(t *testing.T) {
b := newRepublishBed(t, 50)
b.reserve(t, "m")
b.refuse = "the holder is full"
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1)
_, _ = b.Heard(ctx, subject, data, 2)
if len(b.sent) != 2 || len(b.said) != 2 {
t.Errorf("sent %d, said %v: want both events tried and said", len(b.sent), b.said)
}
})
t.Run("a queue past its cap says queue full", func(t *testing.T) {
b := newRepublishBed(t, 50)
b.reserve(t, "m")
b.queue.bytes = PublishQueueMiB << 20
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1)
if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) {
t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said)
}
})
t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) {
b := newRepublishBed(t, 50)
v := b.reserve(t, "m")
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true})
if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil {
t.Fatal(err)
}
if err := b.Resume(ctx); err != nil || len(b.sent) != 1 {
t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err)
}
})
t.Run("the tool queues for a holder known to run", func(t *testing.T) {
b := newRepublishBed(t, 1)
b.reserve(t, "m1")
b.reserve(t, "m2")
if q, err := b.Republish(ctx, "", ""); err != nil || len(q) != 0 {
t.Errorf("no holder known: queued %v (%v), want none", q, err)
}
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1) // Max 1: only m2 sent
b.sent = nil
q, err := b.Republish(ctx, "", "")
if err != nil || fmt.Sprint(q) != "[m2 0.1.0 m1 0.1.0]" || len(b.sent) != 2 {
t.Errorf("the tool with neither queued %v (%v), sent %d: want both versions, with no limit", q, err, len(b.sent))
}
if _, err := b.Republish(ctx, "m1", "0.1.9"); err == nil {
t.Error("a version no reservation holds is not refused")
}
answer, err := AnswerQueue(Reader{Keys: func() ([]string, error) { return nil, nil }}, Reader{Keys: func() ([]string, error) { return b.acted.KeysOf(ctx, AllFilter) },
Get: func(k string) (json.RawMessage, bool, error) { v, _, ok, err := b.acted.Get(ctx, k); return v, ok, err }})
if err != nil || answer.Acted[ActedKey(subject)] != 1 {
t.Errorf("builder_publish_queue acted = %v (%v), want the running's sequence 1", answer.Acted, err)
}
})
}
modules/mesh-delivery-builder/internal/versions/reserve.go
+10 -2
View File
@@ -54,13 +54,21 @@ type ReservedVersion struct {
Repository string `json:"repository,omitempty"`
Commit string `json:"commit,omitempty"`
Tag string `json:"tag,omitempty"`
// Fingerprint is the fingerprint of the catalog entry the reservation was made from, so a publish again reads that
// entry (design 47 G A-76).
Fingerprint string `json:"fingerprint,omitempty"`
// Unstored are the artifacts a holder refused as not stored, with its words: said once, and not published again on
// events (design 47 G §588); builder_build_again builds the key again, which reserves anew.
Unstored map[string]string `json:"unstored,omitempty"`
}
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
// written at, and whether the module has a Go package, whose version is tagged.
type Source struct {
Repository, Commit string
GoPackage bool
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
Fingerprint string
GoPackage bool
}
// ReservedKey is the key of module's reservation of series (MAJOR.MINOR).
@@ -135,7 +143,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
}
}
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(),
Repository: src.Repository, Commit: src.Commit}
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint}
if src.GoPackage {
out.Tag = GoTagOf(out.Version)
}
modules/mesh-delivery-builder/internal/versions/versions_test.go
+9 -58
View File
@@ -73,6 +73,13 @@ func (m *mem) KeysOf(_ context.Context, filter string) ([]string, error) {
out = append(out, k)
continue
}
if strings.Count(filter, ".") == 2 { // a three-token filter, `a.*.*` as the bus reads it
ft, kt := strings.Split(filter, "."), strings.Split(k, ".")
if len(kt) == 3 && kt[0] == ft[0] {
out = append(out, k)
}
continue
}
if strings.HasPrefix(k, prefix) && !strings.Contains(k[len(prefix):], ".") {
out = append(out, k)
}
@@ -540,39 +547,7 @@ func TestTheBuildersVersions(t *testing.T) {
t.Error("80 % is not full")
}
})
t.Run("publish again", func(t *testing.T) {
num := Entry{Name: "m/p", Kind: KindBundle, Version: "0.1.3", Fingerprint: "f1", Digest: "sha256:aa", State: Numbered}
ref := num
ref.State, ref.RefusedDigest = Refused, "sha256:aa"
other := ref
other.RefusedDigest = "sha256:bb"
pub := num
pub.State = Published
gone := pub
gone.State = Removed
tagged := num
tagged.Tag, tagged.Version = "feat-x", "0.1.3-feat-x.1"
for _, c := range []struct {
name string
entries []Entry
fp string
built bool
do string
}{
{"a held build is said numbered again", []Entry{num}, "f1", true, AgainPublish},
{"refused for its own bytes is published again", []Entry{ref}, "f1", true, AgainPublish},
{"a gone file is built again under the same number", []Entry{ref}, "f1", false, AgainBuild},
{"after a published it takes a new number", []Entry{pub}, "f1", true, AgainRenumber},
{"after a removal it takes a new number", []Entry{gone}, "f1", false, AgainRenumber},
{"refused for other bytes is refused", []Entry{other}, "f1", true, ""},
{"a fingerprint not numbered is refused", []Entry{num}, "f2", true, ""},
{"a tagged version is not a trunk fingerprint's", []Entry{tagged}, "f1", true, ""},
} {
a, err := PublishAgain(c.entries, c.fp, c.built)
if (err != nil) != (c.do == "") || a.Do != c.do {
t.Errorf("%s: PublishAgain = %+v, %v; want %q", c.name, a, err, c.do)
}
}
t.Run("fresh claims and the floor", func(t *testing.T) {
ctx := context.Background()
// A fresh claim of a published fingerprint takes the next number; the old one stays.
st := newMem()
@@ -603,32 +578,8 @@ func TestTheBuildersVersions(t *testing.T) {
if all, err := v.All(ctx); err != nil || len(all) != 2 {
t.Errorf("All = %d entries, %v", len(all), err)
}
// The hand acts: a held build is said numbered once more under a fresh dedup id; a gone one is built again; a
// published one takes a new number; a name with versions asks no floor.
if _, _, err := (Acts{Numberer: &Numberer{Versions: v}, Done: func(context.Context, string, string) (bool, error) { return true, nil }}).PublishAgain(ctx, "m/p", "f1", "w", "", time.Unix(4, 0)); err == nil {
t.Error("a version refused for other bytes is published again")
}
_, _ = v.Mark(ctx, "m/p", "0.1.4", func(e *Entry) { e.State = Published })
var rebuilt []string
done := true
acts := Acts{Numberer: &Numberer{Versions: v, Waits: &FloorWaits{}},
Done: func(context.Context, string, string) (bool, error) { return done, nil },
BuildAgain: func(_ context.Context, m, fp, _, _ string) (string, bool) {
rebuilt = append(rebuilt, m+" "+fp)
return "built again", false
},
Reconsider: func(context.Context) {}}
// Nothing says a version numbered (design 47 G §349): a held build is refused before any number is taken.
if _, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(5, 0)); err == nil || len(rebuilt) != 0 {
t.Errorf("publish again of a published, held build answered %v, rebuilt %v; want a refusal", err, rebuilt)
}
if es, _ := v.Of(ctx, "m/p"); len(es) != 2 {
t.Errorf("the refused publish again took a number: %+v", es)
}
done = false
if ag, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(6, 0)); err != nil || ag.Do != AgainRenumber || ag.Entry.Version != "0.1.5" || len(rebuilt) != 1 {
t.Errorf("publish again of a gone build: %+v %v rebuilt %v", ag, err, rebuilt)
}
acts := Acts{Numberer: &Numberer{Versions: v, Waits: &FloorWaits{}}, Reconsider: func(context.Context) {}}
if _, err := acts.FloorAgain(ctx, "m/p"); err == nil {
t.Error("a name with versions asks its floor again")
}
modules/mesh-delivery-builder/manifest_test.go
+29 -13
View File
@@ -3,6 +3,7 @@ package manifest
import (
"bytes"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
@@ -23,6 +24,7 @@ type state struct {
Name string `json:"name"`
History int `json:"history,omitempty"`
KeyTTL bool `json:"key-ttl,omitempty"`
MaxMiB int `json:"max-mib,omitempty"`
}
type artifact struct {
@@ -111,7 +113,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -121,8 +123,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
"mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
t.Errorf("uses = %v, want the three build seats", m.Uses)
if want := []string{"node-build-go", "node-build-image", "node-build-node", "operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) {
t.Errorf("uses = %v, want the three build seats and operator-channel (design 47 G §568: the ask after three event builds)", m.Uses)
}
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
// other tool; it holds no capacity.
@@ -140,10 +142,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
// Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements.
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests",
"mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
"mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
"seat:git.tag",
"seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove",
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no
@@ -159,7 +161,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
"mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules",
"mesh-delivery-indexer.views.packages", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running",
"npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed",
"oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
"oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -169,8 +171,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
@@ -180,8 +182,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days) and the findings' stream", m.Streams)
}
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 1 {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60", m.Config)
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 2 || m.Config["republish-max-versions"].Default != float64(50) {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60 and republish-max-versions of 50 (design 47 G §588, §633)", m.Config)
}
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
@@ -221,8 +223,22 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
t.Errorf("process = %v", p)
}
env, _ := p["env"].(map[string]any)
if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES"] != "${config:toolchain-wait-minutes}" {
t.Errorf("env = %v, want the broker file and the toolchain wait's setting (design 47 G §672)", p["env"])
if len(env) != 3 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_REPUBLISH_MAX_FILE"] != "${dir:state}/republish-max-versions" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
}
for k, v := range env {
if strings.Contains(fmt.Sprint(v), "${config:") {
t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v)
}
}
found := false
for _, r := range m.Resources {
if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" {
found = true
}
}
if !found {
t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71")
}
for _, r := range m.Resources {
if r["id"] == "settings" {
modules/mesh-delivery-builder/module.json
+44 -4
View File
@@ -15,6 +15,7 @@
"builder_announce_again",
"builder_ask_again",
"builder_bounds",
"builder_build",
"builder_build_again",
"builder_builds_since",
"builder_engine_reads",
@@ -22,6 +23,7 @@
"builder_floor_waits",
"builder_old_form",
"builder_publish_again",
"builder_publish_queue",
"builder_reserved",
"builder_release",
"builder_reserved_drop",
@@ -59,6 +61,7 @@
"node-build-node.toolchain-reported",
"node-engine.manifest-reads-changed",
"node-engine.running",
"operator-channel.decided",
"npm-package-registry.kept",
"npm-package-registry.published",
"npm-package-registry.refused",
@@ -82,8 +85,12 @@
"seat:go-package-registry.remove",
"seat:go-package-registry.versions",
"seat:mesh-artifact-sweep.held",
"seat:mesh-catalog.entry",
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.revision",
"seat:mesh-catalog.versions",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-module-assigner.assignments",
"seat:mesh-watchdogs.presence",
"seat:node-build-go.drop",
"seat:node-build-image.drop",
@@ -97,7 +104,11 @@
"mesh-delivery-indexer.indexer_read_candidates",
"mesh-delivery-indexer.indexer_read_manifests",
"mesh-delivery-indexer.indexer_read_modules",
"mesh-delivery-indexer.indexer_read_packages"
"mesh-delivery-indexer.indexer_read_packages",
"seat:container-image-registry.publish",
"seat:go-package-registry.publish",
"seat:npm-package-registry.publish",
"seat:oci-artifact-registry.publish"
],
"state": [
{ "name": "builds", "history": 5 },
@@ -110,8 +121,11 @@
{ "name": "bounds", "history": 1, "key-ttl": true },
{ "name": "queues", "history": 1 },
{ "name": "toolchains-heard", "history": 1 },
{ "name": "waits", "history": 1 },
{ "name": "waits", "history": 1, "max-mib": 4 },
{ "name": "reserved", "history": 1 },
{ "name": "publish-queue", "history": 1, "max-mib": 4 },
{ "name": "acted", "history": 1, "max-mib": 1 },
{ "name": "asked", "history": 1 },
{ "name": "reasks", "history": 1 },
{ "name": "versions", "history": 1 },
{ "name": "first-views", "history": 1 }
@@ -129,6 +143,14 @@
"maximum": 10080,
"trusted": false,
"default": 60
},
"republish-max-versions": {
"type": "number",
"description": "the most versions one event of a registry holder queues to publish again, newest first (design 47 G section 588); the rest wait for the next event or builder_publish_again.",
"minimum": 1,
"maximum": 10000,
"trusted": false,
"default": 50
}
},
"logs": [
@@ -144,7 +166,8 @@
"uses": [
"node-build-go",
"node-build-image",
"node-build-node"
"node-build-node",
"operator-channel"
],
"own-secrets": {
"broker": "${dir:state}/broker"
@@ -166,6 +189,22 @@
"place": ".",
"unknown": "program"
},
{
"id": "toolchain-wait",
"type": "file",
"path": "${dir:state}/toolchain-wait-minutes",
"owner": "mesh-delivery-builder",
"mode": "0600",
"content": "${config:toolchain-wait-minutes}"
},
{
"id": "republish-max",
"type": "file",
"path": "${dir:state}/republish-max-versions",
"owner": "mesh-delivery-builder",
"mode": "0600",
"content": "${config:republish-max-versions}"
},
{
"id": "builder",
"type": "process",
@@ -176,7 +215,8 @@
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES": "${config:toolchain-wait-minutes}"
"MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes",
"MESH_BUILDER_REPUBLISH_MAX_FILE": "${dir:state}/republish-max-versions"
},
"user": "mesh-delivery-builder",
"health": {