diff --git a/modules/mesh-delivery-builder/buildrun/asks.go b/modules/mesh-delivery-builder/buildrun/asks.go index d9a0c35b..85efdecf 100644 --- a/modules/mesh-delivery-builder/buildrun/asks.go +++ b/modules/mesh-delivery-builder/buildrun/asks.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "log" - "strings" "sync" "time" @@ -102,17 +101,18 @@ func (k Asker) dropped(a buildstep.StepAsk, why string) { // PutTry is how long one publish of an ask waits for its stream's answer before it is made again (Put). var PutTry = 10 * time.Second -// Put puts a on its stream: the seat's accept subject of its language and kind. The put is a kept call (novox/hq ADR 0668 +// Put puts a on its stream: the seat's accept subject of its language and kind, under the builder's name as asker (design 47 G +// §481, §600: `accept..builder`). The put is a kept call (novox/hq ADR 0668 // §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it // is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by -// PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it +// PutTry, until ctx ends. A put that still meets one is an error opening `transport fault:` (TransportFault), so the key it // fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does. func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { raw, err := buildstep.EncodeStepAsk(a) if err != nil { return err } - subject, err := buildstep.AskSubject(a.Language, a.Kind) + subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, buildstep.AskerBuilder) if err != nil { return err } @@ -132,7 +132,8 @@ func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { case err == nil: return nil case busretry.TimedOut(err) || busretry.Undelivered(err): - return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err) + seat, _ := buildstep.BuildSeat(a.Language) + return errors.New(TransportFault(FaultClass(err.Error()), seat, "no node", fmt.Sprintf("the put of the %s ask %s: %v", a.Kind, a.ID, err))) } return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err) } @@ -211,7 +212,8 @@ func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, e if step.Passing && step.State == buildstep.StepFailed && (len(agains) > 0 || step.Run > 1) { // The step's own count of runs (ADR 0664 §2), not only the ones this wait saw: a builder that started again // reads the key's last value and none of the runs before it. - step.Reason = fmt.Sprintf(buildstep.TransportFaultWords+"%d runs (%s): %s", max(step.Run, len(agains)+1), strings.Join(append(agains, step.Node), ", "), step.Reason) + seat, _ := buildstep.BuildSeat(a.Language) + step.Reason = TransportFault(FaultClass(step.Reason), seat, step.Node, step.Reason) } return step, nil } @@ -297,6 +299,14 @@ func (k Asker) Passed(step buildstep.Step, err error) (buildstep.Step, error) { // of `*.` in the build module's steps bucket. type JetStreamAsks struct { JS jetstream.JetStream + // Unanswered is told each time the read of an ask's steps meets a transport fault it keeps waiting out (design 47 G §568, ADR + // 0668 §3): the builder arms the call's bound, a key with its own time to live (A-77), and raises call-unanswered when it + // ends unanswered. Answered is told when the read is answered again. Both may be nil. + Unanswered func(module, askID string, fault error) + Answered func(module, askID string) + // OpenWatch opens the watch of keys filtered by pattern in the build module's steps state, from the revision after `after` + // when after is not 0; nil is the bus's. + OpenWatch func(ctx context.Context, module, pattern string, after uint64) (jetstream.KeyWatcher, error) } // PublishAsk is AskPublisher's. @@ -307,46 +317,100 @@ func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data return err } -// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at -// once, and every change after. -func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) { - pattern, err := buildstep.AskStepWatch(askID) - if err != nil { - return nil, err +// openWatch opens the watch of pattern in the steps state of module, from the revision after `after` when it is not 0. +func (s JetStreamAsks) openWatch(ctx context.Context, module, pattern string, after uint64) (jetstream.KeyWatcher, error) { + if s.OpenWatch != nil { + return s.OpenWatch(ctx, module, pattern, after) } kv, err := s.JS.KeyValue(ctx, StepsBucket(module)) if err != nil { return nil, fmt.Errorf("the bucket %s: %w", StepsBucket(module), err) } - w, err := kv.Watch(ctx, pattern) + if after > 0 { + return kv.Watch(ctx, pattern, jetstream.ResumeFromRevision(after)) + } + return kv.Watch(ctx, pattern) +} + +// transportFault says err is a transport fault of ADR 0664 §1's classes or the read's own deadline: nothing was answered. +func transportFault(err error) bool { + return busretry.Undelivered(err) || busretry.TimedOut(err) +} + +// openKept is a kept call: the watch is opened, and opened again while the bus answers nothing, until it is or ctx ends +// (design 47 G §568: a transport fault of the builder's own read ends no build key). Each fault past the call's own asking again +// is told to Unanswered; the answer, to Answered. An error that is no transport fault is the caller's. +func (s JetStreamAsks) openKept(ctx context.Context, module, askID, pattern string, after uint64) (jetstream.KeyWatcher, error) { + for { + w, err := busretry.DoRepeatable("opening the watch of the ask "+askID+"'s step on "+module, func() (jetstream.KeyWatcher, error) { + return s.openWatch(ctx, module, pattern, after) + }, nil) + if err == nil { + if s.Answered != nil { + s.Answered(module, askID) + } + return w, nil + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if !transportFault(err) { + return nil, err + } + if s.Unanswered != nil { + s.Unanswered(module, askID, err) + } + } +} + +// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at +// once, and every change after. A watch that cannot be opened for a transport fault is opened again once the bus answers, and one +// that closes is taken up again from the revision after the last it handed over (design 47 G §568), so a fault fails no key. +func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) { + pattern, err := buildstep.AskStepWatch(askID) + if err != nil { + return nil, err + } + w, err := s.openKept(ctx, module, askID, pattern, 0) if err != nil { return nil, err } out := make(chan buildstep.Step) go func() { defer close(out) - defer w.Stop() + var last uint64 for { - select { - case <-ctx.Done(): - return - case e, ok := <-w.Updates(): - if !ok { - return - } - if e == nil || e.Operation() != jetstream.KeyValuePut { - continue - } - step, err := buildstep.DecodeStep(e.Value()) - if err != nil { - continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's) - } + closed := false + for !closed { select { - case out <- step: case <-ctx.Done(): + _ = w.Stop() return + case e, ok := <-w.Updates(): + if !ok { + closed = true + break + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + last = max(last, e.Revision()) + step, err := buildstep.DecodeStep(e.Value()) + if err != nil { + continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's) + } + select { + case out <- step: + case <-ctx.Done(): + _ = w.Stop() + return + } } } + _ = w.Stop() + if w, err = s.openKept(ctx, module, askID, pattern, last+1); err != nil { + return + } } }() return out, nil diff --git a/modules/mesh-delivery-builder/buildrun/asks_test.go b/modules/mesh-delivery-builder/buildrun/asks_test.go index a460f924..e31fd97e 100644 --- a/modules/mesh-delivery-builder/buildrun/asks_test.go +++ b/modules/mesh-delivery-builder/buildrun/asks_test.go @@ -30,7 +30,8 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { f := &FakeAsks{Answer: func(a buildstep.StepAsk) buildstep.Step { return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, Result: []byte(`{"artifacts":[]}`)} }} - k := Asker{Publisher: f, Waiter: f} + subjects := &subjectsOf{inner: f} + k := Asker{Publisher: subjects, Waiter: f} var seen, also []buildstep.Step // issue 1890: a waiter that asks to see each value of the step sees it // Issue 1921: a second seen on the same context (a check's begin bound inside a try) is added, never in place of the first. both := WithStepSeen(WithStepSeen(context.Background(), func(s buildstep.Step) { seen = append(seen, s) }), @@ -39,6 +40,10 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { if err != nil || step.Node != "ace" || len(f.Asks()) != 1 || len(seen) != 1 || seen[0].Node != "ace" || len(also) != 1 { t.Fatalf("step %+v, %v, asks %d, seen %+v and %+v", step, err, len(f.Asks()), seen, also) } + // Design 47 G §481, §600: the ask is put under the builder's name, accept..builder. + if len(subjects.seen) != 1 || subjects.seen[0] != "mesh.seat.node-build-go.accept."+goAsk("x").Kind+".builder" { + t.Fatalf("the ask was put on %v", subjects.seen) + } f.Answer = func(a buildstep.StepAsk) buildstep.Step { return buildstep.Step{ID: a.ID, Node: "shanks", State: buildstep.StepRefused, Reason: "go 1.27 installed"} } @@ -67,11 +72,11 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { restore() switch { case c.transport: - if err == nil || !strings.HasPrefix(err.Error(), buildstep.TransportFaultWords) { + if err == nil || !strings.HasPrefix(err.Error(), TransportFaultPrefix+" deadline, node-build-go no node, ") { t.Errorf("%s: %v, want it opens with the transport fault's words", name, err) } case c.tries == 1: - if err == nil || p.tries != 1 || strings.Contains(err.Error(), buildstep.TransportFaultWords) { + if err == nil || p.tries != 1 || strings.HasPrefix(err.Error(), TransportFaultPrefix) { t.Errorf("%s: %v after %d tries, want one try and no transport fault", name, err, p.tries) } case err != nil || p.tries != c.tries || len(p.inner.Asks()) != 1 || len(p.ids) != c.tries || p.ids[0] != p.ids[len(p.ids)-1]: @@ -101,7 +106,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { } h.Write(st) } - if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "a transport fault on 3 runs (ace, g14, shanks): lookup proxy: i/o timeout") { + if _, err := (Asker{Publisher: h, Waiter: h, Bound: time.Minute}).Ask(context.Background(), goAsk("mvl0frv18d1972")); err == nil || !strings.Contains(err.Error(), "transport fault: name lookup, node-build-go shanks, lookup proxy: i/o timeout") { t.Errorf("three runs then failed: %v", err) } h = &FakeAsks{} @@ -134,7 +139,7 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) { // how many runs; a drop that failed is said, not discarded. late := &FakeAsks{} late.Write(buildstep.Step{ID: "mvl0frv18d1979", Node: "shanks", State: buildstep.StepFailed, Passing: true, Run: 3, Reason: "lookup proxy: i/o timeout"}) - if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.Contains(st.Reason, "a transport fault on 3 runs") { + if st, err := (Asker{Publisher: late, Waiter: late, Bound: time.Minute}).Wait(context.Background(), goAsk("mvl0frv18d1979")); err != nil || !strings.HasPrefix(st.Reason, "transport fault: name lookup, node-build-go shanks") { t.Errorf("a failed step of run 3 read alone says %q (%v), want it names 3 runs", st.Reason, err) } var said []string @@ -381,3 +386,14 @@ func (p *faultyPuts) PublishAsk(ctx context.Context, subject, id string, data [] } return p.inner.PublishAsk(ctx, subject, id, data) } + +// subjectsOf records the subject each ask is published on. +type subjectsOf struct { + inner AskPublisher + seen []string +} + +func (p *subjectsOf) PublishAsk(ctx context.Context, subject, id string, data []byte) error { + p.seen = append(p.seen, subject) + return p.inner.PublishAsk(ctx, subject, id, data) +} diff --git a/modules/mesh-delivery-builder/buildrun/faults.go b/modules/mesh-delivery-builder/buildrun/faults.go new file mode 100644 index 00000000..e1054451 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/faults.go @@ -0,0 +1,53 @@ +package buildrun + +import ( + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// The words of a key that failed on a transport fault (design 47 G §568, A-76): `transport fault: , , +// `, the class one of ADR 0664 §1's four, the seat and node the step's, the cause the last run's in the build +// module's words. No other failure opens `transport fault:`. + +// TransportFaultPrefix opens the error of a key that failed on a transport fault. +const TransportFaultPrefix = builds.TransportFaultPrefix + +// The classes of a transport fault. +const ( + ClassNoResponders = "no responders" + ClassNameLookup = "name lookup" + ClassConnection = "connection" + ClassDeadline = "deadline" +) + +// FaultClass is the class of a call's error or a step's reason: the words the builder's retake reads a transport fault by +// (buildstep.PassingCause), read as no responders, a name lookup, a deadline or, for every other refused, reset or cut +// connection, a connection. +func FaultClass(cause string) string { + c := strings.ToLower(cause) + has := func(words ...string) bool { + for _, w := range words { + if strings.Contains(c, w) { + return true + } + } + return false + } + switch { + case has("no responders", strings.ToLower(buildstep.HolderSilentWords), "jetstream not enabled"): + return ClassNoResponders + case has("no such host", "name resolution", "lookup ", "could not resolve", "enotfound", "eai_again"): + return ClassNameLookup + case has("deadline", "timeout", "timed out", "etimedout"): + return ClassDeadline + } + return ClassConnection +} + +// TransportFault is the error of a key that failed on a transport fault of the seat's step on node, with the cause of its +// last run. +func TransportFault(class, seat, node, cause string) string { + return TransportFaultPrefix + " " + class + ", " + seat + " " + node + ", " + cause +} diff --git a/modules/mesh-delivery-builder/buildrun/faults_test.go b/modules/mesh-delivery-builder/buildrun/faults_test.go new file mode 100644 index 00000000..528b0699 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/faults_test.go @@ -0,0 +1,23 @@ +package buildrun + +import "testing" + +// Design 47 G §568, A-76: the class of a transport fault is read from the call's error, and its words are one form. +func TestFaultClassAndWords(t *testing.T) { + for cause, class := range map[string]string{ + "nats: no responders available for request": ClassNoResponders, + "the holder is silent: seat x answered": ClassNoResponders, + "lookup proxy: no such host": ClassNameLookup, + "Get https://r: context deadline exceeded": ClassDeadline, + "read tcp: i/o timeout": ClassDeadline, + "dial tcp: connection refused": ClassConnection, + "ssh: kex_exchange_identification: read: reset": ClassConnection, + } { + if got := FaultClass(cause); got != class { + t.Errorf("FaultClass(%q) = %q, want %q", cause, got, class) + } + } + if got, want := TransportFault(ClassDeadline, "node-build-go", "ace", "i/o timeout"), "transport fault: deadline, node-build-go ace, i/o timeout"; got != want { + t.Errorf("TransportFault = %q, want %q", got, want) + } +} diff --git a/modules/mesh-delivery-builder/buildrun/steps_resume_test.go b/modules/mesh-delivery-builder/buildrun/steps_resume_test.go new file mode 100644 index 00000000..cb8c4203 --- /dev/null +++ b/modules/mesh-delivery-builder/buildrun/steps_resume_test.go @@ -0,0 +1,88 @@ +package buildrun + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/busretry" +) + +type fakeEntry struct { + jetstream.KeyValueEntry + rev uint64 + value []byte +} + +func (e fakeEntry) Revision() uint64 { return e.rev } +func (e fakeEntry) Value() []byte { return e.value } +func (e fakeEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut } + +type fakeWatcher struct { + jetstream.KeyWatcher + updates chan jetstream.KeyValueEntry +} + +func (w fakeWatcher) Updates() <-chan jetstream.KeyValueEntry { return w.updates } +func (w fakeWatcher) Stop() error { return nil } + +// Design 47 G §568: the read of an ask's steps is a kept call. A transport fault opening the watch ends no key (the wait is +// not failed): it is opened again once the bus answers, said unanswered meanwhile and answered after; and a watch that closes +// is taken up again from the revision after the last it handed over, so the step that ended meanwhile is heard. +func TestTheWatchOfAStepResumesAfterAFault(t *testing.T) { + t.Cleanup(busretry.Waits(20*time.Millisecond, time.Millisecond, 2*time.Millisecond)) + passed, _ := buildstep.EncodeStep(buildstep.Step{ID: "mvl0frv18d1958", Tool: "build", Node: "ace", State: buildstep.StepPassed}) + running, _ := buildstep.EncodeStep(buildstep.Step{ID: "mvl0frv18d1958", Tool: "build", Node: "ace", State: buildstep.StepRunning}) + var mu sync.Mutex + var opens []uint64 + var unanswered, answered int + asks := JetStreamAsks{ + Unanswered: func(string, string, error) { mu.Lock(); unanswered++; mu.Unlock() }, + Answered: func(string, string) { mu.Lock(); answered++; mu.Unlock() }, + OpenWatch: func(_ context.Context, _, _ string, after uint64) (jetstream.KeyWatcher, error) { + mu.Lock() + defer mu.Unlock() + opens = append(opens, after) + ch := make(chan jetstream.KeyValueEntry, 2) + // The bus answers nothing past the kept call's own asking again (its bound), so the fault is told and the watch tried again. + switch n := len(opens); { + case n <= 40: + return nil, nats.ErrNoResponders + case n == 41: + ch <- fakeEntry{rev: 5, value: running} + close(ch) // the watch closes under the wait + default: + ch <- fakeEntry{rev: 6, value: passed} + } + return fakeWatcher{updates: ch}, nil + }} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + steps, err := asks.WatchAsk(ctx, "mesh-build-go", "mvl0frv18d1958") + if err != nil { + t.Fatalf("a transport fault opening the watch failed the wait: %v", err) + } + var states []string + for s := range steps { + states = append(states, s.State) + if buildstep.Ended(s) { + break + } + } + mu.Lock() + defer mu.Unlock() + if len(states) != 2 || states[0] != buildstep.StepRunning || states[1] != buildstep.StepPassed { + t.Fatalf("steps heard = %v, want running then passed", states) + } + if len(opens) != 42 || opens[41] != 6 { + t.Errorf("opens after revisions %v, want the last taken up again after the last revision heard (6)", opens) + } + if unanswered == 0 || answered != 2 { + t.Errorf("unanswered %d, answered %d, want the fault said and each answer said", unanswered, answered) + } +} diff --git a/modules/mesh-delivery-builder/builds/builds.go b/modules/mesh-delivery-builder/builds/builds.go index fe8d7f7b..0f1b118e 100644 --- a/modules/mesh-delivery-builder/builds/builds.go +++ b/modules/mesh-delivery-builder/builds/builds.go @@ -194,11 +194,15 @@ func FailedUntilChanged(err string) bool { return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) } -// FailedOnTransportFault says whether a failed key's error is a step's end after its third run met a transport fault -// (buildstep.TransportFaultWords): a fault of the network, not of the source, so the builder does not retake it by itself -// (ADR 0664 §3, which changes ADR 0429 clause 18 for this case); a person's hand act or a builder that starts again does. +// TransportFaultPrefix opens the error of a key that failed on a transport fault: `transport fault: , , +// ` (design 47 G §568, A-76). No other failure opens it. +const TransportFaultPrefix = "transport fault:" + +// FailedOnTransportFault says whether a failed key's error opens TransportFaultPrefix: a fault of the network, not of the +// source, so the builder does not retake it on its look (ADR 0664 §3, which ADR 0673 changes): a person's hand act, a builder +// that starts again, or an event that says its cause passed (design 47 G §568) builds it again. func FailedOnTransportFault(err string) bool { - return strings.Contains(err, buildstep.TransportFaultWords) + return strings.HasPrefix(err, TransportFaultPrefix) } // ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and @@ -241,6 +245,11 @@ type BuilderBuild struct { // oci-artifact-registry (design 47 G §671): every artifact of the build is kept there, each with its digest in // Artifacts. A build not kept names none; the builder carries it onto the build's end (BuildLogEntry.KeptOn). KeptOn string `json:"kept-on,omitempty"` + // EventBuilds counts, in the key's newest revision, the revisions an event started since the last one a person started + // (design 47 G §568, ADR 0673): the builder builds a key that failed on a transport fault again on an event for the third + // time at most, then asks the operator. A person's build (builder_build_again, the operator's answer) starts the count + // again. A reader that does not know the field reads none. + EventBuilds int `json:"event-builds,omitempty"` } func (b BuilderBuild) check() error { @@ -361,3 +370,18 @@ func DecodeBuilderBuild(data []byte) (BuilderBuild, error) { // SameBuilderBuild says whether two builds are the same value, so the builder writes only when they differ. func SameBuilderBuild(a, b BuilderBuild) bool { return reflect.DeepEqual(a, b) } + +// TransportFaultOf reads a key's error `transport fault: , , ` (design 47 G §568, A-76): ok is false +// for an error that does not open so. The node of a fault no node ran (the put of an ask) is `no node`. +func TransportFaultOf(err string) (class, seat, node, cause string, ok bool) { + rest, found := strings.CutPrefix(err, TransportFaultPrefix+" ") + if !found { + return "", "", "", "", false + } + parts := strings.SplitN(rest, ", ", 3) + if len(parts) != 3 { + return "", "", "", "", false + } + seat, node, _ = strings.Cut(parts[1], " ") + return parts[0], seat, node, parts[2], seat != "" && node != "" +} diff --git a/modules/mesh-delivery-builder/builds/reference_test.go b/modules/mesh-delivery-builder/builds/reference_test.go index 1713338f..89b6d58a 100644 --- a/modules/mesh-delivery-builder/builds/reference_test.go +++ b/modules/mesh-delivery-builder/builds/reference_test.go @@ -103,13 +103,13 @@ func TestOnlyAPushFailureIsBuiltAgainBeforeTheFingerprintChanges(t *testing.T) { {VersionTaken + "@novox/mesh-sdk@0.1.15 holds another file", false}, {RefusedKind + "a package in go", false}, {"go test: 2 failed", true}, - {"a transport fault on 3 runs (ace, g14, novox)", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's + {"transport fault: deadline, node-build-go ace, i/o timeout", true}, // ADR 0664: not "until changed" (a person's hand act builds it); the builder's own retake is FailedOnTransportFault's } { if got := FailedUntilChanged(c.err); got == c.again { t.Errorf("FailedUntilChanged(%q) = %v", c.err, got) } } - if !FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace): i/o timeout") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") { + if !FailedOnTransportFault("transport fault: connection, node-build-go ace, connection refused") || FailedOnTransportFault("mesh-build-go on ace: a transport fault on 3 runs (ace)") || FailedOnTransportFault("go test: 2 failed") || FailedOnTransportFault(NotStored+"code: 500") { t.Error("FailedOnTransportFault must name the third run's end alone: the builder's own retake holds it (ADR 0664 §3)") } if NotStored != "not stored: " || NotPinned != "not pinned: " || VersionTaken != "version taken: " || RefusedKind != "refused kind: " { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go new file mode 100644 index 00000000..a91ec8a2 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" + + "mesh-delivery-builder/internal/build" +) + +// catalogWithin is how long a call waits for the catalog half's first view of one branch. +var catalogWithin = 30 * time.Second + +// buildTool is builder_build (design 47 G §589): a module's build keys whole. +func buildTool(r *build.BuildReader) stdio.Tool { + return stdio.Tool{Name: build.BuildTool, + Description: "A module's build keys, whole, newest first (at most 20): per key its fingerprint (whole, as the build-input fingerprint the catalog graph " + + "entry carries it), state (queued, building, done or failed), revision and since when, the step and its node, the error whole, event-builds " + + "(the revisions events started since the last a person started; after three the operator is asked) and the open ask; and newest, the watched " + + "branches whose newest catalog graph entry of the module names that fingerprint, read from the catalog half. A module none of whose keys the newest " + + "entry names says so, naming the entry's fingerprint. With fingerprint, that key alone. Which node wants which fingerprint is the assigner's " + + "to answer (the seat mesh-wants). Read-only.", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module"}, + "fingerprint": map[string]any{"type": "string", "description": "one key: its fingerprint, src2:"}}}, + Run: func(args map[string]any) (any, error) { + module, _ := args["module"].(string) + fp, _ := args["fingerprint"].(string) + return r.Build(module, fp) + }} +} + +// newestVia reads the fingerprint of a module's newest catalog graph entry on each of its watched branches, from the catalog +// half through the seat mesh-catalog (the verb registrations for the branches, the verb revision for each branch's view), as +// the delivery view's tools read it: once per call (design 47 G §532, §589, A-87). +func newestVia(ask meshcatalogclient.Call) func(module string) (map[string]string, error) { + return func(module string) (map[string]string, error) { + raw, err := ask(meshcatalogclient.RegistrationsVerb, meshcatalog.RegistrationsInput{Module: module}) + if err != nil { + return nil, fmt.Errorf("the registrations of %s: %w", module, err) + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return nil, fmt.Errorf("%s answered: %s", meshcatalogclient.RegistrationsVerb, said) + } + var regs meshcatalog.RegistrationsAnswer + if err := json.Unmarshal(answer, ®s); err != nil { + return nil, fmt.Errorf("%s answered no registrations: %w", meshcatalogclient.RegistrationsVerb, err) + } + out := map[string]string{} + for _, reg := range regs.Registrations { + if reg.Module != module { + continue + } + for _, branch := range reg.Branches { + c, err := meshcatalogclient.NewCatalog(ask, branch) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), catalogWithin) + err = c.Start(ctx) + cancel() + if err != nil { + return nil, fmt.Errorf("the catalog half of %s has no first view: %w", branch, err) + } + if e, ok := c.Entry(module); ok && e.Fp != "" { + out[branch] = e.Fp + } + } + } + return out, nil + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go new file mode 100644 index 00000000..f18c8ad0 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/build_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Design 47 G §589, A-87: the tools bundle reads each watched branch's newest catalog graph entry of the module from the catalog +// half itself: the verb registrations for the branches, the verb revision for each branch's view. +func TestNewestIsReadFromTheCatalogHalfOfEachWatchedBranch(t *testing.T) { + fp := map[string]string{"main": "src2:" + strings.Repeat("1", 64), "feature": "src2:" + strings.Repeat("2", 64)} + ask := func(key string, body any) (json.RawMessage, error) { + switch key { + case meshcatalogclient.RegistrationsVerb: + return json.Marshal(meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{{Module: "m", Branches: []string{"main", "feature"}}}}) + case meshcatalogclient.RevisionVerb: + in := body.(meshcatalog.RevisionInput) + return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: 3, Kind: meshcatalogclient.KindTrunk}, + Entries: []meshcatalog.CatalogEntry{{Module: "m", Fp: fp[in.Branch]}, {Module: "z", Fp: "src2:" + strings.Repeat("9", 64)}}}) + } + t.Fatalf("asked %s", key) + return nil, nil + } + got, err := newestVia(ask)("m") + if err != nil || len(got) != 2 || got["main"] != fp["main"] || got["feature"] != fp["feature"] { + t.Fatalf("newest = %v, %v, want each branch's entry fingerprint whole", got, err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index dea457ae..b42c7e53 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -54,17 +54,9 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q if a.Refused { return nil, errors.New(a.Answer) } - return map[string]any{"key": key, "name": a.Name, "do": a.Do, "version": a.Version, "answer": a.Answer}, nil + return map[string]any{"key": key, "name": a.Name, "answer": a.Answer}, nil } - nameFP := map[string]any{"type": "object", "required": []string{"name", "fp", "why"}, "properties": map[string]any{ - "name": map[string]any{"type": "string", "description": "the numbered name, /, as builder_versions lists it"}, - "fp": map[string]any{"type": "string", "description": "the fingerprint, src2:"}, - "why": map[string]any{"type": "string", "description": "why, in plain words"}}} return []stdio.Tool{ - {Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, Input: nameFP, - Run: func(args map[string]any) (any, error) { - return handAct(build.NewPublishAgain(str(args, "name"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())) - }}, {Name: versions.FloorAgainTool, Description: versions.FloorAgainDescription, Input: map[string]any{"type": "object", "required": []string{"name", "why"}, "properties": map[string]any{ "name": map[string]any{"type": "string", "description": "the numbered name, /"}, @@ -164,15 +156,16 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q "fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " + "queued), the manifest built, the bases it was built against and each artifact made with its digest and reference; a failed " + "build carries its error and the step it was at. more says the " + - "log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.", + "log held more than one answer carries: ask again from the last build's time. With module, that module's builds only. What the operator's command nox builds reads. Read-only.", Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{ - "since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}}, + "since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}, + "module": map[string]any{"type": "string", "description": "only this module's builds"}}}, Run: func(args map[string]any) (any, error) { at, err := time.Parse(time.RFC3339Nano, str(args, "since")) if err != nil { return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err) } - return build.BuildsSince(since, at) + return build.BuildsSince(since, at, str(args, "module")) }}, {Name: build.AnnounceAgainTool, Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " + @@ -371,7 +364,11 @@ func main() { return e.Value, true, nil }} tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr) + tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")), + Newest: newestVia(stdio.Ask), Steps: r})) tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) + qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState) + tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) ws := stdio.State(build.WaitsState) wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := ws.Get(key) @@ -400,3 +397,14 @@ func main() { os.Exit(1) } } + +// entryOf reads a key with its revision and when it was written. +func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) { + return func(key string) (json.RawMessage, uint64, time.Time, error) { + e, err := s.Get(key) + if err != nil || e == nil { + return nil, 0, time.Time{}, err + } + return e.Value, e.Revision, written(s, key, e.Revision), nil + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go new file mode 100644 index 00000000..9b48df0f --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -0,0 +1,65 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// readerOf is a kept state as the versions package's reader. +func readerOf(s stdio.KeptState) versions.Reader { + return versions.Reader{Keys: s.Keys, Get: func(key string) (json.RawMessage, bool, error) { + e, err := s.Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil + }} +} + +// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: +// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer +// with the versions it queued. +func republishTools(queue, acted versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, + Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's reserved versions"}, + "version": map[string]any{"type": "string", "description": "with module: only this reserved version, MAJOR.MINOR.PATCH"}, + "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewRepublish(str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("published reserved versions again by hand (module %q, version %q)", str(args, "module"), str(args, "version")), + Why: str(args, "why"), Cause: versions.PublishAgainTool, Verb: "publish-again"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was queued: %w", err) + } + raw, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if a.Refused { + return nil, errors.New(a.Answer) + } + return map[string]any{"key": key, "queued": a.Queued, "answer": a.Answer}, nil + }}, + {Name: versions.PublishQueueTool, Description: versions.PublishQueueDescription, + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted) }}, + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go index acdbd9fe..f8cb1e46 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go @@ -115,7 +115,7 @@ func (h heldOnTheBus) seen(ctx context.Context, a buildstep.StepAsk, later bool) // stream is the build seat's stream of language, its name and the subject of kind on it. func (h heldOnTheBus) stream(ctx context.Context, language, kind string) (jetstream.Stream, string, string, error) { - subject, err := buildstep.AskSubject(language, kind) + subject, err := buildstep.AskSubjectBy(language, kind, buildstep.AskerBuilder) if err != nil { return nil, "", "", err } @@ -286,7 +286,7 @@ func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (bu return q, err } for _, kind := range buildstep.AskKinds { - subject, _ := buildstep.AskSubject(language, kind) + subject, _ := buildstep.AskSubjectBy(language, kind, buildstep.AskerBuilder) var ahead []build.Queued if err := onSubject(ctx, stream, subject, func(m build.Queued) bool { ahead = append(ahead, m) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index aba095dd..060b7dcb 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -165,6 +165,18 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil { t.Fatal(err) } + // Design 47 G §600: each queue's worker takes an ask in both forms, main's and the builder's (accept..builder); the + // build modules' helper above filters main's alone, so the worker is declared again with both. + stream, _ := sdkstep.AskStream(language) + for _, kind := range sdkstep.AskKinds { + worker, _ := sdkstep.AskWorker(language, kind) + main, _ := sdkstep.AskSubject(language, kind) + byBuilder, _ := sdkstep.AskSubjectBy(language, kind, sdkstep.AskerBuilder) + if _, err := f.js.CreateOrUpdateConsumer(f.ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{main, byBuilder}, + AckPolicy: jetstream.AckExplicitPolicy, AckWait: 30 * time.Second, MaxDeliver: 5}); err != nil { + t.Fatal(err) + } + } put := func(key string, value any) (uint64, error) { raw, _ := value.(json.RawMessage) return kv.Put(f.ctx, key, raw) @@ -586,18 +598,10 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 { t.Errorf("version.numbered of netcheck/tools-go said %v, want none", got) } - // DLV-B10: builder_publish_again of a published, held build is refused: nothing says a version numbered (§349). for _, a := range []string{"tools-go", "tools-typescript"} { reg.published(t, "netcheck/"+a, "0.1.5", digestOf(a)) } reg.awaitOf(t, "netcheck/tools-go", func(es []versions.Entry) bool { return len(es) == 1 && es[0].State == versions.Published }) - key, again, err := build.NewPublishAgain("netcheck/tools-go", fp, "the holder lost it", "me@shanks", time.Now()) - if err != nil { - t.Fatal(err) - } - if a := reg.answered(t, key, again); !a.Refused { - t.Errorf("builder_publish_again = %+v, want a refusal", a) - } // A numbered version a holder will refuse below, written to the state as the builder numbers one. refusable, _ := json.Marshal(versions.Entry{Name: "netcheck/tools-go", Kind: versions.KindBundle, Series: "0.1", Version: "0.1.6", Module: "netcheck", Fingerprint: "src2:" + strings.Repeat("6", 64), Seat: ociartifactregistry.Seat, State: versions.Numbered, At: time.Now().UTC()}) @@ -1084,7 +1088,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { held := heldOnTheBus{b.js} goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} - subject, _ := sdkstep.AskSubject(sdkstep.AskGo, sdkstep.AskDelivery) + subject, _ := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskDelivery, sdkstep.AskerBuilder) if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index d8678236..3d1d7572 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -7,8 +7,10 @@ package main import ( "cmp" "context" + "errors" "fmt" "io" + "io/fs" "log" "maps" "os" @@ -22,6 +24,8 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" @@ -136,7 +140,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro tools := &buildrun.ToolchainSet{} // Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot // (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity. - asks := buildrun.JetStreamAsks{JS: js} + // The read of an ask's steps is a kept call (design 47 G §568): a fault ends no key, and past five minutes unanswered it is a + // finding; its bound is a key of the state bounds, bound once the bus holds it (set below, with the findings). + stepReads := &stepsRead{bounds: func() jetstream.KeyValue { return nil }, bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf} + asks := buildrun.JetStreamAsks{JS: js, Unanswered: stepReads.unanswered, Answered: stepReads.answered} // The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once, // and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound. // Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the @@ -177,6 +184,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder // answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read. pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf) + stepReads.bounds, stepReads.findings = func() jetstream.KeyValue { return pubBoundsKV.KV() }, found pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf} numberer.Publishing = pubs.arm b := &build.Builder{ @@ -200,13 +208,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.Numbers = build.VersionNumbers{N: numberer} // The builder's toolchain waits (design 47 G §672): its state waits, and wait-held and wait-ended on its stream waits; its // bound is its setting toolchain-wait-minutes. - waitBound := defaultToolchainWait - if v := strings.TrimSpace(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES")); v != "" { - n, err := strconv.Atoi(v) - if err != nil || n < 1 { - return fmt.Errorf("MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES %q is not a number of minutes", v) - } - waitBound = time.Duration(n) * time.Minute + waitBound, err := toolchainWaitOf(getenv("MESH_BUILDER_TOOLCHAIN_WAIT_FILE")) + if err != nil { + return err } waitsBucket := process.AwaitBucket(ctx, js, conn, build.WaitsBucket, "the toolchain waits are kept in memory only, and builder_waits shows none", logger.Printf) b.Waits = newWaitsKept(waitsBucket, func(ctx context.Context, event, id string, body any) error { @@ -221,17 +225,32 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A state bound later numbers the builds looked at without it. versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again. - acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Done: doneVia(b), Reconsider: b.Reconsider, - BuildAgain: func(ctx context.Context, module, fp, why, by string) (string, bool) { - r := b.BuildAgain(ctx, module, fp, why, by, buildAgainBound) - if r.Refused != "" { - return r.Refused, true - } - if r.Failed != "" { - return "built again, and it failed before its asks were put: " + r.Failed, false - } - return fmt.Sprintf("built again as revision %d", r.Revision), false - }} + acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider} + // A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G + // §588): its states publish-queue and acted, bound once the bus holds them. + republishMax, err := republishMaxOf(getenv("MESH_BUILDER_REPUBLISH_MAX_FILE")) + if err != nil { + return err + } + queueKV := process.AwaitBucket(ctx, js, conn, versions.PublishQueueBucket, "no version is published again on an event or by builder_publish_again", logger.Printf) + actedKV := process.AwaitBucket(ctx, js, conn, versions.ActedBucket, "no version is published again on an event or by builder_publish_again", logger.Printf) + republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: versions.PublishQueueBucket}}}, + Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}}, Entry: entryVia(calls.Call), Send: sendVia(calls.Call), + Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf} + acts.Republish = republish.Republish + again := &republishing{rp: republish} + // The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel. + decided := &decisions{subject: oc.SubjectDecided(cred.Module)} + retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: versions.ActedBucket}} + operator := &build.OperatorAsk{Builder: b, Acted: retakeActed, Ctx: ctx, Logf: logger.Printf, + Client: &occ.Client{Module: cred.Module, State: calls.State("asked"), Publish: publishVia(ctx, js), + Record: recordVia(calls.Call, calls.Ask), Subscribe: decided.hear(logger.Printf)}} + if err := operator.Listen(); err != nil { + return fmt.Errorf("hearing the operator's answers: %w", err) + } + retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}} + queueKV.Ready(again.bound(ctx, js, logger.Printf)) + actedKV.Ready(again.bound(ctx, js, logger.Printf)) // A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it @@ -376,6 +395,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} events := func(m jetstream.Msg) { + if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them + return + } + if decided.heard(m) { // the operator's answer to an ask about a key (§568) + return + } + if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568) + return + } if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) return } @@ -422,6 +450,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro unput.expired(ctx, e) return } + if _, _, ok := versions.StepsBoundOf(e.Key()); ok { + stepReads.expired(ctx, e) + return + } if node, ok := enginereads.BoundOf(e.Key()); ok { if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended reads.expiredCall(node) @@ -572,3 +604,28 @@ func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (st } return "", nil } + +// toolchainWaitOf reads the toolchain wait's setting, toolchain-wait-minutes, from the file the manifest declares with the +// content ${config:toolchain-wait-minutes} (design 47 G §672, A-71), at the builder's start: no file, or an empty one, is +// the default; a content that is no number of minutes is an error naming the file. +func toolchainWaitOf(path string) (time.Duration, error) { + if path == "" { + return defaultToolchainWait, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return defaultToolchainWait, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return defaultToolchainWait, nil + } + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return 0, fmt.Errorf("%s holds %q, which is not a number of minutes", path, v) + } + return time.Duration(n) * time.Minute, nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 2082cf87..2c5c1ef5 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -58,7 +58,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream released(ctx, b, kv, e, r, logf) return } - if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop { + if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop { // A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch. go versionActed(ctx, acts, kv, e, r, logf) return @@ -192,10 +192,15 @@ func emitBuildEnded(ctx context.Context, js jetstream.JetStream, cred process.Ac // versionActed answers a reasks value of builder_publish_again, builder_floor_again or builder_old_form's drop, a hand act said in the log. func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) { var err error - if r.Mode == build.ModePublishAgain { - var ag versions.Again - ag, r.Answer, err = acts.PublishAgain(ctx, r.Name, r.FP, r.Why, r.By, r.At) - r.Do, r.Version = ag.Do, ag.Entry.Version + if r.Mode == build.ModeRepublish { + if acts.Republish == nil { + err = errors.New("this builder publishes nothing again") + } else { + r.Queued, err = acts.Republish(ctx, r.Module, r.Version) + if err == nil { + r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them" + } + } } else if r.Mode == build.ModeReservedDrop { if acts.DropReserved == nil { err = errors.New("this builder keeps no reservations") diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go new file mode 100644 index 00000000..7d201fb3 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + "sync/atomic" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "mesh-delivery-builder/internal/versions" +) + +// The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus. + +// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's. +type awaitedQueue struct{ awaitedReserved } + +func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) { + kv, err := s.kv() + if err != nil { + return 0, err + } + st, err := kv.Status(ctx) + if err != nil { + return 0, err + } + return st.Bytes(), nil +} + +// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry. +func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { + return func(_ context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) { + raw, err := call("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbEntry, meshcatalog.EntryInput{Fingerprint: fingerprint}) + if err != nil { + return meshcatalog.CatalogEntry{}, err + } + var a meshcatalog.EntryAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return meshcatalog.CatalogEntry{}, fmt.Errorf("the catalog answered what is no entry: %w", err) + } + return a.Entry, nil + } +} + +// sendVia sends a publish to the registry seat with the digest and no parts (A-73): the holder takes the bytes from the +// artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of +// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). +func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error { + return func(_ context.Context, p versions.Publish) error { + raw, err := call("seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p)) + if err != nil { + return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err) + } + var a gopackageregistry.PublishAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return fmt.Errorf("%s answered what is no publish answer: %w", p.Seat, err) + } + if len(a.Lacks) > 0 { + return fmt.Errorf("%s%s holds %d of the parts of %s %s", versions.NotStoredWords, p.Seat, len(a.Lacks), p.Name, p.Version) + } + return nil + } +} + +// publishBody is the seat's publish input for p: the digest, its kind and name and version, no parts. +func publishBody(p versions.Publish) gopackageregistry.PublishInput { + return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version} +} + +// republishMaxOf reads the setting republish-max-versions from the file the manifest declares (design 47 G §633, A-71's +// form): no file or an empty one is the default; a content that is no positive number is an error naming the file. +func republishMaxOf(path string) (int, error) { + if path == "" { + return versions.DefaultRepublishMax, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return versions.DefaultRepublishMax, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return versions.DefaultRepublishMax, nil + } + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + return 0, fmt.Errorf("%s holds %q, which is not a number of versions", path, v) + } + return n, nil +} + +// republishing starts the publish again once both its states are bound: the queue is taken up again and the last running +// of every node's module is read (§556), acted on where its sequence is newer than the kept one. +type republishing struct { + rp *versions.Republisher + ready atomic.Int32 +} + +func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) { + return func(jetstream.KeyValue) { + if g.ready.Add(1) != 2 { + return + } + if err := g.rp.Resume(ctx); err != nil { + logf("the queued publishes are not taken up again: %v", err) + } + err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(subject string, data []byte, seq uint64) { + if _, err := g.rp.Heard(ctx, subject, data, seq); err != nil { + logf("the running %s is not acted on at the start: %v", subject, err) + } + }) + if err != nil { + logf("the running of the node-engine seat is not read for the publish again at the start: %v", err) + } + } +} + +// heard hands a message of the events consumer to the publish again: one of its subjects whose state cannot take it is left +// for the bus to give again (Nak); every message goes on to the other parts. +func (g *republishing) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) { + meta, err := m.Metadata() + if err != nil { + return false + } + handled, err := g.rp.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream) + if handled && err != nil { + logf("%v; heard again", err) + if nakErr := m.Nak(); nakErr != nil { + logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return true + } + return false +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go new file mode 100644 index 00000000..0532ed69 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/republish_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// TestRepublishSend: design 47 G §588, A-73: a publish again is the registry seat's publish with the digest and no parts; the +// four seats take one wire form, a call's fault or an answer that lacks parts is the holder's refusal in its words, and its +// setting is read from the file the manifest declares. +func TestRepublishSend(t *testing.T) { + p := versions.Publish{Seat: versions.SeatArtifacts, Kind: "bundle", Name: "m/prog", Version: "0.1.3", Digest: "sha256:aa", Fingerprint: "src2:m", Commit: "c", Transfer: "t"} + want, _ := json.Marshal(publishBody(p)) + if strings.Contains(string(want), "parts") { + t.Errorf("the publish names parts: %s", want) + } + for name, v := range map[string]any{"container image": containerimageregistry.PublishInput{Digest: "sha256:aa"}, "oci": ociartifactregistry.PublishInput{Digest: "sha256:aa"}, + "go": gopackageregistry.PublishInput{Digest: "sha256:aa"}, "npm": npmpackageregistry.PublishInput{Digest: "sha256:aa"}} { + var ref, got map[string]any + a, _ := json.Marshal(gopackageregistry.PublishInput{Digest: "sha256:aa"}) + b, _ := json.Marshal(v) + _ = json.Unmarshal(a, &ref) + _ = json.Unmarshal(b, &got) + if len(ref) != len(got) { + t.Errorf("the %s seat's publish input is %s, the go seat's %s: one wire form is sent to all four", name, b, a) + } + } + var asked string + for name, c := range map[string]struct { + answer string + err error + want string + }{ + "held": {`{"held":true,"transfer":"t"}`, nil, ""}, + "lacks": {`{"lacks":["sha256:bb"],"transfer":"t"}`, nil, versions.NotStoredWords}, + "a fault": {"", errors.New("no responders"), "no responders"}, + "no sense": {`[`, nil, "no publish answer"}, + } { + send := sendVia(func(key string, body any) (json.RawMessage, error) { + asked = key + return json.RawMessage(c.answer), c.err + }) + err := send(context.Background(), p) + if asked != "seat:oci-artifact-registry.publish" || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) { + t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want) + } + } + dir := t.TempDir() + for content, want := range map[string]int{"": 50, "7": 7} { + path := filepath.Join(dir, "max") + _ = os.WriteFile(path, []byte(content), 0o600) + if got, err := republishMaxOf(path); err != nil || got != want { + t.Errorf("republishMaxOf(%q) = %d, %v, want %d", content, got, err, want) + } + } + _ = os.WriteFile(filepath.Join(dir, "bad"), []byte("0"), 0o600) + if _, err := republishMaxOf(filepath.Join(dir, "bad")); err == nil { + t.Error("a setting of 0 is accepted") + } + if _, _, err := build.NewRepublish("", "0.1.3", "why", "me", versionsNow()); err == nil { + t.Error("a version with no module is accepted") + } +} + +func versionsNow() time.Time { return time.Unix(1, 0) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go index 7d286318..7dd5921d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -125,6 +125,11 @@ func (rv *reservations) unreleased(ctx context.Context, v versions.ReservedVersi needs: "builder_reserved shows the reservation; builder_publish_again publishes the missing artifacts under it; builder_reserved_drop drops it."}) } +// unreleasedWords raises version-unreleased about a version with the words of a publish again that ended badly. +func (rv *reservations) unreleasedWords(ctx context.Context, v versions.ReservedVersion, why string) { + rv.unreleased(ctx, v, why) +} + // ended is told each change of the state bounds: a reservation's bound whose time ended raises version-unreleased while // the reservation stands built. func (rv *reservations) ended(ctx context.Context, e jetstream.KeyValueEntry) bool { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go new file mode 100644 index 00000000..90db6669 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/retake.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go/jetstream" + + "mesh-delivery-builder/internal/build" +) + +// The builder's retake of a key failed on a transport fault, on the node-engine seat's events (design 47 G §568, ADR 0673): the +// wiring of build.Retaker to the events consumer. +type retaking struct{ rt *build.Retaker } + +// heard hands a message of the events consumer to the retake: one of its subjects whose state cannot take it is left for the bus to +// give again (Nak); every message goes on to the other parts. +func (g *retaking) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) { + meta, err := m.Metadata() + if err != nil { + return false + } + handled, err := g.rt.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream) + if handled && err != nil { + logf("%v; heard again", err) + if nakErr := m.Nak(); nakErr != nil { + logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return true + } + return false +} + +// The builder's three bus adapters for the operator-channel client (design 47 G §568), as the teller has them: its own ask +// published, the router's record read through the seat, its own `decided` heard through the module's hub. + +// publishVia publishes an accept of the seat, de-duplicated by id: the same id twice is one message. +func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string, body any, id string) error { + return func(subject string, body any, id string) error { + data, err := json.Marshal(body) + if err != nil { + return err + } + var opts []jetstream.PublishOpt + if id != "" { + opts = append(opts, jetstream.WithMsgID(id)) + } + _, err = js.Publish(ctx, subject, data, opts...) + return err + } +} + +// routerBucket is the bucket of the router's asks: the state `asks` of the seat operator-channel (the contract's StateAsks) in +// the bucket of the seat's holder on the seat list, which the seat mesh-module-assigner's verb assignments answers with seats +// (design 47 G A-74); no module's name is in the program. +func routerBucket(call func(key string, body any) (json.RawMessage, error)) (string, error) { + raw, err := call("seat:"+mma.Seat+"."+mma.VerbAssignments, mma.AssignmentsInput{Seats: mma.AssignmentsInputSeatsTrue}) + if err != nil { + return "", fmt.Errorf("who holds the seat %s is not asked of the seat %s: %w", oc.Seat, mma.Seat, err) + } + var held struct { + Seats []mma.SeatHolders `json:"seats"` + } + if err := json.Unmarshal(raw, &held); err != nil { + return "", fmt.Errorf("the answer of %s.%s is not its seat's: %w", mma.Seat, mma.VerbAssignments, err) + } + for _, s := range held.Seats { + if s.Seat == oc.Seat && len(s.Holders) > 0 { + return s.Holders[0].Module + "_" + oc.StateAsks, nil + } + } + return "", fmt.Errorf("no module holds the seat %s on the seat list, so the router's record of an ask is not read", oc.Seat) +} + +// recordVia reads the router's record of one of this module's asks: a get of `.` in the bucket the seat's holder +// keeps (record is process.Conn.Ask). +func recordVia(call func(key string, body any) (json.RawMessage, error), record func(bucket, id string) (*stdio.StateEntry, error)) func(id string) (*stdio.StateEntry, error) { + return func(id string) (*stdio.StateEntry, error) { + bucket, err := routerBucket(call) + if err != nil { + return nil, err + } + return record(bucket, id) + } +} + +// decisions holds the handler of the router's `decided` event for this module: the module's one hub has the pattern +// `mesh.seat.*.event.>`, which covers it, so the events handler hands the message here (a hub refuses two patterns that overlap). +type decisions struct { + subject string + handle func(jetstream.Msg) +} + +// hear is the client's Subscribe: it keeps the handler of the warrants the router says to this module. +func (d *decisions) hear(logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error { + return func(pattern string, handle func(stdio.Envelope) error) error { + d.handle = func(m jetstream.Msg) { + if err := handle(stdio.Envelope{Key: pattern, Body: m.Data()}); err != nil { + logf("a warrant could not be handled and is given again: %v", err) + if nakErr := m.Nak(); nakErr != nil { + logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return + } + if err := m.Ack(); err != nil { + logf("a warrant was handled but could not be acknowledged: %v", err) + } + } + return nil + } +} + +// heard takes the message when it is this module's `decided`. +func (d *decisions) heard(m jetstream.Msg) bool { + if d.handle == nil || m.Subject() != d.subject { + return false + } + d.handle(m) + return true +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go new file mode 100644 index 00000000..378a8db2 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + + "mesh-delivery-builder/internal/versions" +) + +// **The builder's read of an ask's steps is a kept call, unanswered past five minutes a finding** (design 47 G §568, A-77, ADR +// 0668 §3): a transport fault opening or taking up the watch of `steps` ends no build key; the read is made again once the bus +// answers. The first fault arms a key of the state bounds (StepsBoundKey) with its own time to live; the bus's marker of its end, +// heard on the watch of the state, raises call-unanswered about the ask while the read is still unanswered; the answer deletes +// the key and clears the finding. No timer. +type stepsRead struct { + bounds func() jetstream.KeyValue + findings *findings + bound time.Duration + now func() time.Time + logf func(string, ...any) + + mu sync.Mutex + open map[string]string // the asks whose read is unanswered, by ask id: the last fault +} + +// unanswered is told each fault of the read of the ask's steps on module: the first arms the bound. +func (s *stepsRead) unanswered(module, askID string, fault error) { + s.mu.Lock() + if s.open == nil { + s.open = map[string]string{} + } + _, armed := s.open[askID] + s.open[askID] = fault.Error() + s.mu.Unlock() + if armed { + return + } + ctx := context.Background() + var err error + if kv := s.bounds(); kv == nil { + err = fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket) + } else if _, err = kv.Create(ctx, versions.StepsBoundKey(module, askID), versions.EncodeBound("the read of the steps of the ask "+askID+" on "+module, s.now().Add(s.bound)), jetstream.KeyTTL(s.bound)); errors.Is(err, jetstream.ErrKeyExists) { + err = nil + } + if err != nil { + s.logf("the bound of the read of the steps of the ask %s is not written, so it is raised now: %v", askID, err) + s.raise(ctx, module, askID, fault.Error(), s.now()) + } +} + +// answered ends the read's bound and clears its finding. +func (s *stepsRead) answered(module, askID string) { + s.mu.Lock() + _, was := s.open[askID] + delete(s.open, askID) + s.mu.Unlock() + if !was { + return + } + ctx := context.Background() + if kv := s.bounds(); kv != nil { + if err := kv.Delete(ctx, versions.StepsBoundKey(module, askID)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + s.logf("the bound of the read of the steps of the ask %s is not deleted: %v", askID, err) + } + } + s.findings.clear(ctx, meshconditionsclient.KindCallUnanswered, meshconditionsclient.ScopeCall, askID) +} + +// expired is told each change of the state bounds: the end of a steps bound raises call-unanswered while its read is unanswered. +func (s *stepsRead) expired(ctx context.Context, e jetstream.KeyValueEntry) { + module, askID, ok := versions.StepsBoundOf(e.Key()) + if !ok || e.Operation() != jetstream.KeyValuePurge { + return + } + s.mu.Lock() + fault, open := s.open[askID] + s.mu.Unlock() + if !open { + return + } + s.raise(ctx, module, askID, fault, s.now().Add(-s.bound)) +} + +func (s *stepsRead) raise(ctx context.Context, module, askID, fault string, since time.Time) { + o := callUnanswered(askID, "the read of the steps of "+module, "", fault, "builder."+meshconditionsclient.KindCallUnanswered, since) + o.needs = "the bus answers the read again by itself; builder_stuck shows the key waiting on the ask." + s.findings.raise(ctx, o) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go new file mode 100644 index 00000000..d47b9c4d --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/stepsread_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + + "mesh-delivery-builder/internal/versions" +) + +// Design 47 G §568, A-77: the first fault of the read of an ask's steps arms a bound key with its own time to live; its end +// while the read is unanswered raises call-unanswered about the ask; the answer deletes the bound and clears the finding. +func TestTheStepsReadIsCallUnansweredAfterItsBound(t *testing.T) { + st := &memFindings{vals: map[string][]byte{}} + f := &findings{w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} + marks := &memBounds{keys: map[string]bool{}} + s := &stepsRead{bounds: func() jetstream.KeyValue { return marks }, findings: f, bound: meshconditionsclient.CallUnansweredAfter, + now: func() time.Time { return time.Unix(600, 0) }, logf: t.Logf} + ask := "mvl0frv18d1958" + callKey := meshconditionsclient.Key(meshconditionsclient.ScopeCall, ask, meshconditionsclient.KindCallUnanswered) + s.unanswered("mesh-build-go", ask, errors.New("nats: no responders available for request")) + if !marks.keys[versions.StepsBoundKey("mesh-build-go", ask)] || st.vals[callKey] != nil { + t.Fatalf("a first fault: bound %v, finding %v; want the bound armed, no finding yet", marks.keys, st.vals[callKey] != nil) + } + s.expired(context.Background(), purged{key: versions.StepsBoundKey("mesh-build-go", ask)}) + if got, err := decodeFinding(st.vals[callKey]); err != nil || got.Kind != meshconditionsclient.KindCallUnanswered || !strings.Contains(got.Summary, "the steps of mesh-build-go") || !strings.Contains(got.Summary, ask) { + t.Fatalf("the bound's end: the finding = %+v, %v", got, err) + } + s.answered("mesh-build-go", ask) + if marks.keys[versions.StepsBoundKey("mesh-build-go", ask)] || st.vals[callKey] != nil { + t.Errorf("the read answered: bound %v, finding left %v", marks.keys, st.vals[callKey] != nil) + } + s.expired(context.Background(), purged{key: versions.StepsBoundKey("mesh-build-go", ask)}) + if st.vals[callKey] != nil { + t.Error("a bound's end after the answer raised call-unanswered") + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go index 62c88a80..bdcf23dd 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go @@ -254,8 +254,8 @@ func (h *toolchainsHeard) agree(ctx context.Context) { } // lastPerSubject reads the last message of every subject of a stream kept last per subject, through an ordered consumer -// made for that read (design 47 G §556), and hands each to each; it answers when the last of them was handed. -func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter string, each func(subject string, data []byte)) error { +// made for that read (design 47 G §556), and hands each to each with its stream sequence; it answers when the last of them was handed. +func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter string, each func(subject string, data []byte, sequence uint64)) error { cons, err := js.OrderedConsumer(ctx, stream, jetstream.OrderedConsumerConfig{FilterSubjects: []string{filter}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) if err != nil { return fmt.Errorf("reading %s: %w", stream, err) @@ -278,14 +278,18 @@ func lastPerSubject(ctx context.Context, js jetstream.JetStream, stream, filter if err != nil { return fmt.Errorf("reading %s: %w", stream, err) } - each(m.Subject(), m.Data()) + seq := uint64(0) + if meta, err := m.Metadata(); err == nil { + seq = meta.Sequence.Stream + } + each(m.Subject(), m.Data(), seq) } return nil } // readRunning is the start's read of every node's module's last running (§556), before the consumer's. func (h *toolchainsHeard) readRunning(ctx context.Context, js jetstream.JetStream) { - if err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(_ string, data []byte) { h.running(ctx, data) }); err != nil { + if err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(_ string, data []byte, _ uint64) { h.running(ctx, data) }); err != nil { h.logf("the running of the node-engine seat is not read at the start, so a build module stopped while this builder was away is forgotten at its next running: %v", err) } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go new file mode 100644 index 00000000..2d63b4ab --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchainwait_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +// A-71: the toolchain wait's setting is read at the start from the file the manifest declares, never from the environment. +func TestToolchainWaitIsReadFromItsFile(t *testing.T) { + dir := t.TempDir() + n := 0 + file := func(content string) string { + n++ + p := filepath.Join(dir, "w"+string(rune('a'+n))) + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return p + } + for _, c := range []struct { + name, path string + want time.Duration + bad bool + }{ + {"no file named", "", defaultToolchainWait, false}, + {"file absent", filepath.Join(dir, "absent"), defaultToolchainWait, false}, + {"empty", file(" \n"), defaultToolchainWait, false}, + {"minutes", file("15\n"), 15 * time.Minute, false}, + {"not a number", file("soon"), 0, true}, + {"zero", file("0"), 0, true}, + } { + got, err := toolchainWaitOf(c.path) + if (err != nil) != c.bad || got != c.want { + t.Errorf("%s: %v, %v; want %v, error %v", c.name, got, err, c.want, c.bad) + } + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index ed1460ab..ff2cf8c6 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/build" @@ -306,22 +305,3 @@ func (f *floorWake) wake(ctx context.Context, kind, seat, module string) { func conditionModules(id string) []string { return strings.FieldsFunc(id, func(r rune) bool { return r == '/' || r == '.' || r == ':' }) } - -// doneVia says whether a build key is done, read from the builder's own builds state: the build's file is held. -func doneVia(b *build.Builder) func(ctx context.Context, module, fp string) (bool, error) { - return func(ctx context.Context, module, fp string) (bool, error) { - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return false, err - } - raw, _, found, err := b.Claimer.Store.Get(ctx, key) - if err != nil || !found { - return false, err - } - cur, err := builds.DecodeBuilderBuild(raw) - if err != nil { - return false, err - } - return cur.State == builds.BuildDone, nil - } -} diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index cf46a190..18e70a23 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -9,6 +9,8 @@ require ( git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-builder/internal/build/buildread.go b/modules/mesh-delivery-builder/internal/build/buildread.go new file mode 100644 index 00000000..1d9ec73a --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/buildread.go @@ -0,0 +1,222 @@ +package build + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + "git.novox.be/novox/mesh-delivery/sdk/buildstep" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// **builder_build answers a module's build keys whole** (design 47 G §589, A-87): with `fingerprint` that key whole, with +// `module` alone the module's keys newest first, at most MostBuilds, each with its fingerprint whole, its state, revision and +// since when, its step and node, its error whole, event-builds and the open ask (§568), and `newest` naming each watched branch +// whose newest catalog graph entry of the module names that fingerprint, read from the catalog half by the tools bundle. + +// BuildTool is the builder's read tool of a module's build keys. +const BuildTool = "builder_build" + +// MostBuilds is how many keys of a module builder_build answers by module alone. +const MostBuilds = 20 + +// BuildRow is one build key as builder_build answers it. +type BuildRow struct { + Key string `json:"key"` + Fingerprint string `json:"fingerprint"` + State string `json:"state"` + Revision uint64 `json:"revision,omitempty"` + Since time.Time `json:"since,omitzero"` + Step string `json:"step,omitempty"` + Node string `json:"node,omitempty"` + Error string `json:"error,omitempty"` + EventBuilds int `json:"event-builds"` + // Ask is the open ask of the operator about the key (§568). + Ask *OpenAsk `json:"ask,omitempty"` + // Newest names each watched branch whose newest catalog graph entry of the module names this fingerprint. + Newest []string `json:"newest,omitempty"` +} + +// OpenAsk is the ask the operator has not answered. +type OpenAsk struct { + ID string `json:"id"` + Asked string `json:"asked"` + Expires string `json:"expires"` +} + +// BuildAnswer is what builder_build answers. +type BuildAnswer struct { + Module string `json:"module"` + Builds []BuildRow `json:"builds"` + // Newest is the fingerprint of the module's newest catalog graph entry on each watched branch. + Newest map[string]string `json:"newest-entries,omitempty"` + // Note says what no key shows: that none of the keys is the newest entry's, naming its fingerprint. + Note string `json:"note,omitempty"` +} + +// BuildReader reads what builder_build answers. +type BuildReader struct { + Builds StateRead + // Entry is a key's value with its revision and when it was written; nil leaves both out. + Entry func(key string) (json.RawMessage, uint64, time.Time, error) + // Asked is the builder's state asked, which holds the asks the operator was asked. + Asked StateRead + // Newest is the fingerprint of the module's newest catalog graph entry on each watched branch, read from the catalog half. + Newest func(module string) (map[string]string, error) + Steps StuckReader + index map[string]map[string]string +} + +// Build answers for module, and for one fingerprint when fingerprint is not empty. +func (r *BuildReader) Build(module, fingerprint string) (BuildAnswer, error) { + out := BuildAnswer{Module: module, Builds: []BuildRow{}} + if module == "" { + return out, errors.New("name the module (module)") + } + all, err := r.Builds.Keys() + if err != nil { + return out, err + } + var rows []BuildRow + for _, key := range all { + mod, fp, err := builds.ParseBuilderBuildKey(key) + if err != nil || mod != module || (fingerprint != "" && fp != fingerprint) { + continue + } + raw, rev, at, err := r.read(key) + if err != nil { + return out, fmt.Errorf("%s cannot be read: %w", key, err) + } + if raw == nil { + continue + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil { + return out, fmt.Errorf("%s holds what is not a build: %w", key, err) + } + row := BuildRow{Key: key, Fingerprint: fp, State: v.State, Revision: rev, Since: at, Step: v.Step, Error: v.Error, EventBuilds: v.EventBuilds} + if _, _, node, _, ok := builds.TransportFaultOf(v.Error); ok && node != "no node" { + row.Node = node + } + if row.Node == "" && v.State == builds.BuildBuilding { + row.Node = r.stepNode(v) + } + rows = append(rows, row) + } + if fingerprint != "" && len(rows) == 0 { + return out, fmt.Errorf("the builder holds no build key of %s at %s", module, fingerprint) + } + sort.Slice(rows, func(i, j int) bool { + if !rows[i].Since.Equal(rows[j].Since) { + return rows[i].Since.After(rows[j].Since) + } + return rows[i].Key > rows[j].Key + }) + if len(rows) > MostBuilds { + rows = rows[:MostBuilds] + } + asks := r.openAsks() + for i := range rows { + rows[i].Ask = asks[rows[i].Key] + } + if r.Newest != nil { + newest, err := r.Newest(module) + if err != nil { + return out, fmt.Errorf("the catalog half's newest entry of %s cannot be read: %w", module, err) + } + out.Newest = newest + named := false + for i := range rows { + for branch, fp := range newest { + if fp == rows[i].Fingerprint { + rows[i].Newest, named = append(rows[i].Newest, branch), true + } + } + slices.Sort(rows[i].Newest) + } + if !named && len(newest) > 0 { + var says []string + for branch, fp := range newest { + says = append(says, branch+" names "+fp) + } + slices.Sort(says) + out.Note = "no build key of " + module + " is the newest entry's: " + strings.Join(says, "; ") + } + } + out.Builds = append(out.Builds, rows...) + return out, nil +} + +// stepNode is the node a building key's step runs on, read from its asks' steps in the build modules' steps; empty when none runs. +func (r *BuildReader) stepNode(v builds.BuilderBuild) string { + if len(v.Asks) == 0 || len(r.Steps.Steps) == 0 { + return "" + } + if r.index == nil { + idx, err := r.Steps.stepIndex() + if err != nil { + return "" + } + r.index = idx + } + languages := make([]string, 0, len(v.Asks)) + for l := range v.Asks { + languages = append(languages, l) + } + slices.Sort(languages) + for _, l := range languages { + bm, _ := buildstep.BuildModuleOf(l) + st, ok := r.Steps.Steps[bm] + k, found := r.index[bm][v.Asks[l]] + if !ok || !found { + continue + } + if raw, _, err := st.Get(k); err == nil && raw != nil { + if step, err := buildstep.DecodeStep(raw); err == nil && step.Node != "" { + return step.Node + } + } + } + return "" +} + +func (r *BuildReader) read(key string) (json.RawMessage, uint64, time.Time, error) { + if r.Entry != nil { + return r.Entry(key) + } + raw, at, err := r.Builds.Get(key) + return raw, 0, at, err +} + +// openAsks are the asks kept in the state asked that nobody answered yet, by the key each is about. +func (r *BuildReader) openAsks() map[string]*OpenAsk { + out := map[string]*OpenAsk{} + if r.Asked.Keys == nil { + return out + } + keys, err := r.Asked.Keys() + if err != nil { + return out + } + for _, k := range keys { + raw, _, err := r.Asked.Get(k) + if err != nil || raw == nil { + continue + } + var kept occ.Kept + if json.Unmarshal(raw, &kept) != nil || kept.Ended != "" || kept.Acted || kept.Ask.About == "" { + continue + } + if t, err := occ.Time(kept.Ask.Expires); err == nil && !t.IsZero() && time.Now().After(t) { + continue + } + out[kept.Ask.About] = &OpenAsk{ID: kept.Ask.ID, Asked: kept.Asked.UTC().Format(time.RFC3339), Expires: kept.Ask.Expires} + } + return out +} diff --git a/modules/mesh-delivery-builder/internal/build/buildread_test.go b/modules/mesh-delivery-builder/internal/build/buildread_test.go new file mode 100644 index 00000000..92be0c20 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/buildread_test.go @@ -0,0 +1,83 @@ +package build + +import ( + "encoding/json" + "sort" + "strings" + "testing" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +const fp2 = "src2:2222222222222222222222222222222222222222222222222222222222222222" + +func stateOf(vals map[string][]byte) StateRead { + return StateRead{ + Keys: func() ([]string, error) { + var ks []string + for k := range vals { + ks = append(ks, k) + } + sort.Strings(ks) + return ks, nil + }, + Get: func(k string) (json.RawMessage, time.Time, error) { return vals[k], time.Time{}, nil }} +} + +// Design 47 G §589: builder_build by module answers its keys newest first with whole fingerprints, event-builds and the open +// ask, and names the watched branches whose newest entry names a key's fingerprint; a module none of whose keys the newest +// entry names says so; with a fingerprint, that key alone. +func TestBuilderBuildAnswersAModulesKeysWhole(t *testing.T) { + failed, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildFailed, EventBuilds: 3, Manifest: mdigest('a'), + Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"}) + done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a'), Artifacts: []builds.BuiltArtifact{{Name: "x", Kind: "file", Digest: "sha256:" + strings.Repeat("a", 64)}}}) + k1, _ := builds.BuilderBuildKey("m", fp1) + k2, _ := builds.BuilderBuildKey("m", fp2) + other, _ := builds.BuilderBuildKey("other", fp1) + vals := map[string][]byte{k1: failed, k2: done, other: done} + ask := occ.Kept{Ask: oc.Ask{ID: "retake-1", About: k1, Expires: occ.FormatTime(time.Now().Add(time.Hour))}, Asked: time.Now()} + askRaw, _ := json.Marshal(ask) + r := &BuildReader{Builds: stateOf(vals), Asked: stateOf(map[string][]byte{"ask.retake-1": askRaw}), + Newest: func(string) (map[string]string, error) { return map[string]string{"main": fp1, "feature": fp2}, nil }} + a, err := r.Build("m", "") + if err != nil || len(a.Builds) != 2 { + t.Fatalf("builds = %+v, %v, want the module's two keys", a, err) + } + var row BuildRow + for _, b := range a.Builds { + if b.Fingerprint == fp1 { + row = b + } + } + if row.Fingerprint != fp1 || len(row.Fingerprint) != len("src2:")+64 || row.State != builds.BuildFailed || row.EventBuilds != 3 || row.Node != "ace" || + !strings.HasPrefix(row.Error, "transport fault: connection, node-build-go ace, dial tcp") || row.Ask == nil || row.Ask.ID != "retake-1" || len(row.Newest) != 1 || row.Newest[0] != "main" { + t.Errorf("the failed key = %+v, want its fingerprint whole, its error whole, event-builds 3, the open ask and newest main", row) + } + if one, err := r.Build("m", fp2); err != nil || len(one.Builds) != 1 || one.Builds[0].Fingerprint != fp2 || one.Builds[0].Newest[0] != "feature" { + t.Errorf("one key = %+v, %v", one, err) + } + r.Newest = func(string) (map[string]string, error) { + return map[string]string{"main": "src2:" + strings.Repeat("c", 64)}, nil + } + if n, err := r.Build("m", ""); err != nil || !strings.Contains(n.Note, "no build key of m is the newest entry's") || !strings.Contains(n.Note, "main names src2:"+strings.Repeat("c", 64)) { + t.Errorf("a module none of whose keys the newest entry names: %+v, %v", n, err) + } +} + +// Design 47 G §589: builder_stuck answers each key's fingerprint whole, as the build-input fingerprint the catalog graph entry carries it. +func TestBuilderStuckAnswersTheFingerprintWhole(t *testing.T) { + building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, Step: "asked", Manifest: mdigest('a')}) + k, _ := builds.BuilderBuildKey("m", fp1) + out, err := StuckReader{Builds: stateOf(map[string][]byte{k: building})}.Stuck("m") + if err != nil { + t.Fatal(err) + } + rows := out["stuck"].([]Stuck) + if len(rows) != 1 || rows[0].FP != fp1 { + t.Fatalf("stuck = %+v, want the fingerprint %s whole", rows, fp1) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/claim.go b/modules/mesh-delivery-builder/internal/build/claim.go index ea041b46..5408e604 100644 --- a/modules/mesh-delivery-builder/internal/build/claim.go +++ b/modules/mesh-delivery-builder/internal/build/claim.go @@ -54,6 +54,9 @@ type ClaimOptions struct { // ByHand is a person's hand act (BuildAgain): a key failed on a transport fault is taken again only by it, or by a builder // that has just started (TakeOver), never by the builder's own retake (ADR 0664 §3). ByHand bool + // ByEvent is the builder's retake on an event that says a transport fault's cause passed (design 47 G §568): it takes a key + // failed on a transport fault, as a new revision that counts one more of EventBuilds. + ByEvent bool // ImageBuilder says whether a node runs a healthy image build module now: a key failed for NoImageBuilderPrefix is taken // again only then. Nil means none does. ImageBuilder func() bool @@ -213,11 +216,18 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string return nil, nil case cur.State == builds.BuildFailed && (builds.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)): return nil, nil - case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver: + case cur.State == builds.BuildFailed && builds.FailedOnTransportFault(cur.Error) && !opt.ByHand && !opt.TakeOver && !opt.ByEvent: return nil, nil case cur.State == builds.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()): return nil, nil case cur.State == builds.BuildFailed: + building.EventBuilds = cur.EventBuilds // a person's build starts the count again, an event's adds one (design 47 G §568) + switch { + case opt.ByHand: + building.EventBuilds = 0 + case opt.ByEvent: + building.EventBuilds++ + } return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) }) case cur.State == builds.BuildQueued && cur.Waits != "": if opt.Published == nil || !opt.Published(cur.Waits) { diff --git a/modules/mesh-delivery-builder/internal/build/claim_test.go b/modules/mesh-delivery-builder/internal/build/claim_test.go index eeda081f..c553addd 100644 --- a/modules/mesh-delivery-builder/internal/build/claim_test.go +++ b/modules/mesh-delivery-builder/internal/build/claim_test.go @@ -166,7 +166,7 @@ func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { take bool }{{ClaimOptions{}, false}, {ClaimOptions{ByHand: true}, true}, {ClaimOptions{TakeOver: true}, true}} { st = newMem() - seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "mesh-build-go on ace: a transport fault on 3 runs (ace, g14, novox): lookup proxy: i/o timeout"}) + seed(t, st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.a", Error: "transport fault: name lookup, node-build-go ace, lookup proxy: i/o timeout"}) if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", o.opt); err != nil || (got != nil) != o.take { t.Errorf("a transport fault on 3 runs with %+v: claim = %v, %v, want taken %v", o.opt, got, err, o.take) } diff --git a/modules/mesh-delivery-builder/internal/build/keyerror_test.go b/modules/mesh-delivery-builder/internal/build/keyerror_test.go new file mode 100644 index 00000000..583d93ac --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/keyerror_test.go @@ -0,0 +1,17 @@ +package build + +import ( + "strings" + "testing" +) + +// Design 47 G §568: a key's error opens `transport fault:` only when every lane failed on one. +func TestKeyErrorOpensTransportFaultOnlyWhenAllLanesDid(t *testing.T) { + tf := "transport fault: deadline, node-build-go ace, i/o timeout" + if got := keyError([]string{tf, "mesh-build-node on g14: the step failed: tsc: 2 errors"}); strings.HasPrefix(got, "transport fault:") { + t.Errorf("beside a real failure the key opens %q", got) + } + if got := keyError([]string{tf, tf}); !strings.HasPrefix(got, "transport fault:") { + t.Errorf("two transport faults open %q", got) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 26dd0fe5..3773ae2e 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -177,7 +177,7 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu for _, t := range plan.Tasks { refs = append(refs, module+":"+t.Artifact) } - src := versions.Source{Repository: m.Repository, Commit: m.Commit} + src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2} for _, t := range plan.Tasks { src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go") } diff --git a/modules/mesh-delivery-builder/internal/build/retake.go b/modules/mesh-delivery-builder/internal/build/retake.go new file mode 100644 index 00000000..22dd339c --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retake.go @@ -0,0 +1,226 @@ +package build + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "mesh-delivery-builder/internal/versions" +) + +// **A key that failed on a transport fault is built again on an event that says its cause passed** (design 47 G §568, ADR 0673 +// §1 to §4, which changes ADR 0664 §3): the node-engine seat's `running` of a module whose catalog graph entry claims the build +// seat the failed step ran on, on any node, and its `manifest-reads-changed` of the node the step ran on with `why` `returned`. +// Each message is acted on once, by its stream sequence (state `acted`). A key is built again once per event and only while its +// newest revision is that failure, as a new revision counting one more of EventBuilds; after MostEventBuilds the next event +// starts nothing and asks the operator once. + +// MostEventBuilds is how many revisions events start, since the last a person started, before the operator is asked. +const MostEventBuilds = 3 + +// Retake builds the failed key again on an event: it starts the look of the key's module at its fingerprint, which claims the +// key as an event's (ClaimOptions.ByEvent). started is false when the key is not failed on a transport fault, when the module +// has another fingerprint now, or when a look holds the key; ask is true when MostEventBuilds are spent, with the key as it +// stands and its revision. +func (b *Builder) Retake(ctx context.Context, key string) (started, ask bool, cur builds.BuilderBuild, rev uint64, err error) { + b.init() + module, fp, err := builds.ParseBuilderBuildKey(key) + if err != nil { + return false, false, cur, 0, err + } + value, rev, found, err := b.Claimer.Store.Get(ctx, key) + if err != nil || !found { + return false, false, cur, 0, err + } + if cur, err = builds.DecodeBuilderBuild(value); err != nil { + return false, false, cur, rev, fmt.Errorf("%s holds what is not a build: %w", key, err) + } + if cur.State != builds.BuildFailed || !builds.FailedOnTransportFault(cur.Error) { + return false, false, cur, rev, nil + } + if cur.EventBuilds >= MostEventBuilds { + return false, true, cur, rev, nil + } + b.mu.Lock() + m, known := b.known[module] + if !known || m.Fp2 != fp || m.Manifest == "" || b.inflight[key] || b.hands[key] != nil { + b.mu.Unlock() + return false, false, cur, rev, nil + } + b.inflight[key], b.events[key] = true, true + b.mu.Unlock() + b.logf("%s: failed (%s); built again on an event, build %d of %d since a person's", key, cur.Error, cur.EventBuilds+1, MostEventBuilds) + b.startOnce(ctx, key, module, m) + return true, false, cur, rev, nil +} + +// Retaker hears the node-engine seat's events and builds again what they say can pass (Builder.Retake). +type Retaker struct { + Builder *Builder + // Acted is the state acted: the last stream sequence acted on for each subject, and the revision the operator was asked at. + Acted versions.ReserveStore + // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). + Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) + // Ask asks the operator about a key whose event builds are spent (the seat operator-channel's ask). + Ask func(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error + Logf func(string, ...any) +} + +func (r *Retaker) logf(f string, a ...any) { + if r.Logf != nil { + r.Logf(f, a...) + } +} + +// retakeKey is the acted key of a subject's sequence, apart from the publish again's. +func retakeKey(subject string) string { return versions.ActedKey("retake:" + subject) } + +// askedKey is the acted key of the revision of a build key the operator was asked at. +func askedKey(key string) string { return versions.ActedKey("retake-asked:" + key) } + +// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq, and acts on +// it once. A subject that is neither is not its; an error leaves it not acted on, so it is read again. +func (r *Retaker) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) { + running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*")) + changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".") + if !running && !changed { + return false, nil + } + key := retakeKey(subject) + raw, rev, found, err := r.Acted.Get(ctx, key) + if err != nil { + return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err) + } + if found { + var a versions.Acted + if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq { + return true, nil + } + } + var want func(seat, node string) bool + if running { + want, err = r.runningWant(ctx, data) + } else { + want, err = r.returnedWant(data) + } + if err == nil && want != nil { + err = r.retakeAll(ctx, want) + } + if err != nil { + return true, err + } + val, _ := json.Marshal(versions.Acted{Sequence: seq}) + if found { + _, err = r.Acted.Update(ctx, key, val, rev) + } else { + _, err = r.Acted.Create(ctx, key, val) + } + if err != nil && !errors.Is(err, versions.ErrConflict) { + return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err) + } + return true, nil +} + +// runningWant is what a running says passed: the build seats the running module's catalog graph entry claims, on any node. +func (r *Retaker) runningWant(ctx context.Context, data []byte) (func(seat, node string) bool, error) { + var ev nodeengine.Running + if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" { + r.logf("a running that does not read is not acted on: %v", err) + return nil, nil + } + if ev.Stopped || ev.Fingerprint == "" { + return nil, nil + } + e, err := r.Entry(ctx, ev.Fingerprint) + if err != nil { + return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err) + } + var seats []string + for _, s := range e.Serves { + if seat, _, _ := strings.Cut(s, "."); strings.HasPrefix(seat, "node-build-") && !slices.Contains(seats, seat) { + seats = append(seats, seat) + } + } + if len(seats) == 0 { + return nil, nil + } + return func(seat, _ string) bool { return slices.Contains(seats, seat) }, nil +} + +// returnedWant is what a manifest-reads-changed says passed: only one with why returned, for the node it names. +func (r *Retaker) returnedWant(data []byte) (func(seat, node string) bool, error) { + var ev nodeengine.ManifestReadsChanged + if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" { + r.logf("a manifest-reads-changed that does not read is not acted on: %v", err) + return nil, nil + } + if ev.Why != nodeengine.ManifestReadsChangedWhyReturned { + return nil, nil + } + return func(_, node string) bool { return node == ev.Node }, nil +} + +// retakeAll builds again every key failed on a transport fault whose seat and node want says passed. +func (r *Retaker) retakeAll(ctx context.Context, want func(seat, node string) bool) error { + keys, err := r.Builder.Claimer.Store.Keys(ctx) + if err != nil { + return fmt.Errorf("the build keys cannot be listed: %w", err) + } + for _, key := range keys { + raw, _, found, err := r.Builder.Claimer.Store.Get(ctx, key) + if err != nil || !found { + continue + } + v, err := builds.DecodeBuilderBuild(raw) + if err != nil || v.State != builds.BuildFailed { + continue + } + _, seat, node, _, ok := builds.TransportFaultOf(v.Error) + if !ok || !want(seat, node) { + continue + } + started, ask, cur, rev, err := r.Builder.Retake(ctx, key) + if err != nil { + return err + } + if started || !ask || r.Ask == nil { + continue + } + if err := r.askOnce(ctx, key, cur, rev); err != nil { + return err + } + } + return nil +} + +// askOnce asks the operator about key once per revision of the key. +func (r *Retaker) askOnce(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error { + ak := askedKey(key) + raw, arev, found, err := r.Acted.Get(ctx, ak) + if err != nil { + return err + } + val := []byte(fmt.Sprint(rev)) + if found && string(raw) == string(val) { + return nil + } + if err := r.Ask(ctx, key, cur, rev); err != nil { + return fmt.Errorf("the operator cannot be asked about %s: %w", key, err) + } + if found { + _, err = r.Acted.Update(ctx, ak, val, arev) + } else { + _, err = r.Acted.Create(ctx, ak, val) + } + if err != nil && !errors.Is(err, versions.ErrConflict) { + return err + } + return nil +} diff --git a/modules/mesh-delivery-builder/internal/build/retake_test.go b/modules/mesh-delivery-builder/internal/build/retake_test.go new file mode 100644 index 00000000..c3b4ba0f --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retake_test.go @@ -0,0 +1,130 @@ +package build + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +type actedStore struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 +} + +func (s *actedStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.vals[k] + return v, s.revs[k], ok, nil +} +func (s *actedStore) Create(_ context.Context, k string, v []byte) (uint64, error) { + return s.Update(context.Background(), k, v, 0) +} +func (s *actedStore) Update(_ context.Context, k string, v []byte, _ uint64) (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.vals == nil { + s.vals, s.revs = map[string][]byte{}, map[string]uint64{} + } + s.vals[k], s.revs[k] = v, s.revs[k]+1 + return s.revs[k], nil +} +func (s *actedStore) KeysOf(context.Context, string) ([]string, error) { return nil, nil } +func (s *actedStore) Delete(_ context.Context, k string) error { delete(s.vals, k); return nil } + +// Design 47 G §568, ADR 0673: a key failed on a transport fault is built again on the node-engine seat's events that say its +// cause passed, once per event, counted in event-builds; after three the next event asks the operator once. +func TestRetakeOnAnEvent(t *testing.T) { + r := newRig(t, goBundle) + // The first look creates the key and its note meets the fault; a retake's claim is an update that succeeds and the note after + // it is the second. + fault := fmt.Errorf("the key: %w", context.DeadlineExceeded) + failOnce := func() { r.st.before = func() { r.st.before = func() { r.st.faults = []error{fault} } } } + r.st.faults = []error{fault} + if v := r.consider(t, "modules/m"); !builds.FailedOnTransportFault(v.Error) { + t.Fatalf("the key = %+v, want failed on a transport fault", v) + } + var asked []string + rt := &Retaker{Builder: r.b, Acted: &actedStore{}, + Entry: func(context.Context, string) (meshcatalog.CatalogEntry, error) { + return meshcatalog.CatalogEntry{Serves: []string{"node-build-go.steps"}}, nil + }, + Ask: func(_ context.Context, key string, _ builds.BuilderBuild, _ uint64) error { + asked = append(asked, key) + return nil + }} + running, _ := json.Marshal(nodeengine.Running{Module: "mesh-build-go", Node: "g14", Fingerprint: "f1"}) + started, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "ace", Why: "started"}) + heard := func(seq uint64, subject string, data []byte) { + t.Helper() + if ok, err := rt.Heard(context.Background(), subject, data, seq); !ok || err != nil { + t.Fatalf("heard %s: %v, %v", subject, ok, err) + } + r.b.Wait() + } + runningSubject := "mesh.seat.node-engine.event.running.g14.mesh-build-go" + readsSubject := "mesh.seat.node-engine.event.manifest-reads-changed.ace" + // A `started` builds nothing, and the same message again builds nothing. + heard(1, readsSubject, started) + if v := valueOf(t, r.st, r.key); v.EventBuilds != 0 || v.State != builds.BuildFailed { + t.Fatalf("a started: the key = %+v", v) + } + for i, ev := range []struct { + seq uint64 + subject string + data []byte + }{{2, runningSubject, running}, {3, runningSubject + ".2", running}, {4, runningSubject + ".3", running}} { + failOnce() + heard(ev.seq, ev.subject, ev.data) + if v := valueOf(t, r.st, r.key); v.EventBuilds != i+1 || !builds.FailedOnTransportFault(v.Error) { + t.Fatalf("event %d: the key = %+v, want event-builds %d failed on a transport fault", i+1, v, i+1) + } + } + heard(4, runningSubject+".3", running) // a sequence heard is not acted on again + if v := valueOf(t, r.st, r.key); v.EventBuilds != 3 || len(asked) != 0 { + t.Fatalf("a sequence heard again: the key = %+v, asked %v", v, asked) + } + heard(5, runningSubject, running) + heard(6, runningSubject, running) + if v := valueOf(t, r.st, r.key); v.EventBuilds != 3 || len(asked) != 1 || asked[0] != r.key { + t.Fatalf("the fourth event: the key = %+v, asked %v, want nothing built and the operator asked once", v, asked) + } + // A person's build starts the count again. + if res := r.b.BuildAgain(context.Background(), "m", fp1, "Build once more", "operator", 2e9); res.Refused != "" { + t.Fatalf("built again by hand: %+v", res) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.EventBuilds != 0 { + t.Fatalf("after a person's build: the key = %+v, want event-builds 0", v) + } +} + +// A manifest-reads-changed with why returned builds again the keys whose step ran on its node, and no other's. +func TestRetakeOnANodeReturned(t *testing.T) { + r := newRig(t, goBundle) + r.consider(t, "modules/m") + seed(t, r.st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.builder", Manifest: mdigest('a'), + Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"}) + rt := &Retaker{Builder: r.b, Acted: &actedStore{}} + for seq, c := range []struct { + node, why string + built bool + }{{"g14", "returned", false}, {"ace", "started", false}, {"ace", "returned", true}} { + data, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: c.node, Why: c.why}) + if _, err := rt.Heard(context.Background(), "mesh.seat.node-engine.event.manifest-reads-changed."+c.node, data, uint64(seq+1)); err != nil { + t.Fatal(err) + } + r.b.Wait() + if v := valueOf(t, r.st, r.key); (v.State == builds.BuildDone) != c.built { + t.Fatalf("%s %s: the key = %+v, want built %v", c.node, c.why, v, c.built) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/retakeask.go b/modules/mesh-delivery-builder/internal/build/retakeask.go new file mode 100644 index 00000000..c59f7bab --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retakeask.go @@ -0,0 +1,162 @@ +package build + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "mesh-delivery-builder/internal/versions" +) + +// **The operator is asked once a key's event builds are spent** (design 47 G §568, ADR 0673 §3): the seat operator-channel's ask, +// level approve, with the options *Build once more* and *Leave it*, `about` the key so a newer ask replaces it (ADR 0666 §12), +// naming the module, the fingerprint and the last error whole. *Build once more* builds one revision as a person's build +// (BuildAgain, which starts the count again); *Leave it*, a decline or the ask's expiry leaves the key failed. + +// The ids of the ask's options. +const ( + OptionBuildOnceMore = "build-once-more" + OptionLeaveIt = "leave-it" +) + +// AskLasts is how long the ask stays open: the longest an ask that approves may (the seat's own bound). +const AskLasts = occ.ApproveLasts + +// askBuildBound is how long the builder waits for the claim and the asks of the build *Build once more* starts. +const askBuildBound = 15 * time.Second + +// OperatorAsk asks the operator about a key and carries out the answer. +type OperatorAsk struct { + // Client is the builder's asker on the seat operator-channel: its ask published under the builder's own name, its state + // `asked` (the asks it kept and the warrants it acted on), the router's record, the `decided` event. + Client *occ.Client + // Acted keeps which key an ask was made for, beside the revision of the key it was made at (Retaker.Acted). + Acted versions.ReserveStore + Builder *Builder + // Ctx is the builder's own context: the build a person's answer starts lives on it. + Ctx context.Context + Now func() time.Time + Logf func(string, ...any) +} + +func (o *OperatorAsk) logf(f string, a ...any) { + if o.Logf != nil { + o.Logf(f, a...) + } +} + +func (o *OperatorAsk) now() time.Time { + if o.Now != nil { + return o.Now() + } + return time.Now() +} + +// AskID is the id of the ask about one revision of a key: the same revision is the same ask, a newer one a new ask. +func AskID(key string, rev uint64) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s@%d", key, rev))) + return "retake-" + hex.EncodeToString(sum[:16]) +} + +// askedOf is the acted key of the key an ask was made for. +func askedOf(id string) string { return versions.ActedKey("retake-ask:" + id) } + +// BuildOnceMoreAct is what *Build once more* binds: the verb and the key's module and fingerprint. +func BuildOnceMoreAct(module, fingerprint string) occ.Act { + return occ.Act{"verb": "build-again", "module": module, "fingerprint": fingerprint} +} + +// Compose is the ask about key as it stands, at revision rev. +func (o *OperatorAsk) Compose(key string, cur builds.BuilderBuild, rev uint64) (oc.Ask, error) { + module, fp, err := builds.ParseBuilderBuildKey(key) + if err != nil { + return oc.Ask{}, err + } + binds, err := occ.ActDigest(BuildOnceMoreAct(module, fp)) + if err != nil { + return oc.Ask{}, err + } + headline := "Build " + module + " once more?" + if r := []rune(headline); len(r) > occ.HeadlineLength { + headline = string(r[:occ.HeadlineLength-1]) + "…" + } + last := []rune(cur.Error) + if len(last) > 120 { + last = append(last[:119], '…') + } + return oc.Ask{ID: AskID(key, rev), Headline: headline, Who: occ.Operator, About: key, + Explanation: fmt.Sprintf("The build of %s failed on a transport fault, and events built it again %d times without it passing.\nLast error: %s", module, cur.EventBuilds, string(last)), + Details: fmt.Sprintf("Key %s\nFingerprint %s\nLast error whole: %s", key, fp, cur.Error), + Expires: occ.FormatTime(o.now().Add(AskLasts)), + OnExpiry: "the key stays failed until builder_build_again or a new fingerprint", + Options: []oc.Option{ + {ID: OptionBuildOnceMore, Label: "Build once more", Does: "the builder builds this fingerprint once more, and counts the build as a person's", Level: occ.Approve, Binds: binds}, + {ID: OptionLeaveIt, Label: "Leave it", Does: "nothing: the key stays failed", Level: occ.Acknowledge}, + }}, nil +} + +// Ask asks the operator about key at revision rev. The same revision asked twice is one ask. +func (o *OperatorAsk) Ask(ctx context.Context, key string, cur builds.BuilderBuild, rev uint64) error { + a, err := o.Compose(key, cur, rev) + if err != nil { + return err + } + k := askedOf(a.ID) + if _, _, found, err := o.Acted.Get(ctx, k); err != nil { + return err + } else if !found { + if _, err := o.Acted.Create(ctx, k, []byte(key)); err != nil && !errors.Is(err, versions.ErrConflict) { + return err + } + } + // An ask published and not marked asked (the write after it failed) is read again: it is one ask. + if e, err := o.Client.State.Get("ask." + a.ID); err != nil { + return err + } else if e != nil { + return nil + } + return o.Client.Ask(a) +} + +// Listen hears the operator's answers: each warrant of this builder's asks is acted on once. +func (o *OperatorAsk) Listen() error { + return o.Client.Listen(o.act, func(h occ.Handled, w oc.Warrant, err error) { + if err != nil { + o.logf("the answer to the ask %s: %s: %v", w.Ask, h, err) + return + } + o.logf("the answer to the ask %s: %s", w.Ask, h) + }) +} + +// act carries out the option the operator chose: Build once more builds the key as a person's build, Leave it nothing. +func (o *OperatorAsk) act(opt oc.Option, w oc.Warrant) error { + if opt.ID != OptionBuildOnceMore { + return nil + } + raw, _, found, err := o.Acted.Get(o.Ctx, askedOf(w.Ask)) + if err != nil { + return err + } + if !found { + return fmt.Errorf("the builder holds no key the ask %s was made for", w.Ask) + } + module, fp, err := builds.ParseBuilderBuildKey(string(raw)) + if err != nil { + return err + } + o.logf("%s: the operator chose %s; built again as a person's build", raw, opt.Label) + go func() { + if res := o.Builder.BuildAgain(o.Ctx, module, fp, "the operator chose "+opt.Label, "operator-channel", askBuildBound); res.Refused != "" { + o.logf("%s: not built again: %s", raw, res.Refused) + } + }() + return nil +} diff --git a/modules/mesh-delivery-builder/internal/build/retakeask_test.go b/modules/mesh-delivery-builder/internal/build/retakeask_test.go new file mode 100644 index 00000000..36a654d8 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/retakeask_test.go @@ -0,0 +1,113 @@ +package build + +import ( + "context" + "encoding/json" + "testing" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" +) + +// askedState is the client's state `asked`. +type askedState struct{ vals map[string][]byte } + +func (s *askedState) Get(key string) (*stdio.StateEntry, error) { + if v, ok := s.vals[key]; ok { + return &stdio.StateEntry{Key: key, Value: v, Revision: 1}, nil + } + return nil, nil +} +func (s *askedState) Create(key string, value any) (uint64, error) { + if _, ok := s.vals[key]; ok { + return 0, stdio.ErrChanged + } + return s.Update(key, value, 0) +} +func (s *askedState) Update(key string, value any, _ uint64) (uint64, error) { + if s.vals == nil { + s.vals = map[string][]byte{} + } + s.vals[key], _ = json.Marshal(value) + return 1, nil +} + +// Design 47 G §568: after three event builds the operator is asked, through the seat operator-channel's ask (level approve, +// *Build once more* and *Leave it*, about the key), and *Build once more* builds the key as a person's build, *Leave it* does not. +func TestTheOperatorIsAskedAndAnswers(t *testing.T) { + for _, option := range []string{OptionLeaveIt, OptionBuildOnceMore} { + t.Run(option, func(t *testing.T) { theOperatorAnswers(t, option) }) + } +} + +func theOperatorAnswers(t *testing.T, option string) { + r := newRig(t, goBundle) + r.consider(t, "modules/m") + seed(t, r.st, builds.BuilderBuild{State: builds.BuildFailed, By: "novox.builder", Manifest: mdigest('a'), EventBuilds: 3, + Error: "transport fault: connection, node-build-go ace, dial tcp: connection refused"}) + var published []oc.Ask + acted := &actedStore{} + asker := &OperatorAsk{Builder: r.b, Acted: acted, Ctx: context.Background(), Now: func() time.Time { return time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) }, + Client: &occ.Client{Module: "mesh-delivery-builder", State: &askedState{}, + Publish: func(subject string, body any, id string) error { + if subject != oc.AcceptSubjectAsk("mesh-delivery-builder") { + t.Errorf("published on %s", subject) + } + published = append(published, body.(oc.Ask)) + return nil + }}} + rt := &Retaker{Builder: r.b, Acted: acted, Ask: asker.Ask, + Entry: nil} + data, _ := json.Marshal(map[string]string{"node": "ace", "why": "returned"}) + for seq := uint64(1); seq <= 2; seq++ { // the same key and revision asked at two events is one ask + if _, err := rt.Heard(context.Background(), "mesh.seat.node-engine.event.manifest-reads-changed.ace", data, seq); err != nil { + t.Fatal(err) + } + r.b.Wait() + } + if len(published) != 1 { + t.Fatalf("asked %d times, want once", len(published)) + } + a := published[0] + if a.About != r.key || len(a.Options) != 2 || a.Options[0].Level != occ.Approve || a.Options[0].Label != "Build once more" || a.Options[1].Label != "Leave it" { + t.Fatalf("the ask = %+v, want about the key with the options Build once more (approve) and Leave it", a) + } + if err := occ.Check(a, time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)); err != nil { + t.Fatalf("the ask does not hold to the seat's checks: %v", err) + } + if err := occ.Glance(a.Explanation); err != nil { + t.Fatal(err) + } + warrant := func(option string) oc.Warrant { + return oc.Warrant{Ask: a.ID, AskDigest: occ.Digest(a), Asker: "mesh-delivery-builder", Outcome: occ.OutcomeChosen, Option: option, Level: levelOf(a, option), + At: occ.FormatTime(time.Date(2026, 11, 11, 12, 5, 0, 0, time.UTC)), By: &oc.Person{Who: occ.Operator, Kind: "telegram", Identity: "op", Verified: "user"}} + } + h, err := asker.Client.Take(warrant(option), asker.act) + if err != nil || h != occ.Acted { + t.Fatalf("%s: %v, %v", option, h, err) + } + for i := 0; i < 50 && option == OptionBuildOnceMore; i++ { // the build the answer starts runs on its own goroutine + r.b.Wait() + if valueOf(t, r.st, r.key).State == builds.BuildDone { + break + } + time.Sleep(time.Millisecond) + } + r.b.Wait() + v := valueOf(t, r.st, r.key) + if option == OptionLeaveIt && (v.State != builds.BuildFailed || v.EventBuilds != 3) { + t.Fatalf("after Leave it the key = %+v, want failed as it was", v) + } + if option == OptionBuildOnceMore && (v.State != builds.BuildDone || v.EventBuilds != 0) { + t.Fatalf("after Build once more the key = %+v, want built as a person's build (event-builds 0)", v) + } +} + +func levelOf(a oc.Ask, option string) string { + o, _ := occ.OptionOf(a, option) + return o.Level +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index ff63d787..cc10b0d1 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -132,6 +132,8 @@ type Builder struct { waiting map[string]map[string]chan string // hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key. hands map[string]*handWatch + // events are the keys an event asks the builder to build again (Retake), until their look claims them. + events map[string]bool // released are the keys builder_release is building now, whatever toolchain their fingerprint names (§276, §672). released map[string]bool // builds are the ends that withdraw the asks of a key's build in flight, by build key (ADR 0664 §4): a newer fingerprint of @@ -162,7 +164,7 @@ func (b *Builder) init() { b.cands, b.via, b.trunkFP = map[string]meshcatalog.CandidatesByGroupValue{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]meshcatalog.ModulesByModuleValue{}, map[string]bool{}, map[string]meshcatalog.ModulesByModuleValue{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} - b.hands = map[string]*handWatch{} + b.hands, b.events = map[string]*handWatch{}, map[string]bool{} b.builds = map[string]context.CancelCauseFunc{} }) } @@ -493,6 +495,8 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB first := !b.tried[key] b.tried[key] = true byHand := b.hands[key] != nil + byEvent := b.events[key] + delete(b.events, key) b.mu.Unlock() var numbering versions.Numbering if trunk && planErr == nil && !nothing { @@ -507,7 +511,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } // A key built again by hand is taken over as it stands too: one left building by a look that ended without its write // (issue 1966) is the hand tool's to finish. - cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, + cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, Stored: b.storedOnce(key)}) if err != nil { b.logf("%s: %v", module, err) @@ -576,7 +580,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB b.logf("%s: left building, the toolchains moved under it", cl.Key) return } - result.By, result.Manifest = b.Claimer.By, m.Manifest + result.By, result.Manifest, result.EventBuilds = b.Claimer.By, m.Manifest, cl.Build.EventBuilds if len(cl.Gone) > 0 && result.State == builds.BuildDone { result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")" } @@ -855,6 +859,12 @@ var keyPrefixes = []string{builds.NotPinned, builds.VersionTaken, builds.Refused // (` on : the step failed: not stored: ...`), the key's error opens with that prefix, so a reader of the // key need not search for it. func keyError(errs []string) string { + // A transport fault is the key's own words only when every lane failed on one: beside a real failure it is said after it, + // so the key is not built again on an event for a failure the source made (design 47 G §568). + errs = slices.Clone(errs) + slices.SortStableFunc(errs, func(a, b string) int { + return boolInt(strings.HasPrefix(a, builds.TransportFaultPrefix)) - boolInt(strings.HasPrefix(b, builds.TransportFaultPrefix)) + }) for _, prefix := range keyPrefixes { for i, e := range errs { if strings.HasPrefix(e, prefix) { @@ -991,11 +1001,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil { // The note and the put are one act (novox/hq ADR 0668 §1; issue 1748 note 27): a note a transport fault kept from // the key fails the key with that fault's words, which builder_build_again takes (ADR 0664 §3). - fault := "" + l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err) if busretry.TimedOut(err) || busretry.Undelivered(err) { - fault = sdkstep.TransportFaultWords + "the note of the ask: " + seat, _ := sdkstep.BuildSeat(language) + l.err = errors.New(buildrun.TransportFault(buildrun.FaultClass(err.Error()), seat, "no node", l.err.Error())) } - l.err = fmt.Errorf("%sthe ask of %s could not be written to the key: %w", fault, l.module, err) return } if err := b.Asks.Put(ctx, ask); err != nil { @@ -1017,6 +1027,9 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes } if step.State != sdkstep.StepPassed { l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason) + if strings.HasPrefix(step.Reason, builds.TransportFaultPrefix) { + l.err = errors.New(step.Reason) // a key's error opens with it (design 47 G §568) + } return } var answer sdkstep.BuildAnswer diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 1c471d37..3955c800 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -270,12 +270,19 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At}) } page.Next = 3 - since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At) + since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, "") if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != builds.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More { t.Errorf("builds since = %+v, %v, want the one end and more", since, err) } + // With module (design 47 G §589) the builds of that module alone. + if other, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, "another"); err != nil || len(other.Builds) != 0 { + t.Errorf("builds since of another module = %+v, %v, want none", other, err) + } + if mine, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, logged[0].Module); err != nil || len(mine.Builds) != 1 { + t.Errorf("builds since of the module = %+v, %v, want its one end", mine, err) + } page.Entries[0].Value = []byte(`{"v":0}`) - if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil { + if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At, ""); err == nil { t.Error("an entry that does not read was skipped, want it refused") } if v.State != builds.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { @@ -1328,6 +1335,27 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { if maxFlight.Load() < 2 { t.Errorf("the values were read one at a time") } + // Design 47 G §484: a step key neither form parses is written in the journal naming it. + var said []string + steps := StateRead{Keys: func() ([]string, error) { + return []string{"ace.mvl0frv18d1957", "builder.ace.mvl0frv18d1958", "a.b.c.d"}, nil + }} + if _, err := (StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{"mesh-build-go": steps}, Now: func() time.Time { return at }, + Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}).Stuck("mod10"); err != nil { + t.Fatal(err) + } + n := 0 + for _, l := range said { + if strings.Contains(l, `"a.b.c.d"`) { + n++ + } + if strings.Contains(l, "ace.mvl0frv18d1957") || strings.Contains(l, "builder.ace.mvl0frv18d1958") { + t.Errorf("a key of one of the two forms is said not to parse: %s", l) + } + } + if n != 1 { + t.Errorf("the key of neither form is named %d times in %q", n, said) + } }) } diff --git a/modules/mesh-delivery-builder/internal/build/since.go b/modules/mesh-delivery-builder/internal/build/since.go index 8875b37d..d459a9b8 100644 --- a/modules/mesh-delivery-builder/internal/build/since.go +++ b/modules/mesh-delivery-builder/internal/build/since.go @@ -2,6 +2,7 @@ package build import ( "fmt" + "slices" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -16,9 +17,9 @@ type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error) // BuildsSince answers builds.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on, // oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again -// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that +// from the last build's time. With module (design 47 G §589) only that module's builds are answered. An entry the log holds that does not read is refused, naming its sequence: a reader that // skipped it would register too little. -func BuildsSince(read LogSince, since time.Time) (builds.BuildsSince, error) { +func BuildsSince(read LogSince, since time.Time, module string) (builds.BuildsSince, error) { out := builds.BuildsSince{Since: since.UTC(), Builds: []builds.BuildLogEntry{}} page, err := read(since, "", BuildsSinceLimit) if err != nil { @@ -38,6 +39,9 @@ func BuildsSince(read LogSince, since time.Time) (builds.BuildsSince, error) { if ended := builds.EndedSince(entries, since); ended != nil { out.Builds = ended } + if module != "" { + out.Builds = slices.DeleteFunc(out.Builds, func(e builds.BuildLogEntry) bool { return e.Module != module }) + } out.More = page.Next != 0 return out, nil } diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 04f3fd1a..5a6666b9 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -57,35 +57,32 @@ type ReAsk struct { Failed string `json:"failed,omitempty"` Revision uint64 `json:"revision,omitempty"` Asks map[string]string `json:"asks,omitempty"` - // Name is the numbered name (`/`) of a ModePublishAgain or ModeFloorAgain value; Do and Version the - // builder's answer: what it did (versions.Again*) and the version published. - Name string `json:"name,omitempty"` - Do string `json:"do,omitempty"` - Version string `json:"version,omitempty"` + // Name is the numbered name (`/`) of a ModeFloorAgain value; Version is a ModeRepublish value's + // request and Queued the builder's answer to it: the versions it queued, ` `. + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Queued []string `json:"queued,omitempty"` } -// ModePublishAgain is the Mode of a reasks value of builder_publish_again (design 51 *Tools of the builder*). -const ModePublishAgain = "publish-again" +// ModeRepublish is the Mode of a reasks value of builder_publish_again (design 47 G §588): Module and Version, each +// optional, name what to publish again; the builder answers with the versions it queued (Queued). +const ModeRepublish = "republish" // ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*). const ModeFloorAgain = "floor-again" -// NewPublishAgain is the value builder_publish_again writes for a name's fingerprint: the key of the name's module and -// fingerprint, and the value. -func NewPublishAgain(name, fp, why, by string, at time.Time) (string, []byte, error) { - module, artifact, ok := strings.Cut(strings.TrimSpace(name), "/") - if !ok || module == "" || artifact == "" { - return "", nil, fmt.Errorf("name %q is not a numbered name, /, as builder_versions lists it", name) - } - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return "", nil, err +// NewRepublish is the value builder_publish_again writes (design 47 G §588): its key, `republish.` +// (a call is one value, never a build key), and the value. Module and version are each optional. +func NewRepublish(module, version, why, by string, at time.Time) (string, []byte, error) { + module, version = strings.TrimSpace(module), strings.TrimSpace(version) + if version != "" && module == "" { + return "", nil, errors.New("a version is published again under its module: give module with version") } if strings.TrimSpace(why) == "" { return "", nil, errors.New("say why it is published again (why)") } - data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModePublishAgain, Module: module, FP: fp, Name: strings.TrimSpace(name)}) - return key, data, err + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeRepublish, Module: module, Version: version}) + return "republish." + strconv.FormatInt(at.UnixNano(), 10), data, err } // NewFloorAgain is the value builder_floor_again writes for a name: the key `floor.`, never a build key, @@ -316,20 +313,9 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { if r.Now != nil { now = r.Now } - // Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it. - stepKeys := map[string]map[string]string{} - for bm, s := range r.Steps { - ks, err := s.Keys() - if err != nil { - return nil, fmt.Errorf("the steps of %s: %w", bm, err) - } - byID := make(map[string]string, len(ks)) - for _, k := range ks { - if _, _, id, err := buildstep.ParseStepKey(k); err == nil { - byID[id] = k - } - } - stepKeys[bm] = byID + stepKeys, err := r.stepIndex() + if err != nil { + return nil, err } // Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in // parallel, at most MaxReads of them. @@ -426,6 +412,32 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) { return answer, nil } +// stepIndex finds the build modules' steps by the ask's id, which their key ends in: one index per build module, no value read +// until an ask names it. +func (r StuckReader) stepIndex() (map[string]map[string]string, error) { + stepKeys := map[string]map[string]string{} + for bm, s := range r.Steps { + ks, err := s.Keys() + if err != nil { + return nil, fmt.Errorf("the steps of %s: %w", bm, err) + } + byID := make(map[string]string, len(ks)) + for _, k := range ks { + _, _, id, err := buildstep.ParseStepKey(k) + if err != nil { + // A key neither form parses is named, never passed over in silence (design 47 G §484). + if r.Logf != nil { + r.Logf("builder_stuck: the key %q of the steps of %s is a step key of neither form, .. or .: %v", k, bm, err) + } + continue + } + byID[id] = k + } + stepKeys[bm] = byID + } + return stepKeys, nil +} + // StuckBuildModules are the build modules whose steps builder_stuck reads. var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule} diff --git a/modules/mesh-delivery-builder/internal/versions/again.go b/modules/mesh-delivery-builder/internal/versions/again.go index 52a3decf..709007be 100644 --- a/modules/mesh-delivery-builder/internal/versions/again.go +++ b/modules/mesh-delivery-builder/internal/versions/again.go @@ -2,63 +2,13 @@ package versions import ( "context" - "errors" "fmt" "strings" "time" ) -// What builder_publish_again does with one fingerprint of a name (design 51 *Tools of the builder*), after a holder failed. -const ( - // AgainPublish is a held build whose recorded version is published again: refused since design 47 G §349 (no version.numbered is said). - AgainPublish = "publish" - // AgainBuild builds the fingerprint again under the same number: the build's file is gone and no published was said - // for the number, so the new digest replaces the recorded one (Numberer.Built). - AgainBuild = "build" - // AgainRenumber takes a new number: a published was said for the number, so other bytes never go under it. - AgainRenumber = "renumber" -) - -// Again is what builder_publish_again does, with the entry it acts on and why. -type Again struct { - Do string `json:"do"` - Entry Entry `json:"entry"` - Why string `json:"why"` -} - -// PublishAgain decides builder_publish_again for fingerprint among a name's entries; built says whether the builder still -// holds the build's file (its done key). It refuses a fingerprint not numbered, and a version refused for other bytes -// than the ones recorded under it. -func PublishAgain(entries []Entry, fingerprint string, built bool) (Again, error) { - fingerprint = strings.TrimSpace(fingerprint) - if fingerprint == "" { - return Again{}, errors.New("fp is required: the fingerprint whose version is published again") - } - for _, e := range latestFirst(entries) { - if e.Tag != "" || e.Fingerprint != fingerprint { - continue - } - switch { - case e.State == Refused && e.RefusedDigest != "" && e.Digest != "" && e.RefusedDigest != e.Digest: - return Again{}, fmt.Errorf("%s@%s was refused for other bytes (%s, recorded %s): it is not published again; a new build takes a new number", - e.Name, e.Version, e.RefusedDigest, e.Digest) - case e.State == Published || e.State == Removed: - return Again{Do: AgainRenumber, Entry: e, Why: fmt.Sprintf("%s@%s was %s, so the fingerprint takes a new number", e.Name, e.Version, e.State)}, nil - case !built: - return Again{Do: AgainBuild, Entry: e, Why: fmt.Sprintf("the build's file is gone and no published was said for %s@%s: built again under the same number", e.Name, e.Version)}, nil - } - return Again{Do: AgainPublish, Entry: e, Why: fmt.Sprintf("%s@%s (%s) is held", e.Name, e.Version, e.State)}, nil - } - return Again{}, fmt.Errorf("fingerprint %s is not numbered for this name: builder_versions lists what is", fingerprint) -} - // The tools' descriptions. const ( - PublishAgainDescription = "Publish one fingerprint's version again after its holder failed (design 51): with the build's file held, the " + - "nothing says a version numbered since design 47 G §349, so a held build is refused; with the file gone, it builds the fingerprint " + - "again under the same number, the new digest replacing the recorded one, since no published was said for it; after a published (or " + - "a removal) it takes a new number. It refuses a fingerprint not numbered and a version the holder refused for other bytes. A hand " + - "act, recorded with its caller and why in the builder's log." FloorAgainDescription = "Ask the floor of one name again now, by hand (design 51 *The floor*): the seat's versions, the forge bridge's " + "bridge_tags and the indexer's indexer_requirements, as the builder asks them by itself when a silent source says something. " + "Answers which sources were silent and whether the floor is now known. A name the builder's versions already hold needs no floor: " + @@ -68,56 +18,16 @@ const ( // Acts are the builder's hand acts on its versions, answered by its program (the tools bundle writes them to reasks). type Acts struct { Numberer *Numberer - // Done says whether the build key of module and fingerprint is done: the build's file is held. - Done func(ctx context.Context, module, fp string) (bool, error) - // BuildAgain builds a failed key again (builder_build_again's path): its answer, or why it refused. - BuildAgain func(ctx context.Context, module, fp, why, by string) (answer string, refused bool) // Reconsider looks at every module again, so a claim asks a name's floor. Reconsider func(ctx context.Context) + // Republish queues the reserved versions builder_publish_again names (design 47 G §588): the versions it queued. + Republish func(ctx context.Context, module, version string) ([]string, error) // DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not. DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error) // DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not. DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error) } -// PublishAgain is builder_publish_again of name's fingerprint: what it did, the version, and its answer; an error refuses. -func (a Acts) PublishAgain(ctx context.Context, name, fp, why, by string, at time.Time) (Again, string, error) { - module, _, _ := strings.Cut(name, "/") - entries, err := a.Numberer.Versions.Of(ctx, name) - if err != nil { - return Again{}, "", err - } - done, err := a.Done(ctx, module, fp) - if err != nil { - return Again{}, "", fmt.Errorf("the build key of %s at %s cannot be read: %w", module, fp, err) - } - ag, err := PublishAgain(entries, fp, done) - if err != nil { - return Again{}, "", err - } - if ag.Do == AgainPublish || (ag.Do == AgainRenumber && done) { - return Again{}, "", errors.New("nothing says a version numbered since design 47 G §349: the version is published under its reservation (§369)") - } - e := ag.Entry - if ag.Do == AgainRenumber { - _, ne, err := a.Numberer.Versions.Number(ctx, Claim{Name: e.Name, Kind: e.Kind, Series: e.Series, Module: e.Module, Fingerprint: e.Fingerprint, - Commit: e.Commit, Revision: e.Revision, Seat: e.Seat, Merge: e.Merge, Fresh: true}) - if err != nil { - return Again{}, "", err - } - ag.Entry = ne - } - answer := ag.Why - if ag.Do == AgainBuild || (ag.Do == AgainRenumber && !done) { - said, refused := a.BuildAgain(ctx, module, fp, why, by) - if refused { - return Again{}, "", fmt.Errorf("%s; and the build again was refused: %s", ag.Why, said) - } - answer += "; " + said - } - return ag, answer, nil -} - // FloorAgain is builder_floor_again of name: the floor counts only for a name the versions hold nothing of. func (a Acts) FloorAgain(ctx context.Context, name string) (string, error) { entries, err := a.Numberer.Versions.Of(ctx, name) diff --git a/modules/mesh-delivery-builder/internal/versions/publish.go b/modules/mesh-delivery-builder/internal/versions/publish.go index 7b401b8a..43457b3b 100644 --- a/modules/mesh-delivery-builder/internal/versions/publish.go +++ b/modules/mesh-delivery-builder/internal/versions/publish.go @@ -37,6 +37,7 @@ const ( publishBoundPrefix = "publish." oldFormBoundPrefix = "old-form." unputBoundPrefix = "unput." + stepsBoundPrefix = "steps." ) // UnputBoundKey is the bound of a build key a look left building because its end could not be written (issue 1966, @@ -44,6 +45,21 @@ const ( // finding call-unanswered about the key. func UnputBoundKey(buildKey string) string { return unputBoundPrefix + buildKey } +// StepsBoundKey is the bound of the builder's read of an ask's steps while the bus does not answer it (design 47 G §568, A-77): +// `steps..`. Its end, the bus's marker of its time to live, while the read is unanswered, is the finding +// call-unanswered about the ask. +func StepsBoundKey(module, askID string) string { return stepsBoundPrefix + module + "." + askID } + +// StepsBoundOf is the build module and ask a steps bound names, and whether key is one. +func StepsBoundOf(key string) (module, askID string, ok bool) { + rest, ok := strings.CutPrefix(key, stepsBoundPrefix) + if !ok { + return "", "", false + } + module, askID, ok = strings.Cut(rest, ".") + return module, askID, ok && module != "" && askID != "" +} + // UnputBoundOf is the build key an unput bound names, and whether key is one. func UnputBoundOf(key string) (string, bool) { return strings.CutPrefix(key, unputBoundPrefix) } diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go new file mode 100644 index 00000000..07dc30cc --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -0,0 +1,646 @@ +package versions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// **A reserved version left unpublished is published again by the builder, on an event or by a person's tool** (design 47 +// G §588, ADR 0674): on the node-engine seat's `running` of a module whose catalog graph entry claims the registry seat +// of the missing artifacts' kind, and on its `manifest-reads-changed` with `why` `returned` of the node such a holder +// runs on. Each message is acted on once, by its stream sequence (state `acted`); each holder has one queue of versions, +// kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no +// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. + +const ( + // PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB. + PublishQueueState = "publish-queue" + PublishQueueBucket = "mesh-delivery-builder_" + PublishQueueState + PublishQueueMiB = 4 + // ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB. + ActedState = "acted" + ActedBucket = "mesh-delivery-builder_" + ActedState + ActedMiB = 1 + // PublishQueueTool is builder_publish_queue. + PublishQueueTool = "builder_publish_queue" + // DefaultRepublishMax is the setting republish-max-versions unless set (§633). + DefaultRepublishMax = 50 + // QueueFull is what version-unreleased says of a version the queue's cap refused. + QueueFull = "queue full" + // NotStoredWords opens what a refused artifact is said as. + NotStoredWords = "not stored: " + // VerbPublish is the registry seats' verb a publish again sends. + VerbPublish = "publish" +) + +// RegistrySeats are the four registry seats a version is published to. +var RegistrySeats = []string{SeatImages, SeatArtifacts, SeatGoPackages, SeatNPM} + +// SeatOfArtifact is the registry seat of one artifact of the catalog graph entry: from its package's ecosystem when the +// entry names one (go, npm, image), else from its kind (SeatOf). +func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) { + if a.Package != nil { + switch a.Package.Ecosystem { + case "go": + return SeatGoPackages, true + case "npm": + return SeatNPM, true + case "image": + return SeatImages, true + } + } + seat, _, ok := SeatOf(a.Kind, a.Language) + return seat, ok +} + +// QueuedPublish is the value of a publish-queue key `..` (each token escaped, §73): the +// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its +// publish has begun. +type QueuedPublish struct { + V int `json:"v"` + Holder string `json:"holder"` + Module string `json:"module"` + Version string `json:"version"` + Artifacts map[string]string `json:"artifacts"` + Queued time.Time `json:"queued"` + Begun bool `json:"begun,omitempty"` +} + +// QueueKey is the key of holder's queued version of module. +func QueueKey(holder, module, version string) string { + return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version) +} + +// Acted is the value of an acted key: the last stream sequence acted on for its subject. +type Acted struct { + Sequence uint64 `json:"sequence"` +} + +// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap. +type QueueStore interface { + ReserveStore + Bytes(ctx context.Context) (uint64, error) +} + +// Publish is one artifact's publish again: what the registry seat's publish takes, without parts. +type Publish struct { + Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string +} + +// Republisher publishes reserved versions again (design 47 G §588). +type Republisher struct { + Reserved *Reserver + Queue QueueStore + Acted ReserveStore + // Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry). + Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) + // Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words. + Send func(ctx context.Context, p Publish) error + // Unreleased says version-unreleased about a version with the holder's words. + Unreleased func(ctx context.Context, v ReservedVersion, why string) + // Max is the setting republish-max-versions: the most versions one event queues. + Max int + Now func() time.Time + Logf func(string, ...any) + // Go starts the sending of a holder's queue; nil starts a goroutine. + Go func(func()) + + mu sync.Mutex + runs map[string]map[string][]string // node -> holder module -> the registry seats it claims + busy map[string]bool +} + +func (r *Republisher) now() time.Time { + if r.Now != nil { + return r.Now().UTC() + } + return time.Now().UTC() +} + +func (r *Republisher) logf(f string, a ...any) { + if r.Logf != nil { + r.Logf(f, a...) + } +} + +// ActedKey is the acted key of a subject. +func ActedKey(subject string) string { return EscapeToken(subject) } + +// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq: it acts +// on it only when its sequence is newer than the one kept for its subject, and keeps it after. A subject that is neither is +// not its; an error leaves the message not acted on, so it is read again. +func (r *Republisher) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) { + running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*")) + changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".") + if !running && !changed { + return false, nil + } + key := ActedKey(subject) + raw, rev, found, err := r.Acted.Get(ctx, key) + if err != nil { + return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err) + } + if found { + var a Acted + if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq { + return true, nil + } + } + if running { + err = r.running(ctx, data) + } else { + err = r.returned(ctx, data) + } + if err != nil { + return true, err + } + val, _ := json.Marshal(Acted{Sequence: seq}) + if found { + _, err = r.Acted.Update(ctx, key, val, rev) + } else { + _, err = r.Acted.Create(ctx, key, val) + } + if err != nil && !errors.Is(err, ErrConflict) { + return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err) + } + return true, nil +} + +// registrySeatsOf are the registry seats the module of the catalog entry fingerprint claims, by their publish verb. +func (r *Republisher) registrySeatsOf(ctx context.Context, fingerprint string) ([]string, error) { + e, err := r.Entry(ctx, fingerprint) + if err != nil { + return nil, err + } + var out []string + for _, seat := range RegistrySeats { + if slices.Contains(e.Serves, seat+"."+VerbPublish) { + out = append(out, seat) + } + } + return out, nil +} + +// running takes a running in: a module stopped is forgotten; one running whose entry claims a registry seat is a holder on +// its node, and its versions are queued. +func (r *Republisher) running(ctx context.Context, data []byte) error { + var ev nodeengine.Running + if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" { + r.logf("a running that does not read is not acted on: %v", err) + return nil + } + if ev.Stopped || ev.Fingerprint == "" { + r.mu.Lock() + delete(r.runs[ev.Node], ev.Module) + r.mu.Unlock() + return nil + } + seats, err := r.registrySeatsOf(ctx, ev.Fingerprint) + if err != nil { + return fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err) + } + r.mu.Lock() + if r.runs == nil { + r.runs = map[string]map[string][]string{} + } + if r.runs[ev.Node] == nil { + r.runs[ev.Node] = map[string][]string{} + } + if len(seats) == 0 { + delete(r.runs[ev.Node], ev.Module) + } else { + r.runs[ev.Node][ev.Module] = seats + } + r.mu.Unlock() + if len(seats) == 0 { + return nil + } + _, err = r.queueEvent(ctx, map[string][]string{ev.Module: seats}) + return err +} + +// returned takes a manifest-reads-changed in: only one with why returned acts, for every holder known to run on its node. +func (r *Republisher) returned(ctx context.Context, data []byte) error { + var ev nodeengine.ManifestReadsChanged + if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" { + r.logf("a manifest-reads-changed that does not read is not acted on: %v", err) + return nil + } + if ev.Why != nodeengine.ManifestReadsChangedWhyReturned { + return nil + } + r.mu.Lock() + holders := map[string][]string{} + for m, seats := range r.runs[ev.Node] { + holders[m] = seats + } + r.mu.Unlock() + if len(holders) == 0 { + return nil + } + _, err := r.queueEvent(ctx, holders) + return err +} + +// view is a reserved version with the artifacts of it a holder would publish. +type view struct { + v ReservedVersion + arts map[string]meshcatalog.Artifact // reference -> its catalog artifact +} + +// queueEvent queues, for each holder (module -> the seats it claims), the unreleased versions whose missing artifacts +// that holder's seats take: newest keys first, at most Max versions in all (one key per module and MAJOR.MINOR is the +// reservation's own). +func (r *Republisher) queueEvent(ctx context.Context, holders map[string][]string) ([]string, error) { + over, err := r.Reserved.Overdue(ctx, nil) + if err != nil { + return nil, err + } + var vs []ReservedVersion + for _, o := range over { + if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok { + vs = append(vs, v) + } + } + return r.queue(ctx, vs, func(seat string) []string { return holdersOf(holders, seat) }, r.Max) +} + +func holdersOf(holders map[string][]string, seat string) []string { + var out []string + for m, seats := range holders { + if slices.Contains(seats, seat) { + out = append(out, m) + } + } + sort.Strings(out) + return out +} + +// queue queues vs newest first, at most max of them (0 is no limit), each artifact to the first holder holdersOf names for +// its seat. It answers the versions queued, ` `. +func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf func(seat string) []string, max int) ([]string, error) { + sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) }) + var queued []string + started := map[string]bool{} + for _, v := range vs { + if max > 0 && len(queued) >= max { + break + } + if v.Fingerprint == "" || v.BuiltAt.IsZero() { + continue + } + e, err := r.Entry(ctx, v.Fingerprint) + if err != nil { + return queued, fmt.Errorf("the catalog entry %s of %s %s cannot be read: %w", v.Fingerprint, v.Module, v.Version, err) + } + byHolder := map[string]map[string]string{} + for _, ref := range v.Missing() { + if _, no := v.Unstored[ref]; no { + continue + } + for _, a := range e.Artifacts { + seat, ok := SeatOfArtifact(a) + if a.Ref != ref || !ok { + continue + } + if hs := holdersOf(seat); len(hs) > 0 { + if byHolder[hs[0]] == nil { + byHolder[hs[0]] = map[string]string{} + } + byHolder[hs[0]][ref] = v.Artifacts[ref] + } + } + } + did := false + for holder, arts := range byHolder { + ok, err := r.enqueue(ctx, v, holder, arts) + if err != nil { + return queued, err + } + did = did || ok + started[holder] = true + } + if did { + queued = append(queued, v.Module+" "+v.Version) + } + } + for holder := range started { + r.drainLater(ctx, holder) + } + return queued, nil +} + +// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the +// section's cap (said as queue full). +func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) { + key := QueueKey(holder, v.Module, v.Version) + if _, _, found, err := r.Queue.Get(ctx, key); err != nil { + return false, err + } else if found { + return false, nil + } + raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.now()}) + used, err := r.Queue.Bytes(ctx) + if err != nil { + return false, fmt.Errorf("the size of %s cannot be read: %w", PublishQueueBucket, err) + } + if used+uint64(len(raw)) > PublishQueueMiB<<20 { + if r.Unreleased != nil { + r.Unreleased(ctx, v, QueueFull) + } + return false, nil + } + if _, err := r.Queue.Create(ctx, key, raw); err != nil { + if errors.Is(err, ErrConflict) { + return false, nil + } + return false, fmt.Errorf("%s cannot be queued: %w", key, err) + } + return true, nil +} + +// Republish is builder_publish_again: with module and version, that reserved version's missing artifacts; with module +// alone, each reserved version of it whose artifacts are not all published; with neither, every version the finding +// version-unreleased names. It queues as the events do (no limit but the cap), for the holders known to run, and answers +// the versions it queued. +func (r *Republisher) Republish(ctx context.Context, module, version string) ([]string, error) { + var vs []ReservedVersion + if module == "" { + over, err := r.Reserved.Overdue(ctx, nil) + if err != nil { + return nil, err + } + for _, o := range over { + if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok { + vs = append(vs, v) + } + } + } else { + all, err := r.Reserved.All(ctx) + if err != nil { + return nil, err + } + for _, v := range all { + if v.Module == module && (version == "" || v.Version == version) && len(v.Missing()) > 0 { + vs = append(vs, v) + } + } + if len(vs) == 0 { + return nil, fmt.Errorf("%s holds no reserved version%s with an artifact unpublished", module, map[bool]string{true: " " + version, false: ""}[version != ""]) + } + } + r.mu.Lock() + known := map[string][]string{} + for _, mods := range r.runs { + for m, seats := range mods { + known[m] = seats + } + } + r.mu.Unlock() + return r.queue(ctx, vs, func(seat string) []string { return holdersOf(known, seat) }, 0) +} + +// Queued are the queued versions, per holder in the order they are sent, and the acted keys with their sequences: +// builder_publish_queue. +type Queued struct { + Queue []QueuedPublish `json:"queue"` + Acted map[string]uint64 `json:"acted"` +} + +// AnswerQueue is builder_publish_queue over the two sections as the tools bundle reads them. +func AnswerQueue(queue, acted Reader) (Queued, error) { + out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}} + keys, err := queue.Keys() + if err != nil { + return out, fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) + } + for _, k := range keys { + raw, ok, err := queue.Get(k) + if err != nil || !ok { + continue + } + var q QueuedPublish + if json.Unmarshal(raw, &q) == nil { + out.Queue = append(out.Queue, q) + } + } + sort.SliceStable(out.Queue, func(i, j int) bool { + if out.Queue[i].Holder != out.Queue[j].Holder { + return out.Queue[i].Holder < out.Queue[j].Holder + } + return out.Queue[i].Queued.Before(out.Queue[j].Queued) + }) + akeys, err := acted.Keys() + if err != nil { + return out, fmt.Errorf("the state %s cannot be listed: %w", ActedState, err) + } + for _, k := range akeys { + raw, ok, err := acted.Get(k) + if err != nil || !ok { + continue + } + var a Acted + if json.Unmarshal(raw, &a) == nil { + out.Acted[k] = a.Sequence + } + } + return out, nil +} + +// PublishQueueDescription is builder_publish_queue's description. +const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588): per registry " + + "seat holder, in the order they are sent, each with its artifacts' digests, when it was queued and whether its publish has begun; and the " + + "keys of the state " + ActedState + ", the last stream sequence the builder acted on for each running and manifest-reads-changed subject. Read-only." + +// PublishAgainDescription is builder_publish_again's description. +const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " + + "version's missing artifacts; with module alone, each reserved version of that module whose artifacts are not all published; with neither, every " + + "version the finding version-unreleased names, newest first. It queues them as the builder does on a registry holder's running, and answers at once " + + "with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " + + "recorded in the hand-act log." + +// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588). +func (r *Republisher) Resume(ctx context.Context) error { + keys, err := r.Queue.KeysOf(ctx, AllFilter) + if err != nil { + return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err) + } + holders := map[string]bool{} + for _, k := range keys { + raw, _, found, err := r.Queue.Get(ctx, k) + if err != nil || !found { + continue + } + var q QueuedPublish + if json.Unmarshal(raw, &q) == nil { + holders[q.Holder] = true + } + } + for h := range holders { + r.drainLater(ctx, h) + } + return nil +} + +func (r *Republisher) drainLater(ctx context.Context, holder string) { + r.mu.Lock() + if r.busy == nil { + r.busy = map[string]bool{} + } + if r.busy[holder] { + r.mu.Unlock() + return + } + r.busy[holder] = true + r.mu.Unlock() + run := func() { + defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }() + r.Drain(ctx, holder) + } + if r.Go != nil { + r.Go(run) + return + } + go run() +} + +// Drain sends holder's queued versions one at a time, oldest first, each bounded by UnreleasedBound; a key is deleted when +// its outcome is said. +func (r *Republisher) Drain(ctx context.Context, holder string) { + for ctx.Err() == nil { + keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*") + if err != nil { + r.logf("the queue of %s cannot be listed: %v", holder, err) + return + } + var items []QueuedPublish + for _, k := range keys { + if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found { + var q QueuedPublish + if json.Unmarshal(raw, &q) == nil { + items = append(items, q) + } + } + } + if len(items) == 0 { + return + } + sort.SliceStable(items, func(i, j int) bool { return items[i].Queued.Before(items[j].Queued) }) + r.send(ctx, items[0]) + } +} + +// send publishes one queued version's artifacts and says the outcome, then deletes its key. +func (r *Republisher) send(ctx context.Context, q QueuedPublish) { + key := QueueKey(q.Holder, q.Module, q.Version) + defer func() { + if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil { + r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err) + } + }() + if !q.Begun { + q.Begun = true + raw, _ := json.Marshal(q) + if _, rev, found, err := r.Queue.Get(ctx, key); err == nil && found { + _, _ = r.Queue.Update(ctx, key, raw, rev) + } + } + v, found, err := r.Reserved.Of(ctx, q.Module, q.Version) + if err != nil || !found { + return // released meanwhile, or dropped + } + e, err := r.Entry(ctx, v.Fingerprint) + if err != nil { + r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)) + return + } + refs := make([]string, 0, len(q.Artifacts)) + for ref := range q.Artifacts { + refs = append(refs, ref) + } + sort.Strings(refs) + for _, ref := range refs { + var art *meshcatalog.Artifact + for i := range e.Artifacts { + if e.Artifacts[i].Ref == ref { + art = &e.Artifacts[i] + } + } + seat, ok := "", false + if art != nil { + seat, ok = SeatOfArtifact(*art) + } + if !ok { + r.say(ctx, v, ref+": the catalog entry names no registry seat for it") + return + } + name := NameOf(q.Module, strings.TrimPrefix(ref, q.Module+":")) + if art.Package != nil && art.Package.Name != "" { + name = art.Package.Name + } + kind := art.Kind + bound, cancel := context.WithTimeout(ctx, UnreleasedBound) // a context's bound, no timer (ADR 0437 §8) + err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint, + Commit: v.Commit, Transfer: "republish-" + q.Module + "-" + v.Version + "-" + strings.TrimPrefix(ref, q.Module+":")}) + cancel() + if err != nil { + words := err.Error() + if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) { + if ok, merr := r.Reserved.MarkUnstored(ctx, q.Module, q.Version, ref, words); merr != nil { + r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr) + } else if ok { + r.say(ctx, v, ref+" "+NotStoredWords+words) + } + continue + } + r.say(ctx, v, ref+": "+words) + } + } +} + +func (r *Republisher) say(ctx context.Context, v ReservedVersion, why string) { + r.logf("%s %s is not published again: %s", v.Module, v.Version, why) + if r.Unreleased != nil { + r.Unreleased(ctx, v, why) + } +} + +// MarkUnstored records that a holder refused artifact of module's reserved version as not stored; it answers whether that +// was new (said once). +func (r *Reserver) MarkUnstored(ctx context.Context, module, version, artifact, words string) (bool, error) { + series, _, err := SplitVersion(version) + if err != nil { + return false, err + } + key := ReservedKey(module, series) + for { + cur, rev, found, err := r.get(ctx, key) + if err != nil || !found || cur.Version != version { + return false, err + } + if _, was := cur.Unstored[artifact]; was { + return false, nil + } + if cur.Unstored == nil { + cur.Unstored = map[string]string{} + } + cur.Unstored[artifact] = words + raw, _ := json.Marshal(cur) + if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return false, fmt.Errorf("%s cannot be written: %w", key, err) + } + return true, nil + } +} diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go new file mode 100644 index 00000000..c7f15d7d --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -0,0 +1,220 @@ +package versions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +type queueMem struct { + *reserveMem + bytes uint64 +} + +func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil } + +// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by +// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record. +type republishBed struct { + *Republisher + reserved *reserveMem + queue *queueMem + acted *reserveMem + now time.Time + sent []Publish + said []string + refuse string // the words Send fails with, empty for none +} + +func newRepublishBed(t *testing.T, max int) *republishBed { + t.Helper() + b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, acted: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)} + res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }} + b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Acted: b.acted, Max: max, Now: func() time.Time { return b.now }, Go: func(f func()) { f() }, + Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) { + if fp == "src2:holder" { + return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil + } + m := strings.TrimPrefix(fp, "src2:") + return meshcatalog.CatalogEntry{Artifacts: []meshcatalog.Artifact{ + {Ref: m + ":prog", Kind: "bundle", Language: "go"}, + {Ref: m + ":lib", Kind: "package", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.novox.be/novox/" + m + "/go"}}}}, nil + }, + Send: func(_ context.Context, p Publish) error { + b.sent = append(b.sent, p) + if b.refuse != "" { + return errors.New(b.refuse) + } + return nil + }, + Unreleased: func(_ context.Context, v ReservedVersion, why string) { + b.said = append(b.said, v.Module+" "+v.Version+": "+why) + }, + Logf: t.Logf} + return b +} + +// reserve reserves and builds a version of module m (its patch given by the reservation order), its build ended long ago. +func (b *republishBed) reserve(t *testing.T, module string) ReservedVersion { + t.Helper() + b.now = b.now.Add(time.Minute) + ctx := context.Background() + v, err := b.Reserved.Reserve(ctx, module, "0.1", []string{module + ":prog", module + ":lib"}, Source{Fingerprint: "src2:" + module}) + if err != nil { + t.Fatal(err) + } + for ref, d := range map[string]string{module + ":prog": "sha256:aa", module + ":lib": "sha256:bb"} { + if _, err := b.Reserved.Built(ctx, module, v.Version, ref, d); err != nil { + t.Fatal(err) + } + } + b.now = b.now.Add(UnreleasedBound + time.Minute) + return v +} + +func running(node, module, fp string, stopped bool) (string, []byte) { + data, _ := json.Marshal(nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Stopped: stopped, At: time.Unix(1, 0)}) + return nodeengine.SubjectRunning(node, module), data +} + +// TestRepublish: design 47 G §588 (A-73, A-76): a reserved version left unpublished is published again, on a registry +// holder's running and on its node's returned, once per message by its stream sequence, as a publish by digest with no parts, +// at most the setting's number per event, newest first; an artifact no store holds is said once and not tried again. +func TestRepublish(t *testing.T) { + ctx := context.Background() + t.Run("running of a holder, once per sequence", func(t *testing.T) { + b := newRepublishBed(t, 50) + v := b.reserve(t, "m") + if v.Fingerprint != "src2:m" { + t.Fatalf("the reservation keeps the entry's fingerprint: %q", v.Fingerprint) + } + subject, data := running("n1", "oci", "src2:holder", false) + for _, seq := range []uint64{5, 4} { // 4 is older than the acted 5: it starts nothing + if _, err := b.Heard(ctx, subject, data, seq); err != nil || len(b.sent) != 1 { + t.Fatalf("seq %d: sent %d (%v), want 1", seq, len(b.sent), err) + } + } + p := b.sent[0] + if p.Seat != SeatArtifacts || p.Name != "m/prog" || p.Kind != "bundle" || p.Version != v.Version || p.Digest != "sha256:aa" || p.Fingerprint != "src2:m" { + t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p) + } + if _, err := b.Heard(ctx, subject, data, 5); err != nil || len(b.sent) != 1 { + t.Errorf("the same sequence read again sent %d, want still 1", len(b.sent)) + } + if _, err := b.Heard(ctx, subject, data, 6); err != nil || len(b.sent) != 2 { + t.Errorf("a newer sequence sent %d, want 2", len(b.sent)) + } + if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 { + t.Errorf("a key stays in the queue after its outcome: %v", keys) + } + }) + t.Run("returned acts, started does not", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + subject, data := running("n1", "oci", "src2:holder", true) // stopped: not a holder yet + _, _ = b.Heard(ctx, subject, data, 1) + subject, data = running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 2) + b.sent = nil + for i, why := range []string{nodeengine.ManifestReadsChangedWhyStarted, "", nodeengine.ManifestReadsChangedWhyReturned} { + ev, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "n1", Why: why}) + handled, err := b.Heard(ctx, nodeengine.SubjectManifestReadsChanged("n1"), ev, uint64(10+i)) + if !handled || err != nil { + t.Fatalf("%q: handled %v, %v", why, handled, err) + } + if want := map[string]int{"started": 0, "": 0, "returned": 1}[why]; len(b.sent) != want { + t.Errorf("why %q: sent %d, want %d", why, len(b.sent), want) + } + } + if handled, _ := b.Heard(ctx, "mesh.seat.node-engine.event.composed.n1", nil, 1); handled { + t.Error("a subject that is neither is taken as its own") + } + }) + t.Run("the setting limits an event, newest first", func(t *testing.T) { + b := newRepublishBed(t, 2) + var versions []string + for _, m := range []string{"m1", "m2", "m3"} { + versions = append(versions, b.reserve(t, m).Module) + } + subject, data := running("n1", "oci", "src2:holder", false) + if _, err := b.Heard(ctx, subject, data, 1); err != nil { + t.Fatal(err) + } + if len(b.sent) != 2 || !strings.HasPrefix(b.sent[0].Name, "m3/") || !strings.HasPrefix(b.sent[1].Name, "m2/") { + t.Errorf("sent %+v, want the two newest, m3 then m2", b.sent) + } + }) + t.Run("an artifact no store holds is said once and not tried again", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + b.refuse = "oci-artifact-registry's publish of m/prog: not stored: no blob sha256:aa" + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) + _, _ = b.Heard(ctx, subject, data, 2) + if len(b.sent) != 1 || len(b.said) != 1 || !strings.Contains(b.said[0], "not stored: ") { + t.Errorf("sent %d, said %v: want one try and one saying of not stored", len(b.sent), b.said) + } + }) + t.Run("a refusal in other words is said each time", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + b.refuse = "the holder is full" + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) + _, _ = b.Heard(ctx, subject, data, 2) + if len(b.sent) != 2 || len(b.said) != 2 { + t.Errorf("sent %d, said %v: want both events tried and said", len(b.sent), b.said) + } + }) + t.Run("a queue past its cap says queue full", func(t *testing.T) { + b := newRepublishBed(t, 50) + b.reserve(t, "m") + b.queue.bytes = PublishQueueMiB << 20 + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) + if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) { + t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said) + } + }) + t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) { + b := newRepublishBed(t, 50) + v := b.reserve(t, "m") + raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true}) + if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil { + t.Fatal(err) + } + if err := b.Resume(ctx); err != nil || len(b.sent) != 1 { + t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err) + } + }) + t.Run("the tool queues for a holder known to run", func(t *testing.T) { + b := newRepublishBed(t, 1) + b.reserve(t, "m1") + b.reserve(t, "m2") + if q, err := b.Republish(ctx, "", ""); err != nil || len(q) != 0 { + t.Errorf("no holder known: queued %v (%v), want none", q, err) + } + subject, data := running("n1", "oci", "src2:holder", false) + _, _ = b.Heard(ctx, subject, data, 1) // Max 1: only m2 sent + b.sent = nil + q, err := b.Republish(ctx, "", "") + if err != nil || fmt.Sprint(q) != "[m2 0.1.0 m1 0.1.0]" || len(b.sent) != 2 { + t.Errorf("the tool with neither queued %v (%v), sent %d: want both versions, with no limit", q, err, len(b.sent)) + } + if _, err := b.Republish(ctx, "m1", "0.1.9"); err == nil { + t.Error("a version no reservation holds is not refused") + } + answer, err := AnswerQueue(Reader{Keys: func() ([]string, error) { return nil, nil }}, Reader{Keys: func() ([]string, error) { return b.acted.KeysOf(ctx, AllFilter) }, + Get: func(k string) (json.RawMessage, bool, error) { v, _, ok, err := b.acted.Get(ctx, k); return v, ok, err }}) + if err != nil || answer.Acted[ActedKey(subject)] != 1 { + t.Errorf("builder_publish_queue acted = %v (%v), want the running's sequence 1", answer.Acted, err) + } + }) +} diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go index 89b0dc32..3a13dff1 100644 --- a/modules/mesh-delivery-builder/internal/versions/reserve.go +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -54,13 +54,21 @@ type ReservedVersion struct { Repository string `json:"repository,omitempty"` Commit string `json:"commit,omitempty"` Tag string `json:"tag,omitempty"` + // Fingerprint is the fingerprint of the catalog entry the reservation was made from, so a publish again reads that + // entry (design 47 G A-76). + Fingerprint string `json:"fingerprint,omitempty"` + // Unstored are the artifacts a holder refused as not stored, with its words: said once, and not published again on + // events (design 47 G §588); builder_build_again builds the key again, which reserves anew. + Unstored map[string]string `json:"unstored,omitempty"` } // Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is // written at, and whether the module has a Go package, whose version is tagged. type Source struct { Repository, Commit string - GoPackage bool + // Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76). + Fingerprint string + GoPackage bool } // ReservedKey is the key of module's reservation of series (MAJOR.MINOR). @@ -135,7 +143,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts } } out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(), - Repository: src.Repository, Commit: src.Commit} + Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint} if src.GoPackage { out.Tag = GoTagOf(out.Version) } diff --git a/modules/mesh-delivery-builder/internal/versions/versions_test.go b/modules/mesh-delivery-builder/internal/versions/versions_test.go index 59b9b7a3..6ed783a7 100644 --- a/modules/mesh-delivery-builder/internal/versions/versions_test.go +++ b/modules/mesh-delivery-builder/internal/versions/versions_test.go @@ -73,6 +73,13 @@ func (m *mem) KeysOf(_ context.Context, filter string) ([]string, error) { out = append(out, k) continue } + if strings.Count(filter, ".") == 2 { // a three-token filter, `a.*.*` as the bus reads it + ft, kt := strings.Split(filter, "."), strings.Split(k, ".") + if len(kt) == 3 && kt[0] == ft[0] { + out = append(out, k) + } + continue + } if strings.HasPrefix(k, prefix) && !strings.Contains(k[len(prefix):], ".") { out = append(out, k) } @@ -540,39 +547,7 @@ func TestTheBuildersVersions(t *testing.T) { t.Error("80 % is not full") } }) - t.Run("publish again", func(t *testing.T) { - num := Entry{Name: "m/p", Kind: KindBundle, Version: "0.1.3", Fingerprint: "f1", Digest: "sha256:aa", State: Numbered} - ref := num - ref.State, ref.RefusedDigest = Refused, "sha256:aa" - other := ref - other.RefusedDigest = "sha256:bb" - pub := num - pub.State = Published - gone := pub - gone.State = Removed - tagged := num - tagged.Tag, tagged.Version = "feat-x", "0.1.3-feat-x.1" - for _, c := range []struct { - name string - entries []Entry - fp string - built bool - do string - }{ - {"a held build is said numbered again", []Entry{num}, "f1", true, AgainPublish}, - {"refused for its own bytes is published again", []Entry{ref}, "f1", true, AgainPublish}, - {"a gone file is built again under the same number", []Entry{ref}, "f1", false, AgainBuild}, - {"after a published it takes a new number", []Entry{pub}, "f1", true, AgainRenumber}, - {"after a removal it takes a new number", []Entry{gone}, "f1", false, AgainRenumber}, - {"refused for other bytes is refused", []Entry{other}, "f1", true, ""}, - {"a fingerprint not numbered is refused", []Entry{num}, "f2", true, ""}, - {"a tagged version is not a trunk fingerprint's", []Entry{tagged}, "f1", true, ""}, - } { - a, err := PublishAgain(c.entries, c.fp, c.built) - if (err != nil) != (c.do == "") || a.Do != c.do { - t.Errorf("%s: PublishAgain = %+v, %v; want %q", c.name, a, err, c.do) - } - } + t.Run("fresh claims and the floor", func(t *testing.T) { ctx := context.Background() // A fresh claim of a published fingerprint takes the next number; the old one stays. st := newMem() @@ -603,32 +578,8 @@ func TestTheBuildersVersions(t *testing.T) { if all, err := v.All(ctx); err != nil || len(all) != 2 { t.Errorf("All = %d entries, %v", len(all), err) } - // The hand acts: a held build is said numbered once more under a fresh dedup id; a gone one is built again; a - // published one takes a new number; a name with versions asks no floor. - if _, _, err := (Acts{Numberer: &Numberer{Versions: v}, Done: func(context.Context, string, string) (bool, error) { return true, nil }}).PublishAgain(ctx, "m/p", "f1", "w", "", time.Unix(4, 0)); err == nil { - t.Error("a version refused for other bytes is published again") - } _, _ = v.Mark(ctx, "m/p", "0.1.4", func(e *Entry) { e.State = Published }) - var rebuilt []string - done := true - acts := Acts{Numberer: &Numberer{Versions: v, Waits: &FloorWaits{}}, - Done: func(context.Context, string, string) (bool, error) { return done, nil }, - BuildAgain: func(_ context.Context, m, fp, _, _ string) (string, bool) { - rebuilt = append(rebuilt, m+" "+fp) - return "built again", false - }, - Reconsider: func(context.Context) {}} - // Nothing says a version numbered (design 47 G §349): a held build is refused before any number is taken. - if _, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(5, 0)); err == nil || len(rebuilt) != 0 { - t.Errorf("publish again of a published, held build answered %v, rebuilt %v; want a refusal", err, rebuilt) - } - if es, _ := v.Of(ctx, "m/p"); len(es) != 2 { - t.Errorf("the refused publish again took a number: %+v", es) - } - done = false - if ag, _, err := acts.PublishAgain(ctx, "m/p", "f1", "w", "me", time.Unix(6, 0)); err != nil || ag.Do != AgainRenumber || ag.Entry.Version != "0.1.5" || len(rebuilt) != 1 { - t.Errorf("publish again of a gone build: %+v %v rebuilt %v", ag, err, rebuilt) - } + acts := Acts{Numberer: &Numberer{Versions: v, Waits: &FloorWaits{}}, Reconsider: func(context.Context) {}} if _, err := acts.FloorAgain(ctx, "m/p"); err == nil { t.Error("a name with versions asks its floor again") } diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 3a9a8c5b..9ed726ed 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -3,6 +3,7 @@ package manifest import ( "bytes" "encoding/json" + "fmt" "io/fs" "os" "path/filepath" @@ -23,6 +24,7 @@ type state struct { Name string `json:"name"` History int `json:"history,omitempty"` KeyTTL bool `json:"key-ttl,omitempty"` + MaxMiB int `json:"max-mib,omitempty"` } type artifact struct { @@ -111,7 +113,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -121,8 +123,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-assigner.assignments", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { - t.Errorf("uses = %v, want the three build seats", m.Uses) + if want := []string{"node-build-go", "node-build-image", "node-build-node", "operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) { + t.Errorf("uses = %v, want the three build seats and operator-channel (design 47 G §568: the ask after three event builds)", m.Uses) } // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. @@ -140,10 +142,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements. // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", - "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", + "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions", "seat:git.tag", - "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", - "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + "seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove", + "seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no @@ -159,7 +161,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.packages", "node-build-go.toolchain-reported", "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", - "oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -169,8 +171,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes. @@ -180,8 +182,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days) and the findings' stream", m.Streams) } - if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 1 { - t.Errorf("config = %+v, want toolchain-wait-minutes of 60", m.Config) + if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 2 || m.Config["republish-max-versions"].Default != float64(50) { + t.Errorf("config = %+v, want toolchain-wait-minutes of 60 and republish-max-versions of 50 (design 47 G §588, §633)", m.Config) } // Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends. if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) { @@ -221,8 +223,22 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("process = %v", p) } env, _ := p["env"].(map[string]any) - if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES"] != "${config:toolchain-wait-minutes}" { - t.Errorf("env = %v, want the broker file and the toolchain wait's setting (design 47 G §672)", p["env"]) + if len(env) != 3 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_REPUBLISH_MAX_FILE"] != "${dir:state}/republish-max-versions" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" { + t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"]) + } + for k, v := range env { + if strings.Contains(fmt.Sprint(v), "${config:") { + t.Errorf("env %s = %v: a setting reaches the process as a file, never its environment (A-71)", k, v) + } + } + found := false + for _, r := range m.Resources { + if r["type"] == "file" && r["path"] == "${dir:state}/toolchain-wait-minutes" && r["content"] == "${config:toolchain-wait-minutes}" { + found = true + } + } + if !found { + t.Errorf("no file resource with the content ${config:toolchain-wait-minutes}: A-71") } for _, r := range m.Resources { if r["id"] == "settings" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f784dcb2..2bd7318e 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -15,6 +15,7 @@ "builder_announce_again", "builder_ask_again", "builder_bounds", + "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", @@ -22,6 +23,7 @@ "builder_floor_waits", "builder_old_form", "builder_publish_again", + "builder_publish_queue", "builder_reserved", "builder_release", "builder_reserved_drop", @@ -59,6 +61,7 @@ "node-build-node.toolchain-reported", "node-engine.manifest-reads-changed", "node-engine.running", + "operator-channel.decided", "npm-package-registry.kept", "npm-package-registry.published", "npm-package-registry.refused", @@ -82,8 +85,12 @@ "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", + "seat:mesh-catalog.entry", + "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", + "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", @@ -97,7 +104,11 @@ "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", - "mesh-delivery-indexer.indexer_read_packages" + "mesh-delivery-indexer.indexer_read_packages", + "seat:container-image-registry.publish", + "seat:go-package-registry.publish", + "seat:npm-package-registry.publish", + "seat:oci-artifact-registry.publish" ], "state": [ { "name": "builds", "history": 5 }, @@ -110,8 +121,11 @@ { "name": "bounds", "history": 1, "key-ttl": true }, { "name": "queues", "history": 1 }, { "name": "toolchains-heard", "history": 1 }, - { "name": "waits", "history": 1 }, + { "name": "waits", "history": 1, "max-mib": 4 }, { "name": "reserved", "history": 1 }, + { "name": "publish-queue", "history": 1, "max-mib": 4 }, + { "name": "acted", "history": 1, "max-mib": 1 }, + { "name": "asked", "history": 1 }, { "name": "reasks", "history": 1 }, { "name": "versions", "history": 1 }, { "name": "first-views", "history": 1 } @@ -129,6 +143,14 @@ "maximum": 10080, "trusted": false, "default": 60 + }, + "republish-max-versions": { + "type": "number", + "description": "the most versions one event of a registry holder queues to publish again, newest first (design 47 G section 588); the rest wait for the next event or builder_publish_again.", + "minimum": 1, + "maximum": 10000, + "trusted": false, + "default": 50 } }, "logs": [ @@ -144,7 +166,8 @@ "uses": [ "node-build-go", "node-build-image", - "node-build-node" + "node-build-node", + "operator-channel" ], "own-secrets": { "broker": "${dir:state}/broker" @@ -166,6 +189,22 @@ "place": ".", "unknown": "program" }, + { + "id": "toolchain-wait", + "type": "file", + "path": "${dir:state}/toolchain-wait-minutes", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:toolchain-wait-minutes}" + }, + { + "id": "republish-max", + "type": "file", + "path": "${dir:state}/republish-max-versions", + "owner": "mesh-delivery-builder", + "mode": "0600", + "content": "${config:republish-max-versions}" + }, { "id": "builder", "type": "process", @@ -176,7 +215,8 @@ ], "env": { "MESH_BROKER_FILE": "${dir:state}/broker", - "MESH_BUILDER_TOOLCHAIN_WAIT_MINUTES": "${config:toolchain-wait-minutes}" + "MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes", + "MESH_BUILDER_REPUBLISH_MAX_FILE": "${dir:state}/republish-max-versions" }, "user": "mesh-delivery-builder", "health": { diff --git a/sdk/buildstep/ask.go b/sdk/buildstep/ask.go index 55a63241..6281a77d 100644 --- a/sdk/buildstep/ask.go +++ b/sdk/buildstep/ask.go @@ -137,7 +137,8 @@ func BuildModuleOf(language string) (string, error) { return "mesh-build-" + language, nil } -// AskSubject is the subject a build seat accepts a kind of ask on: the seat's accept subject. +// AskSubject is the subject a build seat accepts a kind of ask on: the seat's accept subject of main's form, which names no +// asker (design 47 G §600, §483). func AskSubject(language, kind string) (string, error) { seat, err := BuildSeat(language) if err != nil { @@ -149,20 +150,44 @@ func AskSubject(language, kind string) (string, error) { return "mesh.seat." + seat + ".accept." + kind, nil } -// ParseAskSubject is the inverse of AskSubject. +// AskerBuilder is the builder's name as an asker: the last token of the subject its asks are put on and the first of its +// step keys (design 47 G §600, §481). +const AskerBuilder = "builder" + +// AskSubjectBy is the subject a build seat accepts a queue's ask of asker on: `mesh.seat..accept..`, the +// asker's name the last token (design 47 G §481, §600). An empty asker is main's subject (AskSubject). +func AskSubjectBy(language, kind, asker string) (string, error) { + subject, err := AskSubject(language, kind) + if err != nil || asker == "" { + return subject, err + } + if !askerOK.MatchString(asker) { + return "", fmt.Errorf("buildstep: %q is not an asker's name", asker) + } + return subject + "." + asker, nil +} + +// ParseAskSubject is the inverse of AskSubject and of AskSubjectBy: the language and the queue of either form. func ParseAskSubject(subject string) (language, kind string, err error) { + language, kind, _, err = ParseAskSubjectBy(subject) + return language, kind, err +} + +// ParseAskSubjectBy is the inverse of AskSubjectBy: the language, the queue and the asker, which is empty for main's form. +func ParseAskSubjectBy(subject string) (language, kind, asker string, err error) { rest, ok := strings.CutPrefix(subject, "mesh.seat.node-build-") if !ok { - return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) + return "", "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) } language, kind, ok = strings.Cut(rest, ".accept.") if !ok { - return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) + return "", "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject", subject) } - if want, err := AskSubject(language, kind); err != nil || want != subject { - return "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject: %v", subject, err) + kind, asker, _ = strings.Cut(kind, ".") + if want, err := AskSubjectBy(language, kind, asker); err != nil || want != subject { + return "", "", "", fmt.Errorf("buildstep: %q is not a build seat's accept subject: %v", subject, err) } - return language, kind, nil + return language, kind, asker, nil } // AskStream is the work-queue stream of a language's build seat. diff --git a/sdk/buildstep/ask_test.go b/sdk/buildstep/ask_test.go index b394f66f..88558bee 100644 --- a/sdk/buildstep/ask_test.go +++ b/sdk/buildstep/ask_test.go @@ -594,3 +594,29 @@ func TestFetchBuildsResolvedAndToolchainDigestAreRead(t *testing.T) { t.Errorf("toolchain digest: %+v", tc) } } + +// §481, §600: a queue's ask of an asker is put on accept.., and both forms parse. +func TestAskSubjectBy(t *testing.T) { + got, err := AskSubjectBy(AskGo, AskDelivery, AskerBuilder) + if err != nil || got != "mesh.seat.node-build-go.accept.delivery.builder" { + t.Fatalf("%q %v", got, err) + } + for _, c := range []struct{ subject, language, kind, asker string }{ + {got, "go", "delivery", "builder"}, + {"mesh.seat.node-build-node.accept.check", "node", "check", ""}, + } { + l, k, a, err := ParseAskSubjectBy(c.subject) + if err != nil || l != c.language || k != c.kind || a != c.asker { + t.Errorf("%s: %q %q %q %v", c.subject, l, k, a, err) + } + if l, k, err := ParseAskSubject(c.subject); err != nil || l != c.language || k != c.kind { + t.Errorf("%s: ParseAskSubject %q %q %v", c.subject, l, k, err) + } + } + if _, err := AskSubjectBy(AskGo, AskDelivery, "Bad.Name"); err == nil { + t.Error("a bad asker has a subject") + } + if _, _, _, err := ParseAskSubjectBy("mesh.seat.node-build-go.accept.delivery.builder.extra"); err == nil { + t.Error("a subject of four tokens after accept parses") + } +}