J25 A-293: a placed module's failed newest build said as refused in by-entries mode, on the builder's build-ended event
Reading the catalog half by fingerprint, the node-engine no longer reads the builder's builds, so #224's refused key was never written there and a reader waited on a build that will never be done. The engine now hears the builder's build.ended stream, read whole at start then followed (as the assigner hears it); the last trunk end of a placed module that ended failed is written as the refused key with the event's words, a later done end deletes it, an end naming a branch moves nothing. The manifest consumes mesh-delivery-builder.build.ended.**. Builds: design 47 G A-293 (with §421, §628), ADR 0593 §6, issue 1966 U1/U8 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
@@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
@@ -105,3 +107,82 @@ func (b OverNATS) HearRunning(consumer string, heard func([]byte)) (func(), erro
|
||||
}
|
||||
|
||||
var _ RunningBus = OverNATS{}
|
||||
|
||||
// The builder's event build.ended (design 47 G A-293, as the assigner hears it, A-181, §556): the builder's stream of build
|
||||
// ends keeps the last message of each module and fingerprint, and the node reads it whole at each start, then follows it, through
|
||||
// one ordered consumer it makes for the run: what ended, and a failed build's words, are known from the stream however long ago
|
||||
// they were said, never from the builder's builds. The engine's manifest consumes `mesh-delivery-builder.build.ended.**`.
|
||||
|
||||
// BuildEndsStream is the builder's stream of build ends as the bus names it, BuildEndsSubject every event it carries.
|
||||
var (
|
||||
BuildEndsStream = catalogue.StreamBusName(builds.BuildsStream)
|
||||
BuildEndsSubject = "mesh.stream.mesh-delivery-builder.event." + builds.BuildEndedEvent + ".>"
|
||||
)
|
||||
|
||||
// BuildEndsBus is a link that hears the builder's build ends.
|
||||
type BuildEndsBus interface {
|
||||
// HearBuildEnds calls heard with the last message of every subject, then readEnded once they are all read, then heard with
|
||||
// each after them, until ctx ends (nil) or the stream is lost (its error).
|
||||
HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error
|
||||
}
|
||||
|
||||
func hearBuildEnds(ctx context.Context, js nats.JetStreamContext, heard func([]byte), readEnded func()) error {
|
||||
sub, err := js.SubscribeSync(BuildEndsSubject, nats.OrderedConsumer(), nats.DeliverLastPerSubject(), nats.BindStream(BuildEndsStream))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the stream %s cannot be read: %w", BuildEndsStream, err)
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
info, err := sub.ConsumerInfo()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the stream %s cannot be read: %w", BuildEndsStream, err)
|
||||
}
|
||||
total := info.Delivered.Consumer + info.NumPending
|
||||
for got := uint64(0); ; got++ {
|
||||
if got == total {
|
||||
readEnded()
|
||||
}
|
||||
msg, err := sub.NextMsgWithContext(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("the stream %s is not followed any more (%d read): %w", BuildEndsStream, got, err)
|
||||
}
|
||||
heard(msg.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *natsLink) HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error {
|
||||
return hearBuildEnds(ctx, l.js, heard, readEnded)
|
||||
}
|
||||
|
||||
// HearBuildEnds reads the stream as the link's session does, over its connection.
|
||||
func (b OverNATS) HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error {
|
||||
return hearBuildEnds(ctx, b.JS, heard, readEnded)
|
||||
}
|
||||
|
||||
var (
|
||||
_ BuildEndsBus = (*natsLink)(nil)
|
||||
_ BuildEndsBus = OverNATS{}
|
||||
)
|
||||
|
||||
// FollowBuildEnds hands every build end the builder said to heard: the last of each module and fingerprint first, with
|
||||
// readEnded called after them (until then the caller acts on none), then each after them, until ctx ends. A body that is no
|
||||
// end of a module at a fingerprint is passed over. An error is the stream or the link not being there, which the caller tries
|
||||
// again.
|
||||
func (q *Session) FollowBuildEnds(ctx context.Context, heard func(builds.BuildEnded), readEnded func()) error {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
h, ok := bus.(BuildEndsBus)
|
||||
if bus == nil || !ok {
|
||||
return ErrNoLink
|
||||
}
|
||||
return h.HearBuildEnds(ctx, func(raw []byte) {
|
||||
var e builds.BuildEnded
|
||||
if json.Unmarshal(raw, &e) == nil && e.Module != "" && e.Fingerprint != "" {
|
||||
heard(e)
|
||||
}
|
||||
}, readEnded)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
@@ -45,6 +46,17 @@ func (l *runningLink) HearRunning(_ string, heard func([]byte)) (func(), error)
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
func (l *runningLink) HearBuildEnds(_ context.Context, heard func([]byte), readEnded func()) error {
|
||||
for _, raw := range l.first {
|
||||
heard(raw)
|
||||
}
|
||||
readEnded()
|
||||
for _, raw := range l.after {
|
||||
heard(raw)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
|
||||
t.Helper()
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
@@ -84,6 +96,21 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
|
||||
if err != nil || strings.Join(order, " ") != "a/x b/y read" {
|
||||
t.Fatalf("followed %v: %v, want a/x b/y read", order, err)
|
||||
}
|
||||
|
||||
// Design 47 G A-293: the builder's build ends likewise, the last of each subject first, then the read's end, then the
|
||||
// rest, a body that is no end of a module at a fingerprint passed over.
|
||||
if err := (&Session{Membership: m}).FollowBuildEnds(context.Background(), func(builds.BuildEnded) {}, func() {}); !errors.Is(err, ErrNoLink) {
|
||||
t.Fatalf("build ends followed with no link: %v", err)
|
||||
}
|
||||
rl.first = [][]byte{[]byte(`{"module":"x","fingerprint":"f1","outcome":"failed","error":"no go"}`), []byte(`not json`), []byte(`{"module":"x"}`)}
|
||||
rl.after = [][]byte{[]byte(`{"module":"y","fingerprint":"f2","outcome":"done","branch":"b"}`)}
|
||||
order = nil
|
||||
err = s.FollowBuildEnds(context.Background(), func(e builds.BuildEnded) {
|
||||
order = append(order, e.Module+"@"+e.Fingerprint+":"+e.Outcome+":"+e.Error+":"+e.Branch)
|
||||
}, func() { order = append(order, "read") })
|
||||
if err != nil || strings.Join(order, " ") != "x@f1:failed:no go: read y@f2:done::b" {
|
||||
t.Fatalf("build ends followed %v: %v", order, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A link opening says the node's facts, off the link's own loop.
|
||||
|
||||
@@ -91,8 +91,19 @@ func (c *Composer) publishComposed(ctx context.Context) {
|
||||
func (c *Composer) newestFPs(ctx context.Context, at *published) (out map[string]string, failed map[string][2]string) {
|
||||
if c.byEntries() {
|
||||
// The newest build is the indexer's (and the builder's done key says it is built): neither is read, and the node
|
||||
// follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone.
|
||||
return maps.Clone(at.fps), nil
|
||||
// follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. A placed module's newest
|
||||
// build that ended is the last trunk end the builder said of it (NoteBuildEnded): a failed one is refused (A-293).
|
||||
failed = map[string][2]string{}
|
||||
c.mu.Lock()
|
||||
if c.endsRead {
|
||||
for _, module := range at.placed {
|
||||
if e, ok := c.ends[module]; ok && e.Outcome == builds.BuildFailed {
|
||||
failed[module] = [2]string{e.Fingerprint, e.Error}
|
||||
}
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return maps.Clone(at.fps), failed
|
||||
}
|
||||
out = map[string]string{}
|
||||
failed = map[string][2]string{}
|
||||
@@ -153,7 +164,10 @@ func (c *Composer) writeComposed(ctx context.Context, at *published) (wrote, del
|
||||
wrote++
|
||||
c.saidComposed(key, values[key])
|
||||
}
|
||||
for _, key := range c.pastKeeping() {
|
||||
for _, key := range append(c.pastKeeping(), c.endedRefusals(values)...) {
|
||||
if _, kept := c.pubKept[key]; !kept {
|
||||
continue // past keeping and ended both
|
||||
}
|
||||
if err := c.env.ComposedState.Delete(ctx, key); err != nil {
|
||||
return wrote, deleted, fmt.Errorf("%s cannot be deleted: %w", key, err)
|
||||
}
|
||||
@@ -271,6 +285,38 @@ func (c *Composer) composedValues(ctx context.Context, at *published) (map[strin
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// endedRefusals is, of the keys the state holds, each refusal a failed end of a module wrote (newestFPs) once that module's
|
||||
// last trunk end is done, at any fingerprint (design 47 G A-293): the state never says refused for a done build. A key this
|
||||
// pass writes (values), or one a want names, is not one. Under pubWrite.
|
||||
func (c *Composer) endedRefusals(values map[string][]byte) []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if !c.byEntries() || !c.endsRead {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for module, last := range c.ends {
|
||||
if last.Outcome != builds.BuildDone {
|
||||
continue
|
||||
}
|
||||
for fp := range c.endFPs[module] {
|
||||
key, err := engcomp.Key(c.env.Node, module, fp)
|
||||
if err != nil || values[key] != nil || c.wants[module].Fp == fp {
|
||||
continue
|
||||
}
|
||||
kept, ok := c.pubKept[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if comp, err := engcomp.Decode(kept.Value); err == nil && comp.Refused != "" {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// composedKeyOf is a composition's key in the state.
|
||||
func composedKeyOf(comp engcomp.Composed) (string, error) {
|
||||
if comp.Module == "" {
|
||||
|
||||
@@ -2,6 +2,8 @@ package selfcompose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
@@ -12,6 +14,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/buckets"
|
||||
"github.com/novox/mesh-host/internal/buckets/bucketstest"
|
||||
engcomp "github.com/novox/mesh-host/internal/compositions"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -258,4 +261,46 @@ func TestTheNodePublishesItsCompositions(t *testing.T) {
|
||||
if past := keep.pastKeeping(); len(past) != 0 {
|
||||
t.Errorf("past keeping with a want naming the oldest: %v, want none", past)
|
||||
}
|
||||
|
||||
// Design 47 G A-293: reading the catalog half by fingerprint, a placed module's newest build that ended is the last trunk
|
||||
// build.ended heard of it; a failed one is said refused with the event's words once the first read ended, an end naming a
|
||||
// branch moves nothing, a module with no end (queued or running) stays unpublished, and a later done end, at any
|
||||
// fingerprint, deletes the refused key.
|
||||
byEnv := b.env()
|
||||
byEnv.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("nothing is asked")
|
||||
})
|
||||
byState := &fakeComposed{kv: map[string]ComposedEntry{}}
|
||||
byEnv.ComposedState = byState
|
||||
by := New(byEnv)
|
||||
byAt := &published{fps: map[string]string{}, placed: []string{"broken", "waiting", "branched"}}
|
||||
words := `refused kind: package of language "go"`
|
||||
by.NoteBuildEnded(builds.BuildEnded{Module: "broken", Fingerprint: bfp, Outcome: builds.BuildFailed, Error: words})
|
||||
by.NoteBuildEnded(builds.BuildEnded{Module: "branched", Fingerprint: wfp, Outcome: builds.BuildFailed, Error: "no", Branch: "feature"})
|
||||
for _, step := range []struct {
|
||||
name string
|
||||
do func()
|
||||
holds map[string]string // by key, the refusal it says
|
||||
}{
|
||||
{"before the first read ended", func() {}, map[string]string{}},
|
||||
{"once the first read ended", by.BuildEndsRead, map[string]string{brokenKey: words}},
|
||||
{"a later done end", func() {
|
||||
by.NoteBuildEnded(builds.BuildEnded{Module: "broken", Fingerprint: wfp, Outcome: builds.BuildDone})
|
||||
}, map[string]string{}},
|
||||
} {
|
||||
step.do()
|
||||
if _, _, err := by.writeComposed(ctx, byAt); err != nil {
|
||||
t.Fatalf("%s: %v", step.name, err)
|
||||
}
|
||||
byState.mu.Lock()
|
||||
got := map[string]string{}
|
||||
for k, v := range byState.kv {
|
||||
comp, _ := engcomp.Decode(v.Value)
|
||||
got[k] = comp.Refused
|
||||
}
|
||||
byState.mu.Unlock()
|
||||
if !maps.Equal(got, step.holds) {
|
||||
t.Errorf("%s: the state holds %v, want %v", step.name, got, step.holds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -204,6 +204,12 @@ type Composer struct {
|
||||
// node's own consumer (NoteRunning), under mu; heardRead is whether the first read of the stream ended (RunningRead).
|
||||
heard map[string]map[string]heardRun
|
||||
heardRead bool
|
||||
// ends is, by module, the last build end naming no branch the builder said of it (NoteBuildEnded), its newest build that
|
||||
// ended, under mu; endsRead is whether the first read of the builder's stream ended (BuildEndsRead) (design 47 G A-293).
|
||||
ends map[string]builds.BuildEnded
|
||||
endsRead bool
|
||||
// endFPs is, by module, every fingerprint a trunk end was heard of: the keys a failed end may have written.
|
||||
endFPs map[string]map[string]bool
|
||||
// engine is the node-engine's own module as the last composing of the node's input found it (Scope.Engine), under mu.
|
||||
engine string
|
||||
// rehand asks for a pass that hands again what the agent waits for (Waiting).
|
||||
@@ -2443,6 +2449,50 @@ func (c *Composer) RunningLost() {
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// NoteBuildEnded takes the builder's event build.ended (design 47 G A-293): of a module, the last end naming no branch (the
|
||||
// trunk's) stands as its newest build that ended, a failed one said as a refusal in the composition state, a done one ending
|
||||
// that refusal; an end naming a branch moves nothing. Once the first read of the stream ended, an end of a module placed here
|
||||
// asks a pass.
|
||||
func (c *Composer) NoteBuildEnded(e builds.BuildEnded) {
|
||||
if e.Module == "" || e.Fingerprint == "" || e.Branch != "" {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
if c.ends == nil {
|
||||
c.ends, c.endFPs = map[string]builds.BuildEnded{}, map[string]map[string]bool{}
|
||||
}
|
||||
if c.endFPs[e.Module] == nil {
|
||||
c.endFPs[e.Module] = map[string]bool{}
|
||||
}
|
||||
c.endFPs[e.Module][e.Fingerprint] = true
|
||||
if had, ok := c.ends[e.Module]; ok && had.Fingerprint == e.Fingerprint && had.Outcome == e.Outcome && had.Error == e.Error {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
c.ends[e.Module] = e
|
||||
ask := c.endsRead
|
||||
c.mu.Unlock()
|
||||
if ask {
|
||||
c.askRehand()
|
||||
}
|
||||
}
|
||||
|
||||
// BuildEndsRead says the first read of the builder's stream ended: from now the composer acts on the ends it heard, and a
|
||||
// pass is asked.
|
||||
func (c *Composer) BuildEndsRead() {
|
||||
c.mu.Lock()
|
||||
c.endsRead = true
|
||||
c.mu.Unlock()
|
||||
c.askRehand()
|
||||
}
|
||||
|
||||
// BuildEndsLost says the builder's stream is not followed any more: what was heard is not acted on until a new read ends.
|
||||
func (c *Composer) BuildEndsLost() {
|
||||
c.mu.Lock()
|
||||
c.endsRead = false
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *Composer) askRehand() {
|
||||
select {
|
||||
case c.rehand <- struct{}{}:
|
||||
|
||||
Reference in New Issue
Block a user