From 28c3b27fecf640117d74ed301eca937db8ad001f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 13:54:50 +0100 Subject: [PATCH 01/12] DLV-M3: the teller reads the assigner's asks by its read tool's first view and its views family, never its bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0615 §1: sdk/firstview is a process's picture of an owner's state, read as the bucket it replaces (Get, Keys, Watch; every write refused), asked through ._read_ with values and kept from the owner's views family; firstviewtest serves an owner in a whole-module test. The teller declares invokes assigner_read_asks, consumes views.asks, states first-views and findings, and the tools teller_first_views and teller_first_view_again. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-teller-tools/main.go | 7 +- .../mesh-delivery-teller/integration_test.go | 10 +- .../cmd/mesh-delivery-teller/main.go | 40 +- modules/mesh-delivery-teller/go.mod | 4 + modules/mesh-delivery-teller/manifest_test.go | 27 +- modules/mesh-delivery-teller/module.json | 15 +- sdk/firstview/firstview.go | 532 ++++++++++++++++++ sdk/firstview/firstviewtest/owner.go | 109 ++++ sdk/go.mod | 5 +- 9 files changed, 728 insertions(+), 21 deletions(-) create mode 100644 sdk/firstview/firstview.go create mode 100644 sdk/firstview/firstviewtest/owner.go diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller-tools/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller-tools/main.go index a7e035c4..f5b5e57d 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller-tools/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller-tools/main.go @@ -13,6 +13,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-teller/internal/ends" ) @@ -102,7 +103,11 @@ func askEnd(s stdio.KeptState, bound time.Duration) asker { } func main() { - if err := stdio.Serve("", Tools(askEnd(stdio.State(ends.State), 25*time.Second), time.Now)); err != nil { + // teller_first_views and teller_first_view_again: where the teller's first view of the assigner's asks stands, and + // asking it again by hand (ADR 0615, design 25 § 12.2). + tools := append(Tools(askEnd(stdio.State(ends.State), 25*time.Second), time.Now), + views.StatusTools("mesh-delivery-teller", "teller_first_views", "teller_first_view_again", stdio.State(views.StatusState), time.Now)...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-teller-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go index f7e015f0..5ea0dc00 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "git.novox.be/novox/mesh-delivery/sdk/firstview/firstviewtest" "os" "path/filepath" "strings" @@ -184,7 +185,10 @@ func fullConf(dir string) string { seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_"+messengerAsks+".$KV."+messengerAsks+".mesh-delivery-teller.>") // The durable consumer on EVENTS the controller makes for the seat it uses, pulled and acknowledged (issue 1126). teller = append(teller, "$JS.API.CONSUMER.INFO.EVENTS."+tellerConsumer, "$JS.API.CONSUMER.MSG.NEXT.EVENTS."+tellerConsumer, "$JS.ACK.EVENTS."+tellerConsumer+".>") - teller = append(append(teller, readOf(assignerAsks)...), readOf(viewAnswers)...) // issue 1297 B10: the manifest's reads + teller = append(teller, readOf(viewAnswers)...) // issue 1297 B10: the manifest's reads + // The assigner's asks by its read tool and its views family (ADR 0615, DLV-M3), and the teller's own first-views and + // findings; the bucket itself is no longer the teller's to read. + teller = append(append(append(teller, firstviewtest.Grants(delivery.AssignerModule, "asks", tellerConsumer)...), keepsOf(firstViewsBucket)...), keepsOf(findingsBucket)...) view := append(append(append(readOf(tellerAnswers), readOf(gateAsksKV)...), readOf(assignerAsks)...), keepsOf(viewAnswers)...) assigner := keepsOf(assignerAsks) tools := append(keepsOf(gateAsksKV), readOf(tellerAnswers)...) @@ -251,7 +255,7 @@ func startFull(t *testing.T) *fullBed { } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) - for name, history := range map[string]uint8{gateAsksKV: delivery.AsksHistory, tellerAnswers: delivery.AnswersHistory, tellerAsked: 1, ends.Bucket: 1, messengerAsks: 1, assignerAsks: delivery.AssignerAsksHistory, viewAnswers: delivery.ViewAnswersHistory} { + for name, history := range map[string]uint8{gateAsksKV: delivery.AsksHistory, tellerAnswers: delivery.AnswersHistory, tellerAsked: 1, ends.Bucket: 1, messengerAsks: 1, assignerAsks: delivery.AssignerAsksHistory, viewAnswers: delivery.ViewAnswersHistory, firstViewsBucket: 1, findingsBucket: 1} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil { t.Fatal(err) } @@ -267,6 +271,8 @@ func startFull(t *testing.T) *fullBed { AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } + // The assigner as the owner of its asks: its read tool and its views family (ADR 0615). + firstviewtest.Serve(t, ctx, js, ctl, delivery.AssignerModule, "asks", tellerConsumer) return &fullBed{srv: srv, ctl: ctl, js: js, ctx: ctx} } diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go index 8557c036..183f57ef 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go @@ -18,9 +18,13 @@ import ( "syscall" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/sdk/firstview" + "git.novox.be/novox/mesh-sdk/go/asker" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -83,18 +87,28 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not // there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer). failed := make(chan error, 4) - assignerKV, viewKV := process.OptionalBucket(ctx, js, delivery.AssignerAsksBucket, teller, logger.Printf), process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf) + // The assigner's asks are its state, read by the first view of its read tool and then its views family, never its + // bucket (ADR 0615 §1, DLV-M3): the picture reads as the bucket did. + calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return fmt.Errorf("the teller's connection for its tool calls: %w", err) + } + defer calls.Close() + reader := &firstview.Reader{Module: delivery.TellerModule, Call: calls.Call, Logf: logger.Printf} + findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "a first view unanswered past its bound is said in the log alone", logger.Printf) + reader.Findings = findings.KV + assignerKV := jetstream.KeyValue(reader.Follow(delivery.AssignerModule, assignerAsksState)) + reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "teller_first_views reads nothing and teller_first_view_again is not heard", logger.Printf)) + viewKV := process.OptionalBucket(ctx, js, delivery.ViewAnswersBucket, teller, logger.Printf) // The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906). if viewKV != nil { t.View = kvGate{ctx: ctx, kv: viewKV} } - if assignerKV != nil { - t.Assigner = kvGate{ctx: ctx, kv: assignerKV} - go func() { - failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil) - }() - } - if assignerKV != nil && viewKV != nil { + t.Assigner = kvGate{ctx: ctx, kv: assignerKV} + go func() { + failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil) + }() + if viewKV != nil { go func() { failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil) }() @@ -131,5 +145,15 @@ const teller = "the asks and answers it holds are not carried until the teller i // raised (manifest state entry asked, novox/hq ADR 0437 §4). const askedBucket = delivery.TellerModule + "_asked" +// The teller's own states of its first view of the assigner's asks (views.StatusState) and of its findings, where the +// condition keeper raises a first view unanswered past its bound (ADR 0615 §6). +var ( + firstViewsBucket = delivery.TellerModule + "_" + views.StatusState + findingsBucket = delivery.TellerModule + "_findings" +) + +// assignerAsksState is the assigner's state of its asks (delivery.AssignerAsksBucket). +const assignerAsksState = "asks" + // waiting is said once at start. const waiting = "waiting for the gate's asks, whose bucket does not exist yet; the teller is started again with the gate, and nothing is read, written or published until then" diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index ee19602f..add68f57 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -20,3 +20,7 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) + +require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 + +replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk diff --git a/modules/mesh-delivery-teller/manifest_test.go b/modules/mesh-delivery-teller/manifest_test.go index 74420f55..8de6b9e8 100644 --- a/modules/mesh-delivery-teller/manifest_test.go +++ b/modules/mesh-delivery-teller/manifest_test.go @@ -40,6 +40,8 @@ type manifest struct { Uses []string `json:"uses"` State []state `json:"state"` Reads []string `json:"reads"` + Invokes []string `json:"invokes"` + Consumes []string `json:"consumes"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -80,13 +82,26 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1 } // answers is the teller's answer to each ask; asked is what it asked of the router, which condition it raised and which // gate key and revision each ask was made at (issue 1343); ends is the ends asked by hand with teller_end_ask (issue 774). - if want := []state{{Name: "answers", History: 5}, {Name: "asked", History: 1}, {Name: "ends", History: 1}}; !reflect.DeepEqual(m.State, want) { + // first-views is where its first view of the assigner's asks stands and findings its condition past the bound (ADR 0615). + if want := []state{{Name: "answers", History: 5}, {Name: "asked", History: 1}, {Name: "ends", History: 1}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - // The assigner's asks and the view's answers to them are read so the view's answer fills an ask that needs no proof (1297/B10). - if want := []string{"mesh-delivery-assigner.asks", "mesh-delivery-gate.asks", "mesh-delivery-view.answers"}; !reflect.DeepEqual(sorted(m.Reads), want) { + // The view's answers to the assigner's asks are read so the view's answer fills an ask that needs no proof (1297/B10); the + // assigner's asks are its state, asked by its read tool and followed by its views family, never read (ADR 0615 §1, DLV-M3). + if want := []string{"mesh-delivery-gate.asks", "mesh-delivery-view.answers"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } + for _, r := range m.Reads { + if strings.HasPrefix(r, "mesh-delivery-indexer.") || strings.HasPrefix(r, "mesh-delivery-assigner.") { + t.Errorf("reads %s: the indexer's and the assigner's state is read by their tools and views families alone", r) + } + } + if want := []string{"mesh-delivery-assigner.assigner_read_asks"}; !reflect.DeepEqual(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } + if want := []string{"mesh-delivery-assigner.views.asks"}; !reflect.DeepEqual(m.Consumes, want) { + t.Errorf("consumes = %v, want %v", m.Consumes, want) + } if want := []string{"operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) { t.Errorf("uses = %v, want %v", m.Uses, want) } @@ -97,11 +112,11 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 774: teller_end_ask, served by its tools bundle through the state ends; the process serves none. - if want := []string{"teller_end_ask"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"teller_end_ask", "teller_first_views", "teller_first_view_again"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - // No consumes, invokes, emits, streams, bus or seat (ADR 0437 §4). - for _, k := range []string{"consumes", "invokes", "streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus"} { + // No emits, streams, bus or seat (ADR 0437 §4). + for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the teller has none", k) } diff --git a/modules/mesh-delivery-teller/module.json b/modules/mesh-delivery-teller/module.json index 6d814c0e..be0af507 100644 --- a/modules/mesh-delivery-teller/module.json +++ b/modules/mesh-delivery-teller/module.json @@ -3,15 +3,24 @@ "version": "1", "slug": "teller", "tools": [ - "teller_end_ask" + "teller_end_ask", + "teller_first_views", + "teller_first_view_again" + ], + "invokes": [ + "mesh-delivery-assigner.assigner_read_asks" + ], + "consumes": [ + "mesh-delivery-assigner.views.asks" ], "state": [ { "name": "answers", "history": 5 }, { "name": "asked", "history": 1 }, - { "name": "ends", "history": 1 } + { "name": "ends", "history": 1 }, + { "name": "first-views", "history": 1 }, + { "name": "findings", "history": 1 } ], "reads": [ - "mesh-delivery-assigner.asks", "mesh-delivery-gate.asks", "mesh-delivery-view.answers" ], diff --git a/sdk/firstview/firstview.go b/sdk/firstview/firstview.go new file mode 100644 index 00000000..57d82e26 --- /dev/null +++ b/sdk/firstview/firstview.go @@ -0,0 +1,532 @@ +// Package firstview is a process's pictures of the indexer's and the assigner's states it used to read (novox/hq ADR +// 0615 §1, design 25 § 12.2), for the mesh-delivery parts that run as processes (the builder, the checker, the teller, +// the view, the watchman): each state is asked once through the owner's ordinary read tool with `values` (the first +// view), then kept from the owner's views family, asked again only on a gap or by hand (mesh-sdk go/views Follower). +// +// A Picture reads as the key-value bucket the process used to bind (jetstream.KeyValue: Get, Keys, ListKeys, Watch, +// WatchAll), so a reader's code is unchanged; every write is refused, for the owner's state is the owner's alone. The +// picture is kept in memory: the owner's state is the truth and a restart asks again (ADR 0615 §4). +// +// It is the process side of mesh-sdk go/views Reader (the stdio side) and moves there whole when mesh-sdk takes it. +package firstview + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// ErrReadOnly is every write to a picture: the state is its owner's. +var ErrReadOnly = errors.New("a first view is read-only: the state is its owner's") + +// ReadTool is the owner's ordinary read tool of state, as a reader's `invokes` names it (design 25 § 12.2): +// `._read_`, short the owner's name past its last dash-separated `mesh-delivery-` prefix +// (indexer, assigner) and the state's dashes as underscores. +func ReadTool(owner, state string) string { + short := strings.TrimPrefix(owner, "mesh-delivery-") + return owner + "." + short + "_read_" + strings.ReplaceAll(state, "-", "_") +} + +// Reader is one process's pictures of its owners' states. +type Reader struct { + // Module is the reading module, named in the finding of a first view unanswered past its bound. + Module string + // Call asks a module's tool (process.Conn.Call): the read tools are asked with `values`. + Call func(key string, body any) (json.RawMessage, error) + // Findings is the reader's own state `findings`, where the condition keeper raises and clears the condition of a + // first view unanswered past its bound (ADR 0615 §6); nil says it in the log only. + Findings func() jetstream.KeyValue + Logf func(string, ...any) + // Now, Bound and Wait are for tests: zero are the clock, views.Bound and five seconds. + Now func() time.Time + Bound time.Duration + Wait time.Duration + + mu sync.Mutex + pics map[string]*Picture // by bucket name _ + order []string +} + +func (r *Reader) logf(format string, args ...any) { + if r.Logf != nil { + r.Logf(format, args...) + } +} + +// Follow adds the owner's state to follow through its read tool (ReadTool). Call it before Start. +func (r *Reader) Follow(owner, state string) *Picture { + r.mu.Lock() + defer r.mu.Unlock() + if r.pics == nil { + r.pics = map[string]*Picture{} + } + bucket := owner + "_" + state + if p := r.pics[bucket]; p != nil { + return p + } + tool := ReadTool(owner, state) + p := &Picture{bucket: bucket, ctx: context.Background()} + call := r.Call + ask := views.AskVia(func(key string, body any) (json.RawMessage, error) { + m, _ := body.(map[string]any) + if m == nil { + m = map[string]any{} + } + m["values"] = true + if call == nil { + return nil, errors.New("this reader asks no tool") + } + return call(key, m) + }, tool) + p.f = &views.Follower{Owner: owner, Tool: tool, State: state, Ask: ask, Logf: r.Logf, Now: r.Now, Bound: r.Bound, Wait: r.Wait, + Raise: func(s views.Status) { r.finding(s, true) }, Clear: func(s views.Status) { r.finding(s, false) }, + Changed: p.changed} + r.pics[bucket] = p + r.order = append(r.order, bucket) + return p +} + +// Picture is the picture of the bucket the process used to bind, by its bus name (`_`); nil when not +// followed. +func (r *Reader) Picture(bucket string) *Picture { + r.mu.Lock() + defer r.mu.Unlock() + return r.pics[bucket] +} + +// Followers are the followers, in the order they were added. +func (r *Reader) Followers() []*views.Follower { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]*views.Follower, 0, len(r.order)) + for _, b := range r.order { + out = append(out, r.pics[b].f) + } + return out +} + +func (r *Reader) finding(s views.Status, raised bool) { + if raised { + r.logf("the first view %s of %s's %s is unanswered past its bound: %s", s.Tool, s.Owner, s.State, s.Error) + } + if r.Findings == nil { + return + } + kv := r.Findings() + if kv == nil { + return + } + at := time.Now() + if r.Now != nil { + at = r.Now() + } + key, value, err := views.UnansweredFinding(r.Module, s, at) + if err == nil && raised { + _, err = kv.Put(context.Background(), key, value) + } else if err == nil { + err = kv.Delete(context.Background(), key) + } + if err != nil { + r.logf("views: the finding of the first view %s is not written: %v", s.Tool, err) + } +} + +// Hear is the handler of the owner's views family on the reader's durable consumer (cred.EventConsumer on the owner's +// views stream): each event to the follower of its state. +func (r *Reader) Hear(ctx context.Context, owner string) func(jetstream.Msg) { + fs := map[string]*views.Follower{} + for _, f := range r.Followers() { + if f.Owner == owner { + fs[f.State] = f + } + } + return views.Hear(ctx, fs, r.logf) +} + +// Owners are the owners followed, sorted. +func (r *Reader) Owners() []string { + var out []string + for _, f := range r.Followers() { + if !slices.Contains(out, f.Owner) { + out = append(out, f.Owner) + } + } + slices.Sort(out) + return out +} + +// Ask asks every first view until answered, each on a goroutine of its own, so the process starts at once and waits for +// an owner that is down, saying so (ADR 0615 §6). Said, when set, keeps each status (views.KeepStatus). +func (r *Reader) Ask(ctx context.Context, said func(views.Status)) { + for _, f := range r.Followers() { + if said != nil { + f.Said = said + } + go func() { + if err := f.Start(ctx); err != nil { + r.logf("the first view %s was not answered: %v", f.Tool, err) + } + }() + } +} + +// Ready is closed once every followed state had its first view. +func (r *Reader) Ready() <-chan struct{} { + fs := r.Followers() + done := make(chan struct{}) + go func() { + for _, f := range fs { + <-f.Ready() + } + close(done) + }() + return done +} + +// Picture is one owner's state as the process reads it: a read-only jetstream.KeyValue over a views.Follower. Its +// revisions are the picture's own, one per change it took, so a watcher sees them rise; they are not the owner's. +type Picture struct { + jetstream.KeyValue // never set: every method a reader may call is the picture's own + + bucket string + ctx context.Context + f *views.Follower + + mu sync.Mutex + seq uint64 + revs map[string]uint64 + watchers []*watcher +} + +// Follower is the picture's follower (its status, a gap or a hand ask). +func (p *Picture) Follower() *views.Follower { return p.f } + +func (p *Picture) changed(key string, value json.RawMessage, deleted bool, at time.Time) { + p.mu.Lock() + p.seq++ + if p.revs == nil { + p.revs = map[string]uint64{} + } + p.revs[key] = p.seq + op := jetstream.KeyValuePut + if deleted { + op, value = jetstream.KeyValueDelete, nil + } + e := entry{bucket: p.bucket, key: key, value: value, rev: p.seq, at: at, op: op} + ws := slices.Clone(p.watchers) + p.mu.Unlock() + for _, w := range ws { + w.send(e) + } +} + +// answered is nil once the first view is answered, else the error a read before it gets (never a wait). +func (p *Picture) answered() error { + select { + case <-p.f.Ready(): + return nil + default: + return fmt.Errorf("no first view of %s yet: %s has not answered", p.bucket, p.f.Tool) + } +} + +func (p *Picture) ready(ctx context.Context) error { + select { + case <-p.f.Ready(): + return nil + case <-ctx.Done(): + return fmt.Errorf("no first view of %s yet (%s has not answered): %w", p.bucket, p.f.Tool, ctx.Err()) + } +} + +// Bucket is the bus name of the owner's state the picture stands for. +func (p *Picture) Bucket() string { return p.bucket } + +// Get is one key; before the first view is answered it is an error, never a wait. +func (p *Picture) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + if err := p.answered(); err != nil { + return nil, err + } + v, ok := p.f.Get(key) + if !ok { + return nil, jetstream.ErrKeyNotFound + } + p.mu.Lock() + rev := p.revs[key] + p.mu.Unlock() + return entry{bucket: p.bucket, key: key, value: v, rev: rev, op: jetstream.KeyValuePut}, nil +} + +// Keys is every key; none is jetstream.ErrNoKeysFound, as the bus says it; before the first view is answered it is an +// error, never a wait. +func (p *Picture) Keys(_ context.Context, _ ...jetstream.WatchOpt) ([]string, error) { + if err := p.answered(); err != nil { + return nil, err + } + var keys []string + for k := range p.f.Values() { + keys = append(keys, k) + } + if len(keys) == 0 { + return nil, jetstream.ErrNoKeysFound + } + slices.Sort(keys) + return keys, nil +} + +// ListKeys is Keys on a channel. +func (p *Picture) ListKeys(ctx context.Context, opts ...jetstream.WatchOpt) (jetstream.KeyLister, error) { + keys, err := p.Keys(ctx, opts...) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, err + } + ch := make(chan string, len(keys)) + for _, k := range keys { + ch <- k + } + close(ch) + return lister{ch}, nil +} + +// WatchAll is Watch of every key. +func (p *Picture) WatchAll(ctx context.Context, opts ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + return p.Watch(ctx, ">", opts...) +} + +// Watch hands, once the first view is answered, every current value of the keys matching (a subject pattern: `*` one +// token, `>` the rest), then nil, then each change, as the bus's watch does. Options are not read: a picture holds no +// history, and deletes are always handed. +func (p *Picture) Watch(ctx context.Context, keys string, _ ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + w := &watcher{ch: make(chan jetstream.KeyValueEntry, 64), match: keys, wake: make(chan struct{}, 1), stop: make(chan struct{})} + go func() { + defer close(w.ch) + if p.ready(ctx) != nil { + return + } + p.mu.Lock() + p.watchers = append(p.watchers, w) + p.mu.Unlock() + defer p.unwatch(w) + vals := p.f.Values() + names := make([]string, 0, len(vals)) + for k := range vals { + names = append(names, k) + } + slices.Sort(names) + p.mu.Lock() + first := make([]jetstream.KeyValueEntry, 0, len(names)+1) + for _, k := range names { + if matches(keys, k) { + first = append(first, entry{bucket: p.bucket, key: k, value: vals[k], rev: p.revs[k], op: jetstream.KeyValuePut}) + } + } + p.mu.Unlock() + first = append(first, nil) + w.run(ctx, first) + }() + return w, nil +} + +func (p *Picture) unwatch(w *watcher) { + p.mu.Lock() + defer p.mu.Unlock() + p.watchers = slices.DeleteFunc(p.watchers, func(x *watcher) bool { return x == w }) +} + +// The writes and the reads a picture cannot answer are refused, never handed to a bucket. +func (p *Picture) Put(context.Context, string, []byte) (uint64, error) { return 0, ErrReadOnly } +func (p *Picture) PutString(context.Context, string, string) (uint64, error) { return 0, ErrReadOnly } +func (p *Picture) Create(context.Context, string, []byte, ...jetstream.KVCreateOpt) (uint64, error) { + return 0, ErrReadOnly +} +func (p *Picture) Update(context.Context, string, []byte, uint64) (uint64, error) { + return 0, ErrReadOnly +} +func (p *Picture) Delete(context.Context, string, ...jetstream.KVDeleteOpt) error { return ErrReadOnly } +func (p *Picture) Purge(context.Context, string, ...jetstream.KVDeleteOpt) error { return ErrReadOnly } +func (p *Picture) PurgeDeletes(context.Context, ...jetstream.KVPurgeOpt) error { return ErrReadOnly } +func (p *Picture) GetRevision(context.Context, string, uint64) (jetstream.KeyValueEntry, error) { + return nil, errors.New("a first view holds no history") +} +func (p *Picture) History(context.Context, string, ...jetstream.WatchOpt) ([]jetstream.KeyValueEntry, error) { + return nil, errors.New("a first view holds no history") +} +func (p *Picture) WatchFiltered(ctx context.Context, keys []string, opts ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) { + if len(keys) != 1 { + return nil, errors.New("a first view watches one pattern") + } + return p.Watch(ctx, keys[0], opts...) +} +func (p *Picture) ListKeysFiltered(ctx context.Context, filters ...string) (jetstream.KeyLister, error) { + keys, err := p.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, err + } + ch := make(chan string, len(keys)) + for _, k := range keys { + if len(filters) == 0 || slices.ContainsFunc(filters, func(f string) bool { return matches(f, k) }) { + ch <- k + } + } + close(ch) + return lister{ch}, nil +} +func (p *Picture) Status(context.Context) (jetstream.KeyValueStatus, error) { + return nil, errors.New("a first view has no bucket status: the owner's read tool says its fill") +} + +// matches says whether key matches a subject pattern of the bus's watch. +func matches(pattern, key string) bool { + if pattern == "" || pattern == ">" { + return true + } + pt, kt := strings.Split(pattern, "."), strings.Split(key, ".") + for i, t := range pt { + if t == ">" { + return len(kt) > i + } + if i >= len(kt) || (t != "*" && t != kt[i]) { + return false + } + } + return len(pt) == len(kt) +} + +// watcher hands a watch's entries in order: the current values first, then each change heard, never blocking the +// follower (the changes wait in a queue of its own). +type watcher struct { + ch chan jetstream.KeyValueEntry + match string + wake chan struct{} + stop chan struct{} + once sync.Once + + mu sync.Mutex + queue []jetstream.KeyValueEntry +} + +func (w *watcher) send(e entry) { + if !matches(w.match, e.key) { + return + } + w.mu.Lock() + w.queue = append(w.queue, e) + w.mu.Unlock() + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *watcher) run(ctx context.Context, first []jetstream.KeyValueEntry) { + out := func(e jetstream.KeyValueEntry) bool { + select { + case w.ch <- e: + return true + case <-ctx.Done(): + return false + case <-w.stop: + return false + } + } + for _, e := range first { + if !out(e) { + return + } + } + for { + w.mu.Lock() + q := w.queue + w.queue = nil + w.mu.Unlock() + for _, e := range q { + if !out(e) { + return + } + } + select { + case <-w.wake: + case <-ctx.Done(): + return + case <-w.stop: + return + } + } +} + +func (w *watcher) Updates() <-chan jetstream.KeyValueEntry { return w.ch } + +func (w *watcher) Stop() error { + w.once.Do(func() { close(w.stop) }) + return nil +} + +type lister struct{ ch chan string } + +func (l lister) Keys() <-chan string { return l.ch } +func (l lister) Stop() error { return nil } + +type entry struct { + bucket, key string + value []byte + rev uint64 + at time.Time + op jetstream.KeyValueOp +} + +func (e entry) Bucket() string { return e.bucket } +func (e entry) Key() string { return e.key } +func (e entry) Value() []byte { return e.value } +func (e entry) Revision() uint64 { return e.rev } +func (e entry) Created() time.Time { return e.at } +func (e entry) Delta() uint64 { return 0 } +func (e entry) Operation() jetstream.KeyValueOp { return e.op } + +// ViewsStream is the bus name of owner's views family (views.IndexerStream, views.AssignerStream): `-views` +// as the owner's old `streams` section names it. +func ViewsStream(owner string) string { + return catalogue.StreamBusName(strings.TrimPrefix(owner, "mesh-delivery-") + "-views") +} + +// Start hears each owner's views family on the process's durable consumer of it (its manifest's `consumes` +// `.views.`), keeps each status in the process's state `first-views` and hears the hand asks the tools +// bundle's `_first_view_again` writes there (views.StatusTools), then asks every first view (Ask). A consumer +// the bus does not have yet is said: the picture is then the first view's alone, asked again on a gap or by hand. +func (r *Reader) Start(ctx context.Context, js jetstream.JetStream, cred process.Account, status *process.AwaitedBucket) { + for _, owner := range r.Owners() { + stream := ViewsStream(owner) + cons, err := cred.EventConsumer(ctx, js, stream) + if err == nil { + _, err = process.HearEvents(cons, r.Hear(ctx, owner), r.logf) + } + if err != nil { + r.logf("the consumer %s on %s is not heard (%v): %s's states are the first view's alone, asked again by hand, until the %s starts again", + cred.EventConsumerName(), stream, err, owner, r.Module) + } + } + var said func(views.Status) + if status != nil { + status.Ready(func(kv jetstream.KeyValue) { + if err := views.HearAgain(ctx, kv, r.Followers(), r.logf); err != nil { + r.logf("views: the hand asks of %s's first views are not heard (%v) until it starts again", r.Module, err) + } + }) + said = func(s views.Status) { + if kv := status.KV(); kv != nil { + views.KeepStatus(ctx, kv, r.logf)(s) + } + } + } + r.Ask(ctx, said) +} diff --git a/sdk/firstview/firstviewtest/owner.go b/sdk/firstview/firstviewtest/owner.go new file mode 100644 index 00000000..761e13ad --- /dev/null +++ b/sdk/firstview/firstviewtest/owner.go @@ -0,0 +1,109 @@ +// Package firstviewtest is an owner of a state as a reader's whole-module test needs it (novox/hq ADR 0615): the +// owner's read tool answering the first view from its bucket at its mark, and its views family said by the real +// views emitter onto its views stream, which the reader's durable consumer reads. +package firstviewtest + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + + "git.novox.be/novox/mesh-delivery/sdk/firstview" +) + +// Serve is owner's state on the bus with the owner's side of it: the bucket `_` (made when absent, history +// 1), the owner's marks, its views stream with a durable consumer per reader (`_`), the emitter saying +// each change, and the read tool (firstview.ReadTool) answering on nc. It returns the bucket the test writes. +func Serve(t testing.TB, ctx context.Context, js jetstream.JetStream, nc *nats.Conn, owner, state string, readers ...string) jetstream.KeyValue { + t.Helper() + bucket := owner + "_" + state + kv, err := js.KeyValue(ctx, bucket) + if err != nil { + if kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil { + t.Fatal(err) + } + } + marks, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: owner + "_" + views.MarksState, History: 1}) + if err != nil { + t.Fatal(err) + } + stream := firstview.ViewsStream(owner) + if _, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{process.ModuleStreamSubject(owner, views.EventPrefix+">")}}); err != nil { + t.Fatal(err) + } + for _, r := range readers { + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: r, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + } + em := views.Emitter{Owner: owner, Subject: func(e string) string { return process.ModuleStreamSubject(owner, e) }, + Publish: views.JetStreamPublish(js), Marks: marks} + if err := em.Start(ctx, state, kv, nil); err != nil { + t.Fatal(err) + } + tool := strings.TrimPrefix(firstview.ReadTool(owner, state), owner+".") + sub, err := nc.Subscribe("mesh.mod."+owner+".call."+tool+".>", func(m *nats.Msg) { + var in struct { + After string `json:"after"` + } + _ = json.Unmarshal(m.Data, &in) + a, err := views.FirstView(owner, state, Source{ctx, kv}, Source{ctx, marks}, in.After, time.Now()) + out := map[string]any{"result": a, "node": "n0"} + if err != nil { + out = map[string]any{"error": err.Error()} + } + raw, _ := json.Marshal(out) + _ = m.Respond(raw) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + return kv +} + +// Grants are the subjects a reader's account publishes to follow owner's state (its `invokes` and `consumes`): the +// read tool's call subjects, its durable consumer's info, next and acknowledgements on the owner's views stream. +func Grants(owner, state, durable string) []string { + tool := strings.TrimPrefix(firstview.ReadTool(owner, state), owner+".") + stream := firstview.ViewsStream(owner) + return []string{"mesh.mod." + owner + ".call." + tool + ".>", "mesh.mod." + owner + ".call." + tool, + "$JS.API.CONSUMER.INFO." + stream + "." + durable, "$JS.API.CONSUMER.MSG.NEXT." + stream + "." + durable, + "$JS.ACK." + stream + "." + durable + ".>"} +} + +// Source is a bucket as a views.Source. +type Source struct { + Ctx context.Context + KV jetstream.KeyValue +} + +// Keys is every key; none is no error. +func (s Source) Keys() ([]string, error) { + keys, err := s.KV.Keys(s.Ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +// Get is one key's value and revision. +func (s Source) Get(key string) (*views.Entry, bool, error) { + e, err := s.KV.Get(s.Ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return &views.Entry{Value: e.Value(), Revision: e.Revision()}, true, nil +} diff --git a/sdk/go.mod b/sdk/go.mod index 7e692d40..c6bae45b 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -2,4 +2,7 @@ module git.novox.be/novox/mesh-delivery/sdk go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + github.com/nats-io/nats.go v1.54.0 +) From 1ce394db97028044dc9e3158f75c217642da5354 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 13:56:31 +0100 Subject: [PATCH 02/12] DLV-M6: the enrolment follows the assigner's placement by its read tool's first view and its views family, never its state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0615 §1: views.Reader asks mesh-delivery-assigner.assigner_read_placement and hears views.placement; the holder reads placement from the picture and every other state through the runtime. The manifest drops the read, invokes the tool, consumes the event, keeps findings and serves enrolment_first_views and enrolment_first_view_again. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-enrolment/enrolment_test.go | 28 ++++++++++ .../cmd/mesh-enrolment/holder.go | 4 +- .../mesh-enrolment/cmd/mesh-enrolment/main.go | 56 +++++++++++++++++-- .../cmd/mesh-enrolment/tokens_test.go | 20 +++++-- modules/mesh-enrolment/module.json | 14 +++-- 5 files changed, 108 insertions(+), 14 deletions(-) diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go index bb157f87..1c6513ea 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/enrolment_test.go @@ -20,6 +20,7 @@ func TestIssueTokensAndTheManifest(t *testing.T) { {"S54/AC1 tokens answers each stage and relays the sealed token of one node", tokensAnswersEachStageAndRelaysTheSealedTokenOfOneNode}, {"S54/AC2 the manifest says what the code does", theManifestSaysWhatTheCodeDoes}, {"S56 the bus node's loaded list is read from the grants part's pass", theLoadedListIsReadFromTheGrantsPass}, + {"DLV-M6 the placement is read from its first view, never from the assigner's state", thePlacementIsReadFromItsFirstView}, } { t.Run(c.name, c.run) } @@ -59,3 +60,30 @@ func theLoadedListIsReadFromTheGrantsPass(t *testing.T) { t.Error("no pass composed read as an answer") } } + +// thePlacementIsReadFromItsFirstView: the holder's placement comes from the picture of the assigner's placement (its first +// view and views family, ADR 0615 §1), every other state through the runtime; the runtime is never asked for placement. +func thePlacementIsReadFromItsFirstView(t *testing.T) { + pics := memPictures{PlacementRead: {"joiner.node": `{"v":1,"node":"joiner","decided":{"address":"10.9.0.11"},"modules":[]}`}} + rest := memRead{PlacementRead: {"stale.node": `{"v":1,"node":"stale","modules":[]}`}, "node-engine.have": {}} + h := &Holder{Read: followedReader{pics: pics, rest: rest}} + p, err := h.placements() + if err != nil { + t.Fatal(err) + } + if _, ok := p["joiner"]; !ok || len(p) != 1 { + t.Errorf("placements = %v, want joiner's alone, from the first view", p) + } + if keys, err := h.Read.Keys("node-engine.have"); err != nil || len(keys) != 0 { + t.Errorf("node-engine.have read as %v, %v: want it through the runtime", keys, err) + } +} + +// memPictures is the followed states (views.Reader), `.` then key. +type memPictures map[string]map[string]string + +func (p memPictures) Follows(state string) bool { _, ok := p[state]; return ok } +func (p memPictures) Keys(state string) ([]string, error) { + return memRead(p).Keys(state) +} +func (p memPictures) Get(state, key string) ([]byte, bool, error) { return memRead(p).Get(state, key) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go index 35c52701..3251ef43 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/holder.go @@ -189,13 +189,13 @@ func (h *Holder) openEnrolments() (map[string]Row, map[string]Enrolment, error) // placements reads every node's placement, assembled from its keys. func (h *Holder) placements() (map[string]delivery.Placement, error) { - keys, err := h.Read.Keys("mesh-delivery-assigner.placement") + keys, err := h.Read.Keys(PlacementRead) if err != nil { return nil, fmt.Errorf("placement cannot be listed: %w", err) } held := map[string][]byte{} for _, k := range keys { - if raw, found, err := h.Read.Get("mesh-delivery-assigner.placement", k); err != nil { + if raw, found, err := h.Read.Get(PlacementRead, k); err != nil { return nil, fmt.Errorf("placement key %s cannot be read: %w", k, err) } else if found { held[k] = raw diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go index 50169542..778c48d3 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/main.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/main.go @@ -5,6 +5,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -13,7 +14,9 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/views" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -84,14 +87,58 @@ func (runtimeReader) Get(state, key string) ([]byte, bool, error) { return e.Value, true, nil } +// pictures are the states the enrolment follows rather than reads (views.Reader): the assigner's placement (ADR 0615 §1). +type pictures interface { + Follows(state string) bool + Keys(state string) ([]string, error) + Get(state, key string) ([]byte, bool, error) +} + +// followedReader reads a followed state from its picture and every other state through rest. +type followedReader struct { + pics pictures + rest Reader +} + +func (r followedReader) Keys(state string) ([]string, error) { + if r.pics != nil && r.pics.Follows(state) { + return r.pics.Keys(state) + } + return r.rest.Keys(state) +} + +func (r followedReader) Get(state, key string) ([]byte, bool, error) { + if r.pics != nil && r.pics.Follows(state) { + return r.pics.Get(state, key) + } + return r.rest.Get(state, key) +} + +// PlacementRead is the assigner's placement as the enrolment reads it, and PlacementTool the assigner's read tool the +// manifest invokes for its first view (ADR 0615, DLV-M6). +const ( + PlacementRead = delivery.AssignerModule + "." + delivery.PlacementState + PlacementTool = delivery.AssignerModule + ".assigner_read_placement" +) + +// followed is the enrolment's pictures of the assigner's placement, started at once: its first view asked through +// PlacementTool, then the assigner's views family; past its bound the finding in the enrolment's own findings. +func followed() *views.Reader { + v := &views.Reader{Module: Module, Logf: func(f string, a ...any) { logf("["+Module+"] "+f, a...) }, Findings: views.OwnFindings{}, + Acts: handacts.Recorder{Ask: stdio.Ask}} + v.Follow(delivery.AssignerModule, delivery.PlacementState, PlacementTool) + v.Start(context.Background()) + return v +} + // SeatEventSubject is where the seat says an event: mesh.seat..event.. func SeatEventSubject(event string) string { return "mesh.seat." + Seat + ".event." + event } // newHolder is the holder over the runtime. -func newHolder() *Holder { +func newHolder(v *views.Reader) *Holder { call := Call(stdio.Ask) rec := handacts.Recorder{Ask: stdio.Ask} - rd := runtimeReader{} + rd := followedReader{pics: v, rest: runtimeReader{}} client := asker.New("asked") return &Holder{ Store: runtimeStore{}, Read: rd, Vault: vaultOver{call: call, read: rd}, Asker: client, @@ -169,10 +216,11 @@ func loadedVia(rd Reader) func(string) (bool, error) { } func main() { - h := newHolder() + v := followed() + h := newHolder(v) client := h.Asker.(*asker.Client) h.Run(func() error { return client.Listen(h.Warranted, h.Said) }) - if err := stdio.Serve("", Tools(h)); err != nil { + if err := stdio.Serve("", append(Tools(h), v.Tools("enrolment")...)); err != nil { logf("%v", err) os.Exit(1) } diff --git a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go index 668f1931..5141f2af 100644 --- a/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go +++ b/modules/mesh-enrolment/cmd/mesh-enrolment/tokens_test.go @@ -82,9 +82,11 @@ func theManifestSaysWhatTheCodeDoes(t *testing.T) { Scope string `json:"scope"` Serves []string `json:"serves"` } `json:"claims"` - Emits []string `json:"emits"` - Reads []string `json:"reads"` - State []json.RawMessage `json:"state"` + Emits []string `json:"emits"` + Reads []string `json:"reads"` + Invokes []string `json:"invokes"` + Tools []string `json:"tools"` + State []json.RawMessage `json:"state"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -96,10 +98,20 @@ func theManifestSaysWhatTheCodeDoes(t *testing.T) { if slices.Contains(m.Reads, "mesh-controller.lease") { t.Error("the manifest reads the controller's lease, which the code no longer reads (issue 1126)") } + // The assigner's placement is followed, never read (ADR 0615 §1, DLV-M6): its read tool invoked, its views family consumed. + for _, r := range m.Reads { + if strings.HasPrefix(r, "mesh-delivery-indexer.") || strings.HasPrefix(r, "mesh-delivery-assigner.") { + t.Errorf("reads %s: the indexer's and the assigner's state is read by their tools and views families alone", r) + } + } + if !slices.Contains(m.Invokes, PlacementTool) || !slices.Equal(m.Tools, []string{"enrolment_first_views", "enrolment_first_view_again"}) { + t.Errorf("invokes %v, tools %v: want %s and the first-view tools", m.Invokes, m.Tools, PlacementTool) + } var heard []string for _, e := range EndEvents { heard = append(heard, Seat+"."+e) } + heard = append(heard, "mesh-delivery-assigner.views.placement") if !slices.Equal(m.Consumes, heard) { t.Errorf("consumes %v, the end events are %v", m.Consumes, heard) } @@ -112,7 +124,7 @@ func theManifestSaysWhatTheCodeDoes(t *testing.T) { t.Errorf("the verb %s has no tool", v) } } - if len(m.State) != 2 || string(m.State[0]) == "" || !slices.ContainsFunc(m.State, func(s json.RawMessage) bool { + if len(m.State) != 3 || string(m.State[0]) == "" || !slices.ContainsFunc(m.State, func(s json.RawMessage) bool { var d struct { Name string `json:"name"` TTL bool `json:"key-ttl"` diff --git a/modules/mesh-enrolment/module.json b/modules/mesh-enrolment/module.json index 4aed2463..2dfb4404 100644 --- a/modules/mesh-enrolment/module.json +++ b/modules/mesh-enrolment/module.json @@ -20,7 +20,8 @@ "mesh-enrolment.node-enrolled", "mesh-enrolment.enrolment-refused", "mesh-enrolment.enrolment-revoked", - "mesh-enrolment.enrolment-expired" + "mesh-enrolment.enrolment-expired", + "mesh-delivery-assigner.views.placement" ], "uses": [ "operator-channel" @@ -31,10 +32,10 @@ "seat:mesh-placement.propose", "seat:mesh-vault.enrolment-token", "seat:mesh-vault.node-account", - "seat:mesh-vault.sign" + "seat:mesh-vault.sign", + "mesh-delivery-assigner.assigner_read_placement" ], "reads": [ - "mesh-delivery-assigner.placement", "mesh-grants.user-list", "mesh-vault.signers", "node-engine.have" @@ -47,7 +48,12 @@ { "name": "asked", "history": 5 - } + }, + "findings" + ], + "tools": [ + "enrolment_first_views", + "enrolment_first_view_again" ], "own-secrets": { "broker": "${dir:state}/broker" From 72d8e2155f9065ee869025940b98e587215c4408 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:06:04 +0100 Subject: [PATCH 03/12] DLV-M5: the watchman follows the assigner's wants and the indexer's modules by their read tools' first views, read once answered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0615 §1: the watchman invokes assigner_read_wants and indexer_read_modules, consumes their views families and never reads either bucket; it keeps first-views and writes an unanswered first view as a finding of its own. The lead's ruling: an owner that is down never holds the start; each state is read on a goroutine once its first view is answered, and what it holds (seeded pulses, stuck keys, the findings it is the source of) is reconciled then (watch.Awaits, ReadLate), never dropped at the start's end. Tools watchman_first_views and watchman_first_view_again. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-watchman-tools/main.go | 14 ++- .../integration_test.go | 15 +++ .../cmd/mesh-delivery-watchman/main.go | 112 ++++++++++++------ modules/mesh-delivery-watchman/go.mod | 4 + .../internal/watch/watch.go | 71 +++++++++-- .../internal/watch/watch_test.go | 48 ++++++-- .../mesh-delivery-watchman/manifest_test.go | 23 +++- modules/mesh-delivery-watchman/module.json | 17 ++- 8 files changed, 239 insertions(+), 65 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go index b3a0c2cf..9e3cb313 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman-tools/main.go @@ -10,8 +10,10 @@ import ( "os" "sort" "strings" + "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-watchman/internal/socket" "mesh-delivery-watchman/internal/watch" @@ -28,18 +30,22 @@ const ( VerbDeadLetters = "dead-letters" VerbResync = "resync" ToolFindings = "watchman_findings" + // ToolFirstViews and ToolFirstViewAgain are where the watchman's first views of the assigner's wants and the indexer's + // modules stand, and asking one again by hand (novox/hq ADR 0615, DLV-M5). + ToolFirstViews = "watchman_first_views" + ToolFirstViewAgain = "watchman_first_view_again" ) // Verbs are the seat's verbs, in the manifest's order. var Verbs = []string{VerbSignals, VerbStuck, VerbPresence, VerbDeadLetters, VerbResync} // ModuleTools are the module's own tools, in the manifest's order. -var ModuleTools = []string{ToolFindings} +var ModuleTools = []string{ToolFindings, ToolFirstViews, ToolFirstViewAgain} // Reads names, per tool, the states it reads whole; Acts, per tool, the program's act it starts. The clause-12 test holds // that every state the manifest keeps is read by one, and every act the program takes by hand is started by one. var ( - Reads = map[string][]string{VerbSignals: {"pulse"}, VerbStuck: {"stuck"}, VerbPresence: {"presence"}, ToolFindings: {"findings"}} + Reads = map[string][]string{VerbSignals: {"pulse"}, VerbStuck: {"stuck"}, VerbPresence: {"presence"}, ToolFindings: {"findings"}, ToolFirstViews: {views.StatusState}} Acts = map[string][]string{VerbDeadLetters: {socket.AskDeadLetters}, VerbResync: {socket.AskResync}} ) @@ -124,7 +130,7 @@ func Tools(states func(string) state, ask func(socket.Ask) (json.RawMessage, err } resync := func(map[string]any) (any, error) { return ask(socket.Ask{Ask: socket.AskResync}) } verb := func(v string) string { return Seat + "." + v } - return []stdio.Tool{ + return append([]stdio.Tool{ {Name: verb(VerbSignals), Input: keyArgs, Run: signals, Description: "The signals table's rows the watchman holds (S1, S8, S9, S10, S11, S21 and the haves' health), each a pulse key " + "whose time to live is its bound and whose expiry is its finding (novox/hq ADR 0564 §4), and every pulse the watchman " + @@ -157,7 +163,7 @@ func Tools(states func(string) state, ask func(socket.Ask) (json.RawMessage, err Description: "The watchman's findings (novox/hq ADR 0564 §2): each thing it observes wrong now, .., with its " + "severity, subject, evidence and the row that observed it; the condition keeper raises and clears the conditions from " + "them. Give key for one, prefix to narrow. Read-only."}, - } + }, views.StatusTools(watch.Observer, ToolFirstViews, ToolFirstViewAgain, stdio.State(views.StatusState), time.Now)...) } func main() { diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index 5f94991b..0077f109 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/sdk/firstview/firstviewtest" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -57,6 +58,14 @@ accounts { } ` +// serveOwners serves the assigner's wants and the indexer's modules as their owners do, by their read tools and views +// families (DLV-M5, ADR 0615): the watchman (n1) follows both, never their buckets. +func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) { + t.Helper() + firstviewtest.Serve(t, ctx, js, nc, delivery.AssignerModule, wantsState, "n1_"+delivery.WatchmanModule) + firstviewtest.Serve(t, ctx, js, nc, delivery.IndexerModule, modulesState, "n1_"+delivery.WatchmanModule) +} + // TestWatchmanWhole starts the program the way the node does (its entry, run, with its environment and a broker file) // against an embedded bus with the system exports and JetStream, and proves the four things the issue names. func TestWatchmanWhole(t *testing.T) { // 1135/AC8 @@ -109,6 +118,7 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, {Bucket: watch.FindingsBucket, History: 5}, + {Bucket: firstViewsBucket, History: 1}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -116,6 +126,7 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8 } kvs[c.Bucket] = kv } + serveOwners(t, ctx, js, ctl) // The controller's consumer on DEAD_LETTER_NOTICES, the dead letters' streams and a source stream with one message a // consumer will give up on (1952/WM5); no lease bucket at all (issue 1126). @@ -355,6 +366,7 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, {Bucket: watch.FindingsBucket, History: 5}, + {Bucket: firstViewsBucket, History: 1}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -362,6 +374,7 @@ func TestAnAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909 } kvs[c.Bucket] = kv } + serveOwners(t, ctx, js, ctl) const id = "m0000009ffffff" body, err := delivery.EncodeStep(delivery.Step{ID: id, Tool: "build_go_test", Node: "n1", State: delivery.StepRunning, Started: time.Now().UTC()}) if err != nil { @@ -440,6 +453,7 @@ func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { {Bucket: delivery.AssignerWantsBucket, History: 20}, {Bucket: delivery.HaveBucket, History: 20}, {Bucket: watch.FindingsBucket, History: 5}, + {Bucket: firstViewsBucket, History: 1}, } { kv, err := js.CreateKeyValue(ctx, c) if err != nil { @@ -447,6 +461,7 @@ func TestAWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { } kvs[c.Bucket] = kv } + serveOwners(t, ctx, js, ctl) fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64) fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64) putWant := func(node, module, fp string) { diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 8ec5142d..183e5fd8 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -18,6 +18,9 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + + "git.novox.be/novox/mesh-delivery/sdk/firstview" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -122,6 +125,26 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV } } +// readLate reads an owner's state the watchman follows by its first view (pic), on a goroutine of its own once that is +// answered, so an owner that is down never holds the start: every key it holds now, then the start's end for its bucket +// (watch.ReadLate), then each change until ctx ends. +func readLate(ctx context.Context, w *watch.Watchman, pic *firstview.Picture, bucket string, each func(jetstream.KeyValueEntry)) error { + select { + case <-ctx.Done(): + return nil + case <-pic.Follower().Ready(): + } + sw, err := settle(ctx, pic, each) + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + w.ReadLate(bucket) + return follow(ctx, sw, each) +} + // follow hands each change of a watcher to each until ctx ends; a watch that ends is an error, and the process // manager starts the program again. func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) error { @@ -147,6 +170,25 @@ func stepTaken(w *watch.Watchman, language string, e jetstream.KeyValueEntry) { w.Step(language, e.Key(), e.Value(), opOf(e)) } +// createdOr is when an entry was written, or at when its picture does not say: the first view of an owner's state names +// no key's time (novox/hq ADR 0615), so a want that stood before the start waits from the start. +func createdOr(e jetstream.KeyValueEntry, at time.Time) time.Time { + if t := e.Created(); !t.IsZero() { + return t + } + return at +} + +// The watchman's own states of its first views (views.StatusState); its findings of a first view unanswered past its +// bound go to its findings bucket with the others (ADR 0615 §6). The owners' states it follows are the assigner's wants +// and the indexer's modules (DLV-M5). +var firstViewsBucket = delivery.WatchmanModule + "_" + views.StatusState + +const ( + wantsState = "wants" + modulesState = "modules" +) + // run is the program: it returns when ctx ends or the bus cannot be used. func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { logger := log.New(logOut, "mesh-delivery-watchman: ", log.LstdFlags) @@ -168,9 +210,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer conn.Close() // Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants - // and the node-engine's haves only make the pulses of a move: without one the watchman watches presence and the build - // seats' steps, and says that moves are not watched (issue 1945). - var kvs struct{ presence, pulse, stuck, findings, wants, haves jetstream.KeyValue } + // and the node-engine's haves only make the pulses of a move: without the haves the watchman watches presence and the + // build seats' steps, and says that moves are not watched (issue 1945). + var kvs struct{ presence, pulse, stuck, findings, haves jetstream.KeyValue } for _, b := range []struct { name string into *jetstream.KeyValue @@ -179,14 +221,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } } - kvs.wants = process.OptionalBucket(ctx, js, delivery.AssignerWantsBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) kvs.haves = process.OptionalBucket(ctx, js, delivery.HaveBucket, "a want makes no pulse, so a move that stops is not seen, until the watchman starts again", logger.Printf) - if kvs.wants == nil || kvs.haves == nil { - kvs.wants, kvs.haves = nil, nil // a want is a pulse only against the have that ends it + // The assigner's wants and the indexer's modules are their owners' states, read by the first view of their read tools and + // then their views families, never their buckets (ADR 0615 §1, DLV-M5). An owner that is down never holds the start: each + // is read on a goroutine once its first view is answered, and until then the unanswered first view is the watchman's + // finding. + calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return fmt.Errorf("the watchman's connection for its tool calls: %w", err) } + defer calls.Close() + reader := &firstview.Reader{Module: delivery.WatchmanModule, Call: calls.Call, Logf: logger.Printf, + Findings: func() jetstream.KeyValue { return kvs.findings }} + wants := reader.Follow(delivery.AssignerModule, wantsState) + modules := reader.Follow(delivery.IndexerModule, modulesState) w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now) w.Logf(logger.Printf) + // What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end. + w.Awaits(delivery.AssignerWantsBucket) + w.Awaits(delivery.ModulesBucket) // A test shortens how long a move may take; the mesh never sets it. if v := strings.TrimSpace(getenv("MESH_DELIVERY_PULSE_BOUND")); v != "" { d, err := time.ParseDuration(v) @@ -232,18 +286,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // What the node agents run, then the assigner's wants (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent - // until the have names its fingerprint, so the haves are read first and a want already applied makes no pulse. - var haveSeen, wantSeen jetstream.KeyWatcher + // What the node agents run (issue 1906, ADR 0539 §19): a want is a pulse owed by its node's agent until the have names its + // fingerprint, so the haves are read at the start and the wants after them, once their first view is answered. + var haveSeen jetstream.KeyWatcher if kvs.haves != nil { if haveSeen, err = settle(ctx, kvs.haves, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }); err != nil { process.NotWatched(logger.Printf, delivery.HaveBucket, err, "moves are not watched until the watchman starts again") - } else if wantSeen, err = settle(ctx, kvs.wants, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }); err != nil { - process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again") - haveSeen.Stop() haveSeen = nil - } else { - w.ReadWhole(delivery.AssignerWantsBucket) } } // The builder's builds (ADR 0603 §8): a key queued behind a package is a wait, mirrored from the time it was written. @@ -358,17 +407,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { _ = follow(ctx, hw, healthEach) }() }) } - // The indexer's modules (ADR 0578 §1): a module whose value carries a refusal of its manifest is a finding; read - // before the findings are reconciled, each change heard on the watch. - verdictEach := func(e jetstream.KeyValueEntry) { w.ModuleVerdict(e.Key(), e.Value(), opOf(e)) } - var verdictWatch jetstream.KeyWatcher - if kv, err := js.KeyValue(ctx, delivery.ModulesBucket); err == nil { - if verdictWatch, err = settle(ctx, kv, verdictEach); err != nil { - process.NotWatched(logger.Printf, delivery.ModulesBucket, err, "no refused manifest is said until the watchman starts again") - } - } else { - process.NotWatched(logger.Printf, delivery.ModulesBucket, err, "no refused manifest is said until the watchman starts again") - } dead := &deadKeeper{ctx: ctx, js: js, w: w, logf: logger.Printf} w.SetRecount(dead.count) dead.count() @@ -384,8 +422,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if healthWatch != nil { go func() { errs <- follow(ctx, healthWatch, healthEach) }() } - if verdictWatch != nil { - go func() { errs <- follow(ctx, verdictWatch, verdictEach) }() + // The first views of the wants and the modules, asked now and read each on its own goroutine once answered (ADR 0615). + reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "watchman_first_views reads nothing and watchman_first_view_again is not heard", logger.Printf)) + // The indexer's modules (ADR 0578 §1): a module whose value carries a refusal of its manifest is a finding. + verdictEach := func(e jetstream.KeyValueEntry) { w.ModuleVerdict(e.Key(), e.Value(), opOf(e)) } + go func() { errs <- readLate(ctx, w, modules, delivery.ModulesBucket, verdictEach) }() + if haveSeen != nil { + go func() { + errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) + }() + go func() { + errs <- readLate(ctx, w, wants, delivery.AssignerWantsBucket, func(e jetstream.KeyValueEntry) { + w.Want(e.Key(), e.Value(), opOf(e), createdOr(e, time.Now())) + }) + }() } if provWatch != nil { go func() { @@ -402,14 +452,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, pulses, func(e jetstream.KeyValueEntry) { w.Pulse(e.Key(), opOf(e), e.Created()) }) }() - if haveSeen != nil && wantSeen != nil { - go func() { - errs <- follow(ctx, haveSeen, func(e jetstream.KeyValueEntry) { w.Have(e.Key(), e.Value(), opOf(e)) }) - }() - go func() { - errs <- follow(ctx, wantSeen, func(e jetstream.KeyValueEntry) { w.Want(e.Key(), e.Value(), opOf(e), e.Created()) }) - }() - } if buildSeen != nil { go func() { errs <- follow(ctx, buildSeen, func(e jetstream.KeyValueEntry) { w.Build(e.Key(), e.Value(), opOf(e), e.Created()) }) diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 9a0d188f..e404582a 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -22,3 +22,7 @@ require ( ) replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun + +require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 + +replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index 0e8b7925..35908f75 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -101,6 +101,10 @@ type Watchman struct { standing map[string]map[string]bool readWhole map[string]bool reconciled bool + // awaited is each bucket read only once its owner's first view is answered, after the start's end (Awaits): its seeded + // pulses, stuck keys and findings wait for its own end (ReadLate), so a restart while its owner is down neither drops + // nor raises again what it holds. + awaited map[string]bool } // New is a watchman writing to out, reading the time from now. @@ -111,7 +115,7 @@ func New(out Out, now func() time.Time) *Watchman { haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}} } // Logf sets where a write that failed is said; it is silent until set. @@ -294,7 +298,7 @@ func (w *Watchman) Reconcile() { w.mu.Lock() defer w.mu.Unlock() for pk, p := range w.pulses { - if p.seeded { + if p.seeded && !w.awaited[p.bucket] { w.deletePulse(pk) } } @@ -319,7 +323,7 @@ func (w *Watchman) ReadWhole(bucket string) { // stands records a mirrored key read standing during the start. func (w *Watchman) stands(bucket, key string) { - if w.reconciled { + if w.reconciled && !w.awaited[bucket] { return } if w.standing[bucket] == nil { @@ -330,15 +334,61 @@ func (w *Watchman) stands(bucket, key string) { func (w *Watchman) closeGone() { for sk, s := range w.stuck { - if s.Moved || !w.readWhole[s.Bucket] || w.standing[s.Bucket][s.Key] { + if s.Moved || w.awaited[s.Bucket] || !w.readWhole[s.Bucket] || w.standing[s.Bucket][s.Key] { continue } - s.Moved = true - w.say("the stuck key %s is closed: its key %s is no longer in %s (it aged out of the bucket or was deleted while nobody heard), so nothing waits for it since %s", - sk, s.Key, s.Bucket, s.Since.UTC().Format(time.RFC3339)) - w.putStuck(sk, s) + w.closeStuck(sk, s) } - w.standing, w.readWhole, w.reconciled = map[string]map[string]bool{}, map[string]bool{}, true + standing := map[string]map[string]bool{} + for b := range w.awaited { + standing[b] = w.standing[b] + } + w.standing, w.readWhole, w.reconciled = standing, map[string]bool{}, true +} + +func (w *Watchman) closeStuck(sk string, s delivery.Stuck) { + s.Moved = true + w.say("the stuck key %s is closed: its key %s is no longer in %s (it aged out of the bucket or was deleted while nobody heard), so nothing waits for it since %s", + sk, s.Key, s.Bucket, s.Since.UTC().Format(time.RFC3339)) + w.putStuck(sk, s) +} + +// Awaits says bucket (an owner's state the watchman reads through its first view: the assigner's wants, the indexer's +// modules; novox/hq ADR 0615) is read only once that first view is answered, which may be after the start's end: Reconcile +// leaves its seeded pulses, its stuck keys and the seeded findings it is the source of to ReadLate. Called before Reconcile. +func (w *Watchman) Awaits(bucket string) { + w.mu.Lock() + defer w.mu.Unlock() + w.awaited[bucket] = true +} + +// ReadLate ends the start for an awaited bucket, called once every key it holds was read standing: as Reconcile does for +// the others, a seeded pulse of it no key made again is deleted, a stuck key of it whose key is gone is closed, and a +// seeded finding it is the source of and nothing observed again is deleted. +func (w *Watchman) ReadLate(bucket string) { + w.mu.Lock() + defer w.mu.Unlock() + if !w.awaited[bucket] { + return + } + for pk, p := range w.pulses { + if p.seeded && p.bucket == bucket { + w.deletePulse(pk) + } + } + for sk, s := range w.stuck { + if !s.Moved && s.Bucket == bucket && !w.standing[bucket][s.Key] { + w.closeStuck(sk, s) + } + } + for key := range w.seededFindings { + if f, err := DecodeFinding(w.findings[key]); err == nil && f.Source == bucket { + delete(w.seededFindings, key) + w.unfind(key) + } + } + delete(w.awaited, bucket) + delete(w.standing, bucket) } // reconcileFindings ends the start for the findings: one read at the start that what the watchman read since did not @@ -350,6 +400,9 @@ func (w *Watchman) reconcileFindings() { kept[due.Finding.Key()] = true } for key := range w.seededFindings { + if f, err := DecodeFinding(w.findings[key]); err == nil && w.awaited[f.Source] { + continue // ended by ReadLate + } delete(w.seededFindings, key) if kept[key] || signalKind(w.findings[key]) { continue diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index ed96fa6a..29c2759a 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -668,12 +668,17 @@ func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) { standing func(w *Watchman) // what the start reads standing readWhole bool closed bool + late bool // read after the start's end (Awaits, ReadLate) }{ - {"an ask's step aged out of the steps bucket", checksKey, "askold1", func(*Watchman) {}, true, true}, - {"an ask's step still running", checksKey, "askold1", func(w *Watchman) { w.Step("go", "n1.askold1", runningStep("askold1", t0), Put) }, true, false}, - {"the steps bucket could not be read", checksKey, "askold1", func(*Watchman) {}, false, false}, - {"a want deleted while nobody heard", delivery.AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true}, - {"a want still standing", delivery.AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false}, + {"an ask's step aged out of the steps bucket", checksKey, "askold1", func(*Watchman) {}, true, true, false}, + {"an ask's step still running", checksKey, "askold1", func(w *Watchman) { w.Step("go", "n1.askold1", runningStep("askold1", t0), Put) }, true, false, false}, + {"the steps bucket could not be read", checksKey, "askold1", func(*Watchman) {}, false, false, false}, + {"a want deleted while nobody heard", delivery.AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true, false}, + {"a want still standing", delivery.AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false, false}, + // DLV-M5 (ADR 0615): the wants are read from the assigner's first view, which may come after the start's end; what + // they hold waits for it, and is then reconciled as at the start. + {"a want deleted while nobody heard, read from a late first view", delivery.AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true, true}, + {"a want still standing, read from a late first view", delivery.AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false, true}, } { t.Run(row.name, func(t *testing.T) { w, o, c := setup() @@ -684,11 +689,36 @@ func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) { if _, ok := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]; !ok { t.Fatal("the seeded stuck key raised no finding") } - row.standing(w) - if row.readWhole { - w.ReadWhole(row.bucket) + if row.late { + // A pulse of another key of the bucket, and a finding the bucket is the source of, stood at the start. + pk := pulseKeyOf(t, row.bucket, "want.n9.gone") + w.SeedPulse(pk) + fv, err := EncodeFinding(Finding{Scope: ScopeModule, ID: "gone", Kind: KindManifestRefused, Severity: Warning, Source: row.bucket, Summary: "refused", Since: t0}) + if err != nil { + t.Fatal(err) + } + fk := FindingKey(ScopeModule, "gone", KindManifestRefused) + o.findings[fk], _ = DecodeFinding(fv) + w.SeedFinding(fk, fv) + w.Awaits(row.bucket) + w.Reconcile() + _, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)] + _, seeded := o.findings[fk] + if o.stuck[sk].Moved || !open || !seeded || slices.Contains(o.pulseDeletes, pk) { + t.Fatalf("the start's end took what an awaited bucket holds: moved %v, stuck open %v, finding %v, pulses deleted %v", o.stuck[sk].Moved, open, seeded, o.pulseDeletes) + } + row.standing(w) + w.ReadLate(row.bucket) + if _, seeded := o.findings[fk]; seeded || !slices.Contains(o.pulseDeletes, pk) { + t.Errorf("the late read kept the finding (%v) or the pulse (deleted %v) nothing made again", seeded, o.pulseDeletes) + } + } else { + row.standing(w) + if row.readWhole { + w.ReadWhole(row.bucket) + } + w.Reconcile() } - w.Reconcile() _, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)] if got := o.stuck[sk].Moved; got != row.closed || open == row.closed { t.Errorf("moved %v, finding open %v; want closed %v", got, open, row.closed) diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index a577361e..5cb3db48 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -51,6 +51,8 @@ type manifest struct { Version string `json:"version"` Slug string `json:"slug"` Reads []string `json:"reads"` + Invokes []string `json:"invokes"` + Consumes []string `json:"consumes"` State []state `json:"state"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` @@ -84,24 +86,37 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 if m.Module != "mesh-delivery-watchman" || m.Moved || m.Slug != "watchman" { t.Errorf("module %q moved %v (it has no template, so it is not moved) slug %q", m.Module, m.Moved, m.Slug) } - want := []state{{Name: "presence", History: 20}, {Name: "pulse", KeyTTL: true}, {Name: "stuck", History: 20}, {Name: "findings", History: 5}} + want := []state{{Name: "presence", History: 20}, {Name: "pulse", KeyTTL: true}, {Name: "stuck", History: 20}, {Name: "findings", History: 5}, {Name: "first-views", History: 1}} if !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } reads := slices.Clone(m.Reads) slices.Sort(reads) if !reflect.DeepEqual(reads, []string{"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", - "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-indexer.modules", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { + "mesh-delivery-builder.builds", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } + // DLV-M5 (ADR 0615 §1): the assigner's wants and the indexer's modules are their owners' states, asked by their read + // tools and followed by their views families, never read. + for _, r := range m.Reads { + if strings.HasPrefix(r, "mesh-delivery-indexer.") || strings.HasPrefix(r, "mesh-delivery-assigner.") { + t.Errorf("reads %s: the indexer's and the assigner's state is read by their tools and views families alone", r) + } + } + if want := []string{"mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_read_modules"}; !reflect.DeepEqual(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.modules"}; !reflect.DeepEqual(m.Consumes, want) { + t.Errorf("consumes = %v, want %v", m.Consumes, want) + } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). if !reflect.DeepEqual(m.Claims, []claim{{Name: "mesh-watchdogs", Scope: "mesh", Serves: []string{"signals", "stuck", "presence", "dead-letters", "resync"}}}) { t.Errorf("claims = %+v", m.Claims) } - if !reflect.DeepEqual(m.Tools, []string{"watchman_findings"}) { + if !reflect.DeepEqual(m.Tools, []string{"watchman_findings", "watchman_first_views", "watchman_first_view_again"}) { t.Errorf("tools = %v", m.Tools) } - for _, k := range []string{"emits", "context", "items", "consumes", "provides"} { + for _, k := range []string{"emits", "context", "items", "provides"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the watchman has none", k) } diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 4c85f5d7..5d0dc5ec 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -19,21 +19,30 @@ "mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps", - "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", - "mesh-delivery-indexer.modules", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have" ], + "invokes": [ + "mesh-delivery-assigner.assigner_read_wants", + "mesh-delivery-indexer.indexer_read_modules" + ], + "consumes": [ + "mesh-delivery-assigner.views.wants", + "mesh-delivery-indexer.views.modules" + ], "tools": [ - "watchman_findings" + "watchman_findings", + "watchman_first_views", + "watchman_first_view_again" ], "state": [ { "name": "presence", "history": 20 }, { "name": "pulse", "key-ttl": true }, { "name": "stuck", "history": 20 }, - { "name": "findings", "history": 5 } + { "name": "findings", "history": 5 }, + { "name": "first-views", "history": 1 } ], "own-secrets": { "broker": "${dir:state}/broker" From 28dd86b4513927a0259254411ed7e45ef9df541a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:09:22 +0100 Subject: [PATCH 04/12] DLV-M1: the builder follows the indexer's modules, manifests and candidates and the assigner's placement by their read tools' first views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0615 §1: the builder invokes indexer_read_modules, _manifests, _candidates and assigner_read_placement, consumes their views families and reads none of the four buckets; it keeps first-views and findings. The lead's ruling: an owner that is down never holds the start; the modules are built once the modules', manifests' and candidates' first views are answered, the placement is followed into the toolchain gate once its first view is, and until then the gate passes every report on as before the placement settled. Tools builder_first_views and builder_first_view_again. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-builder-tools/main.go | 7 +- .../mesh-delivery-builder/integration_test.go | 15 +- .../cmd/mesh-delivery-builder/main.go | 166 ++++++++++++------ modules/mesh-delivery-builder/go.mod | 4 + .../mesh-delivery-builder/manifest_test.go | 19 +- modules/mesh-delivery-builder/module.json | 26 ++- 6 files changed, 164 insertions(+), 73 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 56516c70..650113cb 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-builder/internal/build" ) @@ -264,7 +265,11 @@ func main() { a, err := build.DecodeReAsk(v) return err == nil && !a.Answered.IsZero() }) - if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now)); err != nil { + // builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's + // states stand, and asking one again by hand (ADR 0615, DLV-M1). + tools := append(Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now), + views.StatusTools(delivery.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), time.Now)...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 96c9d54d..2b9e6c38 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -27,6 +27,7 @@ import ( runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-delivery/sdk/firstview/firstviewtest" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -270,6 +271,13 @@ func startBedWith(t *testing.T, closed bool) *bed { b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory) b.source = mk(delivery.SourceBucket, delivery.SourceHistory) mk(delivery.ManifestsBucket, delivery.ManifestsHistory) + // The indexer and the assigner as the owners of the states the builder follows: their read tools and views families + // (ADR 0615, DLV-M1); n1 and n2 each have their durable consumer. + readers := []string{"n1_" + delivery.BuilderModule, "n2_" + delivery.BuilderModule} + for _, st := range []string{modulesState, manifestsState, candidatesState} { + firstviewtest.Serve(t, ctx, js, ctl, delivery.IndexerModule, st, readers...) + } + firstviewtest.Serve(t, ctx, js, ctl, delivery.AssignerModule, delivery.PlacementState, readers...) b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory) b.instances = mk(instancesBucket, 1) // The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects. @@ -791,7 +799,12 @@ func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) { } defer sub.Close() var calls atomic.Int64 - s, err := sub.Subscribe("mesh.mod.*.call.>", func(*nats.Msg) { calls.Add(1) }) + // A build module's tool, never the owners' read tools the builder's first views ask (DLV-M1). + s, err := sub.Subscribe("mesh.mod.*.call.>", func(m *nats.Msg) { + if strings.HasPrefix(m.Subject, "mesh.mod.mesh-build-") { + calls.Add(1) + } + }) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index bdfdb4a3..f0d2308f 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -25,6 +25,9 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + + "git.novox.be/novox/mesh-delivery/sdk/firstview" "mesh-delivery-builder/internal/build" ) @@ -61,16 +64,23 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - modules, err := process.EssentialBucket(ctx, js, "builder", delivery.ModulesBucket, "know any fingerprint to build") + // The indexer's modules, manifests and candidates and the assigner's placement are their owners' states, read by the + // first view of their read tools and then their views families, never their buckets (ADR 0615 §1, DLV-M1). An owner that + // is down never holds the start: each is read on a goroutine once its first view is answered, and until then the + // unanswered first view is the builder's finding. + calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) if err != nil { return err } - manifests, err := process.EssentialBucket(ctx, js, "builder", delivery.ManifestsBucket, "read what a module builds") - if err != nil { - return err - } - // The candidates' fingerprints are built too (issue 1684): a builder that cannot read them builds the trunk's. - candidates := process.OptionalBucket(ctx, js, delivery.CandidatesBucket, "no candidate's fingerprint is built until the builder starts again; the trunk's are", logger.Printf) + defer calls.Close() + reader := &firstview.Reader{Module: delivery.BuilderModule, Call: calls.Call, Logf: logger.Printf} + findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "a first view unanswered past its bound is said in the log alone", logger.Printf) + reader.Findings = findings.KV + modules := reader.Follow(delivery.IndexerModule, modulesState) + manifests := reader.Follow(delivery.IndexerModule, manifestsState) + // The candidates' fingerprints are built too (issue 1684). + candidates := reader.Follow(delivery.IndexerModule, candidatesState) + placement := reader.Follow(delivery.AssignerModule, delivery.PlacementState) bound := defaultStepBound if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" { if bound, err = time.ParseDuration(v); err != nil { @@ -103,12 +113,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf) conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") }) // The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs - // (the seat verb held, novox/hq issue 1937) before it leaves the key done. - calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) - if err != nil { - return err - } - defer calls.Close() + // (the seat verb held, novox/hq issue 1937) before it leaves the key done, on its calls connection. b := &build.Builder{ Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound}, @@ -165,10 +170,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // placement is settled first, so the toolchains settle through the gate already knowing it. gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain - placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf) - if err != nil { - return err - } + // Until the placement's first view is answered the gate passes every report on, as before the placement is settled. + placementErrs := make(chan error, 1) + go func() { + select { + case <-ctx.Done(): + return + case <-placement.Follower().Ready(): + } + pe, err := buildrun.FollowPlacement(ctx, pictures{placement}, gate, b.NodesChanged, logger.Printf) + if err != nil { + placementErrs <- err + return + } + placementErrs <- <-pe + }() if err := watch.start(); err != nil { return err } @@ -202,6 +218,78 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer cc.Stop() } + reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "builder_first_views reads nothing and builder_first_view_again is not heard", logger.Printf)) + // The modules, once their first views and the manifests' are answered: what a module builds is read from the manifests. + go func() { errs <- buildFromModules(ctx, b, modules, manifests, candidates) }() + // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as + // every other input. The builder asks again by itself without it. + // A state not on the bus yet at this start is watched once it appears (issue 1790). + process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) { + eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) } + if rw, err := settle(ctx, kv, eachReAsk); err != nil { + process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again") + } else { + go func() { errs <- follow(ctx, rw, eachReAsk) }() + } + }) + // A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the + // builder alone may read the stream, so it answers there. + process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) { + eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) } + if qw, err := settle(ctx, kv, eachRead); err != nil { + process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again") + } else { + go func() { errs <- follow(ctx, qw, eachRead) }() + } + }) + logger.Printf("started on %s", cred.Node) + select { + case <-ctx.Done(): + b.Wait() + return nil + case err := <-errs: + return err + case err := <-placementErrs: + return err + case err := <-watch.errs: + return err + } +} + +// The builder's own states of its first views (views.StatusState) and of its findings, where the condition keeper raises a +// first view unanswered past its bound (ADR 0615 §6). +var ( + firstViewsBucket = delivery.BuilderModule + "_" + views.StatusState + findingsBucket = delivery.BuilderModule + "_findings" +) + +// The indexer's states the builder follows (delivery.ModulesBucket, ManifestsBucket, CandidatesBucket). +const ( + modulesState = "modules" + manifestsState = "manifests" + candidatesState = "candidates" +) + +// pictures opens the placement's picture where the gate's follower opens its bucket. +type pictures struct{ placement *firstview.Picture } + +func (p pictures) KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) { + if bucket != p.placement.Bucket() { + return nil, fmt.Errorf("the builder reads %s by its owner's first view alone", bucket) + } + return p.placement, nil +} + +// buildFromModules builds from the indexer's modules and candidates once the first views of the modules, the manifests and +// the candidates are answered (never at the start: an indexer that is down holds no other part of it), and follows them. +func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests, candidates *firstview.Picture) error { + for _, p := range []*firstview.Picture{modules, manifests, candidates} { + select { + case <-ctx.Done(): + return nil + case <-p.Follower().Ready(): + } + } each := func(e jetstream.KeyValueEntry) { if opOf(e) { b.Module(ctx, e.Key(), e.Value()) @@ -225,48 +313,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro b.ForgetCandidate(e.Key()) } } - if candidates != nil { - cw, err := settle(ctx, candidates, eachCandidate) - if err != nil { - process.NotWatched(logger.Printf, delivery.CandidatesBucket, err, "no candidate's fingerprint is built until the builder starts again") - } else { - go func() { errs <- follow(ctx, cw, eachCandidate) }() - } - } - // A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as - // every other input. The builder asks again by itself without it. - // A state not on the bus yet at this start is watched once it appears (issue 1790). - process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) { - eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) } - if rw, err := settle(ctx, kv, eachReAsk); err != nil { - process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again") - } else { - go func() { errs <- follow(ctx, rw, eachReAsk) }() - } - }) - // A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the - // builder alone may read the stream, so it answers there. - process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) { - eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) } - if qw, err := settle(ctx, kv, eachRead); err != nil { - process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again") - } else { - go func() { errs <- follow(ctx, qw, eachRead) }() - } - }) - logger.Printf("started on %s", cred.Node) - go func() { errs <- follow(ctx, w, each) }() - select { - case <-ctx.Done(): - b.Wait() - return nil - case err := <-errs: - return err - case err := <-placementErrs: - return err - case err := <-watch.errs: + cw, err := settle(ctx, candidates, eachCandidate) + if err != nil { return err } + go func() { _ = follow(ctx, cw, eachCandidate) }() + return follow(ctx, w, each) } // sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 3131c6f3..ec9702c5 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -25,3 +25,7 @@ require ( ) replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun + +require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 + +replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index f94b1720..dda6d84c 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -93,13 +93,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { @@ -108,13 +108,22 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no // other tool; it holds no capacity. // 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955). - if want := []string{"seat:mesh-artifact-sweep.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { + // DLV-M1 (ADR 0615 §1): the indexer's modules, manifests and candidates and the assigner's placement are asked by their + // read tools and followed by their views families, never read. + for _, r := range m.Reads { + if strings.HasPrefix(r, "mesh-delivery-indexer.") || strings.HasPrefix(r, "mesh-delivery-assigner.") { + t.Errorf("reads %s: the indexer's and the assigner's state is read by their tools and views families alone", r) + } + } + if want := []string{"mesh-delivery-assigner.assigner_read_placement", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", + "mesh-delivery-indexer.indexer_read_modules", "seat:mesh-artifact-sweep.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it // starts; and by the seat mesh-artifact-sweep's artifacts-gone, a blob the sweep found gone (1937/AC4, 1955/L7), the // controller's no longer (1126/B9, ADR 0590). - if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", + "mesh-delivery-assigner.views.placement", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -124,7 +133,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_first_view_again", "builder_first_views", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 4440f866..a99e4e97 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -8,7 +8,9 @@ "builder_build_again", "builder_builds_since", "builder_seat_queue", - "builder_stuck" + "builder_stuck", + "builder_first_views", + "builder_first_view_again" ], "emits": [ "build.asked-again" @@ -16,17 +18,27 @@ "consumes": [ "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", - "mesh-build-node.toolchain.reported" + "mesh-build-node.toolchain.reported", + "mesh-delivery-assigner.views.placement", + "mesh-delivery-indexer.views.candidates", + "mesh-delivery-indexer.views.manifests", + "mesh-delivery-indexer.views.modules" ], "invokes": [ - "seat:mesh-artifact-sweep.held" + "seat:mesh-artifact-sweep.held", + "mesh-delivery-assigner.assigner_read_placement", + "mesh-delivery-indexer.indexer_read_candidates", + "mesh-delivery-indexer.indexer_read_manifests", + "mesh-delivery-indexer.indexer_read_modules" ], "state": [ { "name": "builds", "history": 5 }, { "name": "hearing", "history": 1 }, { "name": "instances", "history": 1 }, { "name": "queues", "history": 1 }, - { "name": "reasks", "history": 1 } + { "name": "reasks", "history": 1 }, + { "name": "first-views", "history": 1 }, + { "name": "findings", "history": 1 } ], "logs": [ { "name": "build-log", "max-age-days": 30 } @@ -37,11 +49,7 @@ "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-assigner.placement", - "mesh-delivery-gate.source", - "mesh-delivery-indexer.candidates", - "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules" + "mesh-delivery-gate.source" ], "uses": [ "node-build-go", From a6fdb121fe4ded3bb792a37b2eda17daff49d155 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:13:28 +0100 Subject: [PATCH 05/12] DLV-M2: the checker follows the indexer's candidates, modules, manifests, packages and predefined keys and the assigner's placement by first views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0615 §1 and ADR 0596: the checker invokes the six read tools, consumes their views families and reads none of the six buckets; this ends its direct read of the packages state (the lead's ruling 2). It keeps first-views and findings and announces both as its own. The lead's ruling: an owner that is down never holds the start; the checks, the tries and the hand asks start once every first view is answered, the placement is followed into the toolchain gate once its first view is. firstview.Reader opens a followed state's picture as a process.BucketOpener (the builder uses it too). Tools checker_first_views and checker_first_view_again. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-builder/main.go | 12 +- .../cmd/mesh-delivery-checker-tools/main.go | 4 + .../mesh-delivery-checker/integration_test.go | 11 +- .../cmd/mesh-delivery-checker/judge.go | 8 +- .../cmd/mesh-delivery-checker/main.go | 212 +++++++++++------- .../mesh-delivery-checker/manifest_test.go | 20 +- modules/mesh-delivery-checker/module.json | 24 +- sdk/firstview/firstview.go | 10 + 8 files changed, 192 insertions(+), 109 deletions(-) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index f0d2308f..b9b21eac 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -178,7 +178,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return case <-placement.Follower().Ready(): } - pe, err := buildrun.FollowPlacement(ctx, pictures{placement}, gate, b.NodesChanged, logger.Printf) + pe, err := buildrun.FollowPlacement(ctx, reader, gate, b.NodesChanged, logger.Printf) if err != nil { placementErrs <- err return @@ -270,16 +270,6 @@ const ( candidatesState = "candidates" ) -// pictures opens the placement's picture where the gate's follower opens its bucket. -type pictures struct{ placement *firstview.Picture } - -func (p pictures) KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) { - if bucket != p.placement.Bucket() { - return nil, fmt.Errorf("the builder reads %s by its owner's first view alone", bucket) - } - return p.placement, nil -} - // buildFromModules builds from the indexer's modules and candidates once the first views of the modules, the manifests and // the candidates are answered (never at the start: an indexer that is down holds no other part of it), and follows them. func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests, candidates *firstview.Picture) error { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index fd39fe86..f01e94b0 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-checker/internal/check" ) @@ -326,6 +327,9 @@ func main() { return err }} tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...) + // checker_first_views and checker_first_view_again: where the checker's first views of the indexer's and the assigner's + // states stand, and asking one again by hand (ADR 0615, DLV-M2). + tools = append(tools, views.StatusTools(delivery.CheckerModule, "checker_first_views", "checker_first_view_again", stdio.State(views.StatusState), time.Now)...) if err := stdio.Serve("", logged(tools)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 719773f5..6e804bf2 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -28,6 +28,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-delivery/sdk/firstview/firstviewtest" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -440,6 +441,12 @@ func startBed(t *testing.T) *bed { } b.buckets[name] = kv } + // The indexer and the assigner as the owners of the states the checker follows: their read tools and views families + // (ADR 0615, DLV-M2). + for _, st := range []string{candidatesState, modulesState, manifestsState, delivery.PackagesState, delivery.IndexerPredefinedState} { + firstviewtest.Serve(t, ctx, js, ctl, delivery.IndexerModule, st, "n1_"+delivery.CheckerModule) + } + firstviewtest.Serve(t, ctx, js, ctl, delivery.AssignerModule, delivery.PlacementState, "n1_"+delivery.CheckerModule) // 1929/AC1: the indexer's value of the repository's module, with no TypeScript manifest, so the Go build module alone // judges the repository (an unknown repository is judged by both, issue 1887 A12). known, err := delivery.EncodeModule(delivery.Module{Repository: repo, Commit: trunk, Path: "modules/m", None2: "the fingerprint is computed with its chain"}) @@ -1071,7 +1078,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1 } if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") || !said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || - !said.Keeps("asks") || said.Readable("asks") || len(said.States) != 7 { - t.Fatalf("the checker said %+v, want its seven states, asks and hearing its own, with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said) + !said.Keeps("asks") || said.Readable("asks") || !said.Keeps("first-views") || said.Readable("first-views") || !said.Keeps("findings") || len(said.States) != 9 { + t.Fatalf("the checker said %+v, want its nine states, asks, hearing and first-views its own (DLV-M2), with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said) } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 4b1974e3..acd86ce7 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -51,14 +51,10 @@ type judgeDeps struct { } // keptPredefined reads the indexer's key predefined.list (delivery.PredefinedKept), bound once its state appears. -type keptPredefined struct{ b *process.AwaitedBucket } +type keptPredefined struct{ kv jetstream.KeyValue } func (k keptPredefined) PredefinedList(ctx context.Context) (json.RawMessage, error) { - kv := k.b.KV() - if kv == nil { - return nil, process.ErrNotYet - } - e, err := kv.Get(ctx, delivery.PredefinedListKey) + e, err := k.kv.Get(ctx, delivery.PredefinedListKey) if errors.Is(err, jetstream.ErrKeyNotFound) { return nil, nil } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index b2c409f0..eda2e94f 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -24,6 +24,9 @@ import ( "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + + "git.novox.be/novox/mesh-delivery/sdk/firstview" "mesh-delivery-checker/internal/check" ) @@ -61,6 +64,20 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti return d, nil } +// The checker's own states of its first views (views.StatusState) and of its findings, where the condition keeper raises a +// first view unanswered past its bound (ADR 0615 §6); and the indexer's states it follows (delivery.CandidatesBucket, +// ModulesBucket, ManifestsBucket). +var ( + firstViewsBucket = delivery.CheckerModule + "_" + views.StatusState + findingsBucket = delivery.CheckerModule + "_findings" +) + +const ( + candidatesState = "candidates" + modulesState = "modules" + manifestsState = "manifests" +) + // run is the program: it returns when ctx ends or the bus cannot be used. // keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module, // which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view. @@ -77,6 +94,10 @@ var keptStates = []announce.State{ // The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage). {Name: "tries", Readable: true}, {Name: "verdicts", Readable: true}, + // Where its first views of the indexer's and the assigner's states stand (read by checker_first_views alone), and its + // findings of one unanswered past its bound, which the condition keeper observes as it does every reader's (ADR 0615 §6). + {Name: views.StatusState}, + {Name: "findings"}, } func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { @@ -90,11 +111,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer conn.Close() - // Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository: - // a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945). + // Its own states: a check without one of them would be unwritten, so each stops the start, naming its grant (issue 1945). buckets := map[string]jetstream.KeyValue{} for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket, - delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket, check.TriesBucket} { + check.TriesBucket} { if buckets[name], err = process.EssentialBucket(ctx, js, "checker", name, "judge or keep a check"); err != nil { return err } @@ -111,6 +131,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := caller.Keeps(keptStates...); err != nil { logger.Printf("what the checker keeps could not be said on the bus, so readers of owed, previews and verdicts are not granted until it is: %v", err) } + // The candidates it checks, the modules and manifests that say which build modules judge a repository, the published + // packages, the predefined keys and the placement are their owners' states (the indexer's, the assigner's), read by the + // first view of their read tools and then their views families, never their buckets (ADR 0615 §1, ADR 0596, DLV-M2). + // An owner that is down never holds the start: the checks start once every first view is answered, and until then the + // unanswered first view is the checker's finding. + reader := &firstview.Reader{Module: delivery.CheckerModule, Call: caller.Call, Logf: logger.Printf} + reader.Findings = process.AwaitBucket(ctx, js, conn, findingsBucket, "a first view unanswered past its bound is said in the log alone", logger.Printf).KV + candidates := reader.Follow(delivery.IndexerModule, candidatesState) + modules := reader.Follow(delivery.IndexerModule, modulesState) + manifests := reader.Follow(delivery.IndexerModule, manifestsState) + packages := reader.Follow(delivery.IndexerModule, delivery.PackagesState) + predefined := reader.Follow(delivery.IndexerModule, delivery.IndexerPredefinedState) + placement := reader.Follow(delivery.AssignerModule, delivery.PlacementState) stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", check.DefaultStepBound) if err != nil { @@ -183,7 +216,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro verdicts := &manifestVerdicts{} compositions := check.EngineCompositions{ Asker: engineAsks{call: caller.CallOn}, - Placed: placedNodes{kv: func(at context.Context) (jetstream.KeyValue, error) { return js.KeyValue(at, delivery.PlacementBucket) }}.Placed, + Placed: placedNodes{kv: func(context.Context) (jetstream.KeyValue, error) { return placement, nil }}.Placed, Bound: answerBound, Sandbox: seatSandbox{asker: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}}, Verdicts: verdicts.all, @@ -196,23 +229,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro logger.Printf("%s of %s was written and the part that reads it could not be told: %v", event, key, err) } } - repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf, verdicts: verdicts} + repos := &repoModules{manifests: kvStore{manifests}, logf: logger.Printf, verdicts: verdicts} compositions.Composer = func() string { return repos.CommitOf(check.NodeEngineModule) } compositions.Manifests = repos - // The indexer's packages state (ADR 0593 §2): followed once the bus has it, so a Go check builds every mesh package at - // its published version; until then a check builds each at the version its go.mod names. + // The indexer's packages state (ADR 0593 §2), so a Go check builds every mesh package at its published version: read + // once the first views are answered, before any check starts (afterFirstViews). published := &publishedPackages{} - process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "a Go check builds each mesh package at the version its go.mod names", - logger.Printf).Ready(func(kv jetstream.KeyValue) { - w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - logger.Printf("%s cannot be watched, so a Go check builds each mesh package at the version its go.mod names: %v", delivery.PackagesBucket, err) - return - } - go func() { - _ = follow(ctx, w, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) }) - }() - }) ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6) // A group's tagged versions (design 51 *Tagged versions*): tag-wanted on the checker's stream asks, and the package // registry seats' published heard on the checker's durable consumer of the shared events stream. Where that consumer is @@ -239,9 +261,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf, announce: announce, compositions: compositions, tags: tags, - nodes: openNodeNames(ctx, js, logger.Printf), - predefined: keptPredefined{process.AwaitBucket(ctx, js, conn, delivery.IndexerModule+"_"+delivery.IndexerPredefinedState, - "a change's manifests are judged with no predefined keys, each fallback as its value declares it", logger.Printf)}, + nodes: openNodeNames(ctx, js, logger.Printf), + predefined: keptPredefined{predefined}, }) ch.Announce = announce tryer.Judge = ch.Judge @@ -252,10 +273,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // placement is settled first, so the toolchains settle through the gate already knowing it. gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain - placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, ch.NodesChanged, logger.Printf) - if err != nil { - return err - } + // Until the placement's first view is answered the gate passes every report on, as before the placement is settled. + placementErrs := make(chan error, 1) + go func() { + select { + case <-ctx.Done(): + return + case <-placement.Follower().Ready(): + } + pe, err := buildrun.FollowPlacement(ctx, reader, gate, ch.NodesChanged, logger.Printf) + if err != nil { + placementErrs <- err + return + } + placementErrs <- <-pe + }() if err := watch.start(); err != nil { return err } @@ -338,65 +370,89 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil } lazy.ensure() - // The modules say which repository holds a TypeScript module, so the Node build module judges it too. - mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return err - } - go func() { - errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { - before := repos.CommitOf(check.NodeEngineModule) - repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) - if after := repos.CommitOf(check.NodeEngineModule); after != before { - ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6) - } - }) - }() - each := func(e jetstream.KeyValueEntry) { - if opOf(e) { - ch.Candidate(ctx, e.Key(), e.Value()) - } else { - ch.Forget(e.Key()) + afterFirstViews := func() error { + // The modules say which repository holds a TypeScript module, so the Node build module judges it too. + pw, err := settle(ctx, packages, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err } - } - w, err := settle(ctx, buckets[delivery.CandidatesBucket], each) - if err != nil { - return err - } - // A key left running for heads no candidate holds is closed; the candidates are all known now. - if err := ch.Sweep(ctx); err != nil { - return err - } - // The tries: a queued one starts, one left running is taken over, an ended one past its keeping is removed. - tw, err := settle(ctx, buckets[check.TriesBucket], func(jetstream.KeyValueEntry) {}) - if err != nil { - return err - } - tryer.Reconsider(ctx) - go func() { - errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) { + go func() { + errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) }) + }() + mw, err := settle(ctx, modules, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return err + } + go func() { + errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { + before := repos.CommitOf(check.NodeEngineModule) + repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) + if after := repos.CommitOf(check.NodeEngineModule); after != before { + ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6) + } + }) + }() + each := func(e jetstream.KeyValueEntry) { if opOf(e) { - tryer.Put(ctx, e.Key(), e.Value()) - } - }) - }() - // The checker's own verdicts, updates only, in one watch: one checker_ask_again wrote asks its check again - // (Checker.VerdictWritten); a deleted key of heads a candidate holds is judged again (Checker.VerdictDeleted, issue 1733). - vw, err := buckets[delivery.CheckerVerdictsBucket].WatchAll(ctx, jetstream.UpdatesOnly()) - if err != nil { - return fmt.Errorf("the checker's verdicts cannot be watched, so no check can be asked again by hand and a deleted verdict would never be written again: %w", err) - } - go func() { - errs <- follow(ctx, vw, func(e jetstream.KeyValueEntry) { - if opOf(e) { - ch.VerdictWritten(ctx, e.Key(), e.Value()) + ch.Candidate(ctx, e.Key(), e.Value()) } else { - ch.VerdictDeleted(ctx, e.Key()) + ch.Forget(e.Key()) } - }) + } + w, err := settle(ctx, candidates, each) + if err != nil { + return err + } + // A key left running for heads no candidate holds is closed; the candidates are all known now. + if err := ch.Sweep(ctx); err != nil { + return err + } + // The tries: a queued one starts, one left running is taken over, an ended one past its keeping is removed. + tw, err := settle(ctx, buckets[check.TriesBucket], func(jetstream.KeyValueEntry) {}) + if err != nil { + return err + } + tryer.Reconsider(ctx) + go func() { + errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) { + if opOf(e) { + tryer.Put(ctx, e.Key(), e.Value()) + } + }) + }() + // The checker's own verdicts, updates only, in one watch: one checker_ask_again wrote asks its check again + // (Checker.VerdictWritten); a deleted key of heads a candidate holds is judged again (Checker.VerdictDeleted, issue 1733). + vw, err := buckets[delivery.CheckerVerdictsBucket].WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return fmt.Errorf("the checker's verdicts cannot be watched, so no check can be asked again by hand and a deleted verdict would never be written again: %w", err) + } + go func() { + errs <- follow(ctx, vw, func(e jetstream.KeyValueEntry) { + if opOf(e) { + ch.VerdictWritten(ctx, e.Key(), e.Value()) + } else { + ch.VerdictDeleted(ctx, e.Key()) + } + }) + }() + go func() { errs <- follow(ctx, w, each) }() + return nil + } + reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "checker_first_views reads nothing and checker_first_view_again is not heard", logger.Printf)) + // The checks, the tries and the hand asks, once every first view is answered (the lead's ruling for DLV-M2): a check + // judged without the candidates, the modules, the manifests, the published packages, the predefined keys or the + // placement would be another check. + go func() { + select { + case <-ctx.Done(): + return + case <-reader.Ready(): + } + if err := afterFirstViews(); err != nil { + errs <- err + } }() logger.Printf("started on %s", cred.Node) - go func() { errs <- follow(ctx, w, each) }() select { case <-ctx.Done(): ch.Wait() diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index c70b5bc6..41491b4e 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -102,19 +102,27 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("module = %q", m.Module) } // delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets. - if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) { + if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates", - "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.packages", "mesh-delivery-indexer.predefined", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } + // DLV-M2 (ADR 0615 §1, ADR 0596): the indexer's candidates, modules, manifests, packages and predefined keys and the + // assigner's placement are asked by their read tools and followed by their views families, never read. + for _, r := range m.Reads { + if strings.HasPrefix(r, "mesh-delivery-indexer.") || strings.HasPrefix(r, "mesh-delivery-assigner.") { + t.Errorf("reads %s: the indexer's and the assigner's state is read by their tools and views families alone", r) + } + } // 1887/A5b: the checker reads nothing of the controller's plans-building (ADR 0483 superseded by ADR 0525 §7), calls no // build module's tool and holds no capacity: it puts check asks on the Go and Node build seats (ADR 0525 §1, §4). // The forge's reads for a mesh-sdk pin (ADR 0484 clause 9), and of the node seat node-engine its one verb // compose-candidate, asked of every placed node (ADR 0593 §7), never the seat's other verbs. if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches", + "mesh-delivery-assigner.assigner_read_placement", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", + "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "seat:node-engine.compose-candidate"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want the forge's reads and compose-candidate", m.Invokes) } @@ -123,7 +131,9 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 } // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. // And a group's check by the package registry seats' published of its tagged versions (design 51 *Tagged versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", + "mesh-delivery-assigner.views.placement", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", + "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -141,7 +151,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. // The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check // is asked again by hand (checker_ask_again). - if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"checker_ask_again", "checker_first_view_again", "checker_first_views", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code. diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 8c054afd..df8cfd0c 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -17,6 +17,12 @@ "go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", + "mesh-delivery-assigner.views.placement", + "mesh-delivery-indexer.views.candidates", + "mesh-delivery-indexer.views.manifests", + "mesh-delivery-indexer.views.modules", + "mesh-delivery-indexer.views.packages", + "mesh-delivery-indexer.views.predefined", "npm-package-registry.published" ], "streams": [ @@ -30,22 +36,18 @@ { "name": "owed", "history": 5 }, { "name": "previews", "history": 5 }, { "name": "tries", "history": 1 }, - { "name": "verdicts", "history": 5 } + { "name": "verdicts", "history": 5 }, + { "name": "first-views", "history": 1 }, + { "name": "findings", "history": 1 } ], "reads": [ "mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", - "mesh-delivery-indexer.candidates", - "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", - "mesh-delivery-indexer.packages", - "mesh-delivery-indexer.predefined", "mesh-delivery-watchman.presence", "node-engine.have" ], @@ -59,6 +61,8 @@ "checker_previews", "checker_pull", "checker_status", + "checker_first_views", + "checker_first_view_again", "checker_try", "checker_try_end", "checker_try_read", @@ -69,6 +73,12 @@ "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches", + "mesh-delivery-assigner.assigner_read_placement", + "mesh-delivery-indexer.indexer_read_candidates", + "mesh-delivery-indexer.indexer_read_manifests", + "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_read_packages", + "mesh-delivery-indexer.indexer_read_predefined", "seat:node-engine.compose-candidate" ], diff --git a/sdk/firstview/firstview.go b/sdk/firstview/firstview.go index 57d82e26..3795dca0 100644 --- a/sdk/firstview/firstview.go +++ b/sdk/firstview/firstview.go @@ -105,6 +105,16 @@ func (r *Reader) Picture(bucket string) *Picture { return r.pics[bucket] } +// KeyValue opens a followed state's picture by its bus name, where a process opened the bucket (process.BucketOpener, +// as buildrun.FollowPlacement takes it); any other bucket is refused, for the reader reads its owners' states by their +// first views alone. +func (r *Reader) KeyValue(_ context.Context, bucket string) (jetstream.KeyValue, error) { + if p := r.Picture(bucket); p != nil { + return p, nil + } + return nil, fmt.Errorf("%s reads %s by its owner's first view alone, and does not follow it", r.Module, bucket) +} + // Followers are the followers, in the order they were added. func (r *Reader) Followers() []*views.Follower { r.mu.Lock() From c6c66873f454774860a771fb32d793b59573f102 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:14:37 +0100 Subject: [PATCH 06/12] DLV-M6r5: the judge takes a claimed seat's module with tools of its own beside its seat's verbs The lead's ruling 5: one row in the judge's table, the enrolment's shape (a claim serving the seat's verbs and the first-view tools in its own tools list); the judge accepts it as it stands. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- modules/mesh-delivery-indexer/judge/judge_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/mesh-delivery-indexer/judge/judge_test.go b/modules/mesh-delivery-indexer/judge/judge_test.go index 27caf578..c59d5c37 100644 --- a/modules/mesh-delivery-indexer/judge/judge_test.go +++ b/modules/mesh-delivery-indexer/judge/judge_test.go @@ -35,6 +35,13 @@ func TestTheJudge(t *testing.T) { "two": `{"module":"two","version":"1","seats":[{"name":"thing","scope":"mesh"}]}`, "ok": `{"module":"ok","version":"1"}`}, want: map[string]string{"one": "already declares", "two": "already declares"}}, + // The lead's ruling 5 for A2h: a module that claims a seat and serves its verbs keeps tools of its own beside them + // (the enrolment's first-view tools, DLV-M6), and the judge takes it. + {name: "a claimed seat's module with tools of its own", + raws: map[string]string{ + "def": `{"module":"def","version":"1","seats":[{"name":"enrolment","scope":"mesh","serves":["issue","again"]}]}`, + "enrolment": `{"module":"enrolment","version":"1","claims":[{"name":"enrolment","scope":"mesh","serves":["issue","again"]}],` + + `"tools":["enrolment_first_views","enrolment_first_view_again"]}`}}, } { raws := map[string][]byte{} for k, v := range row.raws { From 8afd34525d33256f467f6b0f3a4c0e1b8679cbc9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:24:17 +0100 Subject: [PATCH 07/12] DLV-M4, DLV-V1: the view reads the indexer's and the assigner's twelve states through their read tools, never their buckets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0615 §1, design 25 § 12.2: the manifest drops its twelve reads of the two owners' states and invokes their twelve read tools. The process follows the six it watches (placement, wants, asks, candidates, modules, taken) by the first view and then the views families (consumes), through firstview pictures, so its watch code is unchanged; an owner that is down holds no other watch, and the unanswered first view is the view's finding (states first-views and findings; tools view_first_views, view_first_view_again). The tools keep their names, arguments and answers and ask the owners' read tools on each call (view.OwnerRead; a read of many keys is one paged values read). DLV-V1: a table subtest answers each tool through the owner's own Read over a fake state and matches what reading the state answered. The owners' read tools answer no history of a key: indexer_module_history, indexer_heard and view_tiers' times say so (an error, never none) until the read tool takes a history argument (a request to the join). Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-view-tools/main.go | 43 ++++-- .../cmd/mesh-delivery-view-tools/main_test.go | 1 + .../mesh-delivery-view-tools/owners_test.go | 131 ++++++++++++++++++ .../mesh-delivery-view/integration_test.go | 28 +++- .../cmd/mesh-delivery-view/main.go | 54 +++++++- .../mesh-delivery-view/internal/view/owner.go | 111 +++++++++++++++ .../mesh-delivery-view/internal/view/read.go | 13 ++ .../internal/view/status.go | 3 + modules/mesh-delivery-view/manifest_test.go | 23 ++- modules/mesh-delivery-view/module.json | 42 ++++-- 10 files changed, 410 insertions(+), 39 deletions(-) create mode 100644 modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go create mode 100644 modules/mesh-delivery-view/internal/view/owner.go diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 746dc902..af1a071e 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -14,8 +14,10 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/sdk/firstview" "git.novox.be/novox/mesh-delivery/sdk/graph" "git.novox.be/novox/mesh-sdk/go/provisions" + "git.novox.be/novox/mesh-sdk/go/views" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -311,20 +313,30 @@ func read(state stdio.KeptState) view.StateRead { }} } +// ownersRead is the view's reading of the indexer's and the assigner's states: each through its owner's read tool, asked +// by the bundle on each call (ADR 0615 §1, design 25 § 12.2: the view keeps its tools and answers them by asking the +// owners), never their buckets. +func ownersRead(ask func(key string, body any) (json.RawMessage, error)) view.Reader { + of := func(owner, state string) view.StateRead { return view.OwnerRead(ask, firstview.ReadTool(owner, state)) } + return view.Reader{State: of(delivery.IndexerModule, "modules"), CandidatesState: of(delivery.IndexerModule, "candidates"), + WantsState: of(delivery.AssignerModule, "wants"), DifferencesState: of(delivery.AssignerModule, "differences"), + HeardState: of(delivery.IndexerModule, "heard"), + PredefinedState: of(delivery.IndexerModule, delivery.IndexerPredefinedState), + ConfigCandidatesState: of(delivery.IndexerModule, delivery.IndexerConfigCandidatesState), + PackagesState: of(delivery.IndexerModule, delivery.PackagesState), + GraphState: of(delivery.IndexerModule, graph.State)} +} + func main() { - r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")), - WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")), - ReachState: read(stdio.State("reach")), - HeardState: read(stdio.State("mesh-delivery-indexer.heard")), - SignersState: read(stdio.State("mesh-vault.signers")), - ConditionsState: read(stdio.State(view.StatusConditionsState)), AttemptsState: read(stdio.State(view.StatusAttemptsState)), - PresenceState: read(stdio.State(view.StatusPresenceState)), StuckState: read(stdio.State(view.StatusStuckState)), - PredefinedState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerPredefinedState)), - ConfigCandidatesState: read(stdio.State(delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState)), - PackagesState: read(stdio.State(delivery.IndexerModule + "." + delivery.PackagesState)), - GraphState: read(stdio.State(graph.Read)), SourceState: read(stdio.State(delivery.GateModule + ".source")), - ProvisionsState: read(stdio.State(provisions.ProvisionsRead)), - BuildLog: stdio.Log(delivery.BuildLogRead).Since} + r := ownersRead(stdio.Ask) + r.Builds, r.HavesState = read(stdio.State("mesh-delivery-builder.builds")), read(stdio.State("node-engine.have")) + r.ReachState = read(stdio.State("reach")) + r.SignersState = read(stdio.State("mesh-vault.signers")) + r.ConditionsState, r.AttemptsState = read(stdio.State(view.StatusConditionsState)), read(stdio.State(view.StatusAttemptsState)) + r.PresenceState, r.StuckState = read(stdio.State(view.StatusPresenceState)), read(stdio.State(view.StatusStuckState)) + r.SourceState = read(stdio.State(delivery.GateModule + ".source")) + r.ProvisionsState = read(stdio.State(provisions.ProvisionsRead)) + r.BuildLog = stdio.Log(delivery.BuildLogRead).Since u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")), Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())} for _, lang := range delivery.AskLanguages { @@ -332,7 +344,10 @@ func main() { u.Steps[lang] = read(stdio.State(module + "." + delivery.StepsState)) u.Pauses[lang] = read(stdio.State(module + "." + delivery.PausesState)) } - if err := stdio.Serve("", Tools(r, u)); err != nil { + // view_first_views and view_first_view_again: where the view's process stands with its first views of the indexer's and + // the assigner's states (ADR 0615 §6), and a re-read by hand. + tools := append(Tools(r, u), views.StatusTools(delivery.ViewModule, "view_first_views", "view_first_view_again", stdio.State(views.StatusState), time.Now)...) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index 1520dfc2..217e3425 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -56,6 +56,7 @@ const ( func TestModuleToolAnswersTheStoredValue(t *testing.T) { t.Run("indexer_packages answers the published modules", packagesToolAnswersThePublishedModules) t.Run("view_tiers names each module's wait", tiersToolSaysEachModulesWait) + t.Run("each tool answers through the owners' read tools as before", ownersAnswerTheToolsAsBefore) r := fake{"a": withFP2, "b": withNone}.reader() for key, want := range map[string]string{"a": withFP2, "b": withNone} { out, err := run(t, r, view.ModuleReadTool, map[string]any{"module": key}) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go new file mode 100644 index 00000000..e3b27130 --- /dev/null +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/owners_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/views" + + "git.novox.be/novox/mesh-delivery/sdk/firstview" + "git.novox.be/novox/mesh-delivery/sdk/graph" + + "mesh-delivery-view/internal/view" +) + +// source is a fake owner state as a views.Source: what the owner's read tool answers from. +type source map[string]string + +func (s source) Keys() ([]string, error) { + var out []string + for k := range s { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s source) Get(key string) (*views.Entry, bool, error) { + v, ok := s[key] + if !ok { + return nil, false, nil + } + return &views.Entry{Value: json.RawMessage(v), Revision: uint64(len(key))}, true, nil +} + +// ownersAnswerTheToolsAsBefore is DLV-V1: every view tool over the indexer's and the assigner's states answers through +// the owners' read tools (the real owner side, views.Read, over a fake state) what it answered reading the states. +func ownersAnswerTheToolsAsBefore(t *testing.T) { + cand := `{"v":1,"group":"feat/x","base":3,"heads":"h","pulls":[{"modules":["zsh"],"fps":{"zsh":{"fp2":"` + fp2B + `","manifest":"m"}}}]}` + pkgKey, err := delivery.PackagesKey("git.novox.be/novox/mesh-sdk/go") + if err != nil { + t.Fatal(err) + } + states := map[string]source{ + delivery.IndexerModule + ".modules": {"a": withFP2, "b": withNone}, + delivery.IndexerModule + ".candidates": {"feat/x": cand, "fix": `{"v":1,"group":"fix","base":3,"heads":"h","pulls":[{"modules":["netcheck"]}]}`}, + delivery.AssignerModule + ".wants": {"want.ace.zsh": `{"fp":"` + fp2A + `"}`, "want.novox.zsh": `{"fp":"` + fp2B + `"}`, + "held." + keyB: `{"node":"ace","words":"failed"}`}, + delivery.AssignerModule + ".differences": {"ace.zsh": `{"unexplained":true}`, "novox.zsh": `{"rule":3}`}, + delivery.IndexerModule + "." + delivery.IndexerPredefinedState: {delivery.PredefinedListKey: `{"v":1,"keys":["dir:state"]}`}, + delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState: {"novox.r.7": `{"v":1,"pull":"novox/r#7"}`}, + delivery.IndexerModule + "." + delivery.PackagesState: {pkgKey: `{"v":1,"module":"git.novox.be/novox/mesh-sdk/go","commit":"c1"}`}, + delivery.IndexerModule + "." + graph.State: {}, + } + // The direct reading, as the tools read the states before (the reads the manifest no longer has). + st := func(name string) view.StateRead { return fake(states[name]).reader().State } + direct := view.Reader{State: st(delivery.IndexerModule + ".modules"), CandidatesState: st(delivery.IndexerModule + ".candidates"), + WantsState: st(delivery.AssignerModule + ".wants"), DifferencesState: st(delivery.AssignerModule + ".differences"), + PredefinedState: st(delivery.IndexerModule + "." + delivery.IndexerPredefinedState), + ConfigCandidatesState: st(delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState), + PackagesState: st(delivery.IndexerModule + "." + delivery.PackagesState), + GraphState: st(delivery.IndexerModule + "." + graph.State)} + // The owners' read tools, answered by the owner's own Read over the same state; any other address is refused. + tools := map[string]string{} + for name := range states { + owner, state, _ := strings.Cut(name, ".") + tools[firstview.ReadTool(owner, state)] = name + } + asked := map[string]int{} + ask := func(key string, body any) (json.RawMessage, error) { + name, ok := tools[key] + if !ok { + return nil, fmt.Errorf("no tool %s", key) + } + asked[key]++ + args, _ := body.(map[string]any) + k, _ := args["key"].(string) + all, _ := args["values"].(bool) + after, _ := args["after"].(string) + owner, state, _ := strings.Cut(name, ".") + a, err := views.Read(owner, state, states[name], nil, k, all, after, time.Unix(0, 0)) + if err != nil { + return nil, err + } + return json.Marshal(a) + } + owned := ownersRead(ask) + for _, row := range []struct { + tool string + args map[string]any + }{ + {view.ModuleReadTool, map[string]any{"module": "a"}}, + {view.ModuleReadTool, map[string]any{"module": "gone"}}, + {view.ModulesReadTool, map[string]any{}}, + {view.ModulesReadTool, map[string]any{"none2": true}}, + {view.CandidateReadTool, map[string]any{"group": "feat/x"}}, + {view.CandidatesReadTool, map[string]any{}}, + {view.CandidatesReadTool, map[string]any{"module": "zsh"}}, + {view.WantsReadTool, map[string]any{}}, + {view.WantsReadTool, map[string]any{"node": "ace"}}, + {view.WantReadTool, map[string]any{"node": "ace", "module": "zsh"}}, + {view.DifferencesReadTool, map[string]any{"node": "novox"}}, + {view.PredefinedReadTool, map[string]any{}}, + {view.ConfigCandidatesReadTool, map[string]any{}}, + {view.ConfigCandidatesReadTool, map[string]any{"key": "novox.r.7"}}, + {view.PackagesReadTool, map[string]any{"module": "git.novox.be/novox/mesh-sdk/go"}}, + {view.PackagesReadTool, nil}, + } { + want, werr := run(t, direct, row.tool, row.args) + got, gerr := run(t, owned, row.tool, row.args) + wj, _ := json.Marshal(want) + gj, _ := json.Marshal(got) + if string(wj) != string(gj) || fmt.Sprint(werr) != fmt.Sprint(gerr) { + t.Errorf("%s %v through the owner's read tool = %s, %v; reading the state it was %s, %v", row.tool, row.args, gj, gerr, wj, werr) + } + } + // Each owner state the tools read was asked of its owner's read tool. + for tool, name := range tools { + if name != delivery.IndexerModule+"."+graph.State && asked[tool] == 0 { + t.Errorf("%s was never asked", tool) + } + } + // A history the owner's read tool does not answer is said, never answered as none. + if _, err := run(t, owned, view.ModuleHistoryTool, map[string]any{"module": "a"}); err == nil || !strings.Contains(err.Error(), "no history argument") { + t.Errorf("indexer_module_history with an owner answering no history: %v", err) + } +} diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index 3cbf53e8..44cb4fac 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -20,9 +20,22 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-delivery/sdk/firstview/firstviewtest" + "mesh-delivery-view/internal/view" ) +// serveOwners serves the six states of the indexer and the assigner the view's process follows by their first views +// (ADR 0615 §1, DLV-M4): each owner's read tool and views family, with the view's durable consumer (`n_mesh-delivery-view`). +// A bucket the test made is kept; the others are made empty. +func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) { + t.Helper() + for _, bucket := range ownersBuckets { + owner, state, _ := strings.Cut(bucket, "_") + firstviewtest.Serve(t, ctx, js, nc, owner, state, "n_"+delivery.ViewModule) + } +} + // TestProcessWholeSeesTheBuckets starts the program the way the node does (its entry, run, with its environment and a // broker file) against an embedded bus holding the three buckets, and asks its socket. func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 @@ -99,6 +112,7 @@ func TestProcessWholeSeesTheBuckets(t *testing.T) { // 1134/AC8 env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -198,6 +212,7 @@ func TestProcessWholeShowsWhatTheWatchmanSays(t *testing.T) { // 1168/AC5 env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -264,6 +279,7 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -273,11 +289,11 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124 break } } - cands, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.CandidatesBucket, History: delivery.CandidatesHistory}) + cands, err := js.KeyValue(ctx, delivery.CandidatesBucket) if err != nil { t.Fatal(err) } - mods, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ModulesBucket, History: delivery.ModulesHistory}) + mods, err := js.KeyValue(ctx, delivery.ModulesBucket) if err != nil { t.Fatal(err) } @@ -350,6 +366,7 @@ func TestProcessWholeShowsTheOpenAsksAndDropsTheAnsweredOnes(t *testing.T) { // env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -440,10 +457,11 @@ func TestProcessWholeListsEveryDifference(t *testing.T) { // 1297/B8 AC env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerWantsBucket, History: delivery.AssignerWantsHistory}) + kv, err := js.KeyValue(ctx, delivery.AssignerWantsBucket) if err != nil { t.Fatal(err) } @@ -537,6 +555,7 @@ func TestProcessWholeWritesTheAnswerToAnAskNeedingNoProof(t *testing.T) { // 129 env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -688,6 +707,7 @@ func TestProcessWholeShowsTheBacklogOfBuildAsks(t *testing.T) { // 1887/A8a AC env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -755,6 +775,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) @@ -851,6 +872,7 @@ func TestProcessWholeSaysModulesInMotion(t *testing.T) { env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": filepath.Join(dir, "run", "cli.sock"), "MESH_DELIVERY_WEB": "127.0.0.1:0"} runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) + serveOwners(t, ctx, js, nc) go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() t.Cleanup(func() { stop(); <-done }) time.Sleep(2 * time.Second) // the view has read what stood: nothing diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go index a9953e7a..ae328646 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go @@ -25,6 +25,9 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" + + "git.novox.be/novox/mesh-delivery/sdk/firstview" "mesh-delivery-view/internal/view" ) @@ -106,13 +109,31 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err }) model.SetBacklog(view.BacklogOver(js, 5*time.Second)) + // The indexer's and the assigner's states are their owners': read by the first view of the owner's read tool and then + // its views family, never their buckets (ADR 0615 §1, DLV-M4). An owner that is down never holds the start: a watch of + // its picture waits for the first view on its own goroutine, and until then the unanswered first view is the view's + // finding (ADR 0615 §6). + calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer calls.Close() + reader := &firstview.Reader{Module: delivery.ViewModule, Call: calls.Call, Logf: logger.Printf} + reader.Findings = process.AwaitBucket(ctx, js, conn, findingsBucket, "a first view unanswered past its bound is said in the log alone", logger.Printf).KV + for _, bucket := range ownersBuckets { + owner, state, _ := strings.Cut(bucket, "_") + reader.Follow(owner, state) + } + open := buckets{js: js, reader: reader} for _, bucket := range []string{delivery.HaveBucket, delivery.PlacementBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket, delivery.BuilderBuildsBucket, delivery.AssignerWantsBucket, delivery.SourceBucket, delivery.AssignerAsksBucket, delivery.ViewAnswersBucket, delivery.TakenBucket} { - go follow(ctx, js, bucket, false, model, logger) + go follow(ctx, open, bucket, false, model, logger) } // The modules' and the source's history give each fingerprint's first commit and each head's source revision (ADR 0556). + // The modules' picture holds no history: from its first view on, each change it takes is the history (finding of DLV-M4). for _, bucket := range []string{delivery.ModulesBucket, delivery.SourceBucket} { - go follow(ctx, js, bucket, true, model, logger) + go follow(ctx, open, bucket, true, model, logger) } + reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "view_first_views reads nothing and view_first_view_again is not heard", logger.Printf)) select { case <-ctx.Done(): case err := <-webUp: @@ -152,9 +173,34 @@ func putReach(ctx context.Context, js jetstream.JetStream, w view.ReachWrite) er return err } +// The view's own states of its first views (views.StatusState) and of its findings, where the condition keeper raises a +// first view unanswered past its bound (ADR 0615 §6). +var ( + firstViewsBucket = delivery.ViewModule + "_" + views.StatusState + findingsBucket = delivery.ViewModule + "_findings" +) + +// ownersBuckets are the indexer's and the assigner's states the view's process follows by their first views +// (`_`); the view's tools ask the other six through the owners' read tools on each call. +var ownersBuckets = []string{delivery.PlacementBucket, delivery.AssignerWantsBucket, delivery.AssignerAsksBucket, + delivery.CandidatesBucket, delivery.ModulesBucket, delivery.TakenBucket} + +// buckets opens what the view watches: an owner's state as its picture (firstview), any other bucket on the bus. +type buckets struct { + js jetstream.JetStream + reader *firstview.Reader +} + +func (b buckets) KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error) { + if p := b.reader.Picture(bucket); p != nil { + return p, nil + } + return b.js.KeyValue(ctx, bucket) +} + // follow watches a bucket with its last values first and then each change, for as long as ctx lives: a bucket that is // not there yet, or a watch that ends, is tried again. -func follow(ctx context.Context, js jetstream.JetStream, bucket string, history bool, model *view.Model, logger *log.Logger) { +func follow(ctx context.Context, js process.BucketOpener, bucket string, history bool, model *view.Model, logger *log.Logger) { for ctx.Err() == nil { if err := watchOnce(ctx, js, bucket, history, model); err != nil && ctx.Err() == nil { logger.Printf("cannot watch %s, trying again: %v", bucket, err) @@ -166,7 +212,7 @@ func follow(ctx context.Context, js jetstream.JetStream, bucket string, history } } -func watchOnce(ctx context.Context, js jetstream.JetStream, bucket string, history bool, model *view.Model) error { +func watchOnce(ctx context.Context, js process.BucketOpener, bucket string, history bool, model *view.Model) error { kv, err := js.KeyValue(ctx, bucket) if err != nil { return err diff --git a/modules/mesh-delivery-view/internal/view/owner.go b/modules/mesh-delivery-view/internal/view/owner.go new file mode 100644 index 00000000..938dd138 --- /dev/null +++ b/modules/mesh-delivery-view/internal/view/owner.go @@ -0,0 +1,111 @@ +package view + +import ( + "encoding/json" + "fmt" + "time" +) + +// OwnerRead reads an owner's state through the owner's ordinary read tool (design 25 § 12.2, ADR 0615 §1, DLV-M4): the +// view's tools answer the indexer's and the assigner's states by asking it, never by reading their buckets. ask is the +// bundle's ask of another module's tool (mesh-sdk stdio Ask), tool the read tool as the manifest's `invokes` names it +// (`._read_`). With no argument the tool answers the keys; with key, the key's value and revision; +// with values, the values whole by key in pages; with key and history, the values it keeps for the key with their times. +func OwnerRead(ask func(key string, body any) (json.RawMessage, error), tool string) StateRead { + entry := func(key string) (json.RawMessage, uint64, error) { + raw, err := ask(tool, map[string]any{"key": key}) + if err != nil { + return nil, 0, fmt.Errorf("%s: %w", tool, err) + } + var a struct { + Found bool `json:"found"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, 0, fmt.Errorf("%s answered what is not a key's value: %w", tool, err) + } + if !a.Found || len(a.Value) == 0 { + return nil, 0, nil + } + return a.Value, a.Revision, nil + } + return StateRead{ + Keys: func() ([]string, error) { + raw, err := ask(tool, map[string]any{}) + if err != nil { + return nil, fmt.Errorf("%s: %w", tool, err) + } + var a struct { + Keys []string `json:"keys"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s answered what is not a state's keys: %w", tool, err) + } + return a.Keys, nil + }, + GetEntry: entry, + Get: func(key string) (json.RawMessage, error) { + v, _, err := entry(key) + return v, err + }, + All: func() (map[string]json.RawMessage, error) { + out := map[string]json.RawMessage{} + for after, pages := "", 0; ; pages++ { + if pages == maxOwnerPages { + return nil, fmt.Errorf("%s answered more than %d pages", tool, maxOwnerPages) + } + body := map[string]any{"values": true} + if after != "" { + body["after"] = after + } + raw, err := ask(tool, body) + if err != nil { + return nil, fmt.Errorf("%s: %w", tool, err) + } + var a struct { + Values map[string]json.RawMessage `json:"values"` + Next string `json:"next"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s answered what is not a page of values: %w", tool, err) + } + for k, v := range a.Values { + out[k] = v + } + if a.Next == "" || a.Next == after { + return out, nil + } + after = a.Next + } + }, + History: func(key string) ([]Revision, error) { + raw, err := ask(tool, map[string]any{"key": key, "history": true}) + if err != nil { + return nil, fmt.Errorf("%s: %w", tool, err) + } + var a struct { + History *[]struct { + Revision uint64 `json:"revision"` + At time.Time `json:"at"` + Op string `json:"op"` + Value json.RawMessage `json:"value"` + } `json:"history"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("%s answered what is not a key's history: %w", tool, err) + } + if a.History == nil { + return nil, fmt.Errorf("%s answers no history of a key: its owner's read tool takes no history argument yet", tool) + } + out := make([]Revision, 0, len(*a.History)) + for _, h := range *a.History { + out = append(out, Revision{Revision: h.Revision, At: h.At, Op: h.Op, Value: h.Value}) + } + return out, nil + }, + } +} + +// maxOwnerPages bounds one read of a state's values page by page (a page is at most 768 KiB, mesh-sdk views PageBytes). +const maxOwnerPages = 256 diff --git a/modules/mesh-delivery-view/internal/view/read.go b/modules/mesh-delivery-view/internal/view/read.go index 11c31b07..d76ee9ca 100644 --- a/modules/mesh-delivery-view/internal/view/read.go +++ b/modules/mesh-delivery-view/internal/view/read.go @@ -33,6 +33,9 @@ type StateRead struct { GetEntry func(key string) (json.RawMessage, uint64, error) // History, when set, gives every value the state keeps for a key with its time (indexer_module_history, indexer_heard). History func(key string) ([]Revision, error) + // All, when set, gives every value by key in one read (an owner's read tool's pages, OwnerRead), so a read of many keys + // is not a call per key. + All func() (map[string]json.RawMessage, error) } // Reader answers the read tools from a modules state. @@ -579,6 +582,16 @@ func parallel(n int, read func(i int) error) error { // values reads the stored value of each key, in parallel, in the order of keys (nil for a key with none). func (s StateRead) values(keys []string) ([]json.RawMessage, error) { out := make([]json.RawMessage, len(keys)) + if s.All != nil { + all, err := s.All() + if err != nil { + return nil, err + } + for i, k := range keys { + out[i] = all[k] + } + return out, nil + } err := parallel(len(keys), func(i int) error { raw, err := s.Get(keys[i]) out[i] = raw diff --git a/modules/mesh-delivery-view/internal/view/status.go b/modules/mesh-delivery-view/internal/view/status.go index d8a08373..edc4d1ae 100644 --- a/modules/mesh-delivery-view/internal/view/status.go +++ b/modules/mesh-delivery-view/internal/view/status.go @@ -81,6 +81,9 @@ type StatusMoving struct { } func values(s StateRead) (map[string]json.RawMessage, error) { + if s.All != nil { + return s.All() + } if s.Keys == nil || s.Get == nil { return nil, fmt.Errorf("not read here") } diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 4e65b88f..7ccdba13 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -51,6 +51,8 @@ type manifest struct { Version string `json:"version"` Slug string `json:"slug"` Reads []string `json:"reads"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` Uses []string `json:"uses"` Tools []string `json:"tools"` Emits []string `json:"emits"` @@ -87,17 +89,29 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 if m.Module != "mesh-delivery-view" { t.Errorf("module = %q", m.Module) } - want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.placement", "mesh-delivery-assigner.wants", "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.config-candidates", "mesh-delivery-indexer.graph", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.packages", "mesh-delivery-indexer.predefined", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-provisioner.provisions", "mesh-vault.signers", "node-engine.have"} // view_tiers reads the module graph and the provisioner's keys (ADR 0603 §6); view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 + want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-conditions.conditions", "mesh-config.resolved", "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "mesh-healers.attempts", "mesh-provisioner.provisions", "mesh-vault.signers", "node-engine.have"} // no reads of the indexer's or the assigner's states (ADR 0615 §1, DLV-M4); view_tiers reads the module graph and the provisioner's keys (ADR 0603 §6); view_status reads the keeper's conditions and the healers' attempts (1952/V1); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9 got := slices.Clone(m.Reads) slices.Sort(got) if !reflect.DeepEqual(got, want) { t.Errorf("reads = %v, want exactly %v", got, want) } - for _, k := range []string{"consumes", "provides", "claims"} { + for _, k := range []string{"provides", "claims"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the view writes only its answers", k) } } + // The indexer's and the assigner's states are asked of their owners' read tools (design 25 § 12.2), the six the process + // follows then heard on their views families (ADR 0615 §1, §5; DLV-M4). + if want := []string{"mesh-delivery-assigner.assigner_read_asks", "mesh-delivery-assigner.assigner_read_differences", "mesh-delivery-assigner.assigner_read_placement", + "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_config_candidates", + "mesh-delivery-indexer.indexer_read_graph", "mesh-delivery-indexer.indexer_read_heard", "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_taken"}; !reflect.DeepEqual(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } + if want := []string{"mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.placement", "mesh-delivery-assigner.views.wants", + "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { + t.Errorf("consumes = %v, want %v", m.Consumes, want) + } // The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves. if !reflect.DeepEqual(m.Emits, []string{"delivered", "moving", "settled", "stuck"}) { t.Errorf("emits = %v, want exactly delivered, moving, settled and stuck (1906/D9, D10)", m.Emits) @@ -108,7 +122,8 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The states the view writes: the operator's answers to the asks that need no proof (ADR 0528 clause 6, 1297/B9), which the // teller reads, and how far each repository reached (ADR 0556 clause 2, issue 1280), which the gate reads. - if want := []state{{Name: "answers", History: 5}, {Name: "reach", History: 1}}; !reflect.DeepEqual(m.State, want) { + // Its first views of the owners' states and the finding of one unanswered past its bound (ADR 0615 §6). + if want := []state{{Name: "answers", History: 5}, {Name: "reach", History: 1}, {Name: "first-views", History: 1}, {Name: "findings", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -119,7 +134,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "indexer_packages", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard", "view_tiers"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "indexer_packages", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard", "view_tiers", "view_first_views", "view_first_view_again"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } for _, a := range m.Build.Artifacts { diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 55bbbfb6..6ff8b9ec 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -24,7 +24,9 @@ "view_reach", "indexer_module_history", "indexer_heard", - "view_tiers" + "view_tiers", + "view_first_views", + "view_first_view_again" ], "emits": [ "delivered", @@ -34,7 +36,31 @@ ], "state": [ { "name": "answers", "history": 5 }, - { "name": "reach", "history": 1 } + { "name": "reach", "history": 1 }, + { "name": "first-views", "history": 1 }, + { "name": "findings", "history": 1 } + ], + "consumes": [ + "mesh-delivery-assigner.views.asks", + "mesh-delivery-assigner.views.placement", + "mesh-delivery-assigner.views.wants", + "mesh-delivery-indexer.views.candidates", + "mesh-delivery-indexer.views.modules", + "mesh-delivery-indexer.views.taken" + ], + "invokes": [ + "mesh-delivery-assigner.assigner_read_asks", + "mesh-delivery-assigner.assigner_read_differences", + "mesh-delivery-assigner.assigner_read_placement", + "mesh-delivery-assigner.assigner_read_wants", + "mesh-delivery-indexer.indexer_read_candidates", + "mesh-delivery-indexer.indexer_read_config_candidates", + "mesh-delivery-indexer.indexer_read_graph", + "mesh-delivery-indexer.indexer_read_heard", + "mesh-delivery-indexer.indexer_read_modules", + "mesh-delivery-indexer.indexer_read_packages", + "mesh-delivery-indexer.indexer_read_predefined", + "mesh-delivery-indexer.indexer_read_taken" ], "reads": [ "mesh-build-go.pauses", @@ -44,23 +70,11 @@ "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", - "mesh-delivery-assigner.asks", - "mesh-delivery-assigner.differences", - "mesh-delivery-assigner.placement", - "mesh-delivery-assigner.wants", "mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", - "mesh-delivery-indexer.candidates", - "mesh-delivery-indexer.config-candidates", - "mesh-delivery-indexer.graph", - "mesh-delivery-indexer.heard", - "mesh-delivery-indexer.modules", - "mesh-delivery-indexer.packages", - "mesh-delivery-indexer.predefined", - "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-conditions.conditions", "mesh-healers.attempts", From 890b8f36e940f39c8a51dc903dd8a75cd1b4df2c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:28:18 +0100 Subject: [PATCH 08/12] DLV-C4: the checker names each change of a package series a pull request makes in its status line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 32 (A package, its series): the review of a change sees every series it raises. For each module a pull request touches, every artifact its trunk manifest and its head's manifest both declare whose series differs is named as "/ series → " in the pull request's mesh/check line, never dropped when the line is cut; a manifest that cannot be read is said in the same place. Rows in TestTheIndexersJudgeJudgesTheChangesManifests (raised: named; kept: not named); the raised row failed first. gofmt of the integration test's imports (M2). Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../mesh-delivery-checker/integration_test.go | 2 +- .../internal/check/case.go | 2 + .../internal/check/examiner_test.go | 17 ++- .../internal/check/judge.go | 1 + .../internal/check/line.go | 17 +-- .../internal/check/series.go | 107 ++++++++++++++++++ 6 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 modules/mesh-delivery-checker/internal/check/series.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 6e804bf2..0c4b668f 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -27,8 +27,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" - "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-delivery/sdk/firstview/firstviewtest" + "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index 3f80136f..2e1a6373 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -45,6 +45,8 @@ type Case struct { notesMu sync.Mutex // warnings are, by pull request, what part of its check did not run (Case.warn). warnings map[string]string + // seriesOf are, by pull request, the changes of a package series it makes (series.go), named in its status line. + seriesOf map[string][]string notes []string // asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks). asks []string diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index b237c86f..ecda3707 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1035,8 +1035,12 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // mail's tool while mail awaits web's database: both left out, so neither is refused (ADR 0614 §3, §6). cycle string trunkCycle bool + // series is the series of web's package the change raises it to, and the words its status line names (DLV-C4). + series, seriesWords string }{ {name: "accepted"}, + {name: "a package series raised", series: "0.2", seriesWords: "web/pkg series 0.1 → 0.2"}, + {name: "a package series kept", series: "0.1"}, {name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"}, {name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps", fails: "refuses the manifest of mail: mail reads a state web no longer keeps"}, @@ -1077,6 +1081,13 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { } b.ex.Read = fixedManifests{strings.Repeat("a", 64): mail, strings.Repeat("b", 64): trunkWeb, web: changed} } + if row.series != "" { + pkg := func(series string) *catalogue.Build { + return &catalogue.Build{Artifacts: []catalogue.Artifact{{Name: "pkg", Kind: catalogue.ArtifactPackage, Language: "go", Series: series}}} + } + b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): {Module: "web", Build: pkg("")}, + web: {Module: "web", Build: pkg(row.series)}} + } var judged []delivery.ComposeJob answer := b.asks.Answer b.asks.Answer = func(a delivery.StepAsk) delivery.Step { @@ -1094,7 +1105,11 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { p := pullOf(repoA, 7, head1) p.Modules = []string{"web"} p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}} - v := b.judge(t, caseOf(t, p)).Verdict + cs := caseOf(t, p) + v := b.judge(t, cs).Verdict + if line := Line(v, cs, cs.Pulls[0], nil, nil, nil); row.seriesWords != "" && !strings.Contains(line, row.seriesWords) || row.seriesWords == "" && strings.Contains(line, "series") { + t.Errorf("%s: the line %q, want it to name %q", row.name, line, row.seriesWords) + } if len(judged) != 1 || judged[0].Fetch["repository"] != DeliveryRepository || judged[0].Fetch["commit"] != trunk || judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir { t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v) diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index d3db5775..70ebb497 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -269,6 +269,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { // Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR // 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02). tests := st.judgeAll(ctx) + st.seriesChanges(ctx) // DLV-C4: named in the verdict's line, whatever the verdict if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil { return st.finish(ctx, *v), nil } diff --git a/modules/mesh-delivery-checker/internal/check/line.go b/modules/mesh-delivery-checker/internal/check/line.go index 153b5b61..dc81ac58 100644 --- a/modules/mesh-delivery-checker/internal/check/line.go +++ b/modules/mesh-delivery-checker/internal/check/line.go @@ -117,18 +117,20 @@ type pieces struct { carried string // warning says that a part of the check did not run, for instance that an older build module could not refuse a compiled program; never dropped. warning string + // series names each change of a package series the pull request makes (design 32, DLV-C4); never dropped. + series string } // pullKey names a pull request of a case: `#`. func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) } func (p pieces) join(drop int) string { - parts := []string{p.head, p.commits, p.carried, p.warning, p.counts, p.judged} + parts := []string{p.head, p.commits, p.carried, p.warning, p.counts, p.judged, p.series} switch { case drop < 3 && p.times != "": parts = append(parts, p.times) case drop == 3 && p.shortTimes != "": - parts = []string{p.head, p.commits, p.carried, p.warning, p.shortCounts, p.judged, p.shortTimes} + parts = []string{p.head, p.commits, p.carried, p.warning, p.shortCounts, p.judged, p.series, p.shortTimes} } if p.over != "" { parts = append(parts, p.over) @@ -244,6 +246,7 @@ func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces { var pc pieces pc.judged = judged[pullKey(p)] + pc.series = seriesWords(cs, p) if cs != nil { pc.warning = cs.Warning(pullKey(p)) } @@ -356,7 +359,7 @@ func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []st pc := parts(v, cs, p, builds, slowest, judged) if v.State == delivery.VerdictFail { // The head gives way first to the commits and what is never dropped: its count and "and N more" stay whole. - kept := joinSet(pc.commits, pc.carried, pc.warning, pc.judged) + kept := joinSet(pc.commits, pc.carried, pc.warning, pc.judged, pc.series) pc.head = failedHead(v, MaxLine-utf8.RuneCountInString(" | "+kept)) } for drop := 0; drop <= 4; drop++ { @@ -364,13 +367,13 @@ func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []st return s } } - if pc.judged != "" { + if pc.judged != "" || pc.series != "" { for _, counts := range []string{pc.shortCounts, ""} { - if s := joinSet(pc.head, pc.commits, counts, pc.judged); utf8.RuneCountInString(s) <= MaxLine { + if s := joinSet(pc.head, pc.commits, counts, pc.judged, pc.series); utf8.RuneCountInString(s) <= MaxLine { return s } } - rest := " | " + joinSet(pc.commits, pc.judged) + rest := " | " + joinSet(pc.commits, pc.judged, pc.series) room := MaxLine - utf8.RuneCountInString(rest) - 1 if head := []rune(pc.head); room > 0 && len(head) > room { return string(head[:room]) + "…" + rest @@ -378,7 +381,7 @@ func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []st } if v.State == delivery.VerdictFail { // A failure's head, sized above, and the commits fit: the counts and the composition are in the report. - if s := joinSet(pc.head, pc.commits, pc.carried, pc.warning, pc.judged); utf8.RuneCountInString(s) <= MaxLine { + if s := joinSet(pc.head, pc.commits, pc.carried, pc.warning, pc.judged, pc.series); utf8.RuneCountInString(s) <= MaxLine { return s } } diff --git a/modules/mesh-delivery-checker/internal/check/series.go b/modules/mesh-delivery-checker/internal/check/series.go new file mode 100644 index 00000000..786d90bb --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/series.go @@ -0,0 +1,107 @@ +package check + +import ( + "context" + "fmt" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// A change of a package series is named in the verdict, so the review of the change sees it (design 32 *A package, its +// series*; DLV-C4): for each module a pull request touches, every artifact both its trunk manifest and its head's +// manifest declare whose series differs, as `/ series → `. An artifact that names none +// is in catalogue.DefaultSeries. A manifest that cannot be read is said in the same words, never left out. + +// seriesChanges names, on the case per pull request, each change of an artifact's series the pull request makes. +func (st *state) seriesChanges(ctx context.Context) { + if st.j.Read == nil { + return + } + var trunk map[string]string + if st.j.Manifests != nil { + trunk = st.j.Manifests.ManifestDigests() + } + for _, p := range st.c.Pulls { + var words []string + for _, module := range p.Modules { + if trunk[module] == "" { + continue // new on this pull request: no series is changed + } + was, err := st.j.Read.Manifest(ctx, trunk[module]) + if err != nil { + words = append(words, fmt.Sprintf("%s: the trunk's series not read (%v)", module, err)) + continue + } + var now catalogue.Manifest + switch digest := p.Fps[module].Manifest; { + case digest == trunk[module]: + continue + case digest != "": + if now, err = st.j.Read.Manifest(ctx, digest); err != nil { + words = append(words, fmt.Sprintf("%s: the head's series not read (%v)", module, err)) + continue + } + case st.j.Forge != nil: + var found bool + if now, _, found, err = st.manifestAtHead(ctx, p, module); err != nil { + words = append(words, fmt.Sprintf("%s: the head's series not read (%v)", module, err)) + continue + } else if !found { + continue + } + default: + continue + } + words = append(words, seriesMoved(module, was, now)...) + } + if len(words) > 0 { + st.c.series(pullKey(p), words) + } + } +} + +// seriesMoved are the artifacts of module whose series the head's manifest changes from the trunk's. +func seriesMoved(module string, was, now catalogue.Manifest) []string { + if was.Build == nil || now.Build == nil { + return nil + } + before := map[string]string{} + for _, a := range was.Build.Artifacts { + before[a.Name] = a.SeriesOf() + } + var out []string + for _, a := range now.Build.Artifacts { + if old, ok := before[a.Name]; ok && old != a.SeriesOf() { + out = append(out, fmt.Sprintf("%s/%s series %s → %s", module, a.Name, old, a.SeriesOf())) + } + } + slices.Sort(out) + return out +} + +// series keeps, of a pull request (pullKey), the words of the series changes it makes. +func (c *Case) series(key string, words []string) { + c.notesMu.Lock() + defer c.notesMu.Unlock() + if c.seriesOf == nil { + c.seriesOf = map[string][]string{} + } + c.seriesOf[key] = words +} + +// Series are the series changes a pull request (pullKey) makes, as the check named them; none when it makes none. +func (c *Case) Series(key string) []string { + c.notesMu.Lock() + defer c.notesMu.Unlock() + return slices.Clone(c.seriesOf[key]) +} + +// seriesWords is a pull request's series changes for its status line. +func seriesWords(c *Case, p Pull) string { + if c == nil { + return "" + } + return strings.Join(c.Series(pullKey(p)), ", ") +} From 30ee47caa608eaf421b21040145f3eb8c433cb11 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:32:41 +0100 Subject: [PATCH 09/12] DLV-C5: the checker asks the seat mesh-config's verb renames for every change that brings or drops a renamed-from Design 32 (A config value renamed): the checker invokes seat:mesh-config.renames, never the module. A change bringing renamed-from to a holder that serves no renames (nothing answers the optional verb) is refused naming it; dropping a renamed-from while its window is open is refused naming the nodes holding it open; a check that cannot ask fails, saying so. The pull request's trunk and head manifests are read once for this and for the series changes (manifestPair). Rows in TestTheIndexersJudgeJudgesTheChangesManifests with the verb faked from the seat's definition (its fixture's answer): not served, served, open on g14, closed, an ask that fails, no seat to ask; the four refusals failed first. Manifest row for the invokes. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-checker/judge.go | 3 +- .../cmd/mesh-delivery-checker/main.go | 1 + .../cmd/mesh-delivery-checker/renames.go | 36 +++++ modules/mesh-delivery-checker/go.mod | 1 + .../internal/check/examiner_test.go | 42 ++++++ .../internal/check/judge.go | 5 +- .../internal/check/renames.go | 127 ++++++++++++++++++ .../internal/check/series.go | 67 ++++----- .../mesh-delivery-checker/manifest_test.go | 2 +- modules/mesh-delivery-checker/module.json | 1 + 10 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/renames.go create mode 100644 modules/mesh-delivery-checker/internal/check/renames.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index acd86ce7..bbeef436 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -25,6 +25,7 @@ import ( // judgeDeps are the parts the judge of a check is made of. type judgeDeps struct { asks check.Asks + renames check.Renames tools *buildrun.ToolchainSet trunks *check.Trunks kept check.KV @@ -155,7 +156,7 @@ func newJudge(d judgeDeps) check.Judge { nodes = d.nodes } ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Renames: d.renames, Logf: d.logf} if d.nodes != nil && d.builds != nil && d.repos != nil { ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf} } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index eda2e94f..9f216641 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -263,6 +263,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro announce: announce, compositions: compositions, tags: tags, nodes: openNodeNames(ctx, js, logger.Printf), predefined: keptPredefined{predefined}, + renames: seatRenames{call: caller.Call}, }) ch.Announce = announce tryer.Judge = ch.Judge diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/renames.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/renames.go new file mode 100644 index 00000000..dca90323 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/renames.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/nats-io/nats.go" + + "git.novox.be/novox/mesh-contracts/go/seats/meshconfig" + + "mesh-delivery-checker/internal/check" +) + +// seatRenames asks the seat mesh-config's verb renames as the checker's own account (its `invokes` +// seat:mesh-config.renames; design 32). A call nobody answers is a holder serving no renames (the verb is optional), said +// as check.ErrRenamesNotServed with the bus's words; any other error is an ask that could not be made. +type seatRenames struct { + call func(key string, body any) (json.RawMessage, error) +} + +func (s seatRenames) Renames(_ context.Context, module string) (meshconfig.RenamesAnswer, error) { + raw, err := s.call(check.RenamesSeat, meshconfig.RenamesInput{Module: module}) + if errors.Is(err, nats.ErrNoResponders) { + return meshconfig.RenamesAnswer{}, fmt.Errorf("%w (nothing answers it: %v)", check.ErrRenamesNotServed, err) + } + if err != nil { + return meshconfig.RenamesAnswer{}, err + } + var a meshconfig.RenamesAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return meshconfig.RenamesAnswer{}, fmt.Errorf("%s answered what is not its answer: %w", check.RenamesSeat, err) + } + return a, nil +} diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 929f8473..8781e9a9 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -12,6 +12,7 @@ require git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-2 require ( git.novox.be/novox/mesh-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/meshconfig v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index ecda3707..264cf0d8 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -19,6 +19,7 @@ import ( "unicode/utf8" gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + "git.novox.be/novox/mesh-contracts/go/seats/meshconfig" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/composer" @@ -1037,7 +1038,18 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { trunkCycle bool // series is the series of web's package the change raises it to, and the words its status line names (DLV-C4). series, seriesWords string + // renames is what the change does to web's renamed-from ("bring", "drop") and what the seat mesh-config's verb + // renames answers ("served", "open" on g14, "not served", "unasked" with no seat to ask, "error"); DLV-C5. + renames, seat string }{ + {name: "a rename brought, the verb not served", renames: "bring", seat: "not served", + fails: "web (novox/mesh-delivery#7) renames new-key from old-key: refused, the holder of the seat mesh-config does not serve its verb renames"}, + {name: "a rename brought, the verb served", renames: "bring", seat: "served"}, + {name: "a renamed-from dropped while its window is open", renames: "drop", seat: "open", + fails: "web (novox/mesh-delivery#7) drops renamed-from old-key of new-key while its rename window is open on g14"}, + {name: "a renamed-from dropped once its window closed", renames: "drop", seat: "served"}, + {name: "a rename the check cannot ask of", renames: "bring", seat: "error", fails: "the check cannot ask seat:mesh-config.renames: the bus said no"}, + {name: "a rename with no seat to ask", renames: "drop", seat: "unasked", fails: "this checker cannot ask seat:mesh-config.renames"}, {name: "accepted"}, {name: "a package series raised", series: "0.2", seriesWords: "web/pkg series 0.1 → 0.2"}, {name: "a package series kept", series: "0.1"}, @@ -1088,6 +1100,28 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): {Module: "web", Build: pkg("")}, web: {Module: "web", Build: pkg(row.series)}} } + if row.renames != "" { + renamed := map[string]catalogue.SettingDeclaration{"new-key": {Type: catalogue.ConfigString, RenamedFrom: "old-key"}} + plain := map[string]catalogue.SettingDeclaration{"new-key": {Type: catalogue.ConfigString}} + trunkWeb, headWeb := catalogue.Manifest{Module: "web", Settings: plain}, catalogue.Manifest{Module: "web", Settings: renamed} + if row.renames == "drop" { + trunkWeb, headWeb = headWeb, trunkWeb + } + b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): trunkWeb, web: headWeb} + // The verb faked from its definition: the answer of the seat's fixture (mesh-contracts seats/mesh-config). + fake := fakeRenames{} + switch row.seat { + case "not served": + fake.err = fmt.Errorf("%w (nothing answers it)", ErrRenamesNotServed) + case "error": + fake.err = errors.New("the bus said no") + case "open": + fake.answer = meshconfig.RenamesAnswer{Renames: []meshconfig.RenamesAnswerRenamesItem{{Module: "web", From: "old-key", To: "new-key", At: "2026-11-04T12:00:00Z", Open: []string{"g14"}}}} + } + if row.seat != "unasked" { + b.ex.Renames = fake + } + } var judged []delivery.ComposeJob answer := b.asks.Answer b.asks.Answer = func(a delivery.StepAsk) delivery.Step { @@ -1134,3 +1168,11 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { t.Error("which merge changes the judge") } } + +// fakeRenames is the seat mesh-config's verb renames as its definition answers it. +type fakeRenames struct { + answer meshconfig.RenamesAnswer + err error +} + +func (f fakeRenames) Renames(context.Context, string) (meshconfig.RenamesAnswer, error) { return f.answer, f.err } diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 70ebb497..a8e98824 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -128,6 +128,9 @@ type Examiner struct { // Manifests are the indexer's manifest digests by module: the catalogue a change's manifests are judged in by the // indexer's judge (manifests_judge.go, novox/hq ADR 0578 §2); nil judges no manifest. Manifests TrunkManifests + // Renames asks the seat mesh-config's verb renames for a change that brings or drops a `renamed-from` (renames.go, + // design 32); nil fails such a change, saying it cannot ask. + Renames Renames // Read reads a kept manifest by its digest: the change's module graph is worked out from them (graph_cycle.go, novox/hq // ADR 0603 §2); nil works out no graph. Read ManifestReader @@ -270,7 +273,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { // 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02). tests := st.judgeAll(ctx) st.seriesChanges(ctx) // DLV-C4: named in the verdict's line, whatever the verdict - if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil { + if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx), st.judgeRenames(ctx))); v != nil { return st.finish(ctx, *v), nil } return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil diff --git a/modules/mesh-delivery-checker/internal/check/renames.go b/modules/mesh-delivery-checker/internal/check/renames.go new file mode 100644 index 00000000..0d8bfc67 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/renames.go @@ -0,0 +1,127 @@ +package check + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "strings" + + "git.novox.be/novox/mesh-contracts/go/seats/meshconfig" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A config value renamed (design 32 *A config value renamed*, DLV-C5): the checker asks the seat mesh-config's verb +// renames, never the module, for every change that brings `renamed-from` or drops one. A holder that does not serve the +// verb refuses `renamed-from`, naming that; while a rename's window is open, dropping its `renamed-from` is refused, +// naming the nodes holding it open; a check that cannot ask fails, saying so. + +// RenamesSeat is the verb the checker asks, as its manifest's `invokes` names it. +var RenamesSeat = "seat:" + meshconfig.Seat + "." + meshconfig.VerbRenames + +// ErrRenamesNotServed is the answer of Renames when the seat's holder serves no verb renames. +var ErrRenamesNotServed = errors.New("the holder of the seat " + meshconfig.Seat + " does not serve its verb " + meshconfig.VerbRenames) + +// Renames asks the seat mesh-config's verb renames for a module's open renames. +type Renames interface { + Renames(ctx context.Context, module string) (meshconfig.RenamesAnswer, error) +} + +// renameChange is what a pull request's head does to a module's `renamed-from` against the trunk: the declarations it +// brings (key from old name) and the ones it drops (key, old name). +type renameChange struct { + brought, dropped map[string]string +} + +// renamesOf compares a module's config declarations at the trunk and at the head. +func renamesOf(was, now catalogue.Manifest) renameChange { + rc := renameChange{brought: map[string]string{}, dropped: map[string]string{}} + for key, d := range now.Settings { + if d.RenamedFrom != "" && was.Settings[key].RenamedFrom != d.RenamedFrom { + rc.brought[key] = d.RenamedFrom + } + } + for key, d := range was.Settings { + if d.RenamedFrom != "" && now.Settings[key].RenamedFrom != d.RenamedFrom { + rc.dropped[key] = d.RenamedFrom + } + } + return rc +} + +// judgeRenames fails the check for a rename the seat refuses, a window a change would drop, or an ask that cannot be made. +func (st *state) judgeRenames(ctx context.Context) *delivery.Verdict { + if st.c.Try != nil { + return nil + } + var out *delivery.Verdict + failed := func(words string) { + f := delivery.FailedFirst{Machine: meshconfig.Seat, Words: words} + if out == nil { + out = &delivery.Verdict{State: delivery.VerdictFail, Failed: f} + return + } + addFailure(out, f) + } + for _, p := range st.c.Pulls { + for _, module := range p.Modules { + was, now, ok, err := st.manifestPair(ctx, p, module) + if err != nil { + failed(fmt.Sprintf("%s (%s): its config renames cannot be judged: %v", module, pullKey(p), err)) + continue + } + if !ok { + was = catalogue.Manifest{} // a module new on the pull request brings every rename it declares + if now, ok = st.headManifest(ctx, p, module); !ok { + continue + } + } + rc := renamesOf(was, now) + if len(rc.brought) == 0 && len(rc.dropped) == 0 { + continue + } + if st.j.Renames == nil { + failed(fmt.Sprintf("%s (%s) changes a renamed-from, and this checker cannot ask %s", module, pullKey(p), RenamesSeat)) + continue + } + answer, err := st.j.Renames.Renames(ctx, module) + switch { + case errors.Is(err, ErrRenamesNotServed): + for _, key := range slices.Sorted(maps.Keys(rc.brought)) { + failed(fmt.Sprintf("%s (%s) renames %s from %s: refused, %v, so the value cannot be moved", module, pullKey(p), key, rc.brought[key], err)) + } + for _, key := range slices.Sorted(maps.Keys(rc.dropped)) { + failed(fmt.Sprintf("%s (%s) drops renamed-from %s of %s: refused, %v, so whether its window is open is not known", module, pullKey(p), rc.dropped[key], key, err)) + } + continue + case err != nil: + failed(fmt.Sprintf("%s (%s) changes a renamed-from, and the check cannot ask %s: %v", module, pullKey(p), RenamesSeat, err)) + continue + } + for _, key := range slices.Sorted(maps.Keys(rc.dropped)) { + for _, r := range answer.Renames { + if r.Module == module && r.From == rc.dropped[key] && r.To == key && len(r.Open) > 0 { + open := slices.Sorted(slices.Values(r.Open)) + failed(fmt.Sprintf("%s (%s) drops renamed-from %s of %s while its rename window is open on %s", module, pullKey(p), r.From, key, strings.Join(open, ", "))) + } + } + } + st.c.note(fmt.Sprintf("%s.%s answered for %s (%s)", meshconfig.Seat, meshconfig.VerbRenames, module, pullKey(p))) + } + } + return out +} + +// headManifest is a module's manifest at the pull request's head, new on it: by its candidate's digest, else the forge's. +func (st *state) headManifest(ctx context.Context, p Pull, module string) (catalogue.Manifest, bool) { + if digest := p.Fps[module].Manifest; digest != "" && st.j.Read != nil { + if st.j.Manifests != nil && st.j.Manifests.ManifestDigests()[module] == digest { + return catalogue.Manifest{}, false // unchanged + } + m, err := st.j.Read.Manifest(ctx, digest) + return m, err == nil + } + return catalogue.Manifest{}, false +} diff --git a/modules/mesh-delivery-checker/internal/check/series.go b/modules/mesh-delivery-checker/internal/check/series.go index 786d90bb..a2e00760 100644 --- a/modules/mesh-delivery-checker/internal/check/series.go +++ b/modules/mesh-delivery-checker/internal/check/series.go @@ -16,45 +16,17 @@ import ( // seriesChanges names, on the case per pull request, each change of an artifact's series the pull request makes. func (st *state) seriesChanges(ctx context.Context) { - if st.j.Read == nil { - return - } - var trunk map[string]string - if st.j.Manifests != nil { - trunk = st.j.Manifests.ManifestDigests() - } for _, p := range st.c.Pulls { var words []string for _, module := range p.Modules { - if trunk[module] == "" { - continue // new on this pull request: no series is changed - } - was, err := st.j.Read.Manifest(ctx, trunk[module]) + was, now, ok, err := st.manifestPair(ctx, p, module) if err != nil { - words = append(words, fmt.Sprintf("%s: the trunk's series not read (%v)", module, err)) + words = append(words, fmt.Sprintf("%s: its series not compared (%v)", module, err)) continue } - var now catalogue.Manifest - switch digest := p.Fps[module].Manifest; { - case digest == trunk[module]: - continue - case digest != "": - if now, err = st.j.Read.Manifest(ctx, digest); err != nil { - words = append(words, fmt.Sprintf("%s: the head's series not read (%v)", module, err)) - continue - } - case st.j.Forge != nil: - var found bool - if now, _, found, err = st.manifestAtHead(ctx, p, module); err != nil { - words = append(words, fmt.Sprintf("%s: the head's series not read (%v)", module, err)) - continue - } else if !found { - continue - } - default: - continue + if ok { + words = append(words, seriesMoved(module, was, now)...) } - words = append(words, seriesMoved(module, was, now)...) } if len(words) > 0 { st.c.series(pullKey(p), words) @@ -62,6 +34,37 @@ func (st *state) seriesChanges(ctx context.Context) { } } +// manifestPair is a touched module's trunk manifest and its manifest at the pull request's head: the head's by the digest +// its candidate names, else read from the forge at the head. ok is false when the module is new on the pull request, +// unchanged, or not read here. +func (st *state) manifestPair(ctx context.Context, p Pull, module string) (was, now catalogue.Manifest, ok bool, err error) { + if st.j.Read == nil || st.j.Manifests == nil { + return was, now, false, nil + } + trunk := st.j.Manifests.ManifestDigests()[module] + digest := p.Fps[module].Manifest + if trunk == "" || digest == trunk { + return was, now, false, nil + } + if was, err = st.j.Read.Manifest(ctx, trunk); err != nil { + return was, now, false, fmt.Errorf("the trunk's manifest not read: %w", err) + } + switch { + case digest != "": + if now, err = st.j.Read.Manifest(ctx, digest); err != nil { + return was, now, false, fmt.Errorf("the head's manifest not read: %w", err) + } + case st.j.Forge != nil: + var found bool + if now, _, found, err = st.manifestAtHead(ctx, p, module); err != nil || !found { + return was, now, false, err + } + default: + return was, now, false, nil + } + return was, now, true, nil +} + // seriesMoved are the artifacts of module whose series the head's manifest changes from the trunk's. func seriesMoved(module string, was, now catalogue.Manifest) []string { if was.Build == nil || now.Build == nil { diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 41491b4e..97565506 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -123,7 +123,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches", "mesh-delivery-assigner.assigner_read_placement", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", - "seat:node-engine.compose-candidate"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + "seat:mesh-config.renames", "seat:node-engine.compose-candidate"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want the forge's reads and compose-candidate", m.Invokes) } if want := []string{"node-build-go", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index df8cfd0c..759ea326 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -79,6 +79,7 @@ "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", + "seat:mesh-config.renames", "seat:node-engine.compose-candidate" ], From 20269818c987cd433f9f7150ebc1730335f44009 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:32:46 +0100 Subject: [PATCH 10/12] DLV-C5: gofmt of the checker's examiner test Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- modules/mesh-delivery-checker/internal/check/examiner_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 264cf0d8..d3519a79 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1175,4 +1175,6 @@ type fakeRenames struct { err error } -func (f fakeRenames) Renames(context.Context, string) (meshconfig.RenamesAnswer, error) { return f.answer, f.err } +func (f fakeRenames) Renames(context.Context, string) (meshconfig.RenamesAnswer, error) { + return f.answer, f.err +} From 5700edf955cd62d6c38253b05c197c9f7b8129fa Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:40:02 +0100 Subject: [PATCH 11/12] DLV-M2r1: the checker fails a pull request whose head's seat definition does not parse, naming the seat and where The lead's ruling on A2i's finding 1 (design 26): the indexer keeps an unparsed definition of a pull request's head as its entry of the state seat-definitions with `unparsed`, and cannot refuse the manifest. The checker follows that state as its seventh first view (invokes indexer_read_seat_definitions, consumes views.seat-definitions; checks start once it is answered too, as M2's six) and fails each pull request with an entry holding `unparsed`, naming the seat, its module, its directory and the indexer's words; a state that cannot be read fails it too. Rows: TestTheIndexersJudgeJudgesTheChangesManifests (fails first without the refusal), the adapter's key match in TestThePublishedVersionsAreTheIndexersPackagesState (fails first matching every key), the manifest's invokes and consumes; the integration bed serves the state (without it every check waits and five tests fail). Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../mesh-delivery-checker/integration_test.go | 2 +- .../cmd/mesh-delivery-checker/judge.go | 66 ++++++++++++++++++- .../cmd/mesh-delivery-checker/judge_test.go | 18 +++++ .../cmd/mesh-delivery-checker/main.go | 10 ++- .../internal/check/examiner_test.go | 14 ++++ .../internal/check/judge.go | 4 +- .../internal/check/seatdefs.go | 55 ++++++++++++++++ .../mesh-delivery-checker/manifest_test.go | 3 +- modules/mesh-delivery-checker/module.json | 2 + 9 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-delivery-checker/internal/check/seatdefs.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 0c4b668f..1e98288d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -443,7 +443,7 @@ func startBed(t *testing.T) *bed { } // The indexer and the assigner as the owners of the states the checker follows: their read tools and views families // (ADR 0615, DLV-M2). - for _, st := range []string{candidatesState, modulesState, manifestsState, delivery.PackagesState, delivery.IndexerPredefinedState} { + for _, st := range []string{candidatesState, modulesState, manifestsState, delivery.PackagesState, delivery.IndexerPredefinedState, seatDefinitionsState} { firstviewtest.Serve(t, ctx, js, ctl, delivery.IndexerModule, st, "n1_"+delivery.CheckerModule) } firstviewtest.Serve(t, ctx, js, ctl, delivery.AssignerModule, delivery.PlacementState, "n1_"+delivery.CheckerModule) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index bbeef436..0aa34c2e 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "sort" + "strconv" "strings" "sync" "time" @@ -26,6 +27,7 @@ import ( type judgeDeps struct { asks check.Asks renames check.Renames + seatDefs check.SeatDefinitions tools *buildrun.ToolchainSet trunks *check.Trunks kept check.KV @@ -69,6 +71,68 @@ func (k keptPredefined) PredefinedList(ctx context.Context) (json.RawMessage, er return kept.List, nil } +// keptSeatDefinitions reads a pull request's entries of the indexer's seat definitions from its first view: each whose +// `unparsed` says why its head's definition does not parse (the entry's shape is the indexer's read tool's answer). +type keptSeatDefinitions struct { + keys func(ctx context.Context) ([]string, error) + get func(ctx context.Context, key string) ([]byte, error) +} + +// seatDefinitionsOf reads the state's picture (a key gone between the keys and its get is none). +func seatDefinitionsOf(kv jetstream.KeyValue) keptSeatDefinitions { + return keptSeatDefinitions{keys: func(ctx context.Context) ([]string, error) { + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err + }, get: func(ctx context.Context, key string) ([]byte, error) { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil + }} +} + +func (k keptSeatDefinitions) Unparsed(ctx context.Context, repository string, pull int) ([]check.UnparsedSeat, error) { + suffix := ".pull." + delivery.EscapeName(repository) + "." + strconv.Itoa(pull) + keys, err := k.keys(ctx) + if err != nil { + return nil, err + } + sort.Strings(keys) + var out []check.UnparsedSeat + for _, key := range keys { + if !strings.HasSuffix(key, suffix) { + continue + } + raw, err := k.get(ctx, key) + if err != nil { + return nil, err + } + if raw == nil { + continue + } + var v struct { + Seat string `json:"seat"` + Module string `json:"module"` + Directory string `json:"directory"` + Unparsed string `json:"unparsed"` + } + if err := json.Unmarshal(raw, &v); err != nil { + return nil, fmt.Errorf("%s is no seat definition entry: %w", key, err) + } + if v.Unparsed != "" { + out = append(out, check.UnparsedSeat{Seat: v.Seat, Module: v.Module, Directory: v.Directory, Why: v.Unparsed}) + } + } + return out, nil +} + // nodeNames reads every node's name from the node-engine's have, one key `.node` per node with a record (novox/hq // ADR 0567 §12), at each ask: a node enrolled since is named in the next check. type nodeNames struct{ kv jetstream.KeyValue } @@ -156,7 +220,7 @@ func newJudge(d judgeDeps) check.Judge { nodes = d.nodes } ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Renames: d.renames, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Renames: d.renames, SeatDefinitions: d.seatDefs, Logf: d.logf} if d.nodes != nil && d.builds != nil && d.repos != nil { ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf} } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index 0b88b7f0..f02d7ed3 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -4,14 +4,18 @@ import ( "context" "encoding/json" "fmt" + "maps" "os" "reflect" + "slices" "strings" "testing" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-checker/internal/check" ) type memManifests map[string][]byte @@ -142,6 +146,20 @@ func TestThePublishedVersionsAreTheIndexersPackagesState(t *testing.T) { if got := p.LatestPackages(); len(got) != 1 || got[sdk] == "" { t.Errorf("after a delete: %v", got) } + t.Run("a pull request's unparsed seat definitions are its entries of the indexer's seat-definitions", func(t *testing.T) { + state := map[string]string{ + "web-api.trunk": `{"seat":"web-api","source":"trunk","digest":"d"}`, + "web-api.pull." + delivery.EscapeName("novox/web") + ".7": `{"seat":"web-api","source":"pull","module":"web-contracts","directory":"go","unparsed":"seat.json: bad"}`, + "mail-api.pull." + delivery.EscapeName("novox/web") + ".7": `{"seat":"mail-api","source":"pull","digest":"e"}`, + "web-api.pull." + delivery.EscapeName("novox/web") + ".17": `{"seat":"web-api","source":"pull","unparsed":"another pull request's"}`, + } + k := keptSeatDefinitions{keys: func(context.Context) ([]string, error) { return slices.Collect(maps.Keys(state)), nil }, + get: func(_ context.Context, key string) ([]byte, error) { return []byte(state[key]), nil }} + got, err := k.Unparsed(context.Background(), "novox/web", 7) + if want := []check.UnparsedSeat{{Seat: "web-api", Module: "web-contracts", Directory: "go", Why: "seat.json: bad"}}; err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("unparsed of novox/web#7 = %+v, %v; want %+v", got, err, want) + } + }) } // A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 9f216641..e0b3de85 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -74,8 +74,11 @@ var ( const ( candidatesState = "candidates" - modulesState = "modules" - manifestsState = "manifests" + // seatDefinitionsState is the indexer's state of the seat definitions (design 26), keyed `.trunk` and + // `.pull..`. + seatDefinitionsState = "seat-definitions" + modulesState = "modules" + manifestsState = "manifests" ) // run is the program: it returns when ctx ends or the bus cannot be used. @@ -144,6 +147,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro packages := reader.Follow(delivery.IndexerModule, delivery.PackagesState) predefined := reader.Follow(delivery.IndexerModule, delivery.IndexerPredefinedState) placement := reader.Follow(delivery.AssignerModule, delivery.PlacementState) + // The seventh: the indexer's seat definitions, so a pull request whose head's definition does not parse fails. + seatDefs := reader.Follow(delivery.IndexerModule, seatDefinitionsState) stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", check.DefaultStepBound) if err != nil { @@ -264,6 +269,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro nodes: openNodeNames(ctx, js, logger.Printf), predefined: keptPredefined{predefined}, renames: seatRenames{call: caller.Call}, + seatDefs: seatDefinitionsOf(seatDefs), }) ch.Announce = announce tryer.Judge = ch.Judge diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index d3519a79..cfd92e1d 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1041,7 +1041,11 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // renames is what the change does to web's renamed-from ("bring", "drop") and what the seat mesh-config's verb // renames answers ("served", "open" on g14, "not served", "unasked" with no seat to ask, "error"); DLV-C5. renames, seat string + // unparsed is the indexer's words of why the pull request's head's seat definition of web-api does not parse. + unparsed string }{ + {name: "a seat definition that does not parse", unparsed: "seats/web-api/seat.json: unexpected end of JSON input", + fails: "novox/mesh-delivery#7: the seat definition of web-api (module web-contracts, go) does not parse: seats/web-api/seat.json: unexpected end of JSON input"}, {name: "a rename brought, the verb not served", renames: "bring", seat: "not served", fails: "web (novox/mesh-delivery#7) renames new-key from old-key: refused, the holder of the seat mesh-config does not serve its verb renames"}, {name: "a rename brought, the verb served", renames: "bring", seat: "served"}, @@ -1122,6 +1126,9 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { b.ex.Renames = fake } } + if row.unparsed != "" { + b.ex.SeatDefinitions = fakeSeatDefinitions{"novox/mesh-delivery#7": {{Seat: "web-api", Module: "web-contracts", Directory: "go", Why: row.unparsed}}} + } var judged []delivery.ComposeJob answer := b.asks.Answer b.asks.Answer = func(a delivery.StepAsk) delivery.Step { @@ -1178,3 +1185,10 @@ type fakeRenames struct { func (f fakeRenames) Renames(context.Context, string) (meshconfig.RenamesAnswer, error) { return f.answer, f.err } + +// fakeSeatDefinitions are the indexer's unparsed seat definitions by pull request (pullKey). +type fakeSeatDefinitions map[string][]UnparsedSeat + +func (f fakeSeatDefinitions) Unparsed(_ context.Context, repository string, pull int) ([]UnparsedSeat, error) { + return f[fmt.Sprintf("%s#%d", repository, pull)], nil +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index a8e98824..7ec487ca 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -128,6 +128,8 @@ type Examiner struct { // Manifests are the indexer's manifest digests by module: the catalogue a change's manifests are judged in by the // indexer's judge (manifests_judge.go, novox/hq ADR 0578 §2); nil judges no manifest. Manifests TrunkManifests + // SeatDefinitions reads the indexer's seat definitions of a pull request (seatdefs.go, design 26); nil reads none. + SeatDefinitions SeatDefinitions // Renames asks the seat mesh-config's verb renames for a change that brings or drops a `renamed-from` (renames.go, // design 32); nil fails such a change, saying it cannot ask. Renames Renames @@ -273,7 +275,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { // 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02). tests := st.judgeAll(ctx) st.seriesChanges(ctx) // DLV-C4: named in the verdict's line, whatever the verdict - if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx), st.judgeRenames(ctx))); v != nil { + if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx), st.judgeRenames(ctx), st.refuseUnparsed(ctx))); v != nil { return st.finish(ctx, *v), nil } return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil diff --git a/modules/mesh-delivery-checker/internal/check/seatdefs.go b/modules/mesh-delivery-checker/internal/check/seatdefs.go new file mode 100644 index 00000000..4669fd27 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/seatdefs.go @@ -0,0 +1,55 @@ +package check + +import ( + "context" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A pull request whose head's seat definition does not parse fails its check, naming the seat and where it lives (design +// 26; the lead's ruling on A2i's finding 1): the indexer keeps such a definition as the pull request's entry of its state +// seat-definitions with `unparsed` set, and cannot refuse the manifest itself, whose digest the trunk may share. + +// UnparsedSeat is one seat definition of a pull request's head that does not parse: the seat (the contracts module's name +// when its index did not parse), its module's directory and the indexer's words of why. +type UnparsedSeat struct { + Seat, Module, Directory, Why string +} + +// SeatDefinitions reads the indexer's seat definitions of one pull request (its state seat-definitions, through its +// first view, DLV-M2). +type SeatDefinitions interface { + Unparsed(ctx context.Context, repository string, pull int) ([]UnparsedSeat, error) +} + +// refuseUnparsed fails the check of each pull request with a seat definition that does not parse; a state that cannot be +// read fails it too, saying so. +func (st *state) refuseUnparsed(ctx context.Context) *delivery.Verdict { + if st.j.SeatDefinitions == nil || st.c.Try != nil { + return nil + } + var out *delivery.Verdict + failed := func(words string) { + f := delivery.FailedFirst{Machine: SeatDefinitionsMachine, Words: words} + if out == nil { + out = &delivery.Verdict{State: delivery.VerdictFail, Failed: f} + return + } + addFailure(out, f) + } + for _, p := range st.c.Pulls { + bad, err := st.j.SeatDefinitions.Unparsed(ctx, p.Repository, p.Number) + if err != nil { + failed(fmt.Sprintf("%s: the indexer's seat definitions of it cannot be read, so none is known to parse: %v", pullKey(p), err)) + continue + } + for _, u := range bad { + failed(fmt.Sprintf("%s: the seat definition of %s (module %s, %s) does not parse: %s", pullKey(p), u.Seat, u.Module, u.Directory, u.Why)) + } + } + return out +} + +// SeatDefinitionsMachine names what refused a seat definition in a verdict's failure. +const SeatDefinitionsMachine = "seat-definitions" diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 97565506..3fdc27ca 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -123,6 +123,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches", "mesh-delivery-assigner.assigner_read_placement", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", + "mesh-delivery-indexer.indexer_read_seat_definitions", "seat:mesh-config.renames", "seat:node-engine.compose-candidate"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want the forge's reads and compose-candidate", m.Invokes) } @@ -133,7 +134,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // And a group's check by the package registry seats' published of its tagged versions (design 51 *Tagged versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-assigner.views.placement", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", - "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.seat-definitions", "npm-package-registry.published"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 759ea326..76129b4c 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -23,6 +23,7 @@ "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.packages", "mesh-delivery-indexer.views.predefined", + "mesh-delivery-indexer.views.seat-definitions", "npm-package-registry.published" ], "streams": [ @@ -79,6 +80,7 @@ "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", + "mesh-delivery-indexer.indexer_read_seat_definitions", "seat:mesh-config.renames", "seat:node-engine.compose-candidate" ], From aafd90b8cfec9bf6f95be84f4567b64fff2f4a2c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 14:42:26 +0100 Subject: [PATCH 12/12] DLV-M5r1: the watchman's want time is the picture's written time once the join's views name it The lead's ruling on A2h's finding 1: J3 builds each first-view key's written time into mesh-sdk views (Answer.Written, a Picture entry's Created) and J2 the tool runner's state reads' at. The watchman and the builder's toolchain gate already read the entry's Created(), so neither changes; this says so where the fallback stands. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-watchman/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 183e5fd8..0e183670 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -170,8 +170,9 @@ func stepTaken(w *watch.Watchman, language string, e jetstream.KeyValueEntry) { w.Step(language, e.Key(), e.Value(), opOf(e)) } -// createdOr is when an entry was written, or at when its picture does not say: the first view of an owner's state names -// no key's time (novox/hq ADR 0615), so a want that stood before the start waits from the start. +// createdOr is when an entry was written, or at when its picture does not say. mesh-sdk's views.Picture (the join's J3) +// names each first-view key's written time as its Created, and the tool runner's state reads carry it (J2), so a want +// that stood before the start waits from when it was written; at is left only for an owner whose answer names no time. func createdOr(e jetstream.KeyValueEntry, at time.Time) time.Time { if t := e.Created(); !t.IsZero() { return t