novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/47-G-findings4' into feat/47-G-dev4-join

Browse Source
This commit is contained in:
jschoubben
2026-11-11 22:25:53 +01:00
parent 598359ff36 b815b54bff
commit c8fb62c3ad
63 changed files with 1605 additions and 658 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/askended.go
+1 -6
View File
@@ -18,7 +18,6 @@ import (
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
@@ -119,11 +118,7 @@ func (cw callWait) keep(ctx context.Context, id, callee string, call func() (jso
if said || clock().Sub(since) < mcc.CallUnansweredAfter || cw.heard == nil || !cw.heard() || cw.findings.Put == nil {
continue
}
o := mcc.CallUnanswered(id, callee, "", err.Error(), "assigner."+mcc.KindCallUnanswered, since)
value, eerr := mcc.EncodeFinding(o.Finding(plc.Module, "", clock().UTC()))
if eerr == nil {
eerr = cw.findings.Put(key, value)
}
eerr := cw.findings.Put(assign.CallUnanswered(id, callee, "", err.Error(), "assigner."+mcc.KindCallUnanswered, since).Finding(clock()))
if eerr != nil {
logf("the finding %s is not written: %v", key, eerr)
continue
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go
+23 -58
View File
@@ -9,99 +9,64 @@ import (
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
mcc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
)
// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195): one
// meshcatalogclient.Catalog per branch, a first view of the verb `revision` and then each `catalog-revision` of that branch,
// each entry taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog
// half, never its own output back (§195).
// meshcatalogclient.Catalog per branch, adopted through the module's views reader (A-26), which asks the first view of the verb
// `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings; each entry is taken
// into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog half, never its
// own output back (§195).
type halves struct {
w *world
call mcc.Call
wake func()
logf func(string, ...any)
ctx context.Context
w *world
call mcc.Call
reader *views.ProcessReader
wake func()
logf func(string, ...any)
ctx context.Context
mu sync.Mutex
cats map[string]*mcc.Catalog
// started is called with each catalog made, for the part that reads their status.
started func(*mcc.Catalog)
// settled is told once when a branch's first view is answered, for the first pass to wait on (the trunk's alone).
settled chan<- struct{}
}
// follow makes the catalog of branch and asks its first view, once; true when it was made now.
// follow adopts the catalog of branch through the reader, once; true when it was made now.
func (h *halves) follow(branch string) bool {
h.mu.Lock()
if h.cats[branch] != nil {
h.mu.Unlock()
return false
}
c, err := mcc.NewCatalog(h.call, branch)
c, _, err := mcc.WireCatalogProcess(h.reader, h.call, branch, func(module string, entry json.RawMessage, deleted bool, _ time.Time) {
h.changed(branch, module, entry, deleted)
})
if err != nil {
h.mu.Unlock()
h.logf("the catalog half of branch %q is not followed: %v", branch, err)
return false
}
c.Follower.Logf = h.logf
c.Follower.Changed = func(module string, entry json.RawMessage, deleted bool, _ time.Time) {
h.changed(branch, module, entry, deleted)
}
if h.cats == nil {
h.cats = map[string]*mcc.Catalog{}
}
h.cats[branch] = c
h.mu.Unlock()
if h.started != nil {
h.started(c)
}
go func() {
if err := c.Start(h.ctx); err != nil {
if h.ctx.Err() == nil {
h.logf("the first view of the catalog half of branch %q is not answered: %v", branch, err)
if branch == trunkBranch && h.settled != nil {
go func() {
select {
case <-c.Ready():
h.settled <- struct{}{}
case <-h.ctx.Done():
}
return
}
if branch == trunkBranch && h.settled != nil {
h.settled <- struct{}{}
}
}()
}()
}
return true
}
// hear gives a catalog-revision to every catalog: each takes its own branch's and passes over another's.
func (h *halves) hear(m jetstream.Msg) {
h.mu.Lock()
cats := make([]*mcc.Catalog, 0, len(h.cats))
for _, c := range h.cats {
cats = append(cats, c)
}
h.mu.Unlock()
for _, c := range cats {
if err := c.Hear(h.ctx, m.Data()); err != nil {
h.logf("%s on branch %q: %v", m.Subject(), c.Branch, err)
}
}
_ = m.Ack()
}
// followers are the first views the catalogs hold.
func (h *halves) followers() []*mcc.Catalog {
h.mu.Lock()
defer h.mu.Unlock()
out := make([]*mcc.Catalog, 0, len(h.cats))
for _, c := range h.cats {
out = append(out, c)
}
return out
}
// changed takes one entry of branch's half into the world, or a module taken out of it.
func (h *halves) changed(branch, module string, raw json.RawMessage, deleted bool) {
defer h.wake()
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+10 -5
View File
@@ -7,6 +7,7 @@ import (
"encoding/json"
"errors"
"fmt"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/keys"
"io"
"maps"
@@ -52,7 +53,6 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
@@ -179,6 +179,10 @@ func startBed(t *testing.T, leaveOut string) *bed {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
t.Fatal(err)
}
// The assigner's own stream of its findings' events (design 47 G §383), which its finding writer says into.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ASSIGNER_FINDINGS", Subjects: []string{"mesh.stream.mesh-delivery-assigner.event.finding-written.*", "mesh.stream.mesh-delivery-assigner.event.finding-cleared.*", "mesh.stream.mesh-delivery-assigner.event.findings-stated.*"}}); err != nil {
t.Fatal(err)
}
// The assigner's durable consumer on EVENTS, as the grants part makes it from the manifest's consumes: a node-engine seat's
// event names the engine's machine last, so the filter is the event with every machine (mesh-grants buscompose
// machineEvent; the grants part's own test holds the derivation).
@@ -508,7 +512,8 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
"seat-holders": {"n1.node-tools"},
"wants": {"n1.mesh-grants"},
})
graphKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, assignment.Module+"."+meshcatalog.Seat+"."+views.BranchState(meshcatalog.StateCatalogRevisions, trunkBranch), views.UnansweredKind)
// The reader raises the finding of the first view through the module's finding writer (A-26), its id escaped as a key's name.
graphKey := views.FindingKey(views.FindingScope, keys.EscapeName(assignment.Module+"."+meshcatalog.Seat+"."+views.BranchState(meshcatalog.StateCatalogRevisions, trunkBranch)), views.UnansweredKind)
graphRaised := func() bool {
e, err := b.buckets[assign.FindingsBucket].Get(b.ctx, graphKey)
return err == nil && strings.Contains(string(e.Value()), "mesh-catalog.revision")
@@ -855,12 +860,12 @@ func (b *bed) conditionsSaid(t *testing.T) map[string]oc.Condition {
c := out[e.Key()]
c.Key, c.State = e.Key(), occ.StateCleared
if e.Operation() == jetstream.KeyValuePut {
f, err := meshconditionsclient.DecodeFinding(e.Value())
if err != nil {
var f mcseat.Finding
if err := json.Unmarshal(e.Value(), &f); err != nil {
t.Fatal(err)
}
c = oc.Condition{Key: e.Key(), State: occ.StateRaised, Severity: string(f.Severity), Explanation: f.Explanation,
Resolved: f.Resolved, Subject: oc.ConditionSubject{Scope: string(f.Subject.Scope), ID: f.Subject.ID, Machine: f.Subject.Machine}}
Subject: oc.ConditionSubject{Scope: string(f.Subject.Scope), ID: f.Subject.ID, Machine: f.Subject.Machine}}
}
out[e.Key()] = c
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+46 -18
View File
@@ -8,7 +8,6 @@ package main
import (
"context"
"errors"
"fmt"
"io"
"log"
@@ -18,6 +17,7 @@ import (
"os"
"os/signal"
"slices"
"strings"
"syscall"
"time"
@@ -25,7 +25,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/announce"
@@ -110,7 +110,7 @@ var keptStates = []announce.State{
{Name: "differences", Readable: true},
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
{Name: mcc.FindingsState, Readable: true},
{Name: assign.FindingsState, Readable: true},
{Name: buckets.PoliciesState, Readable: true},
{Name: mma.StateSeatHolders, Readable: true},
{Name: views.MarksState, Readable: true}, // what its views family said each state up to (views.go)
@@ -235,11 +235,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
thisNode = account.Node
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
findings := findingsOf(ctx, kvs, logger.Printf)
findings, writer := findingsOf(ctx, account, conn, js, said, kvs, logger.Printf)
if writer != nil {
defer writer.Close()
}
// Each change of the states its readers read is said on its views family (ADR 0615 §5, ADR 0619 §2); a watch of it
// that ends stops the program, as a watch of what a pass reads does.
viewErrs := make(chan error, len(viewedStates))
startViews(ctx, js, conn, account, kvs, findings, viewErrs, logger.Printf)
startViews(ctx, js, conn, account, kvs, wireOf(writer), viewErrs, logger.Printf)
go func() {
select {
case <-ctx.Done():
@@ -248,27 +251,52 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
conn.Close()
}
}()
iv := &catalogViews{js: js, account: account, call: said.Call, findings: findings, logf: logger.Printf,
iv := &catalogViews{js: js, account: account, call: said.Call, writer: wireOf(writer), logf: logger.Printf,
status: process.AwaitBucket(ctx, js, conn, firstViewsBucket, "assigner_first_views reads nothing and assigner_first_view_again is not heard", logger.Printf)}
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, iv, logger)
}
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, which the
// condition keeper carries (novox/hq ADR 0590 §3). Without it the conditions are said straight to the router.
func findingsOf(ctx context.Context, kvs kvSet, logf func(string, ...any)) assign.Findings {
func findingsOf(ctx context.Context, account process.Account, conn *nats.Conn, js jetstream.JetStream, said *process.Conn, kvs kvSet, logf func(string, ...any)) (assign.Findings, *heldWriter) {
var f assign.Findings
if kv := kvs[assign.FindingsBucket]; kv != nil {
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
f.Delete = func(key string) error { return kv.Delete(ctx, key) }
f.Keys = func() ([]string, error) {
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
if kvs[assign.FindingsBucket] == nil {
return f, nil
}
return f
state := said.State(assign.FindingsState)
w, err := views.NewHeldWriter(views.WriterConfig[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]{
Module: account.Module, State: state, Node: account.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf})
if err != nil {
logf("the assigner's finding writer cannot be made, so its findings are not written: %v", err)
return f, nil
}
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
f.Put = func(x mcseat.Finding) error { return w.Write(ctx, x) }
f.Delete = func(key string) error {
scope, rest, _ := strings.Cut(key, ".")
i := strings.LastIndex(rest, ".")
if i < 0 {
return fmt.Errorf("%q is no finding's key", key)
}
return w.Clear(ctx, scope, rest[:i], rest[i+1:], "the assigner no longer observes it")
}
f.Keys = state.Keys
return f, w
}
// heldWriter is the assigner's finding writer: its findings are raised as they happen, so the writer holds them (design 47 G A-29).
type heldWriter = views.HeldWriter[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]
// wireOf is w as the views parts' finding writer; nil when the assigner has none, so no part writes through a nil writer.
func wireOf(w *heldWriter) views.WireWriter {
if w == nil {
return nil
}
return w
}
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go
+13 -2
View File
@@ -1,6 +1,9 @@
package main
import (
"encoding/json"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
@@ -31,8 +34,7 @@ import (
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
func TestOpensWhatItsManifestNames(t *testing.T) {
// Review round 2: the assigner's views emitter writes its finding view-unsaid (Emitter.Findings set) when it keeps findings.
findings := assign.Findings{Put: func(string, []byte) error { return nil }, Delete: func(string) error { return nil }}
if viewsEmitter(nil, process.Account{}, nil, findings, nil, t.Logf).Findings == nil {
if viewsEmitter(nil, process.Account{}, nil, &heldWriter{}, t.Logf).Findings == nil {
t.Error("the assigner's views emitter writes no view-unsaid finding")
}
// 1297/B2a: without a credential the program does not start, and says why.
@@ -197,3 +199,12 @@ func (e kvEntry) Created() time.Time { return time.Time{} }
func (w *watchKV) Keys(context.Context, ...jetstream.WatchOpt) ([]string, error) {
return nil, jetstream.ErrNoKeysFound
}
// putInto is an assign.Findings Put that keeps each finding in into, as the finding writer would, by its key and its words.
func putInto(into map[string][]byte) func(mcseat.Finding) error {
return func(f mcseat.Finding) error {
raw, err := json.Marshal(f)
into[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
return err
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+8 -12
View File
@@ -149,13 +149,9 @@ func (p *passer) unansweredReads(known []string, online map[string]bool) {
if p.calls == nil || p.calls.Keeper != p.reads {
p.calls = &enginereads.ReadsCalls{Keeper: p.reads, Bound: mcc.CallUnansweredAfter, Now: p.clock, Logf: p.logf,
Raise: func(_ context.Context, u enginereads.UnansweredCall) error {
o := mcc.CallUnanswered(enginereads.CallID(u.Node), enginereads.Invoke, u.Node, u.Words, "assigner."+mcc.KindCallUnanswered, u.Since)
o := assign.CallUnanswered(enginereads.CallID(u.Node), enginereads.Invoke, u.Node, u.Words, "assigner."+mcc.KindCallUnanswered, u.Since)
o.Needs = "assigner_engine_reads with ask_again asks the node again now and shows its statement."
value, err := mcc.EncodeFinding(o.Finding(plc.Module, "", u.At))
if err == nil {
err = p.findings.Put(o.Key(), value)
}
return err
return p.findings.Put(o.Finding(u.At))
},
Clear: func(_ context.Context, node string) error {
return p.findings.Delete(mcc.Key(mcc.ScopeCall, enginereads.CallID(node), mcc.KindCallUnanswered))
@@ -1126,9 +1122,9 @@ func (p *passer) provisionFindings(failed map[string]meshprovisionerclient.Provi
if p.failedProvisions[key] {
continue
}
value, err := assign.ProvisionFailedFinding(key, pr, now())
f, err := assign.ProvisionFailedFinding(key, pr, now())
if err == nil {
err = p.findings.Put(assign.ProvisionFailedKey(key), value)
err = p.findings.Put(f)
}
if err != nil {
p.logf("the failed provision %s is not said: %v", key, err)
@@ -1178,9 +1174,9 @@ func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool)
if p.missingSaid[key] || p.findings.Put == nil {
continue
}
value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
f, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
if err == nil {
err = p.findings.Put(assign.ProvisionMissingKey(key), value)
err = p.findings.Put(f)
}
if err != nil {
p.logf("the missing provision %s is not said: %v", key, err)
@@ -1226,9 +1222,9 @@ func (p *passer) unplacedFindings() {
if p.unplaced[node] {
continue
}
value, err := assign.UnplacedFinding(node, p.clock())
f, err := assign.UnplacedFinding(node, p.clock())
if err == nil {
err = p.findings.Put(assign.UnplacedKey(node), value)
err = p.findings.Put(f)
}
if err != nil {
p.logf("the unplaced node %s is not said: %v", node, err)
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go
+2 -2
View File
@@ -209,7 +209,7 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
for _, heard := range []bool{false, true} {
clock, findings := now, map[string][]byte{}
cw := callWait{heard: func() bool { return heard }, now: func() time.Time { return clock },
findings: assign.Findings{Put: func(k string, v []byte) error { findings[k] = v; return nil },
findings: assign.Findings{Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil }}}
tries, during := 0, 0
answer, err := cw.keep(context.Background(), asksCallID, channelAsksVerb, func() (json.RawMessage, error) {
@@ -332,7 +332,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
return nil, context.DeadlineExceeded
}
q := &passer{w: w, logf: t.Logf, now: func() time.Time { return clock }, findings: assign.Findings{
Put: func(k string, v []byte) error { held[k] = v; return nil },
Put: putInto(held),
Delete: func(k string) error { delete(held, k); return nil },
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(held)), nil }},
reads: &enginereads.Keeper{Call: silent, Sync: true, Now: func() time.Time { return clock }}}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/seat_tools_test.go
+6 -4
View File
@@ -24,7 +24,6 @@ import (
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -147,6 +146,9 @@ func TestTheSeatsVerbs(t *testing.T) {
}
}
for _, stream := range raw0.Streams {
if strings.HasSuffix(stream.Name, "-findings") { // the part's own finding events, not a state of the seat
continue
}
if !reads["stream "+stream.Name] {
t.Errorf("no verb reads the seat's stream %s", stream.Name)
}
@@ -283,7 +285,7 @@ func TestTheSeatsVerbs(t *testing.T) {
}
findings := map[string][]byte{}
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil }}}
p.unplacedFindings()
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
@@ -471,7 +473,7 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: kvSet{buckets.Provisions: nil}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Put: putInto(findings),
Delete: func(k string) error { delete(findings, k); return nil }}}
failed, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "db", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
Provider: meshprovisionerclient.End{Module: "postgres", Node: "novox"}, State: meshprovisionerclient.Failed, Why: "the role exists already"})
@@ -612,7 +614,7 @@ func wantsVerbs(t *testing.T) {
state, watches[state], watches[state+" by key"], indexer.asks[state])
}
}
for _, state := range []string{haveState, mcc.FindingsState} {
for _, state := range []string{haveState, assign.FindingsState} {
if watches[state] != 1 {
t.Errorf("%s was watched %d times, want once", state, watches[state])
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go
+37 -96
View File
@@ -6,23 +6,19 @@ import (
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
mcat "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"mesh-delivery-assigner/internal/assign"
plc "mesh-delivery-assigner/internal/assignment"
@@ -47,7 +43,7 @@ var readTools = []viewedTool{
{"assigner_read_assignments", mma.StateAssignments, "where each module is placed, by module."},
{"assigner_read_seat_holders", mma.StateSeatHolders, "who holds which seat on which node."},
{"assigner_read_wants", "wants", "what each node should run: want.<node>.<module>."},
{"assigner_read_findings", mcc.FindingsState, "the assigner's findings, which the condition keeper raises and clears."},
{"assigner_read_findings", assign.FindingsState, "the assigner's findings, which the condition keeper raises and clears."},
{"assigner_read_asks", "asks", "the asks the assigner put to a person, and their answers."},
{"assigner_read_differences", "differences", "every difference between a want and what a node runs."},
{"assigner_read_copies", copiesState, "the copies the assigner made of a retired state: when the copy into assignments ended and how many keys it copied."},
@@ -58,7 +54,7 @@ var viewedStates = map[string]string{
mma.StateAssignments: plc.AssignmentsBucket,
mma.StateSeatHolders: plc.SeatHoldersBucket,
"wants": buckets.Wants,
mcc.FindingsState: assign.FindingsBucket,
assign.FindingsState: assign.FindingsBucket,
"asks": buckets.Asks,
"differences": buckets.Differences,
copiesState: copiesBucket,
@@ -83,18 +79,10 @@ func keptSource(name string) views.Source { return views.KeptSource{State: stdio
// startViews says each change of the assigner's viewed states on its views family, once its marks state is bound,
// each state as soon as its bucket is (kvs, or bound once it appears). A watch that ends is sent on errs.
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, findings assign.Findings, errs chan<- error, logf func(string, ...any)) {
tooLarge := func(state, key string, size int) {
if findings.Put == nil {
return
}
if err := viewFindings(findings).Put(views.TooLargeFinding(assignment.Module, state, key, size, time.Now())); err != nil {
logf("views: the finding of %s.%s too large is not written: %v", state, key, err)
}
}
func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, account process.Account, kvs kvSet, writer views.WireWriter, errs chan<- error, logf func(string, ...any)) {
process.AwaitBucket(ctx, js, nc, marksBucket, "no change is said on the views family, so every reader waits on its first view", logf).Ready(func(mk jetstream.KeyValue) {
em := viewsEmitter(js, account, mk, findings, tooLarge, logf)
for _, state := range []string{"asks", "differences", mcc.FindingsState, mma.StateAssignments, mma.StateSeatHolders, "wants", copiesState} {
em := viewsEmitter(js, account, mk, writer, logf)
for _, state := range []string{"asks", "differences", assign.FindingsState, mma.StateAssignments, mma.StateSeatHolders, "wants", copiesState} {
state := state
start := func(kv jetstream.KeyValue) {
if err := em.Start(ctx, state, kv, errs); err != nil {
@@ -111,35 +99,27 @@ func startViews(ctx context.Context, js jetstream.JetStream, nc process.Subscrib
}
// viewsEmitter is the assigner's emitter of its views family: a change not said past views.Bound is its finding
// view-unsaid, in its findings (review round 2); with no findings state it is said in the log only.
func viewsEmitter(js jetstream.JetStream, account process.Account, mk jetstream.KeyValue, findings assign.Findings, tooLarge func(string, string, int), logf func(string, ...any)) views.Emitter {
em := views.Emitter{Owner: assignment.Module, Subject: account.EventSubject, Publish: views.JetStreamPublish(js), Marks: mk, TooLarge: tooLarge, Logf: logf}
if findings.Put != nil && findings.Delete != nil {
em.Findings = viewFindings(findings)
}
return em
}
// viewFindings is the assigner's findings as the views' FindingWriter, in the seat mesh-conditions' layout.
func viewFindings(f assign.Findings) views.FindingWriter {
return sdkfindings.WriterOver(f.Put, f.Delete)
// view-unsaid, in its findings, written through its finding writer (review round 2); with none it is said in the log only.
func viewsEmitter(js jetstream.JetStream, account process.Account, mk jetstream.KeyValue, writer views.WireWriter, logf func(string, ...any)) views.Emitter {
return views.Emitter{Owner: assignment.Module, Subject: account.EventSubject, Publish: views.JetStreamPublish(js), Marks: mk, Findings: writer, Logf: logf}
}
// firstViewsBucket is the assigner's own state of its first views (views.StatusState).
var firstViewsBucket = assignment.Module + "_" + views.StatusState
// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195): asked
// at the start through the seat mesh-catalog's verb `revision` (the trunk's, then each branch a registration names), then
// followed on its event `catalog-revision`, asking once more on a gap; past the bound its finding names the seat and the verb,
// cleared by the answer. Its status is read by assigner_first_views and asked again by assigner_first_view_again.
// catalogViews is the assigner's first view of the catalog half of each watched branch (design 47 G §14, §184, §195, A-26): the
// module's views reader adopts each branch's follower, so it asks the seat mesh-catalog's verb `revision` (the trunk's, then
// each branch a registration names), follows its event `catalog-revision`, asks once more on a gap, and past the bound raises
// the finding naming the seat and the verb, cleared by the answer, in the assigner's findings. Its status is read by
// assigner_first_views and asked again by assigner_first_view_again.
type catalogViews struct {
js jetstream.JetStream
account process.Account
call func(string, any) (json.RawMessage, error)
findings assign.Findings
status *process.AwaitedBucket
logf func(string, ...any)
hub *process.EventHub
js jetstream.JetStream
account process.Account
call func(string, any) (json.RawMessage, error)
writer views.WireWriter
status *process.AwaitedBucket
logf func(string, ...any)
hub *process.EventHub
h *halves
}
@@ -149,70 +129,31 @@ type catalogViews struct {
// seconds). A test shortens them.
var firstViewBound, firstViewWait time.Duration
// start follows the trunk's catalog half and hears the seat's catalog-revision; it answers how many settles to wait for.
// start adopts the trunk's catalog half through the reader and starts the reader; it answers how many settles to wait for.
func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settled chan<- struct{}) int {
finding := func(s views.Status, raised bool) {
if iv.findings.Put == nil {
return
}
f := views.UnansweredFinding(assignment.Module, s, time.Now())
var err error
if raised {
err = viewFindings(iv.findings).Put(f)
} else {
err = viewFindings(iv.findings).Delete(f)
}
if err != nil {
iv.logf("views: the finding of the first view %s is not written: %v", s.Tool, err)
}
reader := &views.ProcessReader{Module: assignment.Module, Call: iv.call, Logf: iv.logf, Hub: iv.hub, Bound: firstViewBound, Wait: firstViewWait}
if iv.writer != nil {
reader.Findings = func() views.WireWriter { return iv.writer }
}
var statusKV jetstream.KeyValue
var mu sync.Mutex
iv.h = &halves{w: w, call: iv.call, wake: wake, logf: iv.logf, ctx: ctx, settled: settled}
iv.h.started = func(c *mcat.Catalog) {
f := c.Follower
f.Bound, f.Wait = firstViewBound, firstViewWait
f.Raise, f.Clear = func(s views.Status) { finding(s, true) }, func(s views.Status) { finding(s, false) }
mu.Lock()
defer mu.Unlock()
if statusKV != nil {
f.Said = views.KeepStatus(ctx, statusKV, iv.logf)
}
}
if iv.status != nil {
iv.status.Ready(func(kv jetstream.KeyValue) {
mu.Lock()
statusKV = kv
mu.Unlock()
var all []*views.Follower
for _, c := range iv.h.followers() {
c.Follower.Said = views.KeepStatus(ctx, kv, iv.logf)
all = append(all, c.Follower)
}
if err := views.HearAgain(ctx, kv, all, iv.logf); err != nil {
iv.logf("views: assigner_first_view_again is not heard (%v) until the assigner starts again", err)
}
})
}
// The hub says when the bus made its consumer: a picture asked before then is asked again (views.Follower.ConsumerMade).
iv.hub.OnBound(func(created time.Time) {
for _, c := range iv.h.followers() {
c.Follower.ConsumerMade(ctx, created)
}
})
// Not heard, the catalog-revision is the finding family-not-heard until it binds (ADR 0615 §5): the handler is registered
// on the module's hub, which the assigner starts once every handler is.
_ = notHeardWriter(iv.findings, iv.logf).Bind(process.EventsStream, "the catalog half is the first views' alone until the assigner starts again", func() error {
return iv.hub.Handle(meshcatalog.SubjectCatalogRevisionAll, iv.h.hear)
})
iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled}
iv.h.follow(trunkBranch)
reader.Start(ctx, iv.js, iv.account, iv.status)
return 1
}
// notHeardWriter writes the assigner's finding of a family it does not hear into its findings (heard.Writer); with no
// findings state it is said in the log alone.
func notHeardWriter(findings assign.Findings, logf func(string, ...any)) heard.Writer {
w := heard.Writer{Observer: assignment.Module, Logf: logf, Put: findings.Put}
w := heard.Writer{Observer: assignment.Module, Logf: logf}
if findings.Put != nil {
w.Put = func(_ string, value []byte) error {
var f mcseat.Finding
if err := json.Unmarshal(value, &f); err != nil {
return err
}
return findings.Put(f)
}
}
if findings.Delete != nil {
w.Delete = func(key string) error {
if keys, err := findings.Keys(); err == nil && !slices.Contains(keys, key) {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go
+1 -2
View File
@@ -16,7 +16,6 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -170,7 +169,7 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
out["published"] = published(values)
}
// The assigner's findings, its said conditions as the condition keeper reads them (§15).
if found, err := s.values(mcc.FindingsState); err != nil {
if found, err := s.values(assign.FindingsState); err != nil {
out["findings-unread"] = err.Error()
} else {
keys := []string{}
modules/mesh-delivery-assigner/internal/assign/asks_test.go
+6 -7
View File
@@ -16,10 +16,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/views"
)
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
@@ -292,11 +292,10 @@ func saidConditionsAreFindings(t *testing.T) {
{SeatWaitsCondition([]string{"assignments"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, meshcatalogclient.SourcePrefix2) + ".backup"},
} {
puts, deletes, said := map[string]mcseat.FindingsByKeyValue{}, []string{}, 0
f := Findings{Put: func(k string, v []byte) error {
got, err := mcc.DecodeFinding(v)
puts[k] = got
return err
puts, deletes, said := map[string]mcseat.Finding{}, []string{}, 0
f := Findings{Put: func(f mcseat.Finding) error {
puts[views.FindingKey(f.Scope, f.ID, f.Kind)] = f
return nil
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, Now: func() time.Time { return at }}
say := f.Say(func(oc.Condition) error { said++; return nil })
if err := say(c.raised); err != nil {
@@ -314,7 +313,7 @@ func saidConditionsAreFindings(t *testing.T) {
t.Fatalf("%s: with no findings bucket said %d (%v), want straight once", c.key, said, err)
}
}
if _, _, _, err := FindingOf(oc.Condition{Key: "another.x", State: occ.StateRaised}, at); err == nil {
if _, _, err := FindingOf(oc.Condition{Key: "another.x", State: occ.StateRaised}, at); err == nil {
t.Error("a condition not the assigner's is a finding")
}
}
modules/mesh-delivery-assigner/internal/assign/findings.go
+68 -18
View File
@@ -11,6 +11,7 @@ import (
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/views"
)
// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says
@@ -21,15 +22,66 @@ import (
// assigner no longer says it straight (never both).
// FindingsBucket is the assigner's findings.
var FindingsBucket = mcc.FindingsBucket(assignment.Module)
// FindingsState is the assigner's state of its findings, and FindingsBucket its bucket.
const FindingsState = "findings"
// FindingOf is a said condition as a finding: its key `<scope>.<id>.<kind>`, from the condition's own key
// `mesh-delivery-assigner.<kind>.<id…>` (the subject's id when the key names none), its words unchanged. raised is false for
// a clear, whose key is to be deleted; value is nil then.
func FindingOf(c oc.Condition, at time.Time) (key string, value []byte, raised bool, err error) {
var FindingsBucket = assignment.Module + "_" + FindingsState
// KeyID is an id as the parts of a key: each dot-separated part made safe for a bus key, as the key of a condition is
// (mcc.Key), so a finding's key keeps the shape its condition had.
func KeyID(id string) string {
key := mcc.Key("x", id, "x")
return strings.TrimSuffix(strings.TrimPrefix(key, "x."), ".x")
}
// Observed is what the assigner observes wrong, before it is the seat mesh-conditions' generated finding.
type Observed struct {
Scope, ID, Kind, Machine string
Also []string
Severity string
Summary, Said string
Headline, Explanation string
Needs, Source string
Actions []mcseat.Action
}
// Finding is o as the seat's generated finding at at (design 47 G §686, A-22): its id made safe as a key's part, the resolved
// words not a field of it, its observer the assigner; the writer completes its `since`.
func (o Observed) Finding(at time.Time) mcseat.Finding {
at = at.UTC()
id := KeyID(o.ID)
said := o.Said
if said == "" {
said = o.Summary
}
sev := o.Severity
if sev == "" {
sev = mcseat.FindingSeverityWarning
}
return mcseat.Finding{Scope: o.Scope, ID: id, Kind: o.Kind, Severity: sev, Summary: o.Summary, Headline: o.Headline,
Explanation: o.Explanation, Needs: o.Needs, Actions: o.Actions, Source: o.Source, Observer: assignment.Module, At: at,
Evidence: []mcseat.Evidence{{At: at, Said: said}},
Subject: mcseat.Subject{Scope: o.Scope, ID: id, Machine: o.Machine, Also: o.Also}}
}
// CallUnanswered is the observation of a kept call to callee (a seat verb's or a tool's key) about id (what the assigner waits
// on: a node, a key), unanswered since since, its last fault said. Its key is `call.<id>.call-unanswered`.
func CallUnanswered(id, callee, machine, fault, source string, since time.Time) Observed {
return Observed{Scope: mcc.ScopeCall, ID: id, Kind: mcc.KindCallUnanswered, Machine: machine, Severity: mcseat.FindingSeverityWarning,
Summary: fmt.Sprintf("the call %s about %s is unanswered since %s", callee, id, since.UTC().Format(time.RFC3339)),
Said: fault, Headline: "A call whose answer is kept is not answered",
Explanation: fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has "+
"not been answered since %s. Its last fault: %s.", callee, since.UTC().Format(time.RFC3339), fault),
Source: source}
}
// FindingOf is a said condition as a finding: from the condition's own key `mesh-delivery-assigner.<kind>.<id…>` (the
// subject's id when the key names none), its words unchanged but the resolved ones. raised is false for a clear, whose
// finding names only its scope, id and kind.
func FindingOf(c oc.Condition, at time.Time) (f mcseat.Finding, raised bool, err error) {
rest, ok := strings.CutPrefix(c.Key, assignment.Module+".")
if !ok || rest == "" {
return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
return f, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
}
kind, id, _ := strings.Cut(rest, ".")
if id == "" {
@@ -39,12 +91,11 @@ func FindingOf(c oc.Condition, at time.Time) (key string, value []byte, raised b
if scope == "" {
scope = mcc.ScopeModule
}
key = mcc.Key(scope, id, kind)
if c.State == occ.StateCleared {
return key, nil, false, nil
return mcseat.Finding{Scope: scope, ID: KeyID(id), Kind: kind}, false, nil
}
if c.State != occ.StateRaised {
return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
return f, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
}
summary := c.Summary
if summary == "" {
@@ -54,16 +105,14 @@ func FindingOf(c oc.Condition, at time.Time) (key string, value []byte, raised b
for _, a := range c.Actions {
actions = append(actions, mcseat.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
}
f := mcc.Observation{Scope: scope, ID: id, Kind: kind, Machine: c.Subject.Machine, Severity: string(c.Severity), Summary: summary,
Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved, Needs: c.Needs, Actions: actions,
Source: "assigner." + kind}.Finding(assignment.Module, "", at)
value, err = mcc.EncodeFinding(f)
return key, value, true, err
return Observed{Scope: scope, ID: id, Kind: kind, Machine: c.Subject.Machine, Severity: string(c.Severity), Summary: summary,
Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs, Actions: actions, Source: "assigner." + kind}.Finding(at), true, nil
}
// Findings is where the assigner's findings go: the bucket's put and delete.
// Findings is where the assigner's findings go: its finding writer's write and clear (design 47 G §689, A-29), by the
// finding's key `<scope>.<id>.<kind>`.
type Findings struct {
Put func(key string, value []byte) error
Put func(f mcseat.Finding) error
Delete func(key string) error
Keys func() ([]string, error)
Now func() time.Time
@@ -80,12 +129,13 @@ func (f Findings) Say(say func(oc.Condition) error) func(oc.Condition) error {
if f.Now != nil {
now = f.Now
}
key, value, raised, err := FindingOf(c, now())
finding, raised, err := FindingOf(c, now())
key := views.FindingKey(finding.Scope, finding.ID, finding.Kind)
switch {
case err != nil:
errs = append(errs, err)
case raised:
if err := f.Put(key, value); err != nil {
if err := f.Put(finding); err != nil {
errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err))
}
default:
modules/mesh-delivery-assigner/internal/assign/policy.go
+13 -14
View File
@@ -10,6 +10,7 @@ import (
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -65,19 +66,17 @@ func ProvisionFailedKey(key string) string {
}
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
func ProvisionFailedFinding(key string, p meshprovisionerclient.Provision, at time.Time) ([]byte, error) {
func ProvisionFailedFinding(key string, p meshprovisionerclient.Provision, at time.Time) (mcseat.Finding, error) {
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
return encode(mcc.Observation{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionFailed, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node),
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionFailed, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node),
Summary: summary, Said: p.Why, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provider provides it.",
Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Source: "assigner." + KindProvisionFailed}, at)
Source: "assigner." + KindProvisionFailed}, at)
}
// encode is the observation as the assigner's finding at at, in the seat's layout.
func encode(o mcc.Observation, at time.Time) ([]byte, error) {
return mcc.EncodeFinding(o.Finding(assignment.Module, "", at))
}
// encode is the observation as the assigner's finding at at.
func encode(o Observed, at time.Time) (mcseat.Finding, error) { return o.Finding(at), nil }
func also(provider, consumer string) []string {
if provider == "" || provider == consumer {
@@ -95,18 +94,18 @@ func ProvisionMissingKey(key string) string {
}
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) {
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) (mcseat.Finding, error) {
parts := strings.SplitN(key, ".", 3)
if len(parts) != 3 {
return nil, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
return mcseat.Finding{}, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
}
node, module, provision := parts[0], parts[1], parts[2]
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
"no provider of it may be placed", module, node, provision, bound, module)
return encode(mcc.Observation{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionMissing, Machine: node,
return encode(Observed{Scope: mcc.ScopeProvider, ID: key, Kind: KindProvisionMissing, Machine: node,
Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
Explanation: occ.NothingToDo + " " + summary + ". It clears once the provision has its key.",
Resolved: provision + " for " + module + " on " + node + " has its key", Source: "assigner." + KindProvisionMissing}, at)
Source: "assigner." + KindProvisionMissing}, at)
}
// KindUnplaced is, from the lease write, a node that runs but no assignments names (ADR 0571 §6).
@@ -118,13 +117,13 @@ func UnplacedKey(node string) string {
}
// UnplacedFinding is the finding of a node no assignments names since the lease write.
func UnplacedFinding(node string, at time.Time) ([]byte, error) {
func UnplacedFinding(node string, at time.Time) (mcseat.Finding, error) {
summary := fmt.Sprintf("%s runs, but no assignments names it: since the controller handed its lease away nothing composes it, "+
"so the assigner writes it no want", node)
return encode(mcc.Observation{Scope: mcc.ScopeMachine, ID: node, Kind: KindUnplaced, Machine: node, Summary: summary,
return encode(Observed{Scope: mcc.ScopeMachine, ID: node, Kind: KindUnplaced, Machine: node, Summary: summary,
Headline: clipHeadline(node + " has no assignments"), Needs: "place it with mesh-module-assigner.propose.",
Explanation: occ.NeedsYou + "place it with mesh-module-assigner.propose. " + summary + ".",
Resolved: node + " is placed", Source: "assigner." + KindUnplaced}, at)
Source: "assigner." + KindUnplaced}, at)
}
func clipHeadline(s string) string {
modules/mesh-delivery-assigner/manifest_test.go
+4 -3
View File
@@ -117,8 +117,9 @@ func TestManifest(t *testing.T) {
// ADR 0635 §5: its views family is its ordinary events, `views.<state>` among its emits, on no stream of its own.
// The seat's four events are said on the seat's own subject (the seat's family), so the module keeps no stream for them
// and lists none among its emits.
if len(m.Streams) != 0 {
t.Errorf("streams = %+v, want none: the seat's events ride the shared events stream", m.Streams)
// Its one stream is its own findings' events (design 47 G A-30, §383), which its finding writer says.
if len(m.Streams) != 1 || m.Streams[0].Name != "mesh-delivery-assigner-findings" || !slices.Equal(m.Streams[0].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
t.Errorf("streams = %+v, want its findings' stream alone: the seat's events ride the shared events stream", m.Streams)
}
for _, e := range meshmoduleassigner.Events() {
if slices.Contains(m.Emits, e) {
@@ -127,7 +128,7 @@ func TestManifest(t *testing.T) {
}
// ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events;
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears.
if !slices.Equal(m.Consumes, []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) {
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) {
t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes)
}
if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
modules/mesh-delivery-assigner/module.json
+9
View File
@@ -42,6 +42,7 @@
"operator-channel"
],
"consumes": [
"mesh-conditions.findings-asked",
"mesh-catalog.catalog-revision",
"mesh-catalog.registration-changed",
"mesh-catalog.seat-definition-changed",
@@ -66,6 +67,14 @@
"seat:node-engine.manifest-reads",
"seat:operator-channel.asks"
],
"streams": [
{
"name": "mesh-delivery-assigner-findings",
"events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"],
"max-age-days": 7,
"max-mib": 16
}
],
"emits": [
"assignments-copied",
"put",
modules/mesh-delivery-builder/buildrun/names.go
+4
View File
@@ -11,3 +11,7 @@ const (
// StepsBucket is the bucket of a build module's state steps.
func StepsBucket(module string) string { return module + "_steps" }
// ToolchainsBucket is the bucket of a build module's state toolchains, which the checker and the indexer read (the builder
// hears the toolchain-reported events and keeps ToolchainsHeardBucket instead; design 47 G §492, §604).
func ToolchainsBucket(module string) string { return module + "_toolchains" }
modules/mesh-delivery-builder/buildrun/standing.go
+28 -7
View File
@@ -103,11 +103,33 @@ func (g *PlacedNodes) Settled() {
g.known = true
}
// assignmentsFollowed is what FollowAssignments feeds: the builder's PlacedNodes, or the ToolchainGate of the checker and the
// indexer. settle says the history is read and answers whether what the part passes on changed; placedTold tells its OnPlaced.
type assignmentsFollowed interface {
Assignments(key string, value []byte, deleted bool, at time.Time) bool
settle() bool
placedTold()
}
func (g *PlacedNodes) settle() bool { g.Settled(); return true }
func (g *PlacedNodes) placedTold() {
if g.OnPlaced != nil {
g.OnPlaced()
}
}
func (g *ToolchainGate) settle() bool { return g.Settled() }
func (g *ToolchainGate) placedTold() {
if g.OnPlaced != nil {
g.OnPlaced()
}
}
// FollowAssignments reads the seat mesh-module-assigner's assignments with its history into gate, says it settled, and follows it; changed
// is told whenever what the gate passes on changed. Where the state cannot be read or watched (not there, or this part
// may not read it) it says so and the gate passes every report on: not fatal, since an indexer, builder or checker that
// stops for it stops every delivery (issue 1945). An error of the watch after that ends on the channel.
func FollowAssignments(ctx context.Context, js process.BucketOpener, gate *PlacedNodes, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) {
func FollowAssignments(ctx context.Context, js process.BucketOpener, gate assignmentsFollowed, changed func(context.Context), logf func(string, ...any)) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, AssignmentsBucket)
if err != nil {
@@ -138,11 +160,10 @@ func FollowAssignments(ctx context.Context, js process.BucketOpener, gate *Place
put(e)
}
}
gate.Settled()
changed(ctx)
if gate.OnPlaced != nil {
gate.OnPlaced()
if gate.settle() {
changed(ctx)
}
gate.placedTold()
go func() {
defer func() { _ = w.Stop() }()
for {
@@ -158,8 +179,8 @@ func FollowAssignments(ctx context.Context, js process.BucketOpener, gate *Place
if e != nil && put(e) {
changed(ctx)
}
if e != nil && gate.OnPlaced != nil {
gate.OnPlaced()
if e != nil {
gate.placedTold()
}
}
}
modules/mesh-delivery-builder/buildrun/toolchaingate.go
+236
View File
@@ -0,0 +1,236 @@
package buildrun
import (
"fmt"
"maps"
"slices"
"sort"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient"
meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module leaves a node: no part writes another module's state, and the module is gone. A report of a node the
// build module no longer runs on is therefore no builder's (issue 1940: mesh-build-node taken off ace on 2026-10-23 left
// ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// ToolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// ToolchainSet): an entry is passed on only while the assignments (the seat mesh-module-assigner's state `assignments`, held by
// the assigner, AssignmentsBucket, read by meshmoduleassignerclient.Placed; ADR 0567, ADR 0549 §3) places its build module on
// its node, and only once it was written since that assignments; it is passed on as deleted otherwise, said once, and
// passed on when its node writes it again. A report written before the assignments is one an earlier run of the module
// left (issue 1940, 2026-10-25 20:56: mesh-build-node put on ace again brought ace.typescript v26.12.0 of 2026-10-23
// back into the agreement before the module ran there). Until the assignments is settled, and while no node's assignments
// is held (before the assigner's adopt writes the first one, or where the state cannot be read), every entry is passed
// on. Builders that run and report different versions still disagree: that refusal is the guard of novox/hq ADR 0465,
// and it stays.
//
// This replaces the copies of the gate the indexer, the builder and the checker each kept over the controller's
// assignments (issue 1940's subtask, issue 774: those go with the controller's lease, ADR 0549 §6).
type ToolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
placed placementsSince
known bool // the assignments is settled
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
// OnPlaced is told, outside the gate's lock, once the assignments's history is read and after each revision of it
// FollowAssignments hears from then on (the counted nodes, Nodes, may have changed). Nil tells nobody.
OnPlaced func()
}
// placementsSince is where the gate reads which modules stand on which node, and since when: the assigner's state
// (meshmoduleassignerclient.Placed, fed by Assignments) or the assignment half (meshwantsclient.Placed, fed by Revision).
type placementsSince interface {
Seeded() bool
Since(node, module string) (time.Time, bool)
ByNode() map[string][]string
}
// NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what
// it leaves out.
func NewToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *ToolchainGate {
g := &ToolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, placed: meshmoduleassignerclient.NewPlaced(),
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// UseModules replaces the build modules whose reports the gate passes on again when the assignments moves (a part that
// finds them from the seat list and the manifests, as the indexer does, gives them at each change), and passes every
// report it holds on again. It answers whether what it passed on changed.
func (g *ToolchainGate) UseModules(modules []string) bool {
g.mu.Lock()
defer g.mu.Unlock()
g.modules = map[string]bool{}
for _, m := range modules {
g.modules[m] = true
}
return g.all()
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *ToolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignments takes one revision of the seat mesh-module-assigner's assignments in (at is when the bus wrote it) and answers whether what it
// passed on changed. A value that does not read is said, and places nothing on its node.
func (g *ToolchainGate) Assignments(key string, value []byte, deleted bool, at time.Time) bool {
g.mu.Lock()
defer g.mu.Unlock()
placed, ok := g.placed.(*meshmoduleassignerclient.Placed)
if !ok {
g.logf("%s: %s is not read, the gate takes the assignment half (design 47)", AssignmentsBucket, key)
return false
}
was := placed.Seeded()
node, changed, err := placed.Put(key, value, deleted, at)
if err != nil {
g.logf("%s: %v (issue 1940)", AssignmentsBucket, err)
}
if !g.known {
return false
}
if was != g.placed.Seeded() {
// The first assignments came, or the last went: every report's standing changed.
return g.all()
}
return g.again(node, changed)
}
// Revision takes one assignment graph revision the part's view holds in (design 47 *Each part acts on its own entries*:
// the placements come from the seat mesh-wants' assignment half, never the assigner's state; assignmentview's Held hands
// each) and answers whether what it passed on changed. The first revision taken settles the gate, as the assignments'
// history read does; a later one passes on again the reports of each node whose build modules' standing changed. A gate
// fed by Revision reads no Assignments.
func (g *ToolchainGate) Revision(r meshwants.AssignmentRevision) bool {
g.mu.Lock()
defer g.mu.Unlock()
placed, ok := g.placed.(*meshwantsclient.Placed)
if !ok {
placed = meshwantsclient.NewPlaced()
g.placed = placed
}
was := placed.Seeded()
changed := placed.Take(r)
if changed == nil {
return false
}
if !g.known || was != placed.Seeded() {
g.known = true
return g.all()
}
moved := false
for _, node := range slices.Sorted(maps.Keys(changed)) {
moved = g.again(node, changed[node]) || moved
}
return moved
}
// Nodes are the nodes the assignments places any module on, sorted: the nodes the mesh counts (a node leaves the count
// when the assignments no longer names it; ADR 0624 §1). None until the assignments is read.
func (g *ToolchainGate) Nodes() []string {
var out []string
for node, modules := range g.placed.ByNode() {
if len(modules) > 0 {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// Settled says the assignments's history is read: from now on only the reports of placed nodes pass, once a node's
// assignments is held. It answers whether what it passed on changed.
func (g *ToolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
return g.all()
}
// all passes on every report again. The caller holds mu.
func (g *ToolchainGate) all() bool {
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// again passes on every report of node by a build module among modules. The caller holds mu.
func (g *ToolchainGate) again(node string, modules []string) bool {
changed := false
for _, module := range modules {
if !g.modules[module] {
continue
}
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := buildstep.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is placed and wrote it since (or the assignments is not
// known), deleted otherwise. The caller holds mu.
func (g *ToolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := buildstep.ParseToolchainKey(key)
why := ""
if err == nil && g.known && g.placed.Seeded() {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: the catalogue does not place the module on its
// node, or the report was written before it did. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *ToolchainGate) leftOut(module, node string, value []byte) string {
since, placed := g.placed.Since(node, module)
if !placed {
return fmt.Sprintf("the catalogue does not place %s on %s, so it is no builder", module, node)
}
tc, err := buildstep.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s was placed on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
modules/mesh-delivery-builder/buildrun/toolchaingate_test.go
+260
View File
@@ -0,0 +1,260 @@
package buildrun
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
)
func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) {
t.Helper()
key, _ := buildstep.ToolchainKey(node, "typescript")
value, err := buildstep.EncodeToolchain(buildstep.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
// placing is a value of the state `assignments` placing modules on node.
func agreedOf(set *ToolchainSet) (string, string) {
tc, none := buildstep.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// Issue 1940, the gate over the catalogue's assignments (issue 774, ADR 0549 §3): a build node's report counts only while
// the assignments places its build module on its node, and only once written since; before the assignments is settled, and
// while no node's assignments is held, every report counts. The live cases: mesh-build-node taken off ace left
// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came
// back before the module ran there.
func TestToolchainGate(t *testing.T) {
old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again
three := []string{"g14", "novox", "shanks"}
start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) {
set := &ToolchainSet{}
said := &[]string{}
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
for _, node := range three {
if seeded {
key, value := placing(t, node, NodeBuildModule)
gate.Assignments(key, value, false, old.Add(-time.Hour))
}
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
gate.Toolchain(NodeBuildModule, key, value, false)
}
return gate, set, said
}
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
gate, set, said := start(t, true)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(NodeBuildModule, key, value, false)
if a := buildstep.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
t.Fatalf("before the assignments is settled every report counts; got %+v", a)
}
k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter
gate.Assignments(k, v, false, old)
if !gate.Settled() {
t.Fatal("settling left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
t.Fatalf("ace's report is said left out once; said %q", *said)
}
if gate.Toolchain(NodeBuildModule, key, value, false) {
t.Fatal("a report of a node not placed changed what was passed on")
}
// Placed on ace with a report written since: it counts, and the builders really disagree.
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
gate.Assignments(k, v, false, old.Add(-time.Minute))
if a := buildstep.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
}
// Taken off ace again (its key deleted): its report goes again.
if !gate.Assignments(k, nil, true, placed) {
t.Fatal("taking ace's assignments away passed nothing on")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
// A report deleted from the bucket goes whether its node is placed or not.
key, _ = gateReport(t, "g14", "v26.12.1", placed)
if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 {
t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript")))
}
})
t.Run("build modules given later pass their reports on when the assignments moves", func(t *testing.T) {
set := &ToolchainSet{}
gate := NewToolchainGate(nil, set.Put, func(string, ...any) {})
gate.Settled()
k, v := placing(t, "novox", GoBuildModule)
gate.Assignments(k, v, false, old)
key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
gate.Toolchain(NodeBuildModule, key, value, false) // ace is not placed: left out
k, v = placing(t, "ace", NodeBuildModule)
if gate.Assignments(k, v, false, placed) || len(set.Entries("typescript")) != 0 {
t.Fatalf("a module the gate was not given passed on at the assignments: %v", set.Entries("typescript"))
}
if !gate.UseModules([]string{NodeBuildModule}) || len(set.Entries("typescript")) != 1 {
t.Fatalf("given mesh-build-node, ace's report placed since is passed on; entries %v", set.Entries("typescript"))
}
})
for _, c := range []struct {
name string
at time.Time // when ace wrote its report
counts bool
}{
{"a report left before the assignments", old, false},
{"a report written since the assignments", placed.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
gate, set, said := start(t, true)
k, v := placing(t, "ace") // ace's key, first heard placing nothing
gate.Assignments(k, v, false, old.Add(-time.Hour))
gate.Settled()
k, v = placing(t, "ace", NodeBuildModule)
if gate.Assignments(k, v, false, placed) {
t.Fatal("assignments with no report passed something on")
}
key, value := gateReport(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// ace's assignments written again with another module beside: its standing does not move, nothing is said again.
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
if gate.Assignments(k, v, false, placed.Add(time.Minute)) {
t.Fatal("the same assignments of mesh-build-node changed what was passed on")
}
if c.counts {
return
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's; got %q, %q", v, none)
}
if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", *said)
}
key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
t.Fatal("ace's new report was not passed on")
}
})
}
t.Run("no assignments held: every report counts", func(t *testing.T) {
gate, set, _ := start(t, false)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(NodeBuildModule, key, value, false)
gate.Settled()
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("before the first assignments file every report counts; got %d", n)
}
k, v := placing(t, "g14", NodeBuildModule)
if !gate.Assignments(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 {
t.Fatalf("the first assignments leaves every node it does not place out; got %d", len(set.Entries("typescript")))
}
if !gate.Assignments(k, nil, true, placed) || len(set.Entries("typescript")) != 4 {
t.Fatal("the last assignments gone: every report counts again")
}
})
// Design 47 *Each part acts on its own entries*: fed by the assignment half's revisions, the gate settles at the first,
// leaves out a node no revision places the build module on, and counts a node placed in a later revision only by a
// report written since that revision's time.
t.Run("fed by the assignment half", func(t *testing.T) {
set := &ToolchainSet{}
said := &[]string{}
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
for _, node := range append([]string{"ace"}, three...) {
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
if node == "ace" {
key, value = gateReport(t, node, "v26.12.0", old)
}
gate.Toolchain(NodeBuildModule, key, value, false)
}
revision := func(n int64, at time.Time, nodes ...string) meshwants.AssignmentRevision {
r := meshwants.AssignmentRevision{Revision: n, Kind: "trunk", At: &at}
for _, node := range nodes {
r.Entries = append(r.Entries, meshwants.NodeEntryRef{Node: node, Module: NodeBuildModule, Digest: "d-" + node})
}
return r
}
if !gate.Revision(revision(7, old.Add(-time.Hour), three...)) || len(set.Entries("typescript")) != 3 {
t.Fatalf("the first revision settles the gate and leaves ace out; %d entries", len(set.Entries("typescript")))
}
if gate.Revision(revision(7, old.Add(-time.Hour), three...)) {
t.Fatal("the same revision again changed what was passed on")
}
if gate.Revision(revision(8, placed, "ace", "g14", "novox", "shanks")); len(set.Entries("typescript")) != 3 {
t.Fatal("ace's report from before revision 8 placed it counts")
}
if len(*said) < 2 || !strings.Contains((*said)[len(*said)-1], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want why ace's old report is left out", *said)
}
key, value := gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
t.Fatal("ace's report written since revision 8 is not passed on")
}
if !gate.Revision(revision(9, placed.Add(time.Hour), "ace", "novox", "shanks")) || len(set.Entries("typescript")) != 3 || len(gate.Nodes()) != 3 {
t.Fatalf("g14 taken off in revision 9: %d entries, nodes %v", len(set.Entries("typescript")), gate.Nodes())
}
})
t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) {
p := meshmoduleassignerclient.NewPlaced()
// ace's node key is the first revision heard of it: its module stands since before.
k, v := placing(t, "ace", NodeBuildModule)
p.Put(k, v, false, old)
if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 {
t.Errorf("a module key says which modules stand: %v", changed)
}
k2, v2 := placing(t, "g14")
p.Put(k2, v2, false, old)
k2, v2 = placing(t, "g14", NodeBuildModule)
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule {
t.Errorf("a module placed says changed %v", changed)
}
if _, _, err := p.Put("novox.node", []byte("not json"), false, old); err == nil {
t.Error("a value that does not read was not said")
}
if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() {
t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok)
}
if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) {
t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok)
}
if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode())
}
})
t.Run("assignments the bus refuses is said, and the part goes on", func(t *testing.T) {
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_assignments")
gate, _, _ := start(t, false)
for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} {
if _, err := FollowAssignments(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil {
t.Errorf("a refused assignments stopped the start: %v", err)
}
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
})
}
modules/mesh-delivery-builder/buildrun/toolchains.go
+36
View File
@@ -91,3 +91,39 @@ const (
ToolchainsTool = "builder_toolchains"
ToolchainsDropTool = "builder_toolchains_drop"
)
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
// A key or a value that is not an entry of the module's language is left out.
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
language, ok := LanguageOf[module]
if !ok {
return false
}
node, keyLanguage, err := buildstep.ParseToolchainKey(key)
if err != nil || keyLanguage != language {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if deleted {
if _, had := s.by[language][node]; !had {
return false
}
delete(s.by[language], node)
return true
}
tc, err := buildstep.DecodeToolchain(value)
if err != nil || tc.Language != language {
return false
}
if s.by == nil {
s.by = map[string]map[string]buildstep.Toolchain{}
}
if s.by[language] == nil {
s.by[language] = map[string]buildstep.Toolchain{}
}
cur, had := s.by[language][node]
s.by[language][node] = tc
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe || cur.Digest != tc.Digest
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/findingwriter.go
+77
View File
@@ -0,0 +1,77 @@
package main
import (
"context"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-checker/internal/check"
)
type findingWriterOf = views.Writer[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
// checkerFindings is the checker's finding writer (design 47 G §461, §689): it owns the finding check-could-not-run beside
// its views reader's, observes the raised ones of its kept checks, and writes both as the generated contract type. The
// writer forms its subjects itself (A-13).
type checkerFindings struct {
w *findingWriterOf
node string
}
// newCheckerFindings makes and starts the writer; observe says the checks that could not run and whose finding is
// written (check.Checker.RaisedCouldNotRun).
func newCheckerFindings(ctx context.Context, cred process.Account, conn *nats.Conn, js jetstream.JetStream, state process.State,
observe func(context.Context) ([]check.Finding, error), logf func(string, ...any)) (*checkerFindings, error) {
c := &checkerFindings{node: cred.Node}
w, err := views.NewWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
Module: cred.Module, State: state, Node: cred.Node, Asked: condseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf,
Kinds: []string{check.CouldNotRunKind},
Observe: func(ctx context.Context) (views.Observed[condseat.Finding], error) {
found, err := observe(ctx)
if err != nil {
return views.Observed[condseat.Finding]{}, err
}
var out views.Observed[condseat.Finding]
for _, f := range found {
out.Findings = append(out.Findings, c.body(f))
}
return out, nil
},
})
if err != nil {
return nil, err
}
c.w = w
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
return c, nil
}
// body lays the checker's finding out as the seat's contract type: a finding of the module, about the module on its node.
func (c *checkerFindings) body(f check.Finding) condseat.Finding {
module, _ := c.w.ModuleNode()
return condseat.Finding{Kind: f.Kind, Scope: views.FindingScope, ID: keys.EscapeName(f.ID), At: f.At, Since: f.At,
Severity: condseat.FindingSeverityWarning, Source: f.Kind, Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation,
Subject: condseat.Subject{Scope: views.FindingScope, ID: module, Machine: c.node}}
}
// Put writes one finding; its `since` stays where its key exists.
func (c *checkerFindings) Put(f check.Finding) error {
return c.w.Write(context.Background(), c.body(f))
}
// Delete clears one finding.
func (c *checkerFindings) Delete(f check.Finding) error {
return c.w.Clear(context.Background(), views.FindingScope, keys.EscapeName(f.ID), f.Kind, "the checker no longer observes it")
}
var _ check.FindingWriter = (*checkerFindings)(nil)
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go
+17 -6
View File
@@ -15,11 +15,10 @@ import (
"os"
"os/signal"
"strings"
"sync/atomic"
"syscall"
"time"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -77,7 +76,6 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
// ModulesBucket, ManifestsBucket).
var (
firstViewsBucket = check.CheckerModule + "_" + views.StatusState
findingsBucket = check.CheckerModule + "_findings"
)
const (
@@ -163,8 +161,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
hub := process.NewEventHub(logger.Printf)
reader := &views.ProcessReader{Module: check.CheckerModule, Call: caller.Call, Logf: logger.Printf, Hub: hub}
seats := seatList{reader: followSeatList(seatlist.ProcessBus(caller.Call, hub), logger.Printf)}
findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "a first view unanswered past its bound is said in the log alone", logger.Printf)
reader.Findings = func() views.FindingWriter { return sdkfindings.Writer(ctx, findings.KV) }
// The finding writer is made once the checker is (its observe reads the checker's kept checks); until then a first view
// unanswered is said in the log alone.
var findings atomic.Pointer[checkerFindings]
reader.Findings = func() views.WireWriter {
if f := findings.Load(); f != nil {
return f.w
}
return nil
}
// Each by the owner's read tool and views family as the owner's manifest names them.
var followed error
followOwner := func(owner, state, tool, stream string) *views.Picture {
@@ -236,13 +241,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Owed: kvStore{buckets[check.CheckerOwedBucket]},
// A check that could not run waits for its part's running, and is the finding check-could-not-run after its bound.
CouldNotRun: awaitedKV{couldNotRunKV},
Findings: sdkfindings.Writer(ctx, findings.KV),
Trunks: trunks,
Bound: checkBound,
Link: link,
Logf: logger.Printf,
Settle: settleWindow,
}
cf, err := newCheckerFindings(ctx, cred, conn, js, caller.State("findings"), ch.RaisedCouldNotRun, logger.Printf)
if err != nil {
return fmt.Errorf("the checker's finding writer: %w", err)
}
defer cf.w.Close()
ch.Findings = cf
findings.Store(cf)
kept := make(chan struct{}, 1)
ch.CouldNotRunKept = func() {
select {
modules/mesh-delivery-checker/go.mod
+1 -2
View File
@@ -44,7 +44,6 @@ replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg v0.0.0-00010101000000-000000000000
)
@@ -53,7 +52,7 @@ replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkerms
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/contracts/go/clients/gopackageregistryclient v0.0.0-00010101000000-000000000000 // indirect
modules/mesh-delivery-checker/internal/check/checker.go
+1 -2
View File
@@ -20,7 +20,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/views"
)
// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678):
@@ -94,7 +93,7 @@ type Checker struct {
CouldNotRun KV
// Findings writes the checker's finding of a check not asked again within CouldNotRunBound (CouldNotRunDue); nil
// writes none.
Findings views.FindingWriter
Findings FindingWriter
// CouldNotRunKept is called each time a check that could not run is kept, so the process waits for its bound anew.
CouldNotRunKept func()
// Bound is how long one check may run; none when 0 (DefaultBound).
modules/mesh-delivery-checker/internal/check/checker_test.go
+8 -6
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/views"
"strings"
"sync"
"testing"
@@ -228,10 +227,10 @@ func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3
}
// findingsKept is a FindingWriter that keeps what is written, by ID.
type findingsKept map[string]views.Finding
type findingsKept map[string]Finding
func (f findingsKept) Put(x views.Finding) error { f[x.ID] = x; return nil }
func (f findingsKept) Delete(x views.Finding) error { delete(f, x.ID); return nil }
func (f findingsKept) Put(x Finding) error { f[x.ID] = x; return nil }
func (f findingsKept) Delete(x Finding) error { delete(f, x.ID); return nil }
// A check that could not hear a part of the mesh says `could not run`, naming the part, and releases its claim; only the
// running of that part asks it again, once per heads and part at one fingerprint; not asked again within an hour it is the
@@ -301,10 +300,13 @@ func TestACheckThatCouldNotRunIsAskedAgainWhenItsPartRuns(t *testing.T) { // N6/
if due := r.ch.CouldNotRunDue(ctx, now.Add(CouldNotRunBound)); !due.IsZero() || found[key].Kind != CouldNotRunKind || !strings.Contains(found[key].Summary, "forge on g14") {
t.Fatalf("at its bound: due %v, findings %+v, want %s naming the part", due, found, CouldNotRunKind)
}
if observed, err := r.ch.RaisedCouldNotRun(ctx); err != nil || len(observed) != 1 || observed[0].Kind != CouldNotRunKind || observed[0].ID != key {
t.Fatalf("observed at its bound = %+v, %v, want its one finding, for the finding writer's statement", observed, err)
}
r.ch.PartRunning(ctx, "g14", "forge", "src2:1")
r.ch.Wait()
if len(found) != 0 {
t.Errorf("checked again, the finding stays: %v", found)
if observed, _ := r.ch.RaisedCouldNotRun(ctx); len(found) != 0 || len(observed) != 0 {
t.Errorf("checked again, the finding stays: %v, observed %v", found, observed)
}
},
"its heads replaced, or its pull request closed, the record and the finding go": func(t *testing.T) {
modules/mesh-delivery-checker/internal/check/couldnotrun.go
+35 -3
View File
@@ -10,7 +10,6 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
"git.novox.be/novox/mesh-sdk/go/views"
)
// A check that failed because a part of the mesh was not heard says `could not run`, naming the part, and releases its
@@ -233,9 +232,42 @@ func (ch *Checker) CouldNotRunDue(ctx context.Context, now time.Time) time.Time
return next
}
// Finding is the checker's finding of a check that could not run, in plain fields: the checker's writer lays it out as the seat
// mesh-conditions' finding (design 47 G §689).
type Finding struct {
Kind, ID, Summary, Headline, Explanation string
At time.Time
}
// FindingWriter writes and clears the checker's findings.
type FindingWriter interface {
Put(f Finding) error
Delete(f Finding) error
}
// RaisedCouldNotRun is every finding the checker observes now (design 47 G §384, §461): the check that could not run
// and whose finding is written. The finding writer states them at its start and when the keeper asks.
func (ch *Checker) RaisedCouldNotRun(ctx context.Context) ([]Finding, error) {
if ch.CouldNotRun == nil {
return nil, nil
}
keys, err := ch.CouldNotRun.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("the checks that could not run cannot be listed: %w", err)
}
sort.Strings(keys)
var out []Finding
for _, key := range keys {
if r, _, ok := ch.couldNotRunOf(ctx, key); ok && r.Raised {
out = append(out, couldNotRunFinding(key, r, r.Since))
}
}
return out, nil
}
// couldNotRunFinding is the finding of the check of key that could not run.
func couldNotRunFinding(key string, r CouldNotRun, at time.Time) views.Finding {
return views.Finding{Observer: CheckerModule, Kind: CouldNotRunKind, ID: key,
func couldNotRunFinding(key string, r CouldNotRun, at time.Time) Finding {
return Finding{Kind: CouldNotRunKind, ID: key,
Summary: fmt.Sprintf("the check of %s (heads %s) could not run since %s: %s was not heard, and has not run since",
r.Group, r.Heads, r.Since.UTC().Format(time.RFC3339), r.Part),
Headline: fmt.Sprintf("The check of %s could not run", r.Group),
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go
+1 -2
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
@@ -179,7 +178,7 @@ func outOfStepTool(state func(string) views.Source) stdio.Tool {
"build of the same module), then each open pull request whose check's verdict named out-of-step files.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
rows, err := index.OutOfStepOf(state(index.SeatDefinitionsState), state(mcc.FindingsState))
rows, err := index.OutOfStepOf(state(index.SeatDefinitionsState), state(index.FindingsState))
if err != nil {
return nil, err
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go
+9
View File
@@ -477,6 +477,9 @@ func waitViews(t *testing.T, ctx context.Context, js jetstream.JetStream, state,
const views0Stream = "EVENTS"
// findingsStream is the bus name of the indexer's stream of its findings' events.
const findingsStream = "INDEXER_FINDINGS"
// acceptedHeard counts what a waiter on manifest-accepted heard.
var acceptedHeard atomic.Int64
@@ -640,6 +643,12 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
t.Fatal(err)
}
// The indexer's own stream of its findings' events (design 47 G §383), which its finding writer says into (A-30).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: findingsStream, Subjects: []string{
"mesh.stream." + index.Module + ".event.finding-written.*", "mesh.stream." + index.Module + ".event.finding-cleared.*",
"mesh.stream." + index.Module + ".event.findings-stated.*"}}); err != nil {
t.Fatal(err)
}
put(ctx, buildLog{ctx: ctx, js: js})
for _, b := range before {
b(ctx, js)
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go
+25 -7
View File
@@ -31,8 +31,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
stdio "git.novox.be/novox/mesh-sdk/go"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index"
)
@@ -401,6 +403,21 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
unjudged := process.AwaitBucket(ctx, js, conn, index.UnjudgedBucket, "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf)
tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf)
findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "no finding of the indexer is written", logger.Printf)
// The indexer's findings are the seat mesh-conditions' generated finding, written through mesh-sdk's held finding writer
// (design 47 G §689, A-29): it holds what is written before the state is bound, and starts once the state is.
writer, err := views.NewHeldWriter(views.WriterConfig[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]{
Module: cred.Module, State: calls.State(index.FindingsState), Node: cred.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logger.Printf})
if err != nil {
return fmt.Errorf("the indexer's finding writer cannot be made: %w", err)
}
defer writer.Close()
findings.Ready(func(jetstream.KeyValue) {
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start.
if err := writer.Start(ctx); err != nil {
logger.Printf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
})
// The seat definitions (design 26, DLV-I17): a state this delivery brings, bound when it appears; until then none is kept.
seatDefs := process.AwaitBucket(ctx, js, conn, index.SeatDefinitionsBucket, "no seat definition is kept", logger.Printf)
marks := process.AwaitBucket(ctx, js, conn, index.MarksBucket, "no change is said on the views family, so every reader waits on its first view", logger.Printf)
@@ -413,7 +430,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
})
readWhenBound(seatReader, seatDefs, marks)
ix := &index.Indexer{Findings: awaitedStore{b: findings}, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
ix := &index.Indexer{Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
@@ -492,11 +509,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The indexer's one pull of its durable on the shared events stream (process.EventHub): every handler is registered
// before it starts.
hub := process.NewEventHub(logger.Printf)
followAssignments(ctx, hub, cred, calls.Call, gate, func(ctx context.Context) {
reader := &views.ProcessReader{Module: index.Module, Call: calls.Call, Logf: logger.Printf, Hub: hub,
Findings: func() views.WireWriter { return writer }}
followAssignments(ctx, reader, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
if err := ix.Recompute(ctx); err != nil {
logger.Printf("writing the modules again after the assignments of a build module changed: %v", err)
}
}, findings, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), logger.Printf)
}, logger.Printf)
tools := newToolchainWatch(ctx, js, toolchainSink{
put: gate.Toolchain,
apply: func(ctx context.Context, module, key string, value []byte, deleted bool) error {
@@ -565,7 +584,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Through the seat git (milestone 47/G W4.15, design 47 *A part that cannot read the forge keeps what it has*): a seat that
// does not answer is waited for at most index.GitWaitBound before the judging below runs on the kept definitions, said in
// indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard.
defs := gitWaits(ctx, ix, findings, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf)
defs := gitWaits(ctx, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf)
seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) })
// Every manifest judged at the start, so a judge that changed with this program judges every one again (ADR 0578
// §1), once the verdicts state is there; each merge judges them again (index.ApplyMessage).
@@ -601,7 +620,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// with the build module, as with the gate's source.
// Each consumer below that is not there yet, or cannot be heard, is the finding family-not-heard naming its stream until
// it binds at a later start (heard.Writer; ADR 0615 §5), so the indexer is not healthy while what it waits on is lost.
notHeard := notHeardWriter(ctx, findings, logger.Printf)
notHeard := notHeardWriter(writer, logger.Printf)
// A consumer that is there and cannot be heard stops the indexer, as before; it is said as the finding first.
var stopped error
hear := func(stream, consequence string, each func(jetstream.Msg)) {
@@ -706,8 +725,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
heard := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)}
// Each change of the states its readers read is said on its views family (ADR 0615 §5), from the mark each state was
// said up to; a first view (the tools bundle) answers at that mark.
startViews(ctx, js, cred, marks,
findings,
startViews(ctx, js, cred, marks, writer,
map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates),
index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined),
index.ConfigCandidatesState: awaited(configCandidates), index.PackagesState: awaited(packages),
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go
+4 -23
View File
@@ -17,6 +17,7 @@ import (
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index"
)
@@ -80,7 +81,7 @@ var testAccount = process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
// Review round 2: the indexer's views emitter writes its finding view-unsaid (Emitter.Findings set).
if em := viewsEmitter(context.Background(), nil, process.Account{}, nil, &process.AwaitedBucket{}, nil, t.Logf); em.Findings == nil {
if em := viewsEmitter(nil, process.Account{}, nil, noWire{}, t.Logf); em.Findings == nil {
t.Error("the indexer's views emitter writes no view-unsaid finding")
}
// Milestone 47/G: the object buckets are the module's own by the name its account runs as, never a fixed name.
@@ -245,25 +246,5 @@ func TestTheSeatsSourceAnswerIsReadAndItsErrorIsAnError(t *testing.T) { // desig
func ptr[T any](v T) *T { return &v }
// findingsKV is a findings bucket of the two calls the indexer's call finding makes.
type findingsKV struct {
jetstream.KeyValue
vals map[string][]byte
}
func (f *findingsKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
f.vals[key] = value
return uint64(len(f.vals)), nil
}
func (f *findingsKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
if _, ok := f.vals[key]; !ok {
return nil, jetstream.ErrKeyNotFound
}
return nil, nil
}
func (f *findingsKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
delete(f.vals, key)
return nil
}
// noWire is a finding writer that is none: a non-nil WireWriter for the test of which emitter is made.
type noWire struct{ views.WireWriter }
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go
+14
View File
@@ -332,6 +332,20 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if rows := outOfStep(2); rows[0].Module != "mesh-contracts" || !strings.Contains(rows[0].Summary, "types.go") || rows[1].Pull != "novox/mesh-contracts#82" {
t.Errorf("indexer_out_of_step = %+v, want the trunk's finding and the pull request's files", rows)
}
// The finding was said by the indexer's finding writer on its own stream (design 47 G §689, A-30): finding-written.
findingWritten := func() bool {
st, err := js.Stream(ctx, findingsStream)
if err != nil {
return false
}
info, err := st.Info(ctx)
return err == nil && info.State.Msgs > 0
}
for deadline := time.Now().Add(20 * time.Second); !findingWritten() && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
}
if !findingWritten() {
t.Error("the out-of-step finding was written into the findings state but its finding-written was not said on the indexer's findings stream")
}
logEnd(builds.BuildDone)
if rows := outOfStep(1); rows[0].Source != index.SourcePull {
t.Errorf("after a clean build end indexer_out_of_step = %+v, want the pull request's files alone", rows)
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go
+75 -86
View File
@@ -1,20 +1,23 @@
package main
import (
"cmp"
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/git"
"strings"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -23,7 +26,7 @@ import (
// findingsBucket is the indexer's own findings, which the condition keeper raises and clears (ADR 0564 §2): a value too
// large for its views event, and its first view of the assigner's assignments unanswered past its bound.
var findingsBucket = mcc.FindingsBucket(index.Module)
var findingsBucket = index.FindingsBucket
// readyOf binds a state's bucket: now for one bound at the start, once it appears for one awaited.
type readyOf func(func(jetstream.KeyValue))
@@ -33,19 +36,11 @@ func boundNow(kv jetstream.KeyValue) readyOf { return func(f func(jetstream.KeyV
func awaited(b *process.AwaitedBucket) readyOf { return b.Ready }
// startViews says each change of the indexer's viewed states on its views family (ADR 0615 §5), once its marks state
// is bound: each state from where its mark says, as soon as its own bucket is bound. A watch that ends is sent on errs.
func startViews(ctx context.Context, js jetstream.JetStream, cred process.Account, marks, findings *process.AwaitedBucket, states map[string]readyOf, errs chan<- error, logf func(string, ...any)) {
tooLarge := func(state, key string, size int) {
kv := findings.KV()
if kv == nil {
return
}
if err := sdkfindings.Writer(ctx, findings.KV).Put(views.TooLargeFinding(index.Module, state, key, size, time.Now())); err != nil {
logf("views: the finding of %s.%s too large is not written: %v", state, key, err)
}
}
// is bound: each state from where its mark says, as soon as its own bucket is bound. A watch that ends is sent on errs. Its
// findings (a value too large, a change not said) go through the indexer's finding writer (design 47 G §687).
func startViews(ctx context.Context, js jetstream.JetStream, cred process.Account, marks *process.AwaitedBucket, writer views.WireWriter, states map[string]readyOf, errs chan<- error, logf func(string, ...any)) {
marks.Ready(func(mk jetstream.KeyValue) {
em := viewsEmitter(ctx, js, cred, mk, findings, tooLarge, logf)
em := viewsEmitter(js, cred, mk, writer, logf)
for _, state := range sortedStates(states) {
states[state](func(kv jetstream.KeyValue) {
if err := em.Start(ctx, state, kv, errs); err != nil {
@@ -57,11 +52,10 @@ func startViews(ctx context.Context, js jetstream.JetStream, cred process.Accoun
}
// viewsEmitter is the indexer's emitter of its views family: a change not said past views.Bound is its finding
// view-unsaid, in its own findings once bound (review round 2).
func viewsEmitter(ctx context.Context, js jetstream.JetStream, cred process.Account, mk jetstream.KeyValue, findings *process.AwaitedBucket,
tooLarge func(string, string, int), logf func(string, ...any)) views.Emitter {
// view-unsaid, written through its finding writer (review round 2).
func viewsEmitter(js jetstream.JetStream, cred process.Account, mk jetstream.KeyValue, writer views.WireWriter, logf func(string, ...any)) views.Emitter {
return views.Emitter{Owner: index.Module, Subject: cred.EventSubject, Publish: views.JetStreamPublish(js),
Marks: mk, TooLarge: tooLarge, Findings: sdkfindings.Writer(ctx, findings.KV), Logf: logf}
Marks: mk, Findings: writer, Logf: logf}
}
func sortedStates(states map[string]readyOf) []string {
@@ -79,30 +73,36 @@ func sortedStates(states map[string]readyOf) []string {
return out
}
// notHeardWriter writes the indexer's finding of a family it does not hear into its findings state, once that is bound.
func notHeardWriter(ctx context.Context, findings *process.AwaitedBucket, logf func(string, ...any)) heard.Writer {
// notHeardWriter writes the indexer's finding of a family it does not hear through its finding writer (design 47 G §689).
func notHeardWriter(w index.FindingWriter, logf func(string, ...any)) heard.Writer {
return heard.Writer{Observer: index.Module, Logf: logf,
Put: func(key string, value []byte) error {
findings.Ready(func(kv jetstream.KeyValue) {
if _, err := kv.Put(ctx, key, value); err != nil {
logf("the finding %s is not written: %v", key, err)
}
})
return nil
Put: func(_ string, value []byte) error {
var f mcseat.Finding
if err := json.Unmarshal(value, &f); err != nil {
return err
}
return w.Write(context.Background(), f)
},
Delete: func(key string) error {
findings.Ready(func(kv jetstream.KeyValue) {
if _, err := kv.Get(ctx, key); err != nil {
return // none stands
}
if err := kv.Delete(ctx, key); err != nil {
logf("the finding %s is not deleted: %v", key, err)
}
})
return nil
scope, rest, _ := strings.Cut(key, ".")
i := strings.LastIndex(rest, ".")
if i < 0 {
return fmt.Errorf("%q is no finding's key", key)
}
return clearHeld(context.Background(), w, scope, rest[:i], rest[i+1:], "the indexer hears it now")
}}
}
// clearHeld clears the finding of scope, id and kind when the writer holds it: none standing says nothing.
func clearHeld(ctx context.Context, w index.FindingWriter, scope, id, kind, why string) error {
for _, f := range w.Held() {
if f.Scope == scope && f.ID == id && f.Kind == kind {
return w.Clear(ctx, scope, id, kind, why)
}
}
return nil
}
// graphState is the module graph's state, named apart from run's bucket of the same name.
const graphState = graph.State
@@ -114,46 +114,38 @@ var firstViewsBucket = index.Module + "_" + views.StatusState
const assignmentsTool = index.AssignerModule + "." + index.AssignmentsState
// followAssignments reads the seat mesh-module-assigner's assignments through the assigner's first view and then its views
// family, never its bucket (ADR 0615 §1, DLV-F1r), into gate: each key as it changes, settled once answered, with
// changed told whenever what the gate passes on changed. It waits for the first answer at most views.Bound, so a
// normal start, the two owners coming up together, computes the fingerprints once; past it the indexer goes on, the gate
// passing every report, and raises its condition naming the assigner and assignments until the answer clears it.
func followAssignments(ctx context.Context, hub *process.EventHub, cred process.Account, call func(string, any) (json.RawMessage, error),
gate *buildrun.ToolchainGate, changed func(context.Context), findings, status *process.AwaitedBucket, logf func(string, ...any)) *views.Follower {
finding := func(s views.Status, raised bool) {
w := sdkfindings.Writer(ctx, findings.KV)
f := views.UnansweredFinding(index.Module, s, time.Now())
var err error
if raised {
err = w.Put(f)
} else {
err = w.Delete(f)
// family, never its bucket (ADR 0615 §1, DLV-F1r), into gate: the module's views reader follows it (design 47 G §532, A-26),
// which asks the first view, hears the views family and raises and clears the finding of a first view unanswered through the
// indexer's finding writer. Each key is taken as it changes, settled once answered, with changed told whenever what the gate
// passes on changed. It waits for the first answer at most views.Bound, so a normal start, the two owners coming up together,
// computes the fingerprints once; past it the indexer goes on, the gate passing every report, and raises its finding naming the
// assigner and assignments until the answer clears it.
func followAssignments(ctx context.Context, reader *views.ProcessReader, status *process.AwaitedBucket, cred process.Account, js jetstream.JetStream,
gate *buildrun.ToolchainGate, changed func(context.Context), logf func(string, ...any)) {
var settled atomic.Bool // the first view answered: the gate's nodes are the counted ones from now on
pic, err := reader.Follow(index.AssignerModule, index.AssignmentsState, assignmentsTool)
if err != nil {
logf("the assignments of the assigner is not followed: %v", err)
return
}
f := pic.Follower()
keep := f.Changed // the picture's own, kept
f.Changed = func(key string, value json.RawMessage, deleted bool, at time.Time) {
if keep != nil {
keep(key, value, deleted, at)
}
if err != nil {
logf("views: the finding of the first view %s is not written: %v", s.Tool, err)
if gate.Assignments(key, value, deleted, at) {
changed(ctx)
}
if settled.Load() && gate.OnPlaced != nil {
gate.OnPlaced() // the counted nodes may have changed (ADR 0624 §1)
}
}
var settled atomic.Bool // the first view answered: the gate's nodes are the counted ones from now on
f := &views.Follower{Owner: index.AssignerModule, Tool: assignmentsTool, State: index.AssignmentsState,
Ask: views.AskVia(call, assignmentsTool), Logf: logf,
Raise: func(s views.Status) { finding(s, true) }, Clear: func(s views.Status) { finding(s, false) },
Changed: func(key string, value json.RawMessage, deleted bool, at time.Time) {
if gate.Assignments(key, value, deleted, at) {
changed(ctx)
}
if settled.Load() && gate.OnPlaced != nil {
gate.OnPlaced() // the counted nodes may have changed (ADR 0624 §1)
}
}}
status.Ready(func(kv jetstream.KeyValue) {
f.Said = views.KeepStatus(ctx, kv, logf)
if err := views.HearAgain(ctx, kv, []*views.Follower{f}, logf); err != nil {
logf("views: indexer_first_view_again is not heard (%v) until the indexer starts again", err)
}
})
answered := make(chan struct{})
go func() {
if err := f.Start(ctx); err != nil {
select {
case <-f.Ready():
case <-ctx.Done():
return
}
if gate.Settled() {
@@ -165,13 +157,7 @@ func followAssignments(ctx context.Context, hub *process.EventHub, cred process.
}
close(answered)
}()
// The assigner's views family, on the module's one hub (the host starts it once every handler is registered); a family the
// hub cannot register is the finding family-not-heard until the indexer starts again (ADR 0615 §5).
// The hub says when the bus made its consumer: a picture asked before then is asked again (views.Follower.ConsumerMade).
hub.OnBound(func(created time.Time) { f.ConsumerMade(ctx, created) })
_ = notHeardWriter(ctx, findings, logf).Bind(process.EventsStream, "the assignments is the first view's alone, asked again by hand, until the indexer starts again", func() error {
return hub.Handle("mesh.mod."+index.AssignerModule+".event."+views.EventPrefix+"*", views.Hear(ctx, map[string]*views.Follower{index.AssignmentsState: f}, logf))
})
reader.Start(ctx, js, cred, status)
wait, cancel := context.WithTimeout(ctx, views.Bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
@@ -179,21 +165,24 @@ func followAssignments(ctx context.Context, hub *process.EventHub, cred process.
case <-wait.Done():
logf("the assigner has not answered assignments within %s: the indexer goes on, every node's toolchain report counting until it answers", views.Bound)
}
return f
}
// gitWaits is the indexer's read of the trunk definitions at its start through the seat git (index.DefinitionsAtStart): its
// wait said as a first view of the seat's (Owner git, State seat-definitions) in the indexer's first views, and its finding
// view-waits written past views.WaitsBound and cleared once the seat answers.
func gitWaits(ctx context.Context, ix *index.Indexer, findings, status *process.AwaitedBucket, logf func(string, ...any)) *index.DefinitionsAtStart {
// view-waits written past views.WaitsBound and cleared once the seat answers, by hand through the finding writer (design 47 G A-28).
func gitWaits(ctx context.Context, ix *index.Indexer, w index.FindingWriter, node string, status *process.AwaitedBucket, logf func(string, ...any)) *index.DefinitionsAtStart {
finding := func(s views.Status, raised bool) {
w := sdkfindings.Writer(ctx, findings.KV)
f := views.ViewWaitsFinding(index.Module, s, time.Now())
id := keys.EscapeName(index.Module + "." + s.Owner + "." + s.State)
var err error
if raised {
err = w.Put(f)
at := time.Now().UTC()
summary := fmt.Sprintf("%s waits for the view %s.%s at revision %s since %s and acts on nothing new until it holds it; it holds %s",
index.Module, s.Owner, s.State, s.WaitsFor, s.WaitingSince.UTC().Format(time.RFC3339), cmp.Or(s.View, "none"))
err = w.Write(ctx, mcseat.Finding{Scope: views.FindingScope, ID: id, Kind: views.ViewWaitsKind, Severity: mcseat.FindingSeverityWarning,
Summary: summary, Headline: index.Module + " waits for a view of " + s.Owner, Source: "views", Observer: index.Module, At: at,
Subject: mcseat.Subject{Scope: views.FindingScope, ID: index.Module, Machine: node}})
} else {
err = w.Delete(f)
err = clearHeld(ctx, w, views.FindingScope, id, views.ViewWaitsKind, "the part no longer observes it")
}
if err != nil {
logf("views: the finding view-waits of the seat git is not written: %v", err)
modules/mesh-delivery-indexer/internal/index/accepted.go
+17 -35
View File
@@ -60,31 +60,24 @@ func (ix *Indexer) sayAccepted(ctx context.Context, a meshcatalog.ManifestAccept
return err
}
// unsaidKey is the key of module's unsaid finding.
func unsaidKey(module string) string {
return mcc.Key(mcc.ScopeModule, Module+".accepted."+module, UnsaidKind)
}
// unsaidID is the id of module's unsaid finding.
func unsaidID(module string) string { return Module + ".accepted." + module }
// unsaid writes module's finding for err, or deletes it with none.
// unsaid raises module's finding for err, or clears it with none.
func (ix *Indexer) unsaid(ctx context.Context, module string, err error) {
if ix.Findings == nil {
return
}
key := unsaidKey(module)
werr := write(ctx, ix.Findings, key, func(value []byte, found bool) ([]byte, bool, error) {
if err == nil {
return nil, found, nil
}
at := ix.now().UTC()
summary := fmt.Sprintf("%s of %s was not published: %v", meshcatalog.EventManifestAccepted, module, err)
data, ferr := mcc.EncodeFinding(mcc.Observation{Scope: mcc.ScopeModule, ID: Module + ".accepted." + module, Kind: UnsaidKind,
Summary: summary,
Headline: "The indexer could not tell the bus module that " + module + "'s manifest was accepted",
Explanation: "The bus module creates and removes a module's bus state from this event. The indexer says it again at its " +
"next judging pass and clears this once it is said; indexer_say_accepted says it now.",
Resolved: "manifest-accepted of " + module + " was said.", Source: UnsaidKind}.Finding(Module, "", at))
return data, false, ferr
})
var werr error
if err == nil {
werr = ix.clearFinding(ctx, mcc.ScopeModule, unsaidID(module), UnsaidKind, "manifest-accepted of "+module+" was said.")
} else {
werr = ix.Findings.Write(ctx, ix.observed(mcc.ScopeModule, unsaidID(module), UnsaidKind,
fmt.Sprintf("%s of %s was not published: %v", meshcatalog.EventManifestAccepted, module, err),
"The indexer could not tell the bus module that "+module+"'s manifest was accepted",
"The bus module creates and removes a module's bus state from this event. The indexer says it again at its "+
"next judging pass and clears this once it is said; indexer_say_accepted says it now.", UnsaidKind))
}
if werr != nil {
ix.logf("judge: the finding of %s for %s is not written: %v", meshcatalog.EventManifestAccepted, module, werr)
}
@@ -149,25 +142,14 @@ func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again b
return fmt.Sprintf("said for %s, manifest %s at revision %d, %d bus-state entries", module, a.Digest, a.Revision, len(a.BusState)), nil
}
// SayUnsaid says again each manifest-accepted whose finding stands (the judging pass's end): cleared once said.
// SayUnsaid says again each manifest-accepted whose finding stands (the judging pass's end): cleared once said. What stands is
// what the finding writer holds (design 47 G A-38).
func (ix *Indexer) SayUnsaid(ctx context.Context) {
if ix.Findings == nil {
return
}
keys, err := ix.Findings.Keys(ctx)
if err != nil {
return
}
for _, key := range keys {
if !strings.HasSuffix(key, "."+UnsaidKind) {
continue
}
value, _, found, err := ix.Findings.Get(ctx, key)
if err != nil || !found {
continue
}
f, err := mcc.DecodeFinding(value)
if err != nil || f.Kind != UnsaidKind {
for _, f := range ix.Findings.Held() {
if f.Kind != UnsaidKind {
continue
}
module := strings.TrimPrefix(f.Subject.ID, Module+".accepted.")
modules/mesh-delivery-indexer/internal/index/candidates.go
+3 -3
View File
@@ -84,9 +84,9 @@ type Indexer struct {
// DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving
// the trunk; nil says nothing.
SayAccepted func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error
// Findings is the indexer's findings state, where a manifest-accepted whose publish failed twice is kept until said
// (accepted.go, DLV-I9b); nil writes none.
Findings Store
// Findings is the indexer's finding writer (design 47 G A-29), where a manifest-accepted whose publish failed twice is kept
// until said (accepted.go, DLV-I9b); nil writes none.
Findings FindingWriter
// Packages is the packages state (PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line.
Packages Store
modules/mesh-delivery-indexer/internal/index/control_records_test.go
+1 -1
View File
@@ -165,7 +165,7 @@ func TestTheControlRecords(t *testing.T) {
m3: {"seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0112", 1)},
head: {"seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0200", 1)},
}
defs, findings := newMem(), newMem()
defs, findings := newMem(), newFindings()
var said []SeatDefinitionChanged
ix := &Indexer{SeatDefinitions: defs, Findings: findings, Logf: t.Logf,
// The forge's listing of a contracts module's `seats/` directory: every blob under it at the commit (ADR 0635 §6).
modules/mesh-delivery-indexer/internal/index/findings.go
+49
View File
@@ -0,0 +1,49 @@
package index
import (
"context"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// FindingsState is the indexer's state of its findings, and FindingsBucket its bucket (design 47 G §383, §687).
const (
FindingsState = "findings"
FindingsBucket = Module + "_" + FindingsState
)
// FindingWriter is the indexer's finding writer, mesh-sdk's held writer of the seat mesh-conditions' generated finding (design
// 47 G A-29, A-38): Write raises a finding, Clear clears one, Held answers the findings it holds, so the indexer reads what
// stands from the writer.
type FindingWriter interface {
Write(ctx context.Context, f mcseat.Finding) error
Clear(ctx context.Context, scope, id, kind, why string) error
Held() []mcseat.Finding
}
// keyID is an id as the parts of a key: each dot-separated part made safe for a bus key as a condition's key is (design 47 G
// A-22), so a finding's key keeps the shape it had.
func keyID(id string) string {
key := mcc.Key("x", id, "x")
return key[len("x.") : len(key)-len(".x")]
}
// observed is the indexer's finding of kind for the subject id as the seat's generated finding: a warning, its id made safe as a
// key's part, the writer completing its `since` (design 47 G §686, A-22).
func (ix *Indexer) observed(scope, id, kind, summary, headline, explanation, source string) mcseat.Finding {
id = keyID(id)
return mcseat.Finding{Scope: scope, ID: id, Kind: kind, Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: headline,
Explanation: explanation, Source: source, Observer: Module, At: ix.now().UTC(), Subject: mcseat.Subject{Scope: scope, ID: id}}
}
// clearFinding clears the finding of scope, id and kind when the writer holds it: a clear of none held says nothing.
func (ix *Indexer) clearFinding(ctx context.Context, scope, id, kind, why string) error {
id = keyID(id)
for _, f := range ix.Findings.Held() {
if f.Scope == scope && f.ID == id && f.Kind == kind {
return ix.Findings.Clear(ctx, scope, id, kind, why)
}
}
return nil
}
modules/mesh-delivery-indexer/internal/index/helpers_test.go
+43
View File
@@ -6,6 +6,10 @@ import (
"maps"
"slices"
"sync"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
)
// rawList is the list a kept predefined value holds, as it was written.
@@ -82,3 +86,42 @@ var errObjectsFullForTest = &objectsFullError{}
type objectsFullError struct{}
func (*objectsFullError) Error() string { return "maximum bytes exceeded" }
// memFindings is the finding writer in memory (FindingWriter): what is written is kept by its key as the finding's JSON, what
// is cleared is gone, and Held answers what is kept.
type memFindings struct {
vals map[string][]byte
err error // every write's error
}
func newFindings() *memFindings { return &memFindings{vals: map[string][]byte{}} }
func (m *memFindings) Write(_ context.Context, f mcseat.Finding) error {
if m.err != nil {
return m.err
}
raw, err := json.Marshal(f)
m.vals[views.FindingKey(f.Scope, f.ID, f.Kind)] = raw
return err
}
func (m *memFindings) Clear(_ context.Context, scope, id, kind, _ string) error {
delete(m.vals, views.FindingKey(scope, id, kind))
return nil
}
func (m *memFindings) Held() []mcseat.Finding {
var out []mcseat.Finding
for _, k := range slices.Sorted(maps.Keys(m.vals)) {
var f mcseat.Finding
if json.Unmarshal(m.vals[k], &f) == nil {
out = append(out, f)
}
}
return out
}
// unsaidKey is the key of module's unsaid finding.
func unsaidKey(module string) string {
return views.FindingKey(mcc.ScopeModule, keyID(unsaidID(module)), UnsaidKind)
}
modules/mesh-delivery-indexer/internal/index/pending_test.go
+2 -4
View File
@@ -638,7 +638,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
} {
b = held()
clock := time.Date(2026, 11, 10, 1, 0, 0, 0, time.UTC)
b.ix.Now, b.ix.Findings = func() time.Time { return clock }, newMem()
b.ix.Now, b.ix.Findings = func() time.Time { return clock }, newFindings()
answers := b.ix.Message
b.ix.held = nil
b.ix.Message = func(context.Context, string, string, int) ([]byte, error) { return nil, c.fault }
@@ -653,10 +653,8 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
// machine away, only a toolchain report heard), none either; the next message heard on the forge, the read still
// failing, writes call-unanswered naming the bridge's call; the merge read and taken in clears it.
finding := func() (string, bool) {
keys, _ := b.ix.Findings.Keys(ctx)
for _, k := range keys {
for k, v := range b.ix.Findings.(*memFindings).vals {
if strings.HasSuffix(k, "."+mcc.KindCallUnanswered) {
v, _, _, _ := b.ix.Findings.Get(ctx, k)
return string(v), true
}
}
modules/mesh-delivery-indexer/internal/index/seatdefs.go
+19 -29
View File
@@ -28,6 +28,7 @@ import (
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
@@ -462,8 +463,7 @@ func (ix *Indexer) twiceFinding(ctx context.Context, repository, module string,
fmt.Sprintf("%s's trunk defines seats another trunk already defines, refused: %s", module, strings.Join(lines, "; ")),
"A seat on "+module+"'s trunk is already defined by another repository",
"A seat has one definition: the one that stood first stays the seat's contract and "+module+"'s is kept nowhere. "+
"Remove the seat from one of the two repositories; the next merge of "+module+" clears this.",
"No seat on "+module+"'s trunk is defined by another repository.")
"Remove the seat from one of the two repositories; the next merge of "+module+" clears this.")
}
// outsideFinding writes the contracts module's seat-outside-its-domain finding for the seats it defines that belong to
@@ -478,8 +478,7 @@ func (ix *Indexer) outsideFinding(ctx context.Context, module string, outside ma
fmt.Sprintf("%s's trunk defines seats of another domain, refused: %s", module, strings.Join(lines, "; ")),
"A seat on "+module+"'s trunk belongs to another domain's contracts module",
"A seat of the mesh's own contracts is taken only from its domain's contracts module (mesh-contracts for a domain with none). "+
"Move the definition there; the next merge of "+module+" without it clears this.",
"Every seat on "+module+"'s trunk is of its domain.")
"Move the definition there; the next merge of "+module+" without it clears this.")
}
// dropPull removes a pull request's entries and its out-of-step files.
@@ -528,32 +527,24 @@ func (ix *Indexer) unparsedFinding(ctx context.Context, module string, unparsed
fmt.Sprintf("%s's trunk holds seat definitions that do not parse: %s", module, strings.Join(lines, "; ")),
"A seat definition on "+module+"'s trunk does not parse",
"The trunk's last definition that parsed stays the seat's contract. A change fixing the definition clears this when it "+
"merges; indexer_seat_definitions_again reads the trunk again now.",
"Every seat definition on "+module+"'s trunk parses.")
"merges; indexer_seat_definitions_again reads the trunk again now.")
}
// finding writes (raise) or deletes the indexer's finding of a kind for a subject id.
func (ix *Indexer) finding(ctx context.Context, id, kind string, raise bool, summary, headline, explanation, resolved string) {
// finding raises (raise) or clears the indexer's finding of a kind for a subject id through its finding writer (design 47 G
// §689, A-29): a finding of the seat mesh-conditions' generated type, its id made safe as a key's part (A-22); the writer says
// again only what changed.
func (ix *Indexer) finding(ctx context.Context, id, kind string, raise bool, summary, headline, explanation string) {
if ix.Findings == nil {
return
}
key := mcc.Key(mcc.ScopeModule, id, kind)
err := write(ctx, ix.Findings, key, func(value []byte, found bool) ([]byte, bool, error) {
if !raise {
return nil, found, nil
}
if found {
if f, err := mcc.DecodeFinding(value); err == nil && f.Summary == summary {
return nil, false, nil
}
}
at := ix.now().UTC()
data, err := mcc.EncodeFinding(mcc.Observation{Scope: mcc.ScopeModule, ID: id, Kind: kind, Summary: summary,
Headline: headline, Explanation: explanation, Resolved: resolved, Source: kind}.Finding(Module, "", at))
return data, false, err
})
var err error
if raise {
err = ix.Findings.Write(ctx, ix.observed(mcc.ScopeModule, id, kind, summary, headline, explanation, kind))
} else {
err = ix.clearFinding(ctx, mcc.ScopeModule, id, kind, "the indexer no longer observes it")
}
if err != nil {
ix.logf("seat definitions: the finding %s is not written: %v", key, err)
ix.logf("seat definitions: the finding %s is not written: %v", views.FindingKey(mcc.ScopeModule, keyID(id), kind), err)
}
}
@@ -584,10 +575,9 @@ func (ix *Indexer) ApplyOutOfStepBuild(ctx context.Context, value []byte) {
fmt.Sprintf("%s's trunk build at %.12s has generated files out of step with their seat definitions: %s", e.Module, e.Commit, strings.Join(files, ", ")),
"Generated types on "+e.Module+"'s trunk are out of step with their seat definitions",
"Its build's test step regenerated the types and found the committed files differ, missing or not written by the "+
"generator (a generator's change does this). A change regenerating them clears this once a build of "+e.Module+" ends clean.",
"A build of "+e.Module+" ended clean: its generated types match their definitions.")
"generator (a generator's change does this). A change regenerating them clears this once a build of "+e.Module+" ends clean.")
case e.Outcome == builds.BuildDone:
ix.finding(ctx, findingID("out-of-step", e.Module), OutOfStepKind, false, "", "", "", "")
ix.finding(ctx, findingID("out-of-step", e.Module), OutOfStepKind, false, "", "", "")
}
}
@@ -699,8 +689,8 @@ func OutOfStepOf(defs, findings views.Source) ([]OutOfStepRow, error) {
if err != nil || !ok || v == nil {
continue
}
f, err := mcc.DecodeFinding(v.Value)
if err != nil || f.Kind != OutOfStepKind {
var f mcseat.Finding
if err := json.Unmarshal(v.Value, &f); err != nil || f.Kind != OutOfStepKind {
continue
}
out = append(out, OutOfStepRow{Source: SourceTrunk, Module: strings.TrimPrefix(f.Subject.ID, Module+".out-of-step."), Summary: f.Summary})
modules/mesh-delivery-indexer/internal/index/unreadcall.go
+23 -25
View File
@@ -2,9 +2,13 @@ package index
import (
"context"
"fmt"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
)
// **A merged message unread past five minutes, the forge heard, is a finding** (issue 1966 note 153; novox/hq ADR 0668 §3).
@@ -21,7 +25,8 @@ const UnreadCallee = ForgeBridgeModule + ".bridge_merged_message"
// unreadCallID is the finding's id for the pending key: the merge it waits on.
func unreadCallID(key string) string { return "merged-message." + key }
// unreadFindings raises the finding of every merge of entries unread past the bound and clears the finding of every other.
// unreadFindings raises the finding of every merge of entries unread past the bound and clears the finding of every other
// that the finding writer holds (design 47 G §689, A-29).
func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) {
if ix.Findings == nil {
return
@@ -31,39 +36,32 @@ func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) {
if e.Value.Unread == "" {
continue
}
unread[mcc.Key(mcc.ScopeCall, unreadCallID(e.Key), mcc.KindCallUnanswered)] = true
id := keyID(unreadCallID(e.Key))
unread[id] = true
since := e.Value.UnreadSince
if since.IsZero() || ix.now().Sub(since) < mcc.CallUnansweredAfter {
continue
}
o := mcc.CallUnanswered(unreadCallID(e.Key), UnreadCallee, "", e.Value.Unread, "indexer."+mcc.KindCallUnanswered, since)
o.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread."
key := o.Key()
if err := write(ctx, ix.Findings, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if f, err := mcc.DecodeFinding(value); err == nil && f.Summary == o.Summary {
return nil, false, nil
}
}
data, err := mcc.EncodeFinding(o.Finding(Module, "", ix.now().UTC()))
return data, false, err
}); err != nil {
ix.logf("take-in: the finding %s is not written: %v", key, err)
f := ix.observed(mcc.ScopeCall, unreadCallID(e.Key), mcc.KindCallUnanswered,
fmt.Sprintf("the call %s about %s is unanswered since %s", UnreadCallee, unreadCallID(e.Key), since.UTC().Format(time.RFC3339)),
"A call whose answer is kept is not answered",
fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has not been answered since %s. Its last fault: %s.",
UnreadCallee, since.UTC().Format(time.RFC3339), e.Value.Unread), "indexer."+mcc.KindCallUnanswered)
f.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread."
f.Evidence = []mcseat.Evidence{{At: since.UTC(), Said: e.Value.Unread}}
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("take-in: the finding %s is not written: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err)
} else {
ix.logf("take-in: %s (%s#%d) is unread since %s and the forge is heard: the finding %s", e.Key, e.Forge.Repository, e.Forge.Number, since.Format("2006-01-02T15:04:05Z07:00"), key)
ix.logf("take-in: %s (%s#%d) is unread since %s and the forge is heard: the finding %s", e.Key, e.Forge.Repository, e.Forge.Number, since.Format("2006-01-02T15:04:05Z07:00"), views.FindingKey(f.Scope, f.ID, f.Kind))
}
}
keys, err := ix.Findings.Keys(ctx)
if err != nil {
return
}
prefix, suffix := mcc.ScopeCall+"."+unreadCallID(""), "."+mcc.KindCallUnanswered
for _, k := range keys {
if !strings.HasPrefix(k, prefix) || !strings.HasSuffix(k, suffix) || unread[k] {
prefix := keyID(strings.TrimSuffix(unreadCallID(""), ".")) + "."
for _, f := range ix.Findings.Held() {
if f.Scope != mcc.ScopeCall || f.Kind != mcc.KindCallUnanswered || !strings.HasPrefix(f.ID, prefix) || unread[f.ID] {
continue
}
if err := write(ctx, ix.Findings, k, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
ix.logf("take-in: the finding %s is not cleared: %v", k, err)
if err := ix.Findings.Clear(ctx, f.Scope, f.ID, f.Kind, "the merge is no longer unread"); err != nil {
ix.logf("take-in: the finding %s is not cleared: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err)
}
}
}
modules/mesh-delivery-indexer/internal/index/verdicts_test.go
+3 -3
View File
@@ -237,7 +237,7 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
}
// DLV-I9b: a failed publish is published again; one failing twice writes the finding naming the module, which the next
// judging pass says again and clears.
ix.Findings = newMem()
ix.Findings = newFindings()
failing = 1
once := hold("once", `{"module":"once","version":"1.0"}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{once: true}); err != nil || accepted[len(accepted)-1].Module != "once" {
@@ -248,13 +248,13 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if _, err := ix.JudgeAll(ctx, map[string]bool{flaky: true}); err != nil {
t.Fatal(err)
}
if value, _, found, _ := ix.Findings.Get(ctx, unsaidKey("flaky")); !found || !strings.Contains(string(value), "flaky") {
if value, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; !found || !strings.Contains(string(value), "flaky") {
t.Fatalf("a publish failing twice wrote no finding naming flaky: %s", value)
}
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
if _, _, found, _ := ix.Findings.Get(ctx, unsaidKey("flaky")); found || accepted[len(accepted)-1].Module != "flaky" ||
if _, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; found || accepted[len(accepted)-1].Module != "flaky" ||
accepted[len(accepted)-1].Digest != flaky {
t.Fatalf("the next pass did not say flaky and clear its finding (found %v, last said %+v)", found, accepted[len(accepted)-1])
}
modules/mesh-delivery-indexer/internal/index/views.go
+2 -4
View File
@@ -1,8 +1,6 @@
package index
import (
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -30,7 +28,7 @@ var ReadTools = []ViewedTool{
{Tool: "indexer_read_graph", State: graph.State, Words: "the module graph: each module's build and provision edges."},
{Tool: "indexer_read_heard", State: HeardState, Words: "the forge messages the indexer heard and what it did with each."},
{Tool: "indexer_read_taken", State: "taken", Words: "each repository's merge the indexer took in last."},
{Tool: "indexer_read_findings", State: mcc.FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."},
{Tool: "indexer_read_findings", State: FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."},
{Tool: "indexer_read_seat_definitions", State: SeatDefinitionsState, Words: "the seat definitions the trunk's and each open pull request's contracts modules hold, resolved, or why one does not parse."},
{Tool: "indexer_read_unjudged", State: UnjudgedState, Words: "the manifests a take-in newly wrote that no judge pass judged yet, by digest, each with its module and merge; indexer_judge judges them."},
}
@@ -50,7 +48,7 @@ var ViewedStates = map[string]string{
PackagesState: PackagesBucket,
HeardState: HeardBucket,
"taken": TakenBucket,
mcc.FindingsState: mcc.FindingsBucket(Module),
FindingsState: FindingsBucket,
SeatDefinitionsState: SeatDefinitionsBucket,
UnjudgedState: UnjudgedBucket,
}
modules/mesh-delivery-indexer/manifest_test.go
+2 -3
View File
@@ -16,7 +16,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/catalogue"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index"
@@ -121,7 +120,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
{Name: views.MarksState, History: 1}, {Name: mcc.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
{Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
{Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1},
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
@@ -194,7 +193,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5)
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, // design 47 G §675
index.EntryIndexTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool) // design 47 G §667, §8
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/findingwriter.go
+33
View File
@@ -0,0 +1,33 @@
package main
import (
"context"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
)
// findingWriter is the runner's mesh-sdk finding writer (design 47 G §461, §689): the runner observes only what its views
// reader finds, so its own observe states none and the writer's statement covers the reader's open findings. The writer
// forms its subjects itself (design 47 G §383, A-13).
func findingWriter(ctx context.Context, cred process.Account, conn *nats.Conn, js jetstream.JetStream, state process.State, logf func(string, ...any)) (*views.Writer[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent], error) {
w, err := views.NewWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
Module: cred.Module, State: state, Node: cred.Node, Asked: condseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf,
Observe: func(context.Context) (views.Observed[condseat.Finding], error) {
return views.Observed[condseat.Finding]{}, nil
},
})
if err != nil {
return nil, err
}
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
return w, nil
}
modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/main.go
+1 -1
View File
@@ -56,7 +56,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
hub := process.NewEventHub(logger.Printf)
rv, err := startViews(ctx, js, conn, cred, hub, calls.Call, logger.Printf)
rv, err := startViews(ctx, js, conn, cred, hub, calls.State("findings"), calls.Call, logger.Printf)
if err != nil {
return err
}
modules/mesh-delivery-pipeline/cmd/mesh-delivery-pipeline/views.go
+11 -81
View File
@@ -2,13 +2,10 @@ package main
import (
"context"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -17,9 +14,6 @@ import (
"mesh-delivery-pipeline/internal/pipeline"
)
// catalogRevisions is every watched branch's catalog-revision, `mesh.seat.mesh-catalog.event.catalog-revision.<branch>`.
const catalogRevisions = "mesh.seat." + catalog.Seat + ".event." + catalog.EventCatalogRevision + ".>"
// runnerViews are the runner's first views (design 47 G §21, §195, §200): the registrations, then the catalog half of
// each watched branch they name, each said in `<module>_first_views` (its state first-views) and, past five minutes
// without a view, as view-waits in its findings.
@@ -30,59 +24,24 @@ type runnerViews struct {
// startViews wires the runner's views on hub and asks each first view; register every other handler on hub before or
// after, then start hub once.
func startViews(ctx context.Context, js jetstream.JetStream, conn process.Subscriber, cred process.Account, hub *process.EventHub,
func startViews(ctx context.Context, js jetstream.JetStream, conn *nats.Conn, cred process.Account, hub *process.EventHub, findingsState process.State,
call meshcatalogclient.Call, logf func(string, ...any)) (*runnerViews, error) {
findings := process.AwaitBucket(ctx, js, conn, mcc.FindingsBucket(cred.Module), "a view waited for past five minutes is said in the log alone", logf)
findings, err := findingWriter(ctx, cred, conn, js, findingsState, logf)
if err != nil {
return nil, err
}
status := process.AwaitBucket(ctx, js, conn, cred.Module+"_"+views.StatusState, "the runner's first views are not said in its first-view tool, nor asked again by hand", logf)
writer := func() views.FindingWriter { return sdkfindings.Writer(ctx, findings.KV) }
writer := func() views.WireWriter { return findings }
reader := &views.ProcessReader{Module: cred.Module, Call: call, Logf: logf, Hub: hub, Findings: writer}
// A watched branch first named by a registration gets its view of the catalog half (CT10), its conditions and its
// status kept as every first view's, asked at once.
wire := func(f *views.Follower) {
f.Logf = logf
f.Raise = func(s views.Status) {
putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), true, logf)
}
f.Clear = func(s views.Status) {
putFinding(writer(), views.UnansweredFinding(cred.Module, s, time.Now()), false, logf)
}
f.RaiseWaits = func(s views.Status) {
logf("waits for %s's view %s since %s", s.Owner, s.WaitsFor, s.WaitingSince.Format(time.RFC3339))
putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), true, logf)
}
f.ClearWaits = func(s views.Status) {
putFinding(writer(), views.ViewWaitsFinding(cred.Module, s, time.Now()), false, logf)
}
f.Said = func(s views.Status) {
if kv := status.KV(); kv != nil {
views.KeepStatus(ctx, kv, logf)(s)
}
}
status.Ready(func(kv jetstream.KeyValue) {
if err := views.HearAgain(ctx, kv, []*views.Follower{f}, logf); err != nil {
logf("the hand asks of the first view %s are not heard (%v) until the runner starts again", f.Tool, err)
}
})
}
var (
mu sync.Mutex
made []*views.Follower
)
// A watched branch first named by a registration gets its view of the catalog half (CT10), adopted through the reader (design
// 47 G §532, A-26): the reader keeps its status, raises and clears its findings through the module's writer, hears the
// seat's catalog-revision on the module's consumer and asks its first view at once.
cv := &pipeline.Views{Logf: logf, Open: func(branch string) (pipeline.BranchView, error) {
c, err := meshcatalogclient.NewCatalog(call, branch)
c, _, err := meshcatalogclient.WireCatalogProcess(reader, call, branch, nil)
if err != nil {
return nil, err
}
wire(c.Follower)
mu.Lock()
made = append(made, c.Follower)
mu.Unlock()
go func() {
if err := c.Start(ctx); err != nil {
logf("the first view of the catalog half of %s was not answered: %v", branch, err)
}
}()
return catalogView{c}, nil
}}
regs, _, err := meshcatalogclient.WireRegistrationsProcess(reader, call, func(reg catalog.Registration, gone bool) {
@@ -93,39 +52,10 @@ func startViews(ctx context.Context, js jetstream.JetStream, conn process.Subscr
if err != nil {
return nil, err
}
if err := hub.Handle(catalogRevisions, func(m jetstream.Msg) {
if err := cv.HearRevision(ctx, m.Data()); err != nil {
logf("%s is not taken: %v", m.Subject(), err)
}
_ = m.Ack()
}); err != nil {
return nil, err
}
// A consumer made after a view was asked does not deliver what was said before it: each branch's view asks again.
hub.OnBound(func(created time.Time) {
mu.Lock()
fs := append([]*views.Follower(nil), made...)
mu.Unlock()
for _, f := range fs {
f.ConsumerMade(ctx, created)
}
})
reader.Start(ctx, js, cred, status)
return &runnerViews{Registrations: regs, Catalog: cv}, nil
}
func putFinding(w views.FindingWriter, f views.Finding, raised bool, logf func(string, ...any)) {
var err error
if raised {
err = w.Put(f)
} else {
err = w.Delete(f)
}
if err != nil {
logf("the finding %s is not written: %v", f.Kind, err)
}
}
// catalogView is a branch's meshcatalogclient.Catalog as the runner reads it.
type catalogView struct{ c *meshcatalogclient.Catalog }
modules/mesh-delivery-pipeline/go.mod
+1 -1
View File
@@ -9,8 +9,8 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/findingwriter.go
+33
View File
@@ -0,0 +1,33 @@
package main
import (
"context"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
)
// findingWriter is the teller's mesh-sdk finding writer (design 47 G §461, §689): the teller observes only what its views
// reader finds, so its own observe states none and the writer's statement covers the reader's open findings. The writer
// forms its subjects itself (design 47 G §383, A-13).
func findingWriter(ctx context.Context, cred process.Account, conn *nats.Conn, js jetstream.JetStream, state process.State, logf func(string, ...any)) (*views.Writer[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent], error) {
w, err := views.NewWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
Module: cred.Module, State: state, Node: cred.Node, Asked: condseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf,
Observe: func(context.Context) (views.Observed[condseat.Finding], error) {
return views.Observed[condseat.Finding]{}, nil
},
})
if err != nil {
return nil, err
}
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
return w, nil
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go
+3
View File
@@ -187,6 +187,9 @@ func fullConf(dir string) string {
seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_"+messengerAsks+".$KV."+messengerAsks+".mesh-delivery-teller.>")
// The durable consumer on EVENTS the controller makes for the seat it uses, pulled and acknowledged (issue 1126).
teller = append(teller, "$JS.API.CONSUMER.INFO.EVENTS."+tellerConsumer, "$JS.API.CONSUMER.MSG.NEXT.EVENTS."+tellerConsumer, "$JS.ACK.EVENTS."+tellerConsumer+".>")
// The finding writer finds the stream of the keeper's findings-asked (design 47 G §384): the bed has none, so it is
// said and the findings stay in the state.
teller = append(teller, "$JS.API.STREAM.NAMES")
teller = append(teller, "mesh.mod.mesh-delivery-teller.event.answered") // the manifest's emits (ADR 0596 §2)
teller = append(teller, readOf(viewAnswers)...) // issue 1297 B10: the manifest's reads
// The assigner's asks by its read tool and its views family (ADR 0615, DLV-M3), and the teller's own first-views and
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go
+6 -3
View File
@@ -11,7 +11,6 @@ import (
"context"
"errors"
"fmt"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"io"
"log"
@@ -104,8 +103,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return calls.Emit(answeredEvent, Answered{Key: key, Ask: askID, Answer: answer}, stdio.EmitOptions{MessageID: answeredID(key, askID)})
}
reader := &views.ProcessReader{Module: tellerModule, Call: calls.Call, Logf: logger.Printf, Hub: hub}
findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "a first view unanswered past its bound is said in the log alone", logger.Printf)
reader.Findings = func() views.FindingWriter { return sdkfindings.Writer(ctx, findings.KV) }
writer, err := findingWriter(ctx, cred, conn, js, calls.State("findings"), logger.Printf)
if err != nil {
return fmt.Errorf("the teller's finding writer: %w", err)
}
defer writer.Close()
reader.Findings = func() views.WireWriter { return writer }
asks, err := reader.Follow(assignerModule, assignerAsksState, assignerModule+".assigner_read_asks")
if err != nil {
return err
modules/mesh-delivery-teller/go.mod
+1 -2
View File
@@ -29,7 +29,6 @@ require (
)
require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
)
@@ -43,7 +42,7 @@ require (
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 // indirect
modules/mesh-delivery-view/cmd/mesh-delivery-view/findingwriter.go
+33
View File
@@ -0,0 +1,33 @@
package main
import (
"context"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
)
// findingWriter is the view's mesh-sdk finding writer (design 47 G §461, §689): the view observes only what its views
// reader finds, so its own observe states none and the writer's statement covers the reader's open findings. The writer
// forms its subjects itself (design 47 G §383, A-13).
func findingWriter(ctx context.Context, cred process.Account, conn *nats.Conn, js jetstream.JetStream, state process.State, logf func(string, ...any)) (*views.Writer[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent], error) {
w, err := views.NewWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
Module: cred.Module, State: state, Node: cred.Node, Asked: condseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf,
Observe: func(context.Context) (views.Observed[condseat.Finding], error) {
return views.Observed[condseat.Finding]{}, nil
},
})
if err != nil {
return nil, err
}
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
return w, nil
}
modules/mesh-delivery-view/cmd/mesh-delivery-view/main.go
+8 -9
View File
@@ -22,7 +22,6 @@ import (
"syscall"
"time"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"git.novox.be/novox/mesh-delivery/sdk/trunk"
@@ -125,8 +124,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer calls.Close()
reader := &views.ProcessReader{Module: viewstate.ViewModule, Call: calls.Call, Logf: logger.Printf}
findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "a first view unanswered past its bound is said in the log alone", logger.Printf)
reader.Findings = func() views.FindingWriter { return sdkfindings.Writer(ctx, findings.KV) }
writer, err := findingWriter(ctx, cred, conn, js, calls.State("findings"), logger.Printf)
if err != nil {
return fmt.Errorf("the view's finding writer: %w", err)
}
defer writer.Close()
reader.Findings = func() views.WireWriter { return writer }
for _, o := range ownersFollowed {
if _, err := reader.Follow(o.owner, o.state, o.tool); err != nil {
return err
@@ -196,12 +199,8 @@ type ReachedEvent struct {
Reach json.RawMessage `json:"reach"`
}
// The view's own states of its first views (views.StatusState) and of its findings, where the condition keeper raises a
// first view unanswered past its bound (ADR 0615 §6).
var (
firstViewsBucket = viewstate.ViewModule + "_" + views.StatusState
findingsBucket = viewstate.ViewModule + "_findings"
)
// The view's own state of its first views (views.StatusState).
var firstViewsBucket = viewstate.ViewModule + "_" + views.StatusState
// ownersFollowed are the indexer's and the assigner's states the view's process follows by their first views, each by its
// owner's read tool (as the view's invokes names it) and views family (its events on the shared events stream); the
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/findingwriter.go
+94
View File
@@ -0,0 +1,94 @@
package main
import (
"context"
"sync"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-watchman/internal/watch"
)
type heldWriterOf = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]
// heldFindings is the watchman's findings through mesh-sdk's held finding writer (design 47 G §461, §689, A-29): the writer
// holds what the watchman has put, states it at its start and when the keeper asks, writes and clears as the watchman puts and
// deletes. Here is only the watchman's own key of each finding, which the watchman deletes by.
type heldFindings struct {
mu sync.Mutex
by map[string]condseat.Finding
w *heldWriterOf
}
// seedFindings reads the findings that stand in the state `findings` at the start, by the watchman's key of each.
func seedFindings(state process.State) (*heldFindings, map[string][]byte, error) {
h := &heldFindings{by: map[string]condseat.Finding{}}
seeds := map[string][]byte{}
keys, err := state.Keys()
if err != nil {
return nil, nil, err
}
for _, k := range keys {
e, err := state.Get(k)
if err != nil {
return nil, nil, err
}
if e == nil {
continue
}
f, err := watch.DecodeFinding(e.Value)
if err != nil {
continue
}
h.by[f.Key()] = f.Contract()
seeds[f.Key()] = e.Value
}
return h, seeds, nil
}
// start makes and starts the held writer, which reads the state's findings itself before it states anything.
func (h *heldFindings) start(ctx context.Context, cred process.Account, conn *nats.Conn, js jetstream.JetStream, state process.State, logf func(string, ...any)) error {
w, err := views.NewHeldWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{
Module: cred.Module, State: state, Node: cred.Node, Asked: condseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logf,
})
if err != nil {
return err
}
h.w = w
// A writer that cannot start (the stream of its asks is not there yet) is said; its findings are then kept in the
// state alone and said by the next start (as the node-engine's is).
if err := w.Start(ctx); err != nil {
logf("the finding writer cannot start, its findings are kept in the state alone: %v", err)
}
return nil
}
// Put writes the finding the watchman holds under key.
func (h *heldFindings) Put(ctx context.Context, key string, value []byte) error {
f, err := watch.DecodeFinding(value)
if err != nil {
return err
}
c := f.Contract()
h.mu.Lock()
h.by[key] = c
h.mu.Unlock()
return h.w.Write(ctx, c)
}
// Delete clears the finding the watchman held under key.
func (h *heldFindings) Delete(ctx context.Context, key string) error {
h.mu.Lock()
c, ok := h.by[key]
delete(h.by, key)
h.mu.Unlock()
if !ok {
return nil
}
return h.w.Clear(ctx, c.Scope, c.ID, c.Kind, "the watchman no longer observes it")
}
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go
+11
View File
@@ -71,6 +71,14 @@ func serveOwners(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *
viewstest.Serve(t, ctx, js, nc, watch.IndexerModule, modulesState, watch.IndexerModule+".indexer_read_modules", "n1_"+watch.WatchmanModule)
}
// findingsStream is the watchman's own stream of its findings' events (design 47 G §383), which its finding writer says into.
func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream) {
t.Helper()
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "WATCHMAN_FINDINGS", Subjects: []string{"mesh.stream." + watch.WatchmanModule + ".event.>"}}); err != nil {
t.Fatal(err)
}
}
// TestWatchmanWhole starts the program the way the node does (its entry, run, with its environment and a broker file)
// against an embedded bus with the system exports and JetStream, and proves the four things the issue names.
func TestWatchmanWhole(t *testing.T) { // 1135/AC8
@@ -132,6 +140,7 @@ func TestWatchmanWhole(t *testing.T) { // 1135/AC8
kvs[c.Bucket] = kv
}
serveOwners(t, ctx, js, ctl)
findingsStream(t, ctx, js)
// The controller's consumer on DEAD_LETTER_NOTICES, the dead letters' streams and a source stream with one message a
// consumer will give up on (1952/WM5); no lease bucket at all (issue 1126).
@@ -383,6 +392,7 @@ func anAskRunningBeforeTheStartGetsItsPulse(t *testing.T) { // 1909
kvs[c.Bucket] = kv
}
serveOwners(t, ctx, js, ctl)
findingsStream(t, ctx, js)
const id = "m0000009ffffff"
body, err := buildstep.EncodeStep(buildstep.Step{ID: id, Tool: "build_go_test", Node: "n1", State: buildstep.StepRunning, Started: time.Now().UTC()})
if err != nil {
@@ -470,6 +480,7 @@ func aWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) {
kvs[c.Bucket] = kv
}
serveOwners(t, ctx, js, ctl)
findingsStream(t, ctx, js)
fpNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
putWant := func(node, module, fp string) {
modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go
+25 -22
View File
@@ -18,8 +18,6 @@ import (
"syscall"
"time"
sdkfindings "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/findings"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -51,12 +49,13 @@ func main() {
}
}
// kvOut writes the watchman's four buckets.
// kvOut writes the watchman's three buckets and its findings, which go through mesh-sdk's finding writer.
type kvOut struct {
ctx context.Context
presence, pulse, stuk, findings jetstream.KeyValue
js jetstream.JetStream
cred process.Account
ctx context.Context
presence, pulse, stuk jetstream.KeyValue
findings *heldFindings
js jetstream.JetStream
cred process.Account
}
// Say publishes an event of the seat mesh-watchdogs, said under the watchman's own account (design 47 G §389, §436).
@@ -74,10 +73,7 @@ func (o *kvOut) Say(subject string, body []byte, id string) error {
return err
}
func (o *kvOut) PutFinding(key string, value []byte) error {
_, err := o.findings.Put(o.ctx, key, value)
return err
}
func (o *kvOut) PutFinding(key string, value []byte) error { return o.findings.Put(o.ctx, key, value) }
func (o *kvOut) DeleteFinding(key string) error { return o.findings.Delete(o.ctx, key) }
@@ -233,11 +229,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Its own presence, pulses and stuck rows are what it keeps: each stops the start, naming its grant. The assigner's wants
// and the node-engine's haves only make the pulses of a move: without the haves the watchman watches presence and the
// build seats' steps, and says that moves are not watched (issue 1945).
var kvs struct{ presence, pulse, stuck, findings, haves jetstream.KeyValue }
var kvs struct{ presence, pulse, stuck, haves jetstream.KeyValue }
for _, b := range []struct {
name string
into *jetstream.KeyValue
}{{watch.PresenceBucket, &kvs.presence}, {watch.PulseBucket, &kvs.pulse}, {watch.StuckBucket, &kvs.stuck}, {watch.FindingsBucket, &kvs.findings}} {
}{{watch.PresenceBucket, &kvs.presence}, {watch.PulseBucket, &kvs.pulse}, {watch.StuckBucket, &kvs.stuck}} {
if *b.into, err = process.EssentialBucket(ctx, js, "watchman", b.name, "say who is online or what stopped moving"); err != nil {
return err
}
@@ -255,10 +251,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The module has one durable consumer of EVENTS: one hub pulls it for the views reader, the indexer's take-in events
// and the seats' events, started once every part is registered.
hub := process.NewEventHub(logger.Printf)
// Its findings: those that stand in its state are read first, so the writer's statement at its start says them and
// deletes none (the watchman reconciles them below, once it has read what it observes).
findingsState := calls.State("findings")
held, seeds, err := seedFindings(findingsState)
if err != nil {
return fmt.Errorf("the watchman's findings cannot be read: %w", err)
}
if err := held.start(ctx, cred, conn, js, findingsState, logger.Printf); err != nil {
return fmt.Errorf("the watchman's finding writer: %w", err)
}
defer held.w.Close()
reader := &views.ProcessReader{Module: watch.WatchmanModule, Call: calls.Call, Logf: logger.Printf, Hub: hub,
Findings: func() views.FindingWriter {
return sdkfindings.Writer(ctx, func() jetstream.KeyValue { return kvs.findings })
}}
Findings: func() views.WireWriter { return held.w }}
wants, err := reader.Follow(watch.AssignerModule, wantsState, watch.AssignerModule+".assigner_read_wants")
if err != nil {
return err
@@ -268,7 +273,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings, js: js, cred: cred}, time.Now)
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: held, js: js, cred: cred}, time.Now)
w.Logf(logger.Printf)
// What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end.
w.Awaits(watch.AssignerWantsBucket)
@@ -282,6 +287,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
w.SetBound(d)
}
for key, value := range seeds {
w.SeedFinding(key, value)
}
// What stands now: presence and stuck as written, pulses as the server holds them, then the checks.
for _, s := range []struct {
kv jetstream.KeyValue
@@ -292,11 +300,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
w.SeedPresence(e.Key(), p.Online, e.Created())
}
}},
{kvs.findings, func(e jetstream.KeyValueEntry) {
if opOf(e) == watch.Put {
w.SeedFinding(e.Key(), e.Value())
}
}},
{kvs.stuck, func(e jetstream.KeyValueEntry) {
if opOf(e) == watch.Put {
w.SeedStuck(e.Key(), e.Value())
modules/mesh-delivery-watchman/go.mod
+1 -1
View File
@@ -47,7 +47,7 @@ replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
require git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 // indirect
modules/mesh-delivery-watchman/internal/watch/busstate.go
+2 -4
View File
@@ -83,8 +83,7 @@ func (w *Watchman) Accepted(a meshcatalog.ManifestAcceptedEvent) {
Explanation: "The indexer accepted a manifest of " + a.Module + " asking the bus module for " + e.Name + ", and the bus " +
"module said neither created nor refused within " + OverdueBound.String() + ": it is down, or behind. " + a.Module +
" fails at its start while the entry's binding is missing (novox/hq ADR 0595 §1).",
Needs: "a look at the bus module's health and its process's log; nats_first_view_again asks it to read the verdicts again",
Resolved: "the bus module said " + e.Name + " of " + a.Module + " created, or a later manifest no longer asks for it"}})
Needs: "a look at the bus module's health and its process's log; nats_first_view_again asks it to read the verdicts again"}})
}
for _, local := range w.busStateEntriesOf(a.Module) {
if asked[local] {
@@ -151,8 +150,7 @@ func (w *Watchman) BrokerRefused(s BrokerSaid) {
Headline: "The bus state " + s.Local + " of " + s.Module + " was refused",
Explanation: "The bus module could not create " + s.Local + " for " + s.Module + ", so no binding is handed and " + s.Module +
" fails at its start, naming the entry (novox/hq ADR 0595 §1): " + why,
Needs: "a pull request that mends the entry in the manifest of " + s.Module + ", or the bus module's cause fixed",
Resolved: "the bus module said " + s.Local + " of " + s.Module + " created"})
Needs: "a pull request that mends the entry in the manifest of " + s.Module + ", or the bus module's cause fixed"})
}
// busStateEntriesOf are the entries of module this watchman waits on or holds a finding of, by local name, sorted.
modules/mesh-delivery-watchman/internal/watch/finding.go
+51 -14
View File
@@ -1,20 +1,23 @@
package watch
import (
"encoding/json"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
)
// The watchman is an observer of the seat mesh-watchdogs (novox/hq ADR 0564 §2, §4): what it observes wrong it writes as a
// finding, one key `<scope>.<id>.<kind>` of its own bucket `findings` in the seat mesh-conditions' layout (meshconditionsclient), only when what
// it says changes, and it deletes the key when it no longer observes it. The condition keeper (mesh-conditions) raises and
// finding, one key `<scope>.<id>.<kind>` of its own state `findings` in the seat mesh-conditions' generated layout, written
// through mesh-sdk's finding writer (design 47 G §689), only when what it says changes, and it deletes the key when it no longer observes it. The condition keeper (mesh-conditions) raises and
// clears the conditions from them; the watchman writes no condition and reads none (Limits 11).
// FindingsBucket is the watchman's findings.
var FindingsBucket = mcc.FindingsBucket(Observer)
var FindingsBucket = Observer + "_findings"
// Observer is the name a finding of the watchman carries.
const Observer = "mesh-delivery-watchman"
@@ -43,7 +46,7 @@ type Finding struct {
Severity string
Summary, Said string
Headline, Explanation string
Needs, Resolved string
Needs string
Source string
Observer string
// Since is when what it says began, as the watchman saw it: the finding's time and its observation's, so the same
@@ -63,22 +66,56 @@ func (f Finding) Key() string {
return FindingKey(f.Scope, f.ID, token)
}
// EncodeFinding is f in the seat's layout, refusing one the keeper could not make a condition of.
func EncodeFinding(f Finding) ([]byte, error) {
return mcc.EncodeFinding(mcc.Observation{Scope: f.Scope, ID: f.ID, Kind: f.Kind, Ref: f.Token, Machine: f.Machine, Also: f.Also,
Severity: f.Severity, Summary: f.Summary, Said: f.Said, Headline: f.Headline, Explanation: f.Explanation,
Resolved: f.Resolved, Needs: f.Needs, Source: f.Source}.Finding(Observer, "", f.Since))
// KeyID is an id as the parts of a key: each dot-separated part made safe for a bus key, as the key of a condition is
// (mcc.Key), so the finding's key keeps the shape its condition had.
func KeyID(id string) string {
parts := strings.Split(id, ".")
for i, p := range parts {
p = unsafeKey.ReplaceAllString(strings.TrimSpace(p), "_")
if p == "" {
p = "_"
}
parts[i] = p
}
return strings.Join(parts, ".")
}
// Contract is f as the seat mesh-conditions' generated finding (design 47 G §686, §689): a ref, where the finding has a
// token other than its kind, goes into the id as `<id>.<ref>` (A-14), and is kept in `ref` so the finding reads back as
// it was written. The resolved words are not a field of the seat (A-22), and its `at` is its `since`, so the same thing
// observed again writes the same bytes.
func (f Finding) Contract() condseat.Finding {
id, ref := KeyID(f.ID), ""
if f.Token != "" && f.Token != f.Kind {
ref = f.Token
id = id + "." + KeyID(ref)
}
v := condseat.Finding{Scope: f.Scope, ID: id, Kind: f.Kind, Ref: ref, Severity: f.Severity, Summary: f.Summary,
Headline: f.Headline, Explanation: f.Explanation, Needs: f.Needs, Source: f.Source, Observer: Observer,
At: f.Since.UTC(), Since: f.Since.UTC(),
Subject: condseat.Subject{Scope: f.Scope, ID: id, Machine: f.Machine, Also: f.Also}}
if f.Said != "" {
v.Evidence = []condseat.Evidence{{At: f.Since.UTC(), Said: f.Said}}
}
return v
}
// EncodeFinding is f in the seat's generated layout.
func EncodeFinding(f Finding) ([]byte, error) { return json.Marshal(f.Contract()) }
// DecodeFinding reads a finding the watchman wrote.
func DecodeFinding(data []byte) (Finding, error) {
h, err := mcc.DecodeFinding(data)
if err != nil {
var h condseat.Finding
if err := json.Unmarshal(data, &h); err != nil {
return Finding{}, err
}
f := Finding{Scope: h.Subject.Scope, ID: h.Subject.ID, Token: h.Ref, Kind: h.Kind, Machine: h.Subject.Machine,
id := h.ID
if h.Ref != "" {
id = strings.TrimSuffix(id, "."+h.Ref)
}
f := Finding{Scope: h.Scope, ID: id, Token: h.Ref, Kind: h.Kind, Machine: h.Subject.Machine,
Also: h.Subject.Also, Severity: h.Severity, Summary: h.Summary, Headline: h.Headline, Explanation: h.Explanation,
Needs: h.Needs, Resolved: h.Resolved, Source: h.Source, Observer: h.Observer, Since: h.At}
Needs: h.Needs, Source: h.Source, Observer: h.Observer, Since: h.Since}
if len(h.Evidence) > 0 {
f.Said = h.Evidence[0].Said
}
@@ -163,7 +200,7 @@ func (w *Watchman) NotHeard(family, consequence string, why error) {
w.say("%s", summary)
w.find(Finding{Scope: ScopeModule, ID: heard.ID(Observer, family), Token: heard.Kind, Kind: heard.Kind, Severity: Warning,
Source: Observer + "." + heard.Kind, Since: w.now().UTC().Truncate(time.Second), Summary: summary, Headline: headline,
Explanation: explanation, Resolved: "the watchman's consumer of " + family + " is bound"})
Explanation: explanation})
}
// Heard ends the finding that the watchman does not hear family.
modules/mesh-delivery-watchman/internal/watch/findings_test.go
+34 -2
View File
@@ -1,6 +1,7 @@
package watch
import (
"encoding/json"
"errors"
"strings"
"sync/atomic"
@@ -9,7 +10,7 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
@@ -325,7 +326,8 @@ func TestTheWatchmansFindings(t *testing.T) {
case !row.noState && row.want == "" && stands:
t.Errorf("%s: the finding stands once bound", row.name)
case row.want != "":
f, err := mcc.DecodeFinding(raw)
var f condseat.Finding
err := json.Unmarshal(raw, &f)
// Review round 2: it says it clears at the part's next start, since Bind runs at the start.
if err != nil || f.Kind != heard.Kind || f.Summary != row.want || f.Observer != "mesh-delivery-indexer" || !strings.Contains(f.Explanation, "next start") {
t.Errorf("%s: the finding %+v (%v), want summary %q", row.name, f, err, row.want)
@@ -820,3 +822,33 @@ func takeInBody(t *testing.T, kind, key string, since time.Time) []byte {
}
return b
}
// Design 47 G §689, A-14, A-22: a token other than the kind goes into the contract finding's id and reads back as it was
// written; the finding's time is its `since`; the resolved words are not on the wire.
func TestAFindingIsTheContractsWithItsRefInItsID(t *testing.T) {
since := time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC)
for _, f := range []Finding{
{Scope: ScopeModule, ID: "web.ace", Token: "unhealthy", Kind: KindUnhealthy, Machine: "ace", Severity: Warning, Source: "health", Since: since, Summary: "s", Said: "said"},
{Scope: ScopeModule, ID: "web", Token: KindBusStateOverdue + "-state", Kind: KindBusStateOverdue, Severity: Warning, Source: "bus", Since: since, Summary: "s"},
{Scope: ScopeMachine, ID: "ace", Kind: KindMachineUnits, Severity: Warning, Source: "units", Since: since, Summary: "s", Needs: "mend"},
} {
c := f.Contract()
wantID := f.ID
if f.Token != "" && f.Token != f.Kind {
wantID = f.ID + "." + f.Token
}
if c.ID != wantID || c.Kind != f.Kind || !c.At.Equal(since) || !c.Since.Equal(since) || c.Subject.ID != wantID {
t.Errorf("%+v: contract %+v, want id %q, kind %q, at and since %v", f, c, wantID, f.Kind, since)
}
raw, err := EncodeFinding(f)
if err != nil {
t.Fatal(err)
}
if back, err := DecodeFinding(raw); err != nil || back.ID != f.ID || back.Token != f.Token || back.Said != f.Said || back.Needs != f.Needs {
t.Errorf("%+v read back as %+v (%v)", f, back, err)
}
if strings.Contains(string(raw), "esolved") {
t.Errorf("%+v: resolved words are on the wire: %s", f, raw)
}
}
}
modules/mesh-delivery-watchman/internal/watch/nodehealth.go
+3 -6
View File
@@ -198,8 +198,7 @@ func unitsOf(node string, u *nodeengine.UnitsHealth) (Finding, bool) {
"mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")),
Said: strings.Join(said, "; "),
Headline: strings.ToUpper(failed[:1]) + failed[1:] + " on " + node,
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node),
Resolved: "No failed services on " + node + " any more"}, true
Needs: fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node)}, true
}
// reloginOf is one finding per module whose accounts wait for a new login.
@@ -251,8 +250,7 @@ func foundOf(node string, dirs []nodeengine.NodeDirectory) []Finding {
Headline: fmt.Sprintf("%s waits for a directory on %s", m, node),
Explanation: fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+
"and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, node, m),
Needs: "hand the directory over from the control-node, as the operator; the details name it and the line to type.",
Resolved: fmt.Sprintf("%s's directory on %s is the mesh's", m, node)})
Needs: "hand the directory over from the control-node, as the operator; the details name it and the line to type."})
}
return out
}
@@ -266,8 +264,7 @@ func fallbackOf(node string, f *nodeengine.LoginFallback) (Finding, bool) {
return Finding{Scope: ScopeMachine, ID: node, Token: KindAccountFallback, Kind: KindAccountFallback, Machine: node, Severity: Warning,
Source: "health-report", Since: f.At, Summary: said, Said: said,
Headline: fmt.Sprintf("%s fell back to its old bus login", node),
Needs: fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node),
Resolved: fmt.Sprintf("%s no longer falls back to its old bus login", node)}, true
Needs: fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node)}, true
}
func orModule(m string) string {
modules/mesh-delivery-watchman/internal/watch/signals.go
+1 -3
View File
@@ -763,7 +763,7 @@ func (w *Watchman) applyCheck(node string) {
}
}
f := Finding{Scope: ScopeMachine, ID: node, Kind: KindApplyFailing, Machine: node, Severity: Warning, Source: "S21", Since: since,
Said: strings.Join(words, "; "), Resolved: node + " applies what it is sent again", Headline: node + " is not applying its update"}
Said: strings.Join(words, "; "), Headline: node + " is not applying its update"}
switch {
case len(applied) == 0:
f.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", node,
@@ -845,7 +845,6 @@ func (w *Watchman) Closed(kind string, c BusClosure) {
f.Headline = clip(capital(who)+" was too slow for the bus", 60)
f.Explanation = capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus closed its connection (%s "+
"in the last hour) and what was on its way was lost. It connects again by itself; this clears an hour after the last time.", who, count))
f.Resolved = "Resolved: " + who + " kept up for an hour"
} else {
reason := strings.ToLower(c.Reason)
f.Said = fmt.Sprintf("the bus closed %s's connection %d: %s", c.User, c.CID, c.Reason)
@@ -854,7 +853,6 @@ func (w *Watchman) Closed(kind string, c BusClosure) {
f.Headline = clip(capital(who)+" was cut off by the bus", 60)
f.Explanation = capital(fmt.Sprintf("The bus closed the connection of %s because it broke the bus's rules (%s, %s in the "+
"last hour), so what it was sending was lost. This clears an hour after the last time.", who, reason, count))
f.Resolved = "Resolved: " + who + " kept to the bus's rules for an hour"
}
w.find(f)
if quiet := AdvisoryQuiet - w.now().Sub(last); quiet > 0 {
modules/mesh-delivery-watchman/manifest_test.go
+11 -11
View File
@@ -47,17 +47,17 @@ type claim struct {
}
type manifest struct {
Module string `json:"module"`
Claims []claim `json:"claims"`
Tools []string `json:"tools"`
Moved bool `json:"moved"`
Version string `json:"version"`
Slug string `json:"slug"`
Reads []string `json:"reads"`
Consumes []string `json:"consumes"`
Invokes []string `json:"invokes"`
State []state `json:"state"`
Streams []struct {
Module string `json:"module"`
Claims []claim `json:"claims"`
Tools []string `json:"tools"`
Moved bool `json:"moved"`
Version string `json:"version"`
Slug string `json:"slug"`
Reads []string `json:"reads"`
Consumes []string `json:"consumes"`
Invokes []string `json:"invokes"`
State []state `json:"state"`
Streams []struct {
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`