Compare commits

..
Author SHA1 Message Date
jschoubben 236fe47e4a mesh-grants: derive that a bundle loads from build.artifacts, as the indexer's manifest has no derived bundles, so route-proxy's helper is carried (issue 1945) 2026-10-26 11:26:20 +01:00
mesh-tools 32bc4f3c48 Merge pull request 'Checker: read every test result of a level and name every failure in the verdict, the try and the status line (issue 1921)' (#62) from fix/1921-every-failure-r into main 2026-10-26 10:18:11 +00:00
mesh-tools ed1125b66d Merge pull request 'Assigner: take a stale ask away only once its held condition's clear was said, so a failed clear is retried (issue 1906)' (#61) from fix/1906-clear-retried into main 2026-10-26 10:09:42 +00:00
jschoubben 366bc750da Assigner: clear a stale hold's condition before taking its ask away, so a failed clear is retried at the next pass (issue 1906)
mesh/delivery merged: source.main moved to novox/mesh-delivery@ed1125b66d91
mesh/check pass 366bc750da87 on ed1125b66d91 | unit 168 run, 489 kept | integration 21 run, 712 kept | composition passed on ace
2026-10-26 11:03:09 +01:00
jschoubben 04a56b7aff Name every failed test in a check's verdict and a try's failed list (issue 1921)
mesh/delivery merged: source.main moved to novox/mesh-delivery@32bc4f3c48b2
mesh/delivery-group group fix/1921-every-failure-r merged: every member is merged
mesh/check checking 04a56b7aff66 on 32bc4f3c48b2
The checker stopped reading a level's test results at the first that did
not pass and kept only its first failure, so a check named one fault per
round. It now reads and counts every result and writes every failed test
with its first line into the verdict's Failures and the try's failed
list (at most 50, then how many more); Failed keeps the first one's
words whole (issue 1901). Pins mesh-sdk fix/1921-every-failure.
2026-10-26 10:58:13 +01:00
19 changed files with 302 additions and 320 deletions
@@ -481,13 +481,19 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]delivery.Module
}
for key := range p.w.read(delivery.AssignerAsksBucket) {
if p.staleAsk(key, trunk) {
// The condition is cleared first and the ask goes only once that was said: the ask is what brings this key back at the
// next pass, so a clear that failed once is made again, with no state of its own (issue 1906, note 29).
k, kerr := delivery.ParseAnyAskKey(key)
held := kerr == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore)
if held && !p.clearHeld(k.Module, k.FP) {
continue
}
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away", key)
p.remove(ctx, delivery.AssignerAsksBucket, key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
// exactly it. Only the condition, which would else ask for an answer that can no longer be given, is cleared, and it is
// raised again if the trunk returns to the fingerprint.
if k, err := delivery.ParseAnyAskKey(key); err == nil && (k.Kind == delivery.AskKindHold || k.Kind == delivery.AskKindRestore) {
p.clearHeld(k.Module, k.FP)
if held {
delete(p.said, assign.HeldCondition(k.Module, k.FP, "", "", false, false).Key+".raised")
}
}
@@ -566,16 +572,18 @@ func (p *passer) answeredWith(key, option string) bool {
}
// clearHeld clears, once, the condition of a hold the operator released.
func (p *passer) clearHeld(module, fp string) {
// It says whether nothing is left to say: the clear was said now or before, or there is no router to say it to.
func (p *passer) clearHeld(module, fp string) bool {
c := assign.HeldCondition(module, fp, "", "", false, false)
if p.say == nil || p.said[c.Key+".cleared"] {
return
return true
}
if err := p.say(c); err != nil {
p.logf("the condition of the released %s could not be cleared, and is cleared at the next pass: %v", module, err)
return
return false
}
p.said[c.Key+".cleared"] = true
return true
}
// recordAsk writes, once, the ask about a move of a module of policy record that waits for the operator's yes.
@@ -606,6 +606,36 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
})
}
t.Run("a clear that failed to be said is said at the next pass, and the ask goes after it", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
var said []asks.Condition
fail := true
p.say = func(c asks.Condition) error {
if fail {
fail = false
return errors.New("router unreachable")
}
said = append(said, c)
return nil
}
a := assign.HoldAsk(holdOld, "web", fpOld, "novox", "w", 1, time.Now())
v, err := delivery.EncodeAssignerAsk(holdOld, a)
if err != nil {
t.Fatal(err)
}
asksKV.values[holdOld] = v
p.w.put(delivery.AssignerAsksBucket, holdOld, v)
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; !there || len(said) != 0 {
t.Fatalf("after a failed clear: ask left = %v, conditions said = %d; want the ask kept and nothing said", there, len(said))
}
p.staleAsks(context.Background(), trunkOf(fpNew))
if _, there := asksKV.values[holdOld]; there || len(said) != 1 || said[0].State != asks.StateCleared {
t.Errorf("at the next pass: ask left = %v, conditions said = %v; want the ask gone and the clear said", there, said)
}
})
t.Run("a lift takes the restore asks of its fingerprint on every node", func(t *testing.T) {
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -4
View File
@@ -1,7 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -0,0 +1,90 @@
package check
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// failingResult is a level's test result in which the named tests failed, each in its own package.
func failingResult(t *testing.T, tests int, failed ...Failure) json.RawMessage {
t.Helper()
raw, err := json.Marshal(Tested{Level: delivery.LevelIntegration, Tests: tests, Elapsed: 2, Failed: failed, Passed: len(failed) == 0})
if err != nil {
t.Fatal(err)
}
return raw
}
func judgeLevel(t *testing.T, tests ...json.RawMessage) (*delivery.Verdict, *state) {
t.Helper()
st := &state{j: &Examiner{}, c: &Case{}, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
v, _ := st.level(t.Context(), &target{pull: Pull{Repository: "novox/x"}}, delivery.LevelAnswer{Level: delivery.LevelIntegration, Tests: tests})
return v, st
}
// Issue 1921: two failing tests in two packages are two entries of the verdict's Failures and two lines of a try's
// failed list, each with its own first line (its cause, issue 1901), whether the build module answered them in one test
// result (one go test run over both packages) or in two (the changed packages first, then the rest); every result is
// counted, the ones after a failure too; and Failed keeps the first one's words whole.
func TestTwoFailingTestsInTwoPackagesAreTwoEntries(t *testing.T) {
a := Failure{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: want 1, got 2", Output: " a_test.go:3: logged\n a_test.go:9: want 1, got 2\n--- FAIL: TestA (0.00s)"}
b := Failure{Package: "x/b", Test: "TestB", Cause: "b_test.go:4: boom", Output: " b_test.go:4: boom\n--- FAIL: TestB (0.00s)"}
for name, tests := range map[string][]json.RawMessage{
"one result": {failingResult(t, 10, a, b)},
"two results": {failingResult(t, 4, a), failingResult(t, 3), failingResult(t, 3, b)},
} {
v, st := judgeLevel(t, tests...)
if v == nil {
t.Fatalf("%s: the level passed", name)
}
want := []string{"integration: x/a.TestA: a_test.go:9: want 1, got 2", "integration: x/b.TestB: b_test.go:4: boom"}
if got := failedLines(*v); strings.Join(got, "\n") != strings.Join(want, "\n") {
t.Errorf("%s: failed lines\n%s\nwant\n%s", name, strings.Join(got, "\n"), strings.Join(want, "\n"))
}
if len(v.Failures) != 2 || v.Failed.Test != "integration: x/a.TestA" || !strings.Contains(v.Failed.Words, "a_test.go:3: logged") {
t.Errorf("%s: verdict %+v", name, v)
}
if run := st.levels[delivery.LevelIntegration].run; run != 10 {
t.Errorf("%s: %d tests counted, want all 10", name, run)
}
if _, err := delivery.EncodeVerdict(delivery.Verdict{Group: "g", Heads: "0123456789ab", State: v.State, Failed: v.Failed, Failures: v.Failures}); err != nil {
t.Errorf("%s: the verdict is not written: %v", name, err)
}
}
}
// Issue 1921: past delivery.MaxFailures the list names the first fifty and how many more, the status line says how many
// failed, and each entry's line is cut to MaxFailureLine.
func TestFailuresAreCappedWithHowManyMore(t *testing.T) {
var fs []Failure
for i := range 60 {
fs = append(fs, Failure{Package: fmt.Sprintf("x/p%d", i), Test: "TestT", Cause: strings.Repeat("y", 500)})
}
v, _ := judgeLevel(t, failingResult(t, 60, fs...))
if len(v.Failures) != delivery.MaxFailures+1 || v.Failures[delivery.MaxFailures].Test != "and 10 more" {
t.Fatalf("%d entries, last %+v", len(v.Failures), v.Failures[len(v.Failures)-1])
}
if len(v.Failures[0].Words) > MaxFailureLine {
t.Errorf("an entry's line is %d bytes", len(v.Failures[0].Words))
}
if n := failedCount(*v); n != 60 {
t.Errorf("failedCount = %d, want 60", n)
}
cs := &Case{Pulls: []Pull{{Repository: "novox/x", Head: "0123456789abcdef", Trunk: "fedcba9876543210"}}}
if line := Line(*v, cs, cs.Pulls[0], nil, nil, nil); !strings.HasPrefix(line, "failed (60 tests): integration: x/p0.TestT") {
t.Errorf("line %q", line)
}
}
// A verdict that names no list (a conflict, a refused step, a verdict written before issue 1921) is one failed line, as
// before.
func TestAVerdictWithoutFailuresIsOneLine(t *testing.T) {
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "g14", Words: "files in conflict: go.mod\nmore"}}
if got := failedLines(v); len(got) != 1 || got[0] != "g14: files in conflict: go.mod" {
t.Fatalf("%q", got)
}
}
@@ -341,7 +341,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
}
}
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
// after it (so one check names every failure), and no later level runs.
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
@@ -357,8 +359,8 @@ func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { //
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
}
if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
@@ -438,8 +438,9 @@ func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *deliver
}
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
// verdict. It answers the level's running time (its tests' and setup's).
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
// running time (its tests' and setup's).
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
level := la.Level
a := st.levels[level]
@@ -461,6 +462,7 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
a.kept += k.Tests
}
elapsed := 0.0
var failedResults []Tested
for _, raw := range la.Tests {
var res Tested
if err := json.Unmarshal(raw, &res); err != nil {
@@ -489,9 +491,12 @@ func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer)
}
}
if !res.Passed {
return levelFailure(t, level, res), elapsed
failedResults = append(failedResults, res)
}
}
if len(failedResults) > 0 {
return levelFailure(t, level, failedResults...), elapsed
}
return nil, elapsed
}
@@ -633,23 +638,60 @@ func firstLine(s string) string {
return ""
}
// failure turns a level that did not pass into a verdict naming what failed first.
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
// MaxFailureLine is the most bytes of a failure's line in a verdict's Failures: the first line of its words, which is
// its cause where the build module told one apart (issue 1901). Failed keeps the first failure's words whole.
const MaxFailureLine = 300
// levelFailure turns a level that did not pass into a verdict: Failed names what failed first with its words whole,
// and Failures every failed test of the level's results with its first line, at most delivery.MaxFailures and an entry
// saying how many more (issue 1921).
func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict {
v := delivery.Verdict{State: delivery.VerdictFail}
switch {
case len(res.Failed) > 0:
f := res.Failed[0]
name := failedName(f)
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
var all []delivery.FailedFirst
for _, res := range results {
if len(res.Failed) == 0 {
why := res.Why
if why == "" {
why = "the " + level + " level did not pass"
}
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why})
continue
}
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
default:
why := res.Why
if why == "" {
why = "the " + level + " level did not pass"
for _, f := range res.Failed {
name := failedName(f)
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
}
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
all = append(all, delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: "the " + level + " level did not pass"})
}
v.Failed = all[0]
v.Failures = failureLines(all)
return &v
}
// failureLines is every failure with the first line of its words, cut to MaxFailureLine, at most delivery.MaxFailures
// and then one entry saying how many more failed.
func failureLines(all []delivery.FailedFirst) []delivery.FailedFirst {
n := min(len(all), delivery.MaxFailures)
out := make([]delivery.FailedFirst, 0, n+1)
for _, f := range all[:n] {
out = append(out, delivery.FailedFirst{Test: f.Test, Machine: f.Machine, Words: clipLine(firstLine(f.Words), MaxFailureLine)})
}
if more := len(all) - n; more > 0 {
out = append(out, delivery.FailedFirst{Test: fmt.Sprintf("and %d more", more)})
}
return out
}
// clipLine cuts a line to at most n bytes on a whole character, marking the cut.
func clipLine(s string, n int) string {
if len(s) <= n {
return s
}
s = strings.ToValidUTF8(s[:n-3], "")
return s + "..."
}
@@ -49,6 +49,34 @@ func firstFailed(v delivery.Verdict) string {
return firstWordsLine(f.Words)
}
// failedLines is everything a verdict names as failed, one line each: every failed test with its first line (issue
// 1921), or what failed first when the verdict names no list (a conflict, a refused step, an older verdict).
func failedLines(v delivery.Verdict) []string {
if len(v.Failures) == 0 {
if f := firstFailed(v); f != "" {
return []string{f}
}
return nil
}
lines := make([]string, 0, len(v.Failures))
for _, f := range v.Failures {
lines = append(lines, firstFailed(delivery.Verdict{Failed: f}))
}
return lines
}
// failedCount is how many tests a verdict names as failed, the ones after delivery.MaxFailures counted too.
func failedCount(v delivery.Verdict) int {
n := len(v.Failures)
if n > delivery.MaxFailures {
var more int
if _, err := fmt.Sscanf(v.Failures[n-1].Test, "and %d more", &more); err == nil {
return delivery.MaxFailures + more
}
}
return n
}
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
func firstWordsLine(w string) string { return firstLine(w) }
@@ -155,6 +183,9 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
pc.head = "pass " + commits
default:
pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
pc.head = fmt.Sprintf("failed (%d tests): %s", n, firstFailed(v))
}
pc.commits = commits
}
var counts []string
@@ -69,7 +69,9 @@ type Try struct {
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
Said string `json:"said,omitempty"`
// Took is the time from its start to its end, as Go writes a duration.
Took string `json:"took,omitempty"`
Took string `json:"took,omitempty"`
// Failed is every test that failed, each with its first line, at most delivery.MaxFailures and then how many more
// (issue 1921); or what else failed first (a conflict, a refused step).
Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"`
@@ -359,9 +361,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
t.State = TryPass
default:
t.State = TryFail
if f := firstFailed(v); f != "" {
t.Failed = []string{f}
}
t.Failed = failedLines(v)
}
if t.Tests != "" {
t.Said += " | only the tests " + t.Tests + " (Go); nothing kept"
@@ -125,15 +125,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
})
}
f.Hear = func(handle func(string, []byte)) error {
for _, event := range grants.Consumes {
event := event
if _, err := nc.Subscribe(grants.EventSubject(event), func(m *nats.Msg) { handle(event, m.Data) }); err != nil {
return fmt.Errorf("hearing %s: %w", event, err)
}
}
return nil
}
logger.Printf("started on the account %s: publishing memberships and creating the declared buckets, logs and streams the bus lacks; the user list, grants and any change to a held stream in shadow", account.User)
f.Run(ctx)
return nil
+22 -101
View File
@@ -16,7 +16,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -49,8 +48,6 @@ type fakeBus struct {
// creates are the streams the part asked to create, refused when refuseCreate.
creates []string
refuseCreate bool
// rootFree answers the controller's root-free verb: nil is no answerer at all (no responders).
rootFree func(machines []string) (any, error)
}
func (b *fakeBus) LastMembership(node, module string) ([]byte, uint64, bool, error) {
@@ -169,15 +166,6 @@ func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
return json.Marshal(map[string]any{"users": out})
case grants.CallStreams:
return json.Marshal(map[string]any{"detail": b.streams})
case grants.CallRootFree:
if b.rootFree == nil {
return nil, errors.New("nats: no responders available for request")
}
out, err := b.rootFree(args["machines"].([]string))
if err != nil {
return nil, err
}
return json.Marshal(out)
}
return nil, fmt.Errorf("no tool %s", key)
}
@@ -312,7 +300,6 @@ func TestGrants(t *testing.T) {
}{
{"reads", m.Reads, grants.Reads},
{"invokes", m.Invokes, grants.Invokes},
{"consumes", m.Consumes, grants.Consumes},
{"emits", m.Emits, grants.Emits},
{"tools", m.Tools, grants.Tools},
} {
@@ -419,6 +406,28 @@ func TestGrants(t *testing.T) {
}
})
}
// A module whose only runtime-loaded part is a helper bundle (route-proxy's route-provider) says `loads` in
// the repository manifest's build.artifacts; the indexer's manifest has no derived Bundles (issue 1945).
for _, row := range []struct {
name, build string
carried bool
}{
{"a bundle that says loads is carried by node-tools", `,"build":{"artifacts":[{"name":"helper","kind":"bundle","language":"go","system":"arch","from":"cmd/h","binary":"h","loads":["h"]}]}`, true},
{"a module with no loading bundle is left to its own account", ``, false},
} {
t.Run(row.name, func(t *testing.T) {
b := fixture()
b.manifestOf("mod-z", `{"module":"mod-z","version":"1","own-secrets":{"broker":"/var/lib/z/broker"}`+row.build+`}`)
b.assign("a", "mod-z")
b.manifestOf("node-tools", `{"module":"node-tools","version":"1","own-secrets":{"broker":"/var/lib/nt/broker"}}`)
b.assign("a", "node-tools")
users := shadowOf(t, b)
tools := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.node-tools" })]
if got := slices.Contains(tools.Subscribe, "mesh.mod.mod-z.tool.>"); got != row.carried {
t.Errorf("a.node-tools subscribes to mod-z's tools = %v, want %v: %v", got, row.carried, tools.Subscribe)
}
})
}
b := fixture()
users := shadowOf(t, b)
y := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "b.mod-y" })]
@@ -427,94 +436,6 @@ func TestGrants(t *testing.T) {
}
})
t.Run("root-free: the controller's word fills the composition, failing closed (issue 1945)", func(t *testing.T) {
const router = `{"module":"messenger","version":"1","runs-as":"messenger","resources":[{"id":"account","type":"user","name":"messenger","shell":"/usr/bin/nologin","home":"/var/lib/messenger"}],"own-secrets":{"broker":"/var/lib/m/broker"},"secrets-owner":"messenger",` +
`"seats":[{"name":"operator-channel","scope":"mesh","serves":["open"]},{"name":"channel","scope":"mesh","kinded":true,"accepts":["show"]}],` +
`"claims":[{"name":"operator-channel","scope":"mesh","serves":["open"]}]}`
const telegram = `{"module":"telegram","version":"1","runs-as":"telegram","resources":[{"id":"account","type":"user","name":"telegram","shell":"/usr/bin/nologin","home":"/var/lib/telegram"}],"own-secrets":{"broker":"/var/lib/t/broker"},"secrets-owner":"telegram",` +
`"claims":[{"name":"channel","scope":"mesh","kind":"telegram","capabilities":["deliver","verified-sender"]}]}`
answer := func(free ...string) func([]string) (any, error) {
return func(machines []string) (any, error) {
var vs []map[string]any
for _, m := range machines {
ok := slices.Contains(free, m)
why := "judged free"
if !ok {
why = "its node-engine judged nothing"
}
vs = append(vs, map[string]any{"machine": m, "free": ok, "why": why})
}
return map[string]any{"machines": vs}, nil
}
}
for _, row := range []struct {
name string
routerOn []string
telegramOn string
ask func([]string) (any, error)
keeps bool
note string
}{
{"router and telegram on one free machine", []string{"a"}, "a", answer("a"), true, ""},
{"router and telegram on two free machines", []string{"a"}, "b", answer("a", "b"), true, ""},
{"the holder's machine is not free", []string{"a"}, "b", answer("a"), false, "b: not free, its node-engine judged nothing"},
{"the router's machine is not free", []string{"a"}, "b", answer("b"), false, "a: not free"},
{"an error is not free", []string{"a"}, "a", func([]string) (any, error) { return nil, errors.New("boom") }, false, "boom"},
{"no answerer is not free", []string{"a"}, "a", nil, false, "no responders"},
{"a machine left out of the answer is not free", []string{"a"}, "a",
func([]string) (any, error) { return map[string]any{"machines": []any{}}, nil }, false, "named no verdict"},
{"the router on two machines frees nothing", []string{"a", "b"}, "a", answer("a", "b"), false, ""},
} {
t.Run(row.name, func(t *testing.T) {
b := fixture()
b.manifestOf("messenger", router)
b.manifestOf("telegram", telegram)
for _, node := range row.routerOn {
b.assign(node, "messenger")
}
b.assign(row.telegramOn, "telegram")
b.rootFree = row.ask
res := partOn(b).Compose()
recs, _ := grants.Records(grants.ReadInputs(b, &grants.Manifests{}))
grants.WithRootFree(&recs, b)
kept := false
for _, k := range buscompose.PlacementsOf(recs, recs.Interchangeable).Kinds {
kept = kept || slices.Contains(k.Capabilities, "verified-sender")
}
if kept != row.keeps {
t.Errorf("verified-sender kept = %v, want %v (said %v)", kept, row.keeps, res.RootNotFree)
}
if !strings.Contains(strings.Join(res.RootNotFree, " | "), row.note) || (row.note == "" && row.keeps && len(res.RootNotFree) != 0) {
t.Errorf("the pass says %v, want it to name %q", res.RootNotFree, row.note)
}
})
}
// A root-not-free condition raised or cleared wakes a pass; another kind does not (never a timer).
wb := fixture()
var handle func(string, []byte)
wf := &grants.Follower{Part: partOn(wb), Watch: &fakeWatcher{opened: make(chan string, 16)}, Logf: func(string, ...any) {},
Hear: func(h func(string, []byte)) error { handle = h; return nil }}
wctx, wcancel := context.WithCancel(context.Background())
wdone := make(chan struct{})
go func() { wf.Run(wctx); close(wdone) }()
waitFor(t, func() bool { return wb.count(grants.CallUsers) > 0 && handle != nil })
passes := func() int { return wb.count(grants.CallUsers) }
before := passes()
handle(grants.EventConditionRaised, []byte(`{"kind":"disk-full","key":"machine.a.disk-full"}`))
time.Sleep(100 * time.Millisecond)
if passes() != before {
t.Errorf("a condition of another kind woke a pass")
}
handle(grants.EventConditionCleared, []byte(`{"kind":"root-not-free","key":"machine.a.root-not-free"}`))
waitFor(t, func() bool { return passes() > before })
wcancel()
<-wdone
if !grants.IsRootNotFree([]byte(`{"kind":"root-not-free"}`)) || !grants.IsRootNotFree([]byte(`{"key":"machine.novox.root-not-free"}`)) ||
grants.IsRootNotFree([]byte(`{"kind":"disk-full","key":"machine.novox.disk-full"}`)) {
t.Errorf("only a root-not-free condition wakes a pass")
}
})
t.Run("a predefined-key fallback is read with no list; every refused manifest listed", func(t *testing.T) {
// mod-p's value falls back to a predefined key, as claude-code's managed_settings does, and no list is on the bus
// (issue 1945); mod-z's manifest carries a key mesh-sdk's reader refuses, and mod-z is assigned nowhere.
+7 -34
View File
@@ -28,20 +28,14 @@ type Follower struct {
Logf func(string, ...any)
// Asked watches the part's own ask to compose now (a tool's compose-now), waking at each; nil watches none.
Asked func(wake func()) error
// Hear hears the controller seat's condition events (Consumes) and hands each, by event name, with its body, to
// handle; nil hears none. A condition of the kind root-not-free raised or cleared wakes a pass: a machine's
// root-free word changed.
Hear func(handle func(event string, body []byte)) error
wake chan struct{}
once sync.Once
mu sync.Mutex
asked bool
heard_ bool
rootSaid string
watched map[string]bool
refused map[string]string
have map[string]string
wake chan struct{}
once sync.Once
mu sync.Mutex
asked bool
watched map[string]bool
refused map[string]string
have map[string]string
// held and notAsserted are what the last pass held back and could not create, said again only when they change.
held string
notAsserted string
@@ -122,18 +116,6 @@ func (f *Follower) open() {
f.asked = true
}
}
if f.Hear != nil && !f.heard_ {
err := f.Hear(func(event string, body []byte) {
if IsRootNotFree(body) {
f.Wake()
}
})
if err != nil {
f.logf("the controller's condition events cannot be heard yet (%v): tried again at the next change", err)
} else {
f.heard_ = true
}
}
for _, state := range Followed {
f.mu.Lock()
open := f.watched[state]
@@ -193,15 +175,6 @@ func (f *Follower) said(res Result) {
}
}
}
if said := strings.Join(res.RootNotFree, "\n"); said != f.rootSaid {
f.rootSaid = said
if len(res.RootNotFree) == 0 {
f.logf("every machine a composition needs a root-free word for is free")
}
for _, n := range res.RootNotFree {
f.logf("root-free: %s (a verified sender is composed as from no root-free machine there)", n)
}
}
for _, a := range res.Asserted {
f.logf("asserted %s", a)
}
@@ -46,7 +46,7 @@ const (
)
// Invokes is every tool the part calls.
var Invokes = []string{CallUsers, CallUserCan, CallStreams, CallRootFree}
var Invokes = []string{CallUsers, CallUserCan, CallStreams}
// Reader reads another part's state, named `<module>.<state>`. A refusal (a read not granted yet, a bucket not there) is
// an error the part names as a gap and composes past: it never stops for one.
@@ -22,7 +22,7 @@ import (
// MembershipGaps are what a membership carries that the part cannot compose from the bus.
var MembershipGaps = []Gap{
{Input: "membership: private network and received contributions", Why: "kept in each node's composition, which only that node reads: carried over from the membership last published, none for a first one"},
{Input: "membership: root-free machines", Why: "asked of the controller's root-free verb at each pass and kept for that pass alone; a machine it does not call free (an error included) is composed as not free, and the pass says why"},
{Input: "membership: root-free machines", Why: "judged by the controller on the bus at its composition, not kept: a verified sender is composed as from no root-free machine"},
}
// Publisher publishes a membership on the assignments stream (membership.Subject), last per subject, and reads the one
+1 -5
View File
@@ -100,10 +100,7 @@ type Result struct {
NotAsserted []string `json:"not_asserted,omitempty"`
// Refused is every manifest the indexer holds that this pass could not read, `manifest of <module>: <why>`: its
// module counts as placed nowhere (issue 1945).
Refused []string `json:"refused,omitempty"`
// RootNotFree is each machine the composition needed a root-free word for and did not get one, `<machine>: not
// free, <why>`: a verified sender is composed as from no root-free machine there (issue 1945).
RootNotFree []string `json:"root_not_free,omitempty"`
Refused []string `json:"refused,omitempty"`
Errors []string `json:"errors,omitempty"`
PublishedOrAsserted string `json:"published_or_asserted"`
}
@@ -175,7 +172,6 @@ func (p *Part) Compose() Result {
res.Refused = append(res.Refused, g.Input+": "+g.Why)
}
recs, left := Records(in)
res.RootNotFree = WithRootFree(&recs, p.Call)
st := UserListState{V: 1, Mode: Shadow, LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()}
if st.LeftOut == nil {
+33 -5
View File
@@ -101,11 +101,7 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
Streams: streamsOf(m), Queues: queueOwnersOf(m), Processes: processesOf(m), Checks: catalogue.HealthChecks(m),
RunsAs: m.RunsAs, Issued: issuedTools(m),
}
for _, b := range m.Bundles {
if len(b.Loads) > 0 {
d.Loads = true
}
}
d.Loads = loadsABundle(m)
if _, reads := m.OwnSecrets["broker"]; !reads {
d.NoAccount = true
}
@@ -129,6 +125,38 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
return d
}
// loadsABundle says the node's tool runtime loads a bundle of the module. The controller reads it from the resolved
// manifest's Bundles, which a build derives and the indexer's manifest does not carry (the builder's build record names
// the repository's manifest digest), so the same answer is derived here from `build.artifacts` by mesh-sdk's rule
// (the build's own rule for what a bundle loads): a bundle that says `loads`, else every bundle of a module that declares tools and that no
// resource of the module runs. Without it a module whose only runtime-loaded part is a helper (route-proxy's
// route-provider) was left out of node-tools' grants (issue 1945).
func loadsABundle(m catalogue.Manifest) bool {
for _, b := range m.Bundles {
if len(b.Loads) > 0 {
return true
}
}
if m.Build == nil {
return false
}
run := map[string]bool{}
for _, r := range m.Resources {
if a, ok := r["artifact"].(string); ok && a != "" {
run[a] = true
}
}
for _, a := range m.Build.Artifacts {
if a.Kind != catalogue.ArtifactBundle {
continue
}
if len(a.Loads) > 0 || (a.Loads == nil && len(m.Tools) > 0 && !run[a.Name]) {
return true
}
}
return false
}
func waitsOf(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaration) []buscompose.Wait {
var out []buscompose.Wait
for _, w := range m.Waits {
@@ -1,122 +0,0 @@
package grants
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// CallRootFree is the controller seat's verb that judges machines root-free (novox/hq ADR 0259 §8): the one judgement
// the composer needs and the part cannot make from the bus (issue 1945). Only the controller answers it, on purpose.
const CallRootFree = "seat:mesh-controller.root-free"
// The controller seat's condition events the part hears (never a timer): a machine's root-free word changes with the
// raising and the clearing of its `root-not-free` condition.
const (
EventConditionRaised = "mesh-controller.condition-raised"
EventConditionCleared = "mesh-controller.condition-cleared"
// KindRootNotFree is the condition kind whose events wake a pass.
KindRootNotFree = "root-not-free"
)
// Consumes is every event the part hears, as its manifest's consumes name them.
var Consumes = []string{EventConditionRaised, EventConditionCleared}
// EventSubject is the bus subject of a controller seat event named `mesh-controller.<event>`.
func EventSubject(event string) string {
return "mesh.seat.mesh-controller.event." + strings.TrimPrefix(event, "mesh-controller.")
}
// IsRootNotFree says a condition event's body names the `root-not-free` kind (its `kind`, or the last part of its
// `key`, `<scope>.<id>.<kind>`). An unreadable body wakes a pass too: a pass is cheap, a missed change is not.
func IsRootNotFree(body []byte) bool {
var c struct {
Kind string `json:"kind"`
Key string `json:"key"`
}
if json.Unmarshal(body, &c) != nil {
return true
}
return c.Kind == KindRootNotFree || (c.Kind == "" && strings.HasSuffix(c.Key, "."+KindRootNotFree)) || (c.Kind == "" && c.Key == "")
}
// RootFreeNodes are the machines a composition needs a root-free word for: where the router's seat is held and where a
// kinded seat's holder declares `verified-sender`. None when no holder declares it. Sorted. Pure.
func RootFreeNodes(r buscompose.Records) []string {
var nodes []string
need := false
for node, declared := range r.Assigned {
for _, d := range declared {
for _, s := range d.Holds {
if s.Name == "operator-channel" && !slices.Contains(nodes, node) {
nodes = append(nodes, node)
}
if s.Kinded && s.Kind != "" && slices.Contains(s.Capabilities, "verified-sender") {
need = true
if !slices.Contains(nodes, node) {
nodes = append(nodes, node)
}
}
}
}
}
if !need {
return nil
}
sort.Strings(nodes)
return nodes
}
// verdict is one machine's answer of the controller's root-free verb.
type verdict struct {
Machine string `json:"machine"`
Free bool `json:"free"`
Why string `json:"why"`
}
// WithRootFree fills r.RootFree for the machines the composition needs by asking the controller once, and answers a
// line for each machine that is not free, `<machine>: <why>`. It fails closed: an error, no answer or a machine
// left out is not free, and says so. Pure but for the one call; the answer is kept for this pass alone.
func WithRootFree(r *buscompose.Records, call Caller) []string {
nodes := RootFreeNodes(*r)
if len(nodes) == 0 {
return nil
}
free := map[string]verdict{}
var failure string
if call == nil {
failure = "no way to ask the controller here"
} else if raw, err := call.Call(CallRootFree, map[string]any{"machines": nodes}); err != nil {
failure = "the controller's root-free word could not be asked: " + err.Error()
} else {
var body struct {
Machines []verdict `json:"machines"`
}
if err := json.Unmarshal(raw, &body); err != nil {
failure = "the controller's root-free answer could not be read: " + err.Error()
}
for _, v := range body.Machines {
free[v.Machine] = v
}
}
r.RootFree = map[string]bool{}
var notes []string
for _, n := range nodes {
v, answered := free[n]
switch {
case answered && v.Free:
r.RootFree[n] = true
case failure != "":
notes = append(notes, fmt.Sprintf("%s: not free, %s", n, failure))
case !answered:
notes = append(notes, fmt.Sprintf("%s: not free, the controller named no verdict for it", n))
default:
notes = append(notes, fmt.Sprintf("%s: not free, %s", n, v.Why))
}
}
return notes
}
+1 -2
View File
@@ -70,9 +70,8 @@ func (p *Part) MembershipOf(node, module string) (any, error) {
}
in := ReadInputs(p.Read, &p.cache)
recs, left := Records(in)
notFree := WithRootFree(&recs, p.Call)
m, ok := Memberships(recs)[node+"."+module]
out := map[string]any{"node": node, "module": module, "gaps": MembershipGaps, "root_free": recs.RootFree, "root_not_free": notFree}
out := map[string]any{"node": node, "module": module, "gaps": MembershipGaps}
if ok {
out["composed"] = m
} else {
+1 -6
View File
@@ -13,15 +13,10 @@
"node-engine.have",
"mesh-vault.bus-accounts"
],
"consumes": [
"mesh-controller.condition-raised",
"mesh-controller.condition-cleared"
],
"invokes": [
"nats.nats_streams",
"nats.nats_user_can",
"nats.nats_users",
"seat:mesh-controller.root-free"
"nats.nats_users"
],
"emits": [
"asserted",