Merge pull request 'Bus state by template: the judge's rules, manifest-accepted, the old-section count and the ledger account (issue 1966, B1)' (#146) from feat/bus-state into main

This commit was merged in pull request #146.
This commit is contained in:
2026-11-03 15:24:36 +00:00
29 changed files with 1163 additions and 42 deletions
@@ -22,9 +22,22 @@ import (
type asker func(key string, value []byte) (index.TakeInAsk, error) type asker func(key string, value []byte) (index.TakeInAsk, error)
// Tools are the indexer's tools. // Tools are the indexer's tools.
func Tools(ask asker, now func() time.Time) []stdio.Tool { func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCount, error)) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{ return []stdio.Tool{
{Name: index.OldSectionsTool,
Description: "How many accepted modules use each old section of a manifest (state, logs, streams, objects), the foundation " +
"(nats, mesh-grants, the indexer) left out: the count the judge refuses a use of a section at 0 by, as modules move " +
"their bus state into `requires` by template. Read from the indexer's oldsections state, kept as the indexer judges " +
"the manifests; an empty answer means the indexer has not judged since it started.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
c, err := counts()
if err != nil {
return nil, err
}
return c.Counts, nil
}},
{Name: index.TakeInTool, {Name: index.TakeInTool,
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " + Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " + "build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
@@ -105,8 +118,22 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
} }
} }
// heldCounts reads the count of the old sections the indexer keeps.
func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
return func() (index.OldSectionsCount, error) {
e, err := s.Get(index.OldSectionsKey)
if err != nil {
return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsBucket, index.OldSectionsKey, err)
}
if e == nil {
return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsBucket, index.OldSectionsKey)
}
return index.DecodeOldSections(e.Value)
}
}
func main() { func main() {
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil { if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, heldCounts(stdio.State(index.OldSectionsState)))); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1) os.Exit(1)
} }
@@ -296,8 +296,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every // The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
// fingerprint takes the version line. // fingerprint takes the version line.
packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf) packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, // How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), // the judge refuses no use of an old section.
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call)} ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call)}
@@ -36,3 +36,29 @@ func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string,
} }
} }
} }
// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4):
// mesh.mod.<module>.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as
// the bus's dedup id (delivery.ManifestAccepted.EventID), so one acceptance is said once.
func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, delivery.ManifestAccepted) {
return func(ctx context.Context, a delivery.ManifestAccepted) {
data, err := json.Marshal(a)
if err != nil {
logf("judge: %s for %s is not said: %v", delivery.ManifestAcceptedEvent, a.Module, err)
return
}
msg := nats.NewMsg(cred.EventSubject(delivery.ManifestAcceptedEvent))
id := a.EventID()
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", delivery.ManifestAcceptedEvent, a.Module, err))
}
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1 h1:T66tg9T/1hHg6KN/bDScHPT41IHkxLAGGO8VJiAOlb8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -54,10 +54,16 @@ type Indexer struct {
Taken Store Taken Store
// Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing. // Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing.
Verdicts Store Verdicts Store
// OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no
// count and the judge refuses no use of an old section.
OldSections Store
// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's. // JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
JudgeID string JudgeID string
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing. // SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
SayRefused func(ctx context.Context, r delivery.ManifestRefused) SayRefused func(ctx context.Context, r delivery.ManifestRefused)
// SayAccepted says the event manifest-accepted for an accepted verdict newly written whose manifest asks for bus state,
// carrying the entries whole (verdicts.go; issue 1966 D3); nil says nothing.
SayAccepted func(ctx context.Context, a delivery.ManifestAccepted)
// Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none, // Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line. // and every fingerprint takes the version line.
Packages Store Packages Store
@@ -0,0 +1,76 @@
package index
import (
"context"
"encoding/json"
"fmt"
"maps"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/judge"
)
// How many accepted modules use each old section (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts them, the
// foundation left out, as it judges, keeps the count in its state oldsections under one key, and the tool
// indexer_old_sections reads it. The judge refuses a manifest of another module that uses a section whose count is 0.
const (
// OldSectionsTool is the tool that reads the count.
OldSectionsTool = "indexer_old_sections"
// OldSectionsState is the indexer's state of the count: the bucket mesh-delivery-indexer_oldsections.
OldSectionsState = "oldsections"
OldSectionsBucket = delivery.IndexerModule + "_" + OldSectionsState
// OldSectionsKey is the one key it holds.
OldSectionsKey = "counts"
)
// OldSectionsCount is the value of the oldsections state: per old section, the accepted modules outside the foundation using it.
type OldSectionsCount struct {
V int `json:"v"`
Counts map[string]int `json:"counts"`
}
// DecodeOldSections reads the oldsections value.
func DecodeOldSections(value []byte) (OldSectionsCount, error) {
var c OldSectionsCount
if err := json.Unmarshal(value, &c); err != nil {
return c, fmt.Errorf("the count of the old sections cannot be read: %w", err)
}
if c.V != 1 {
return c, fmt.Errorf("a count of the old sections of version %d, not 1", c.V)
}
return c, nil
}
// heldOldSections is the count the last pass wrote (the accepted set it held): a module that keeps using a section was
// accepted with it, so it is counted, and a fresh manifest changes nothing in it. Nil when none is held yet or it cannot be
// read; a failed read is said, never taken as 0 (issue 1966 D4).
func (ix *Indexer) heldOldSections(ctx context.Context) judge.OldSectionCounts {
value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey)
if err == nil && found {
var c OldSectionsCount
if c, err = DecodeOldSections(value); err == nil {
return c.Counts
}
}
if err != nil {
ix.logf("judge: the count of the old sections cannot be read (%v), so the zero rule is not applied in this pass", err)
}
return nil
}
// keepOldSections writes the counts under OldSectionsKey when they differ from the ones held.
func (ix *Indexer) keepOldSections(ctx context.Context, counts judge.OldSectionCounts) error {
if ix.OldSections == nil {
return nil
}
return write(ctx, ix.OldSections, OldSectionsKey, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := DecodeOldSections(value); err == nil && maps.Equal(cur.Counts, counts) {
return nil, false, nil
}
}
next, err := json.Marshal(OldSectionsCount{V: 1, Counts: counts})
return next, false, err
})
}
@@ -74,7 +74,13 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
raws[name] = []byte(kept.JSON) raws[name] = []byte(kept.JSON)
digests[name] = digest digests[name] = digest
} }
refused, unnamed := judge.Shelf(raws, ix.predefinedList(ctx), 0) predefined := ix.predefinedList(ctx)
// The old sections end when no module is left in them (issue 1966 D4): the count is the accepted set the last pass held.
var counts judge.OldSectionCounts
if ix.OldSections != nil {
counts = ix.heldOldSections(ctx)
}
refused, unnamed := judge.ShelfCounting(raws, predefined, 0, counts)
for _, p := range unnamed { for _, p := range unnamed {
ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p) ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p)
} }
@@ -83,6 +89,7 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
if id == "" { if id == "" {
id = judge.Identity() id = judge.Identity()
} }
var accepted []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(digests)) { for _, name := range slices.Sorted(maps.Keys(digests)) {
out.Judged++ out.Judged++
v := delivery.ManifestVerdict{Module: name, State: delivery.VerdictAccepted, Judge: id, At: ix.now()} v := delivery.ManifestVerdict{Module: name, State: delivery.VerdictAccepted, Judge: id, At: ix.now()}
@@ -94,6 +101,12 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
v.State, v.Problems = delivery.VerdictRefused, problems v.State, v.Problems = delivery.VerdictRefused, problems
} }
} }
if !v.Refused() {
if m, err := catalogue.Decode(raws[name]); err == nil {
m.Module = name
accepted = append(accepted, m)
}
}
wrote, err := ix.writeVerdict(ctx, digests[name], v) wrote, err := ix.writeVerdict(ctx, digests[name], v)
if errors.Is(err, process.ErrNotYet) { if errors.Is(err, process.ErrNotYet) {
return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsBucket, err) return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsBucket, err)
@@ -109,12 +122,20 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
if ix.SayRefused != nil { if ix.SayRefused != nil {
ix.SayRefused(ctx, delivery.RefusedOf(digests[name], v)) ix.SayRefused(ctx, delivery.RefusedOf(digests[name], v))
} }
} else if ix.SayAccepted != nil {
// A manifest asking for bus state, newly accepted, is said whole to the bus module (issue 1966 D3, ADR 0595 §4).
if m, problems := judge.Manifest(raws[name], predefined); len(problems) == 0 && len(m.BusState) > 0 {
ix.SayAccepted(ctx, delivery.AcceptedOf(digests[name], v, m))
}
} }
} }
if err := ix.carryVerdict(ctx, name, digests[name], v); err != nil { if err := ix.carryVerdict(ctx, name, digests[name], v); err != nil {
ix.logf("judge: modules.%s does not carry its verdict: %v", name, err) ix.logf("judge: modules.%s does not carry its verdict: %v", name, err)
} }
} }
if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted)); err != nil {
ix.logf("judge: the count of the old sections is not kept: %v", err)
}
return out, nil return out, nil
} }
@@ -2,6 +2,8 @@ package index
import ( import (
"context" "context"
"errors"
"fmt"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -118,9 +120,92 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if err := ix.writeOverlay(ctx, map[string]delivery.Module{"old": same}, nil, w); err != nil || !w.fresh[other] { if err := ix.writeOverlay(ctx, map[string]delivery.Module{"old": same}, nil, w); err != nil || !w.fresh[other] {
t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err) t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err)
} }
// 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries
// whole; a verdict already held, and a manifest asking for none, say nothing.
var accepted []delivery.ManifestAccepted
ix.SayAccepted = func(_ context.Context, a delivery.ManifestAccepted) { accepted = append(accepted, a) }
moved := hold("moved", `{"module":"moved","version":"1","requires":[`+
`{"name":"cache","storage":"key-value","template":"state","options":{"history":8}},`+
`{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64}}]}`)
for pass := 0; pass < 2; pass++ {
if _, err := ix.JudgeAll(ctx, map[string]bool{moved: true}); err != nil {
t.Fatal(err)
}
}
if len(accepted) != 1 || accepted[0].Module != "moved" || accepted[0].Digest != moved || accepted[0].Judge != "judge-2" ||
len(accepted[0].BusState) != 2 || accepted[0].BusState[0].Name != "cache" || accepted[0].BusState[1].Template != "log" ||
string(accepted[0].BusState[0].Options["history"]) != "8" {
t.Fatalf("said accepted %+v", accepted)
}
// 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another
// module using logs is refused and the foundation's is accepted.
ix.OldSections = newMem()
hold("user", `{"module":"user","version":"1","state":["x"]}`)
hold("mesh-grants", `{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`)
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
counts := func() map[string]int {
value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey)
if err != nil || !found {
t.Fatalf("no count of the old sections: %v", err)
}
c, err := DecodeOldSections(value)
if err != nil {
t.Fatal(err)
}
return c.Counts
}
if c := counts(); c["state"] != 1 || c["logs"] != 0 || c["streams"] != 0 || c["objects"] != 0 {
t.Fatalf("the old sections count %v, want state 1 and the rest 0", c)
}
late := hold("late", `{"module":"late","version":"1","logs":[{"name":"l"}]}`)
grants := hold("mesh-grants", `{"module":"mesh-grants","version":"2","logs":[{"name":"l"},{"name":"m"}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil {
t.Fatal(err)
}
if v := verdict(late); !v.Refused() || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") {
t.Fatalf("late: %+v", v)
}
if v := verdict(grants); v.Refused() {
t.Fatalf("the foundation's manifest was refused: %+v", v)
}
// 1966/AC11: the sole user of an old section changes its manifest and keeps the section: accepted, since it was accepted
// with it (the count is the accepted set held from the last pass, not recounted from fresh digests).
hold("solo", `{"module":"solo","version":"1","objects":[{"name":"o","max-mib":1}]}`)
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
solo2 := hold("solo", `{"module":"solo","version":"2","objects":[{"name":"o","max-mib":2}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{solo2: true}); err != nil {
t.Fatal(err)
}
if v := verdict(solo2); v.Refused() {
t.Fatalf("the sole user of objects, changing its manifest and keeping objects, was refused: %+v", v)
}
// A count that cannot be read is never taken as 0: the zero rule is not applied in that pass and the log says so.
var logged []string
ix.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
held := ix.OldSections
ix.OldSections = unreadable{held}
retry := hold("late", `{"module":"late","version":"2","logs":[{"name":"l"}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{retry: true}); err != nil {
t.Fatal(err)
}
if v := verdict(retry); v.Refused() || !strings.Contains(strings.Join(logged, "\n"), "the zero rule is not applied") {
t.Fatalf("a count that could not be read: %+v, logged %q", v, logged)
}
ix.OldSections = held
// Without the verdicts state the indexer judges nothing. // Without the verdicts state the indexer judges nothing.
ix.Verdicts = nil ix.Verdicts = nil
if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 { if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 {
t.Fatalf("without the state: %+v %v", j, err) t.Fatalf("without the state: %+v %v", j, err)
} }
} }
// unreadable is a store whose reads fail.
type unreadable struct{ Store }
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
return nil, 0, false, errors.New("the bus did not answer")
}
@@ -0,0 +1,420 @@
package judge
import (
"bytes"
"encoding/json"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The rules for bus state asked for in `requires` (issue 1966 D1; novox/hq ADR 0595 §1 to §3, ADR 0596 §2, ADR 0597 §2,
// design 25 § 12.1). mesh-sdk keeps only the entry's shape (ADR 0578 §7); every bound below is this judge's. An entry is
// judged once, here, and a refusal names the module, the entry and the field.
// optionKind is what an option's value is.
type optionKind int
const (
optionInt optionKind = iota
optionFlag
optionText
optionEvents
)
// optionRule is one option a template takes: its kind, its bound where it is a number, and whether it must be given.
type optionRule struct {
kind optionKind
least, most int
required bool
}
// busTemplate is a template: the storage type it is on, the options it takes, and what it fixes in plain words.
type busTemplate struct {
storage string
options map[string]optionRule
fixed string
}
// busTemplates are the six templates of design 25 § 12.1 and the bounds of its table.
var busTemplates = map[string]busTemplate{
catalogue.TemplateState: {
storage: catalogue.StorageKeyValue,
fixed: "what happens when full, the size of a value and of the bucket, and the copies",
options: map[string]optionRule{
catalogue.OptionPastValues: {kind: optionInt, least: 1, most: catalogue.StateMostHistory},
catalogue.OptionTimeToLive: {kind: optionInt, least: 1, most: 315360000},
catalogue.OptionLifetimePerKey: {kind: optionFlag},
catalogue.OptionPerMachine: {kind: optionFlag},
},
},
catalogue.TemplateEvents: {
storage: catalogue.StorageStream,
fixed: "what happens when full (the oldest is dropped) and the module's own subjects",
options: map[string]optionRule{
"events": {kind: optionEvents, required: true},
catalogue.OptionAgeDays: {kind: optionInt, least: catalogue.StreamLeastDays, most: catalogue.StreamMostDays},
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
catalogue.OptionGrant: {kind: optionText},
},
},
catalogue.TemplateWorkQueue: {
storage: catalogue.StorageStream,
fixed: "what happens when full (a new message is refused), one taker, and no grant",
options: map[string]optionRule{
"events": {kind: optionEvents, required: true},
catalogue.OptionDeliveries: {kind: optionInt, least: catalogue.StreamLeastDeliveries, most: catalogue.StreamMostDeliveries},
catalogue.OptionDuplicates: {kind: optionInt, least: 1, most: catalogue.StreamMostDays * 86400},
catalogue.OptionAgeDays: {kind: optionInt, least: catalogue.StreamLeastDays, most: catalogue.StreamMostDays},
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
},
},
catalogue.TemplateLog: {
storage: catalogue.StorageStream,
fixed: "what happens when full (a new entry is refused), that an entry is never rewritten, and its size",
options: map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
catalogue.OptionAgeDays: {kind: optionInt, least: 1, most: 3650},
},
},
catalogue.TemplateObjects: {
storage: catalogue.StorageObjectStore,
fixed: "what happens when full (a new object is refused) and that objects are kept for good",
options: map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.ObjectsLeastMiB, most: catalogue.ObjectsMostMiB, required: true},
catalogue.OptionGrant: {kind: optionText},
},
},
catalogue.TemplateLedger: {
storage: catalogue.StorageLedger,
fixed: "what happens when full (a new entry is refused) and its state and index buckets",
options: map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
},
},
}
// bareStream is a stream of a module's own configuration, where no template fits: the three settings the judge
// refuses one for missing (design 25 § 12.1), and those it may also give.
var bareStream = map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB, required: true},
catalogue.OptionAgeDays: {kind: optionInt, least: 1, most: 3650, required: true},
catalogue.OptionGrant: {kind: optionText},
"events": {kind: optionEvents},
}
// ledgerConfiguration is what a ledger's own configuration may give: how long its history is kept (design 25 § 12.1).
var ledgerConfiguration = map[string]optionRule{
catalogue.OptionAgeDays: {kind: optionInt, least: 1, most: 3650},
}
// noAge is how a bare stream says it keeps its messages for good.
const noAge = "none"
// ledgerEffects are the four effects an operation may have (ADR 0597 §2).
var ledgerEffects = []string{catalogue.EffectSetFields, catalogue.EffectSetEntry, catalogue.EffectAppend, catalogue.EffectRemove}
// BusStateProblems is what is wrong with one manifest's bus state, judged on the manifest alone (issue 1966 D1).
func BusStateProblems(m catalogue.Manifest) []string {
if len(m.BusState) == 0 {
return nil
}
var problems []string
if !stateName.MatchString(m.Module) {
problems = append(problems, fmt.Sprintf(
"%s keeps bus state, and a module's name is part of its names on the bus, which take one plain name — "+
"no dot (novox/hq ADR 0201, ADR 0595 §1)", m.Module))
}
storageOf := map[string]string{}
oldSection := map[string]string{} // a local name an old section keeps, to that section (issue 1966 D2)
for _, st := range m.State {
oldSection[st.Name] = "state"
}
for _, l := range m.Logs {
oldSection[l.Name] = "logs"
}
for _, st := range m.Streams {
oldSection[st.Name] = "streams"
}
for _, o := range m.Objects {
oldSection[o.Name] = "objects"
}
terminal := catalogue.TerminalKeys(m)
for _, e := range m.BusState {
at := fmt.Sprintf("%s's bus state %q", m.Module, e.Name)
switch {
case strings.Contains(e.Name, "_"):
problems = append(problems, fmt.Sprintf(
"%s: its name holds an underscore, which separates the module from the local name on the bus; use a "+
"hyphen (novox/hq design 25 § 12.1)", at))
case !stateName.MatchString(e.Name):
problems = append(problems, fmt.Sprintf(
"%s: its name is a local name — lower-case letters, digits and hyphens, no dot and no underscore; the "+
"mesh derives the name on the bus (novox/hq ADR 0201)", at))
}
if first, taken := storageOf[e.Name]; taken {
problems = append(problems, fmt.Sprintf(
"%s: the name is used twice (as %s and as %s); each name is one, whatever its storage type (novox/hq "+
"ADR 0595 §1)", at, first, e.Storage))
}
storageOf[e.Name] = e.Storage
if section, kept := oldSection[e.Name]; kept {
problems = append(problems, fmt.Sprintf(
"%s: the name is also kept in the old section %s; one local name is one form, so a module that moves an "+
"entry removes it from the old section (novox/hq ADR 0595 §11)", at, section))
}
problems = append(problems, busEntryProblems(at, e, terminal)...)
}
return problems
}
// busEntryProblems judges one entry's storage type, template, options and configuration.
func busEntryProblems(at string, e catalogue.BusStateEntry, terminal []string) []string {
var problems []string
switch e.Storage {
case catalogue.StorageStream, catalogue.StorageKeyValue, catalogue.StorageObjectStore, catalogue.StorageLedger:
default:
return []string{fmt.Sprintf("%s: the storage type %q is unknown; it is %s", at, e.Storage,
strings.Join([]string{catalogue.StorageStream, catalogue.StorageKeyValue, catalogue.StorageObjectStore, catalogue.StorageLedger}, ", "))}
}
if e.Storage != catalogue.StorageLedger && len(e.Types) > 0 {
problems = append(problems, fmt.Sprintf("%s: types are a ledger's; storage type %s has none", at, e.Storage))
}
if e.Template == "" {
if e.Storage != catalogue.StorageStream {
return append(problems, fmt.Sprintf(
"%s: the storage type %s is asked for by its template; only a stream is asked for bare, with a "+
"configuration of its own (novox/hq design 25 § 12.1)", at, e.Storage))
}
if len(e.Options) > 0 {
problems = append(problems, fmt.Sprintf(
"%s: a bare stream gives its settings in its configuration, not in options", at))
}
return append(problems, settingsProblems(at, "configuration", e.Configuration, bareStream, "a bare stream", terminal, true)...)
}
tpl, known := busTemplates[e.Template]
switch {
case !known:
names := make([]string, 0, len(busTemplates))
for n := range busTemplates {
names = append(names, n)
}
slices.Sort(names)
return append(problems, fmt.Sprintf("%s: the template %q is unknown; it is %s", at, e.Template, strings.Join(names, ", ")))
case tpl.storage != e.Storage:
return append(problems, fmt.Sprintf(
"%s: the template %s is on the storage type %s, not %s", at, e.Template, tpl.storage, e.Storage))
}
problems = append(problems, settingsProblems(at, "options", e.Options, tpl.options, "the template "+e.Template+" (it fixes "+tpl.fixed+")", terminal, false)...)
if e.Template == catalogue.TemplateLedger {
problems = append(problems, settingsProblems(at, "configuration", e.Configuration, ledgerConfiguration, "a ledger's own configuration", terminal, false)...)
problems = append(problems, ledgerTypeProblems(at, e)...)
} else if len(e.Configuration) > 0 {
problems = append(problems, fmt.Sprintf(
"%s: the template %s is given a configuration; a configuration of its own is for a bare stream or a "+
"ledger whose history expires, and a template's settings are its options (novox/hq ADR 0595 §2)", at, e.Template))
}
if e.Template == catalogue.TemplateWorkQueue {
problems = append(problems, workQueueProblems(at, e)...)
}
return problems
}
// settingsProblems judges the given settings against the rules: an unknown one is refused (a fixed setting among them),
// a number outside its bound, a kind mismatch, and a required one missing. A bare stream's when-full and age are read
// here too.
func settingsProblems(at, field string, given map[string]json.RawMessage, rules map[string]optionRule, of string, terminal []string, bare bool) []string {
var problems []string
for _, k := range sortedKeys(given) {
raw := given[k]
if bare && k == catalogue.ConfigWhenFull {
var v string
if json.Unmarshal(raw, &v) != nil || (v != catalogue.WhenFullRefuse && v != catalogue.WhenFullDropOldest) {
problems = append(problems, fmt.Sprintf(
"%s: %s %q is %s; a bare stream says %q or %q", at, field, k, string(raw), catalogue.WhenFullRefuse, catalogue.WhenFullDropOldest))
}
continue
}
rule, ok := rules[k]
if !ok {
if k == catalogue.ConfigWhenFull {
problems = append(problems, fmt.Sprintf(
"%s: %s %q is given to %s; what happens when full is fixed and is never an option (novox/hq ADR 0595 §3)", at, field, k, of))
continue
}
problems = append(problems, fmt.Sprintf(
"%s: %s %q is not a setting of %s (novox/hq ADR 0595 §3)", at, field, k, of))
continue
}
problems = append(problems, valueProblems(at, field, k, raw, rule, terminal, bare)...)
}
for _, k := range sortedKeys(rules) {
if _, given := given[k]; !given && rules[k].required {
problems = append(problems, fmt.Sprintf("%s: %s %q is missing; %s gives it", at, field, k, of))
}
}
if bare {
if _, given := given[catalogue.ConfigWhenFull]; !given {
problems = append(problems, fmt.Sprintf(
"%s: %s %q is missing; a bare stream says %q or %q", at, field, catalogue.ConfigWhenFull, catalogue.WhenFullRefuse, catalogue.WhenFullDropOldest))
}
}
return problems
}
// valueProblems judges one setting's value against its rule.
func valueProblems(at, field, key string, raw json.RawMessage, rule optionRule, terminal []string, bare bool) []string {
switch rule.kind {
case optionInt:
if bare && key == catalogue.OptionAgeDays {
var word string
if json.Unmarshal(raw, &word) == nil && word == noAge {
return nil
}
}
n, ok := wholeNumber(raw)
if !ok {
return []string{fmt.Sprintf("%s: %s %q is %s, which is not a whole number", at, field, key, string(raw))}
}
if n < rule.least || n > rule.most {
extra := ""
if key == catalogue.OptionPastValues && n == 0 {
extra = "; 0 is refused here"
}
return []string{fmt.Sprintf("%s: %s %q is %d, outside its bound of %d to %d%s", at, field, key, n, rule.least, rule.most, extra)}
}
case optionFlag:
var b bool
if json.Unmarshal(raw, &b) != nil {
return []string{fmt.Sprintf("%s: %s %q is %s, which is not true or false", at, field, key, string(raw))}
}
case optionText:
var config string
if json.Unmarshal(raw, &config) != nil || !slices.Contains(terminal, config) {
return []string{fmt.Sprintf(
"%s: %s %q names %s, which is not a trusted config value of the module, set only at the controller's "+
"terminal or on the operator's Approve (novox/hq ADR 0348 §8)", at, field, key, string(raw))}
}
case optionEvents:
var events []string
if json.Unmarshal(raw, &events) != nil || len(events) == 0 {
return []string{fmt.Sprintf("%s: %s %q is %s; it is a list of at least one event", at, field, key, string(raw))}
}
var out []string
for _, ev := range events {
if err := eventPattern(ev, false); err != nil {
out = append(out, fmt.Sprintf("%s: %s %q carries %q: %v", at, field, key, ev, err))
}
}
return out
}
return nil
}
// wholeNumber reads a JSON number that is a whole number.
func wholeNumber(raw json.RawMessage) (int, bool) {
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
var n json.Number
if dec.Decode(&n) != nil {
return 0, false
}
i, err := n.Int64()
return int(i), err == nil
}
// workQueueProblems holds a work queue's duplicate window to its age (design 25 § 12.1: up to its age).
func workQueueProblems(at string, e catalogue.BusStateEntry) []string {
raw, given := e.Options[catalogue.OptionDuplicates]
if !given {
return nil
}
window, ok := wholeNumber(raw)
if !ok {
return nil // valueProblems said so
}
days := catalogue.StreamDefaultDays
if age, ok := wholeNumber(e.Options[catalogue.OptionAgeDays]); ok && age > 0 {
days = age
}
if window > days*86400 {
return []string{fmt.Sprintf(
"%s: options %q is %d seconds, longer than the queue's age of %d days (novox/hq design 25 § 12.1)",
at, catalogue.OptionDuplicates, window, days)}
}
return nil
}
// ledgerTypeProblems judges a ledger's types: named locally and once, and every operation's effect one of the four.
func ledgerTypeProblems(at string, e catalogue.BusStateEntry) []string {
var problems []string
if len(e.Types) == 0 {
problems = append(problems, fmt.Sprintf("%s: a ledger names at least one type (novox/hq ADR 0597 §1)", at))
}
seen := map[string]bool{}
for _, t := range e.Types {
where := fmt.Sprintf("%s: type %q", at, t.Name)
switch {
case !stateName.MatchString(t.Name):
problems = append(problems, fmt.Sprintf("%s: its name is lower-case letters, digits and hyphens", where))
case seen[t.Name]:
problems = append(problems, fmt.Sprintf("%s: the type is named twice", where))
}
seen[t.Name] = true
operations := map[string]bool{}
for _, op := range t.Operations {
if operations[op.Name] {
problems = append(problems, fmt.Sprintf("%s: the operation %q is named twice", where, op.Name))
}
operations[op.Name] = true
if !slices.Contains(ledgerEffects, op.Effect) {
problems = append(problems, fmt.Sprintf(
"%s: the operation %q has the effect %q, outside the four (%s) (novox/hq ADR 0597 §2)",
where, op.Name, op.Effect, strings.Join(ledgerEffects, ", ")))
}
}
}
return problems
}
// BusStateCatalogueProblems are the rules of bus state no single manifest can be judged against: two modules' names on
// the bus alike, and a `reads` naming bus state in the new form, which has no readers (novox/hq ADR 0596 §2).
func BusStateCatalogueProblems(shelf catalogue.Shelf) []string {
var problems []string
owner := map[string]string{} // a name on the bus, upper-cased, to the module asking for it
for _, module := range shelfOrder(shelf) {
for _, o := range shelf[module].Objects {
owner[strings.ToUpper(o.Bucket(module))] = module
}
}
for _, module := range shelfOrder(shelf) {
for _, e := range shelf[module].BusState {
name := catalogue.ObjectsBucket(module, e.Name)
if first, taken := owner[strings.ToUpper(name)]; taken && first != module {
problems = append(problems, fmt.Sprintf(
"%s's bus state %q has the name %s on the bus, which %s's already has or is alike; a name on the bus "+
"has one owner (novox/hq ADR 0201 §4)", module, e.Name, name, first))
continue
}
owner[strings.ToUpper(name)] = module
}
}
for _, module := range shelfOrder(shelf) {
for _, r := range shelf[module].Reads {
from, local, err := catalogue.ReadState(r)
if err != nil || from == module {
continue // StateProblems says why
}
for _, e := range shelf[from].BusState {
if e.Name == local {
problems = append(problems, fmt.Sprintf(
"%s reads %q, which %s has moved to bus state in `requires`; the new form has no readers: ask %s's "+
"tools, or hear its events (novox/hq ADR 0596 §2)", module, r, from, from))
}
}
}
}
return problems
}
@@ -0,0 +1,95 @@
package judge
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// A manifest's bus state, asked for in `requires` by template, is judged once and each refusal names the module, the entry
// and the field (issue 1966 D1, D2; novox/hq design 25 § 12.1, ADR 0595, ADR 0596 §2, ADR 0597 §2).
func TestBusStateIsJudgedOnce(t *testing.T) {
const ledger = `{"name":"records","storage":"ledger","template":"ledger","options":{"max-mib":512},"types":[` +
`{"name":"decision","schema":{"title":{"type":"text"}},"operations":[{"name":"propose","effect":"set-fields","fields":["title"]}]}]}`
manifest := func(entries string) string {
return `{"module":"a","version":"1","requires":[` + entries + `]}`
}
for _, c := range []struct {
name string
raw string
refused []string // words the first problem holds, in order: absent is accepted
}{
// 1966/AC8: a state, a log, objects and a ledger within bounds
{"within bounds", manifest(`{"name":"cache","storage":"key-value","template":"state","options":{"history":8,"key-ttl":true}},` +
`{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64,"max-age-days":365}},` +
`{"name":"files","storage":"object-store","template":"objects","options":{"max-mib":2048}},` + ledger), nil},
// 1966/AC7: each refused case, naming the module, the entry and the field
{"unknown storage type", manifest(`{"name":"x","storage":"tape"}`), []string{"a's bus state \"x\"", "storage type \"tape\""}},
{"unknown template", manifest(`{"name":"x","storage":"stream","template":"pile"}`), []string{"\"x\"", "template \"pile\""}},
{"template on another storage type", manifest(`{"name":"x","storage":"stream","template":"state"}`), []string{"\"x\"", "template state is on the storage type key-value"}},
{"past values 0", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"history":0}}`), []string{"\"x\"", "options \"history\" is 0", "0 is refused"}},
{"past values 65", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"history":65}}`), []string{"\"x\"", "\"history\" is 65", "1 to 64"}},
{"a lifetime that is no number", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"ttl-seconds":"soon"}}`), []string{"\"x\"", "\"ttl-seconds\""}},
{"events age over 30 days", manifest(`{"name":"x","storage":"stream","template":"events","options":{"events":["seen"],"max-age-days":31}}`), []string{"\"x\"", "\"max-age-days\" is 31", "1 to 30"}},
{"events cap over 8192", manifest(`{"name":"x","storage":"stream","template":"events","options":{"events":["seen"],"max-mib":8193}}`), []string{"\"x\"", "\"max-mib\" is 8193"}},
{"events naming none", manifest(`{"name":"x","storage":"stream","template":"events","options":{}}`), []string{"\"x\"", "\"events\" is missing"}},
{"a grant that is no trusted value", manifest(`{"name":"x","storage":"stream","template":"events","options":{"events":["seen"],"grant":"nobody"}}`), []string{"\"x\"", "\"grant\"", "trusted config value"}},
{"deliveries 11", manifest(`{"name":"x","storage":"stream","template":"work-queue","options":{"events":["job"],"max-deliveries":11}}`), []string{"\"x\"", "\"max-deliveries\" is 11", "1 to 10"}},
{"duplicates past the age", manifest(`{"name":"x","storage":"stream","template":"work-queue","options":{"events":["job"],"max-age-days":1,"duplicates":86401}}`), []string{"\"x\"", "\"duplicates\"", "longer than the queue's age of 1 days"}},
{"log age 3651", manifest(`{"name":"x","storage":"stream","template":"log","options":{"max-age-days":3651}}`), []string{"\"x\"", "\"max-age-days\" is 3651"}},
{"objects without a cap", manifest(`{"name":"x","storage":"object-store","template":"objects"}`), []string{"\"x\"", "\"max-mib\" is missing"}},
{"when full given as an option", manifest(`{"name":"x","storage":"stream","template":"log","options":{"when-full":"drop-oldest"}}`), []string{"\"x\"", "\"when-full\"", "fixed"}},
{"another fixed setting given", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"max-value-kib":1024}}`), []string{"\"x\"", "\"max-value-kib\" is not a setting of the template state"}},
{"a configuration beside a template", manifest(`{"name":"x","storage":"key-value","template":"state","configuration":{"max-mib":1}}`), []string{"\"x\"", "configuration"}},
{"a name used twice across storage types", manifest(
`{"name":"x","storage":"key-value","template":"state"},{"name":"x","storage":"stream","template":"log"}`), []string{"\"x\"", "used twice (as key-value and as stream)"}},
{"a local name holding an underscore", manifest(`{"name":"my_x","storage":"key-value","template":"state"}`), []string{"\"my_x\"", "underscore"}},
{"a bare stream without when full", manifest(`{"name":"x","storage":"stream","configuration":{"max-mib":10,"max-age-days":"none"}}`), []string{"\"x\"", "\"when-full\" is missing"}},
{"a bare stream without a cap", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"refuse","max-age-days":3}}`), []string{"\"x\"", "\"max-mib\" is missing"}},
{"a bare stream without an age", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"refuse","max-mib":10}}`), []string{"\"x\"", "\"max-age-days\" is missing"}},
{"a bare stream, complete", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"drop-oldest","max-mib":10,"max-age-days":"none"}}`), nil},
{"a bare stream with a wrong when full", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"crash","max-mib":10,"max-age-days":9}}`), []string{"\"x\"", "\"when-full\"", "\"refuse\" or \"drop-oldest\""}},
{"a ledger effect outside the four", manifest(strings.Replace(ledger, `"set-fields"`, `"upsert"`, 1)), []string{"\"records\"", "effect \"upsert\"", "outside the four"}},
{"a ledger whose history expires", manifest(strings.Replace(ledger, `"options"`, `"configuration":{"max-age-days":30},"options"`, 1)), nil},
} {
_, problems := Manifest([]byte(c.raw), nil)
switch {
case c.refused == nil && len(problems) > 0:
t.Errorf("%s: refused: %v", c.name, problems)
case c.refused != nil && len(problems) == 0:
t.Errorf("%s: accepted", c.name)
case c.refused != nil:
text := strings.Join(problems, "\n")
at := 0
for _, word := range c.refused {
i := strings.Index(text[at:], word)
if i < 0 {
t.Errorf("%s: problems %q lack %q", c.name, text, word)
break
}
at += i
}
}
}
// Between the manifests: two modules' names on the bus alike, and a read of bus state that moved (ADR 0596 §2).
entry := func(name string) []catalogue.BusStateEntry {
return []catalogue.BusStateEntry{{Name: name, Storage: catalogue.StorageKeyValue, Template: catalogue.TemplateState}}
}
for _, c := range []struct {
name string
shelf catalogue.Shelf
want string
}{
{"names alike", catalogue.Shelf{"a_b": {Module: "a_b", BusState: entry("c")}, "a": {Module: "a", BusState: entry("b_c")}}, "has one owner"},
{"a read of moved state", catalogue.Shelf{"owner": {Module: "owner", BusState: entry("cache")},
"reader": {Module: "reader", Reads: []string{"owner.cache"}}}, "reader reads \"owner.cache\", which owner has moved"},
{"a read of state still kept", catalogue.Shelf{"owner": {Module: "owner", State: []catalogue.StateDeclaration{{Name: "cache"}}},
"reader": {Module: "reader", Reads: []string{"owner.cache"}}}, ""},
} {
got := strings.Join(BusStateCatalogueProblems(c.shelf), "\n")
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
t.Errorf("%s: between the manifests: %q, want %q", c.name, got, c.want)
}
}
}
+45 -1
View File
@@ -69,6 +69,12 @@ func problemsOf(err error) []string {
// whose removal from the catalogue ends it (a rule two modules break together refuses both); one no removal ends is in // whose removal from the catalogue ends it (a rule two modules break together refuses both); one no removal ends is in
// unnamed, for the caller to say. A module not in the answer is accepted. // unnamed, for the caller to say. A module not in the answer is accepted.
func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int) (refused map[string][]string, unnamed []string) { func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int) (refused map[string][]string, unnamed []string) {
return ShelfCounting(raws, predefined, longestMachine, nil)
}
// ShelfCounting is Shelf also refusing a manifest of a module outside the foundation that uses an old section whose count
// is 0 (OldSectionProblems; issue 1966 D4); nil counts refuse none.
func ShelfCounting(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int, counts OldSectionCounts) (refused map[string][]string, unnamed []string) {
refused = map[string][]string{} refused = map[string][]string{}
shelf := catalogue.Shelf{} shelf := catalogue.Shelf{}
for name, raw := range raws { for name, raw := range raws {
@@ -81,6 +87,10 @@ func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestM
refused[name] = []string{fmt.Sprintf("the manifest is of module %q, not %q", m.Module, name)} refused[name] = []string{fmt.Sprintf("the manifest is of module %q, not %q", m.Module, name)}
continue continue
} }
if problems := OldSectionProblems(m, counts); len(problems) > 0 {
refused[name] = problems
continue
}
shelf[name] = m shelf[name] = m
} }
between := Between(shelf, longestMachine) between := Between(shelf, longestMachine)
@@ -179,6 +189,7 @@ const Unnamed = "(between manifests)"
func Request(r composer.Request) (composer.Verdict, error) { func Request(r composer.Request) (composer.Verdict, error) {
v := composer.Verdict{V: composer.Version, SDK: Identity()} v := composer.Verdict{V: composer.Version, SDK: Identity()}
seen := map[string]bool{} seen := map[string]bool{}
counts := trunkOldSections(r)
for _, c := range r.Cases { for _, c := range r.Cases {
if seen[c.Name] { if seen[c.Name] {
return composer.Verdict{}, fmt.Errorf("the case %q is named twice", c.Name) return composer.Verdict{}, fmt.Errorf("the case %q is named twice", c.Name)
@@ -191,7 +202,7 @@ func Request(r composer.Request) (composer.Verdict, error) {
} }
raws[name] = m.Raw raws[name] = m.Raw
} }
refused, unnamed := Shelf(raws, r.Predefined, 0) refused, unnamed := ShelfCounting(raws, r.Predefined, 0, counts)
cv := composer.CaseVerdict{Name: c.Name, Manifests: map[string]composer.ManifestRead{}} cv := composer.CaseVerdict{Name: c.Name, Manifests: map[string]composer.ManifestRead{}}
for name := range raws { for name := range raws {
if problems := refused[name]; len(problems) > 0 { if problems := refused[name]; len(problems) > 0 {
@@ -208,6 +219,39 @@ func Request(r composer.Request) (composer.Verdict, error) {
return v, nil return v, nil
} }
// TrunkCase is the case of a request holding the catalogue as the trunk holds it (the checker's JudgeTrunk).
const TrunkCase = "trunk"
// trunkOldSections counts the old sections the modules accepted at the trunk use, as the indexer counts them from its
// verdicts (issue 1966 D4, ADR 0595 §11): a change is refused at the merge check exactly as at take-in (judged once, ADR
// 0578). Nil where the request holds no trunk case, which refuses no use of a section.
func trunkOldSections(r composer.Request) OldSectionCounts {
for _, c := range r.Cases {
if c.Name != TrunkCase {
continue
}
raws := map[string][]byte{}
for name, m := range c.Manifests {
if len(m.Raw) > 0 {
raws[name] = m.Raw
}
}
refused, _ := Shelf(raws, r.Predefined, 0)
var accepted []catalogue.Manifest
for name, raw := range raws {
if len(refused[name]) > 0 {
continue
}
if m, err := catalogue.Decode(raw); err == nil {
m.Module = name
accepted = append(accepted, m)
}
}
return CountOldSections(accepted)
}
return nil
}
var ( var (
identityOnce sync.Once identityOnce sync.Once
identity string identity string
@@ -86,6 +86,27 @@ func TestTheJudge(t *testing.T) {
v.Cases[0].Manifests["b"].Accepted || !strings.Contains(v.Cases[0].Manifests["b"].Refused, `requires "amqp"`) { v.Cases[0].Manifests["b"].Accepted || !strings.Contains(v.Cases[0].Manifests["b"].Refused, `requires "amqp"`) {
t.Fatalf("request: %+v %v", v, err) t.Fatalf("request: %+v %v", v, err)
} }
// 1966/AC11: the checker's judge counts the old sections from the trunk case as the indexer does from its verdicts (the
// foundation left out): a change adding logs where no trunk module uses them is refused, the foundation's accepted, and
// state, which a trunk module uses, is accepted.
old := composer.Request{V: composer.Version, Cases: []composer.Case{
{Name: "change", Manifests: map[string]composer.Manifest{
"user": {Raw: json.RawMessage(`{"module":"user","version":"1","state":["x"]}`)},
"late": {Raw: json.RawMessage(`{"module":"late","version":"1","logs":[{"name":"l"}]}`)},
"newer": {Raw: json.RawMessage(`{"module":"newer","version":"1","state":["y"]}`)},
"mesh-grants": {Raw: json.RawMessage(`{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`)}}},
{Name: "trunk", Manifests: map[string]composer.Manifest{
"user": {Raw: json.RawMessage(`{"module":"user","version":"1","state":["x"]}`)},
"mesh-grants": {Raw: json.RawMessage(`{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`)}}}}}
ov, err := Request(old)
if err != nil {
t.Fatal(err)
}
got := ov.Cases[0].Manifests
if got["late"].Accepted || !strings.Contains(got["late"].Refused, "late uses the old section logs") ||
!got["newer"].Accepted || !got["user"].Accepted || !got["mesh-grants"].Accepted {
t.Errorf("the change's old sections: %+v", got)
}
r.Cases[0].Manifests["c"] = composer.Manifest{Digest: strings.Repeat("a", 64)} r.Cases[0].Manifests["c"] = composer.Manifest{Digest: strings.Repeat("a", 64)}
if _, err := Request(r); err == nil { if _, err := Request(r); err == nil {
t.Fatal("a manifest left as a digest was judged") t.Fatal("a manifest left as a digest was judged")
@@ -0,0 +1,69 @@
package judge
import (
"fmt"
"slices"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The old sections end when no module is left in them (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts, per old
// section, the accepted modules using it, the foundation left out, and the judge refuses a manifest of another module
// that uses a section whose count is 0. The foundation (the bus module, the grants part and the indexer itself) is
// created before the bus module can create anything, so it keeps the old sections for good and is always accepted.
// OldSections are the sections of a manifest that keep bus state in the old forms.
var OldSections = []string{"state", "logs", "streams", "objects"}
// Foundation are the modules that keep using the old sections for good.
var Foundation = []string{"nats", "mesh-grants", "mesh-delivery-indexer"}
// OldSectionCounts is, per old section, how many accepted modules outside the foundation use it. Nil is no count: the
// judge then refuses no use of a section.
type OldSectionCounts map[string]int
// SectionsUsed are the old sections a manifest uses, in OldSections' order.
func SectionsUsed(m catalogue.Manifest) []string {
var used []string
for _, s := range OldSections {
if (s == "state" && len(m.State) > 0) || (s == "logs" && len(m.Logs) > 0) ||
(s == "streams" && len(m.Streams) > 0) || (s == "objects" && len(m.Objects) > 0) {
used = append(used, s)
}
}
return used
}
// CountOldSections counts, per old section, the manifests using it, the foundation's left out; every section is in the
// answer, with 0 where none uses it.
func CountOldSections(manifests []catalogue.Manifest) OldSectionCounts {
counts := OldSectionCounts{}
for _, s := range OldSections {
counts[s] = 0
}
for _, m := range manifests {
if slices.Contains(Foundation, m.Module) {
continue
}
for _, s := range SectionsUsed(m) {
counts[s]++
}
}
return counts
}
// OldSectionProblems are the old sections a manifest uses that no accepted module outside the foundation uses any more.
func OldSectionProblems(m catalogue.Manifest, counts OldSectionCounts) []string {
if counts == nil || slices.Contains(Foundation, m.Module) {
return nil
}
var problems []string
for _, s := range SectionsUsed(m) {
if counts[s] == 0 {
problems = append(problems, fmt.Sprintf(
"%s uses the old section %s, which no other module uses any more (0 modules count); ask for it in "+
"`requires` by template (novox/hq ADR 0595 §11)", m.Module, s))
}
}
return problems
}
+5 -1
View File
@@ -2626,6 +2626,8 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M
// And the object buckets it owns (novox/hq ADR 0592 §5, §9). // And the object buckets it owns (novox/hq ADR 0592 §5, §9).
problems = append(problems, ObjectsProblems(m)...) problems = append(problems, ObjectsProblems(m)...)
problems = append(problems, WorkQueueStreamProblems(m)...) problems = append(problems, WorkQueueStreamProblems(m)...)
// And the bus state it asks for in `requires`, by template (busstate.go, novox/hq ADR 0595; issue 1966 D1).
problems = append(problems, BusStateProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262), or, written in the // And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262), or, written in the
// config form, the whole declaration of each value (config.go, novox/hq ADR 0348). // config form, the whole declaration of each value (config.go, novox/hq ADR 0348).
if m.DeclaresConfig() { if m.DeclaresConfig() {
@@ -4554,6 +4556,8 @@ func CatalogueProblems(shelf catalogue.Shelf) []string {
problems = append(problems, StreamNameProblems(shelf)...) problems = append(problems, StreamNameProblems(shelf)...)
// An object bucket's name has one owner, and another module reads it only through its owner's grant (ADR 0592 §5, §9). // An object bucket's name has one owner, and another module reads it only through its owner's grant (ADR 0592 §5, §9).
problems = append(problems, ObjectsCatalogueProblems(shelf)...) problems = append(problems, ObjectsCatalogueProblems(shelf)...)
// Bus state in `requires`: a name on the bus has one owner, and the new form has no readers (ADR 0595, ADR 0596 §2).
problems = append(problems, BusStateCatalogueProblems(shelf)...)
// A module that hears a restricted stream runs as an account of its own (novox/hq ADR 0376 §2). // A module that hears a restricted stream runs as an account of its own (novox/hq ADR 0376 §2).
problems = append(problems, RestrictedHearingProblems(shelf)...) problems = append(problems, RestrictedHearingProblems(shelf)...)
// A work-queue stream is never asked for by another module's consumes (novox/hq ADR 0436 §6). // A work-queue stream is never asked for by another module's consumes (novox/hq ADR 0436 §6).
@@ -5358,7 +5362,7 @@ func StateProblems(m catalogue.Manifest) []string {
seen[s.Name] = true seen[s.Name] = true
if s.History < 0 || s.History > catalogue.StateMostHistory { if s.History < 0 || s.History > catalogue.StateMostHistory {
problems = append(problems, fmt.Sprintf( problems = append(problems, fmt.Sprintf(
"%s keeps %d values of %q; a key keeps between 1 and %d", m.Module, s.History, s.Name, catalogue.StateMostHistory)) "%s keeps %d values of %q; a key keeps 0 or 1 to %d (0 is taken as 1)", m.Module, s.History, s.Name, catalogue.StateMostHistory))
} }
if s.TTLSeconds < 0 { if s.TTLSeconds < 0 {
problems = append(problems, fmt.Sprintf("%s gives %q a negative lifetime", m.Module, s.Name)) problems = append(problems, fmt.Sprintf("%s gives %q a negative lifetime", m.Module, s.Name))
@@ -54,12 +54,6 @@ func TestABinaryMustSayWhichSystemItIsFor(t *testing.T) {
} }
} }
func TestABinaryThatNamesASystemIsAccepted(t *testing.T) {
if got := buildProblemsOf(t, bundleFor("go", "arch")); got != "" {
t.Fatalf("a compiled bundle naming a system was refused:\n%s", got)
}
}
func TestABlockOutsideTheShapeIsRefused(t *testing.T) { func TestABlockOutsideTheShapeIsRefused(t *testing.T) {
cases := map[string]string{ cases := map[string]string{
`{"seat":"node-bar","kind":"block","content":"[[block]]"}`: "has no data", `{"seat":"node-bar","kind":"block","content":"[[block]]"}`: "has no data",
@@ -913,6 +907,12 @@ func TestALogIsNamedLocallyAndCapped(t *testing.T) {
{`{"module":"a.b","version":"1","logs":["c"]}`, `no dot`}, {`{"module":"a.b","version":"1","logs":["c"]}`, `no dot`},
} { } {
_, err := ParseManifest([]byte(c.manifest)) _, err := ParseManifest([]byte(c.manifest))
if c.says == "" {
if err != nil {
t.Errorf("%s was refused: %v", c.manifest, err)
}
continue
}
if err == nil { if err == nil {
t.Errorf("%s was accepted", c.manifest) t.Errorf("%s was accepted", c.manifest)
continue continue
@@ -2356,18 +2356,19 @@ func TestAPlaceholderOutsideTheHolderIsRefused(t *testing.T) {
} }
} }
// **And a language that runs anywhere must not name one.** A system that decides nothing reads as // **A language that runs anywhere must not name a system.** A system that decides nothing reads as
// though it did, which is the same fault as a restriction that restricts nothing (novox/hq ADR 0045). // though it did, which is the same fault as a restriction that restricts nothing (novox/hq ADR 0045); a compiled
func TestAPortableBundleMayNotNameASystem(t *testing.T) { // bundle names one, and a portable bundle naming none is an ordinary one.
got := buildProblemsOf(t, bundleFor("typescript", "arch")) func TestABundleNamesASystemOnlyWhereItCompiles(t *testing.T) {
if !strings.Contains(got, "runs anywhere") { for _, c := range []struct{ language, system, refused string }{
t.Fatalf("a portable bundle was allowed to name a system:\n%s", got) {"go", "arch", ""},
} {"typescript", "", ""},
} {"typescript", "arch", "runs anywhere"},
} {
func TestAPortableBundleNamingNoSystemIsAccepted(t *testing.T) { got := buildProblemsOf(t, bundleFor(c.language, c.system))
if got := buildProblemsOf(t, bundleFor("typescript", "")); got != "" { if (c.refused == "") != (got == "") || !strings.Contains(got, c.refused) {
t.Fatalf("an ordinary bundle was refused:\n%s", got) t.Errorf("a %s bundle naming %q: %q, want %q", c.language, c.system, got, c.refused)
}
} }
} }
@@ -2882,13 +2883,23 @@ func TestAStateNameIsLocalAndOneToken(t *testing.T) {
{`{"module":"a","version":"1","state":["mesh.servers"]}`, `keeps state "mesh.servers": a state is named locally`}, {`{"module":"a","version":"1","state":["mesh.servers"]}`, `keeps state "mesh.servers": a state is named locally`},
{`{"module":"a","version":"1","state":["my_servers"]}`, `keeps state "my_servers"`}, {`{"module":"a","version":"1","state":["my_servers"]}`, `keeps state "my_servers"`},
{`{"module":"a","version":"1","state":["s","s"]}`, `keeps state "s" twice`}, {`{"module":"a","version":"1","state":["s","s"]}`, `keeps state "s" twice`},
{`{"module":"a","version":"1","state":[{"name":"s","history":65}]}`, `a key keeps between 1 and 64`}, {`{"module":"a","version":"1","state":[{"name":"s","history":65}]}`, `a key keeps 0 or 1 to 64 (0 is taken as 1)`},
{`{"module":"a","version":"1","state":[{"name":"s","history":0}]}`, ``}, // 0 passes, as before (ADR 0579 §1)
// 1966/AC9: one name, one form
{`{"module":"a","version":"1","state":["x"],"requires":[{"name":"x","storage":"key-value","template":"state"}]}`,
`a's bus state "x": the name is also kept in the old section state`},
{`{"module":"a.b","version":"1","state":["s"]}`, `no dot`}, {`{"module":"a.b","version":"1","state":["s"]}`, `no dot`},
{`{"module":"a","version":"1","reads":["bindings"]}`, `a read is <module>.<name>`}, {`{"module":"a","version":"1","reads":["bindings"]}`, `a read is <module>.<name>`},
{`{"module":"a","version":"1","reads":["a.s"]}`, `which is its own state`}, {`{"module":"a","version":"1","reads":["a.s"]}`, `which is its own state`},
{`{"module":"a","version":"1","state":[{"name":"s","shared":true}]}`, `{name, history, ttl-seconds, per-machine, key-ttl}`}, {`{"module":"a","version":"1","state":[{"name":"s","shared":true}]}`, `{name, history, ttl-seconds, per-machine, key-ttl}`},
} { } {
_, err := ParseManifest([]byte(c.manifest)) _, err := ParseManifest([]byte(c.manifest))
if c.says == "" {
if err != nil {
t.Errorf("%s was refused: %v", c.manifest, err)
}
continue
}
if err == nil { if err == nil {
t.Errorf("%s was accepted", c.manifest) t.Errorf("%s was accepted", c.manifest)
continue continue
@@ -84,12 +84,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("module = %q", m.Module) t.Errorf("module = %q", m.Module)
} }
// 1126: placement is the seat mesh-placement's (ADR 0567): the indexer holds no placement state. ADR 0578 §1: it says // 1126: placement is the seat mesh-placement's (ADR 0567): the indexer holds no placement state. ADR 0578 §1: it says
// a refused manifest as manifest-refused, its one event. // a refused manifest as manifest-refused, and (issue 1966 D3, 1966/AC10) a newly accepted one asking for bus state as
if !reflect.DeepEqual(m.Emits, []string{"manifest-refused"}) { // manifest-accepted.
t.Errorf("emits = %v, want manifest-refused alone", m.Emits) if !reflect.DeepEqual(m.Emits, []string{"manifest-refused", "manifest-accepted"}) {
t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits)
} }
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) { if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want) t.Errorf("state = %+v, want %+v", m.State, want)
} }
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
@@ -112,8 +113,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) { if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want) t.Errorf("invokes = %v, want %v", m.Invokes, want)
} }
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11:
if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) { // indexer_old_sections reads the count of the old sections (state oldsections).
if want := []string{"indexer_take_in", "indexer_old_sections"}; !reflect.DeepEqual(m.Tools, want) {
t.Errorf("tools = %v, want %v", m.Tools, want) t.Errorf("tools = %v, want %v", m.Tools, want)
} }
// No streams, no bus, no seat (ADR 0437 §4); it says no event. // No streams, no bus, no seat (ADR 0437 §4); it says no event.
+5 -2
View File
@@ -3,7 +3,8 @@
"version": "1", "version": "1",
"slug": "indexer", "slug": "indexer",
"tools": [ "tools": [
"indexer_take_in" "indexer_take_in",
"indexer_old_sections"
], ],
"consumes": [ "consumes": [
"mesh-build-go.toolchain.reported", "mesh-build-go.toolchain.reported",
@@ -21,10 +22,12 @@
{ "name": "taken", "history": 1 }, { "name": "taken", "history": 1 },
{ "name": "takeins", "history": 1 }, { "name": "takeins", "history": 1 },
{ "name": "verdicts", "history": 5 }, { "name": "verdicts", "history": 5 },
{ "name": "oldsections", "history": 1 },
{ "name": "packages", "history": 5 } { "name": "packages", "history": 5 }
], ],
"emits": [ "emits": [
"manifest-refused" "manifest-refused",
"manifest-accepted"
], ],
"reads": [ "reads": [
"mesh-build-go.toolchains", "mesh-build-go.toolchains",
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-grants
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1 h1:T66tg9T/1hHg6KN/bDScHPT41IHkxLAGGO8VJiAOlb8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+22
View File
@@ -2202,6 +2202,28 @@ func TestGrants(t *testing.T) {
if res := bare.Compose(); res.Accounts != nil || !strings.Contains(strings.Join(res.Errors, " "), "no bus address") { if res := bare.Compose(); res.Accounts != nil || !strings.Contains(strings.Join(res.Errors, " "), "no bus address") {
t.Errorf("a part with no bus address wrote %+v: %v", res.Accounts, res.Errors) t.Errorf("a part with no bus address wrote %+v: %v", res.Accounts, res.Errors)
} }
// 1966/AC20 (issue 1966 D5, D5b; novox/hq ADR 0597 §9): a module asking for a ledger in `requires` is wanted a ledger
// account on each machine it is assigned to, minted by the vault like every kind and held in the tool runner's
// `ledgers`; the account is retired once the module leaves that machine.
l := fixture()
l.manifestOf("mod-x", strings.Replace(xManifest, `"state"`, `"requires":[{"name":"records","storage":"ledger","template":"ledger",`+
`"types":[{"name":"decision","schema":{},"operations":[{"name":"propose","effect":"set-fields"}]}]}],"state"`, 1))
l.assign("b", "mod-x")
l.loaded = shadowOf(t, l)
lp := partOn(l)
lp.Bus = grants.BusAddress{URL: "nats://a.mesh:4222"}
lp.Compose()
for _, user := range []string{"ledger.a.mod-x", "ledger.b.mod-x"} {
w, err := buscompose.ReadWantedAccount(user, l.own[grants.StateAccounts][user])
if err != nil || w.Kind != buscompose.KindLedger || w.OwnSecret() != "ledgers" || w.Module != "mod-x" {
t.Errorf("%s is wanted as %+v (%v)", user, w, err)
}
}
l.unassign("b", "mod-x")
if res := lp.Compose(); res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-x", "ledger.b.mod-x"}) ||
l.own[grants.StateAccounts]["ledger.a.mod-x"] == nil {
t.Errorf("after mod-x left b: %+v", res.Accounts)
}
}) })
// 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash // 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash
@@ -50,7 +50,8 @@ type ClaimedSeat struct {
// WantedAccount is one key of the grants part's state `accounts`: an account the mesh needs, without its password. // WantedAccount is one key of the grants part's state `accounts`: an account the mesh needs, without its password.
type WantedAccount struct { type WantedAccount struct {
User string `json:"user"` User string `json:"user"`
// Kind is KindModule (KindNodeTools for the runtime, named as a module is), KindProcess or KindProvisioner. // Kind is KindModule (KindNodeTools for the runtime, named as a module is), KindProcess, KindProvisioner or
// KindLedger.
Kind Kind `json:"kind"` Kind Kind `json:"kind"`
Node string `json:"node"` Node string `json:"node"`
Module string `json:"module"` Module string `json:"module"`
@@ -65,8 +66,12 @@ type WantedAccount struct {
At time.Time `json:"at"` At time.Time `json:"at"`
} }
// LedgersSecret is the tool runner's own secret holding its machine's ledger accounts, a map of module to credential
// (ADR 0597 §9).
const LedgersSecret = "ledgers"
// mintedKinds are the kinds of account the vault mints (ADR 0561 §1). // mintedKinds are the kinds of account the vault mints (ADR 0561 §1).
var mintedKinds = []Kind{KindModule, KindNodeTools, KindProcess, KindProvisioner} var mintedKinds = []Kind{KindModule, KindNodeTools, KindProcess, KindProvisioner, KindLedger}
// principal is the account as the composer names it. // principal is the account as the composer names it.
func (w WantedAccount) principal() Principal { func (w WantedAccount) principal() Principal {
@@ -112,9 +117,12 @@ func (w WantedAccount) Same(o WantedAccount) bool {
} }
// OwnSecret is the module's own secret the account is held in: `broker`, `broker-<process id>` or `provisioner-broker` // OwnSecret is the module's own secret the account is held in: `broker`, `broker-<process id>` or `provisioner-broker`
// (the controller's names, ADR 0436 §2, ADR 0349 §11). // (the controller's names, ADR 0436 §2, ADR 0349 §11); a ledger account in the tool runner's `ledgers`, which keeps every
// ledger account of its machine, by module (ADR 0597 §9).
func (w WantedAccount) OwnSecret() string { func (w WantedAccount) OwnSecret() string {
switch w.Kind { switch w.Kind {
case KindLedger:
return LedgersSecret
case KindProcess: case KindProcess:
return "broker-" + w.Process return "broker-" + w.Process
case KindProvisioner: case KindProvisioner:
@@ -49,6 +49,7 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
t.Run("only declared streams are created", theGrantsPartCreatesOnlyDeclaredStreams) t.Run("only declared streams are created", theGrantsPartCreatesOnlyDeclaredStreams)
t.Run("a declared stream is created as the controller asserted it", declaredStreamsAreCreatedAsTheControllerAssertsThem) t.Run("a declared stream is created as the controller asserted it", declaredStreamsAreCreatedAsTheControllerAssertsThem)
t.Run("every node's compositions are read by their named readers alone and written by their node", everyNodesCompositions) t.Run("every node's compositions are read by their named readers alone and written by their node", everyNodesCompositions)
t.Run("a module's ledger is written by its ledger account alone", aLedgerIsWrittenByItsLedgerAccountAlone)
// ADR 0567 §3, §9: a node reads every key of the seat's placement and its candidates (it composes with the whole // ADR 0567 §3, §9: a node reads every key of the seat's placement and its candidates (it composes with the whole
// proposal laid over), and none of its open proposals; it writes neither. // proposal laid over), and none of its open proposals; it writes neither.
reads := strings.Join(NodeComposeReads("laptop"), " ") reads := strings.Join(NodeComposeReads("laptop"), " ")
@@ -942,3 +943,75 @@ func everyNodesCompositions(t *testing.T) {
t.Errorf("the view's membership issues %+v", m.State) t.Errorf("the view's membership issues %+v", m.State)
} }
} }
// 1966/AC20, AC21 (issue 1966 D5, ADR 0597 §9): a module asking for a ledger, assigned on two machines, gets one ledger account
// on each, publishing only on that module's ledger histories and snapshots; the account is gone from a machine the module
// leaves; and the module's own account, its process and the machine's tool runner hold no publish on its ledger.
func aLedgerIsWrittenByItsLedgerAccountAlone(t *testing.T) {
assigned := func(nodes ...string) []Principal {
r := Records{Nodes: nodes, Assigned: map[string][]Declared{}}
for _, n := range nodes {
r.Assigned[n] = []Declared{
{Module: RuntimeModule},
{Module: "m", Ledgers: []string{"records", "audit"}, Processes: []ProcessPrincipal{{Process: "w"}}},
{Module: "plain"},
}
}
users, err := Users(r)
if err != nil {
t.Fatal(err)
}
return users
}
ledgers := func(users []Principal) []string {
var out []string
for _, p := range users {
if p.Kind == KindLedger {
out = append(out, p.Username())
}
}
return out
}
both := assigned("a", "b")
if got := ledgers(both); !slices.Equal(got, []string{"ledger.a.m", "ledger.b.m"}) {
t.Fatalf("ledger accounts %v, want ledger.a.m and ledger.b.m", got)
}
if got := ledgers(assigned("a")); !slices.Equal(got, []string{"ledger.a.m"}) {
t.Fatalf("after m left b: ledger accounts %v, want ledger.a.m alone", got)
}
want := []string{"mesh.ledger.m.audit.history.>", "mesh.ledger.m.audit.snapshot.>",
"mesh.ledger.m.records.history.>", "mesh.ledger.m.records.snapshot.>"}
raw := []string{"mesh.ledger.m.records.history.decision/0290", "mesh.ledger.m.records.snapshot.decision/0290"}
for _, p := range both {
perms, err := PermissionsFor(p)
if err != nil {
t.Fatalf("%s: %v", p.Username(), err)
}
if p.Kind == KindLedger {
if !slices.Equal(perms.Publish, want) || len(perms.PublishDeny) != 0 {
t.Errorf("%s may publish %v, want %v alone", p.Username(), perms.Publish, want)
}
continue
}
for _, pattern := range perms.Publish {
for _, subject := range raw {
if SubjectsOverlap(pattern, subject) {
t.Errorf("%s may publish %s, which reaches the ledger subject %s", p.Username(), pattern, subject)
}
}
}
if p.Kind == KindModule || p.Kind == KindNodeTools || p.Kind == KindProcess {
if err := CheckWriters(p, want); err == nil {
t.Errorf("%s was granted a raw publish on a ledger's history", p.Username())
}
}
}
if err := CheckWriters(Principal{Kind: KindLedger, Node: "a", Module: "other", Ledgers: []string{"records"}}, want); err == nil {
t.Error("another module's ledger account was granted m's ledger histories")
}
for _, bad := range []Principal{{Kind: KindLedger, Node: "a", Module: "m"}, {Kind: KindLedger, Node: "a", Module: "m", Ledgers: []string{"a.b"}}} {
if _, err := PermissionsFor(bad); err == nil {
t.Errorf("%+v was composed", bad)
}
}
}
@@ -0,0 +1,57 @@
package buscompose
import (
"fmt"
"slices"
"sort"
"strings"
sdk "git.novox.be/novox/mesh-sdk/go/buscompose"
)
// A module's ledger and the one account that writes it (issue 1966 D5; novox/hq ADR 0597 §9, design 25 § 12.4).
//
// **Only the tool runner's ledger credential publishes on a ledger's history** (ADR 0597 §9): one per module assignment,
// named ledger.<machine>.<module> (KindLedger), scoped to the subjects of that module's ledger histories and snapshots, and
// gone when the module is unassigned from the machine. The module's own account, its processes and the machine's tool
// runner hold no publish there, so a raw publish on a history is refused by the bus. The subjects are named in one place, mesh-sdk's
// buscompose (LedgerHistorySubject, LedgerSnapshotSubject), `mesh.ledger.<module>.<local name>.history|snapshot` and
// one subject per instance below it (design 25 § 12.4).
// LedgerPublishGrants are the publish patterns a module's ledger account is granted: the histories and the snapshots of the
// ledgers named, every instance of each, and nothing else. Sorted.
func LedgerPublishGrants(module string, locals []string) []string {
var out []string
for _, local := range locals {
out = append(out, sdk.LedgerHistorySubject(module, local)+".>", sdk.LedgerSnapshotSubject(module, local)+".>")
}
sort.Strings(out)
return out
}
// writesLedger is a module's ledger account publishing on that module's own ledgers' histories and snapshots, and no other
// principal (ADR 0597 §9).
func writesLedger(p Principal, pattern string) bool {
return p.Kind == KindLedger && slices.Contains(LedgerPublishGrants(p.Module, p.Ledgers), pattern)
}
// ledgerSubjects are the subjects no principal but a ledger account may publish on, for the writers table.
var ledgerSubjects = []string{"mesh.ledger.*.*.history.>", "mesh.ledger.*.*.snapshot.>"}
// ledgerPermissions are a ledger account's whole authority: publishing on its module's ledgers, and its own reply space for
// the stream's acknowledgements. Refused when it names no ledger or a name that is no subject token.
func ledgerPermissions(p Principal) (Permissions, error) {
pub := LedgerPublishGrants(p.Module, p.Ledgers)
if len(p.Ledgers) == 0 {
return Permissions{}, fmt.Errorf("%s asks for no ledger, so its account would publish nowhere", p.Username())
}
for _, l := range p.Ledgers {
if !safeSubject.MatchString(l) || strings.Contains(l, "_") {
return Permissions{}, fmt.Errorf("%s: %q cannot be part of a subject: a permission is a subject pattern, and this would widen it", p.Username(), l)
}
}
if err := CheckWriters(p, pub); err != nil {
return Permissions{}, err
}
return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil
}
@@ -52,6 +52,10 @@ const (
// exactly what its bus says — never root, never the operator account, and never able to answer a tool // exactly what its bus says — never root, never the operator account, and never able to answer a tool
// call or a seat's verb (§3). One per process and machine. // call or a seat's verb (§3). One per process and machine.
KindProcess Kind = "process" KindProcess Kind = "process"
// KindLedger is one module assignment's ledger credential (novox/hq ADR 0597 §9, design 25 § 12.4): held by that
// machine's tool runner, never by the module's own code, and the only account that publishes on that module's ledger
// histories and snapshots. One per module assignment, whose module asks for a ledger in `requires`.
KindLedger Kind = "ledger"
) )
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is // RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
@@ -102,6 +106,9 @@ type Principal struct {
Kind Kind Kind Kind
Node string Node string
Module string Module string
// Ledgers are the local names of the ledgers its module asks for in `requires`, for a KindLedger principal alone
// (issue 1966 D5): its whole authority is publishing on those ledgers' histories and snapshots (LedgerPublishGrants).
Ledgers []string
// Process is the process resource's id, for a KindProcess principal alone (novox/hq ADR 0436 §2): // Process is the process resource's id, for a KindProcess principal alone (novox/hq ADR 0436 §2):
// Username and inbox are <node>.<module>.<process id>. // Username and inbox are <node>.<module>.<process id>.
Process string Process string
@@ -425,6 +432,10 @@ func (p Principal) Username() string {
return p.Login return p.Login
} }
return "node." + p.Node return "node." + p.Node
case KindLedger:
// `ledger.<machine>.<module>`: never `<node>.<module>`, which is the module's own account or the tool runner's,
// so the module's code never holds it.
return "ledger." + p.Node + "." + p.Module
case KindProvisioner: case KindProvisioner:
// Named for the machine, as a node is: one client per machine, whichever module runs it, and never // Named for the machine, as a node is: one client per machine, whichever module runs it, and never
// `<node>.<module>`, which is the module's own account or the tool runner's. // `<node>.<module>`, which is the module's own account or the tool runner's.
@@ -567,6 +578,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil
} }
// A ledger account publishes on its module's ledger histories and snapshots and nothing else (issue 1966 D5).
if p.Kind == KindLedger {
return ledgerPermissions(p)
}
if p.Kind == KindProcess && len(p.Removes) > 0 { if p.Kind == KindProcess && len(p.Removes) > 0 {
pub, err := BusRemovalGrants(p.Removes, p.FilesFor) pub, err := BusRemovalGrants(p.Removes, p.FilesFor)
if err != nil { if err != nil {
@@ -103,6 +103,9 @@ type Declared struct {
// composed whether or not the module itself has an account (NoAccount), because a process account is // composed whether or not the module itself has an account (NoAccount), because a process account is
// delivered in its own secret, broker-<process id>, never the module's `broker`. // delivered in its own secret, broker-<process id>, never the module's `broker`.
Processes []ProcessPrincipal Processes []ProcessPrincipal
// Ledgers are the local names of the ledgers it asks for in `requires` (catalogue.StorageLedger); each assignment of a
// module with any gets a ledger principal of its own (issue 1966 D5, ADR 0597 §9).
Ledgers []string
} }
// ProcessPrincipal is one process of a module that declares bus (novox/hq ADR 0436 §1, §2), as the broker // ProcessPrincipal is one process of a module that declares bus (novox/hq ADR 0436 §1, §2), as the broker
@@ -229,6 +232,11 @@ func Users(r Records) ([]Principal, error) {
out[len(out)-1].FilesFor = sortedCopy(r.Nodes) out[len(out)-1].FilesFor = sortedCopy(r.Nodes)
} }
} }
// **A ledger account per assignment of a module that asks for a ledger** (issue 1966 D5, ADR 0597 §9): held by the
// machine's tool runner, so it is composed whether or not the module has an account of its own.
if len(d.Ledgers) > 0 {
out = append(out, Principal{Kind: KindLedger, Node: node, Module: d.Module, Ledgers: sortedCopy(d.Ledgers)})
}
if runtimeHere && d.Module == RuntimeModule { if runtimeHere && d.Module == RuntimeModule {
continue continue
} }
@@ -236,6 +236,12 @@ var WritersTable = []WriterRow{
ProvisionerAppliedBucket, ProvisionerAppliedBucket,
Others: "the watchman reads it; a failure is that machine's failed apply", Others: "the watchman reads it; a failure is that machine's failed apply",
Subjects: []string{"prov.applied.*.>"}, Writes: ownProvisioner}, Subjects: []string{"prov.applied.*.>"}, Writes: ownProvisioner},
// A ledger's history and its snapshots (novox/hq ADR 0597 §9, issue 1966 D5): the tool runner's ledger credential of
// the owning module alone, so the module's own code, its processes and the runner's own account cannot write one raw.
{State: "a ledger's history and snapshots", Writer: "the tool runner's ledger credential of the owning module",
KeptIn: "the bus, the ledger's history stream and its snapshot subject",
Others: "the owner reads them through the tool runner's calls; nobody else reads them (ADR 0596 §2)",
Subjects: ledgerSubjects, Writes: writesLedger},
} }
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives // CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
@@ -46,8 +46,9 @@ type BusAddress struct {
Fingerprint string Fingerprint string
} }
// WantedAccounts is every account the vault mints that the records compose, keyed by user, with the bus's address. A // WantedAccounts is every account the vault mints that the records compose, keyed by user, with the bus's address: a
// node, the controller, an enrolment user or a person is never one (ADR 0561 §4, §5). Pure. // module's, a process's, a machine's provisioner client's and a module assignment's ledger account (issue 1966 D5b, ADR
// 0597 §9). A node, the controller, an enrolment user or a person is never one (ADR 0561 §4, §5). Pure.
func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[string]buscompose.WantedAccount, error) { func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[string]buscompose.WantedAccount, error) {
principals, err := buscompose.Users(recs) principals, err := buscompose.Users(recs)
if err != nil { if err != nil {
@@ -62,7 +63,7 @@ func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[str
if m, ok := manifestOn(in, p.Node, p.Module); ok { if m, ok := manifestOn(in, p.Node, p.Module); ok {
w.Claims = ClaimsOf(m) w.Claims = ClaimsOf(m)
} }
case buscompose.KindProcess, buscompose.KindProvisioner: case buscompose.KindProcess, buscompose.KindProvisioner, buscompose.KindLedger:
default: default:
continue continue
} }
@@ -139,6 +139,8 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
Served: catalogue.ServedSettings(m), Served: catalogue.ServedSettings(m),
// Its own object buckets (ADR 0592 §5). // Its own object buckets (ADR 0592 §5).
Objects: objectsOf(m), Objects: objectsOf(m),
// The ledgers it asks for in `requires` (issue 1966 D5): each assignment gets a ledger account.
Ledgers: ledgersOf(m),
} }
d.Loads = loadsABundle(m) d.Loads = loadsABundle(m)
if _, reads := m.OwnSecrets["broker"]; !reads { if _, reads := m.OwnSecrets["broker"]; !reads {
@@ -282,6 +284,17 @@ func readsOfObjects(module, node string, reads, refused []string, manifests map[
return states, objects, refusedOut return states, objects, refusedOut
} }
// ledgersOf are the local names of the ledgers a manifest asks for in `requires`.
func ledgersOf(m catalogue.Manifest) []string {
var out []string
for _, e := range m.BusState {
if e.Storage == catalogue.StorageLedger {
out = append(out, e.Name)
}
}
return out
}
func logsOf(m catalogue.Manifest) []buscompose.Log { func logsOf(m catalogue.Manifest) []buscompose.Log {
var out []buscompose.Log var out []buscompose.Log
for _, l := range m.Logs { for _, l := range m.Logs {