Merge pull request 'Bus state by template: the judge's rules, manifest-accepted, the old-section count and the ledger account (issue 1966, B1)' (#146) from feat/bus-state into main

This commit was merged in pull request #146.
This commit is contained in:
2026-11-03 15:24:36 +00:00
29 changed files with 1163 additions and 42 deletions
@@ -22,9 +22,22 @@ import (
type asker func(key string, value []byte) (index.TakeInAsk, error)
// Tools are the indexer's tools.
func Tools(ask asker, now func() time.Time) []stdio.Tool {
func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCount, error)) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: index.OldSectionsTool,
Description: "How many accepted modules use each old section of a manifest (state, logs, streams, objects), the foundation " +
"(nats, mesh-grants, the indexer) left out: the count the judge refuses a use of a section at 0 by, as modules move " +
"their bus state into `requires` by template. Read from the indexer's oldsections state, kept as the indexer judges " +
"the manifests; an empty answer means the indexer has not judged since it started.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
c, err := counts()
if err != nil {
return nil, err
}
return c.Counts, nil
}},
{Name: index.TakeInTool,
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
@@ -105,8 +118,22 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
}
}
// heldCounts reads the count of the old sections the indexer keeps.
func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
return func() (index.OldSectionsCount, error) {
e, err := s.Get(index.OldSectionsKey)
if err != nil {
return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsBucket, index.OldSectionsKey, err)
}
if e == nil {
return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsBucket, index.OldSectionsKey)
}
return index.DecodeOldSections(e.Value)
}
}
func main() {
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, heldCounts(stdio.State(index.OldSectionsState)))); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
@@ -296,8 +296,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
// fingerprint takes the version line.
packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf),
// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
// the judge refuses no use of an old section.
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call)}
@@ -36,3 +36,29 @@ func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string,
}
}
}
// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4):
// mesh.mod.<module>.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as
// the bus's dedup id (delivery.ManifestAccepted.EventID), so one acceptance is said once.
func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, delivery.ManifestAccepted) {
return func(ctx context.Context, a delivery.ManifestAccepted) {
data, err := json.Marshal(a)
if err != nil {
logf("judge: %s for %s is not said: %v", delivery.ManifestAcceptedEvent, a.Module, err)
return
}
msg := nats.NewMsg(cred.EventSubject(delivery.ManifestAcceptedEvent))
id := a.EventID()
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", delivery.ManifestAcceptedEvent, a.Module, err))
}
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1 h1:T66tg9T/1hHg6KN/bDScHPT41IHkxLAGGO8VJiAOlb8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -54,10 +54,16 @@ type Indexer struct {
Taken Store
// Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing.
Verdicts Store
// OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no
// count and the judge refuses no use of an old section.
OldSections Store
// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
JudgeID string
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
SayRefused func(ctx context.Context, r delivery.ManifestRefused)
// SayAccepted says the event manifest-accepted for an accepted verdict newly written whose manifest asks for bus state,
// carrying the entries whole (verdicts.go; issue 1966 D3); nil says nothing.
SayAccepted func(ctx context.Context, a delivery.ManifestAccepted)
// Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none,
// and every fingerprint takes the version line.
Packages Store
@@ -0,0 +1,76 @@
package index
import (
"context"
"encoding/json"
"fmt"
"maps"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/judge"
)
// How many accepted modules use each old section (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts them, the
// foundation left out, as it judges, keeps the count in its state oldsections under one key, and the tool
// indexer_old_sections reads it. The judge refuses a manifest of another module that uses a section whose count is 0.
const (
// OldSectionsTool is the tool that reads the count.
OldSectionsTool = "indexer_old_sections"
// OldSectionsState is the indexer's state of the count: the bucket mesh-delivery-indexer_oldsections.
OldSectionsState = "oldsections"
OldSectionsBucket = delivery.IndexerModule + "_" + OldSectionsState
// OldSectionsKey is the one key it holds.
OldSectionsKey = "counts"
)
// OldSectionsCount is the value of the oldsections state: per old section, the accepted modules outside the foundation using it.
type OldSectionsCount struct {
V int `json:"v"`
Counts map[string]int `json:"counts"`
}
// DecodeOldSections reads the oldsections value.
func DecodeOldSections(value []byte) (OldSectionsCount, error) {
var c OldSectionsCount
if err := json.Unmarshal(value, &c); err != nil {
return c, fmt.Errorf("the count of the old sections cannot be read: %w", err)
}
if c.V != 1 {
return c, fmt.Errorf("a count of the old sections of version %d, not 1", c.V)
}
return c, nil
}
// heldOldSections is the count the last pass wrote (the accepted set it held): a module that keeps using a section was
// accepted with it, so it is counted, and a fresh manifest changes nothing in it. Nil when none is held yet or it cannot be
// read; a failed read is said, never taken as 0 (issue 1966 D4).
func (ix *Indexer) heldOldSections(ctx context.Context) judge.OldSectionCounts {
value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey)
if err == nil && found {
var c OldSectionsCount
if c, err = DecodeOldSections(value); err == nil {
return c.Counts
}
}
if err != nil {
ix.logf("judge: the count of the old sections cannot be read (%v), so the zero rule is not applied in this pass", err)
}
return nil
}
// keepOldSections writes the counts under OldSectionsKey when they differ from the ones held.
func (ix *Indexer) keepOldSections(ctx context.Context, counts judge.OldSectionCounts) error {
if ix.OldSections == nil {
return nil
}
return write(ctx, ix.OldSections, OldSectionsKey, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := DecodeOldSections(value); err == nil && maps.Equal(cur.Counts, counts) {
return nil, false, nil
}
}
next, err := json.Marshal(OldSectionsCount{V: 1, Counts: counts})
return next, false, err
})
}
@@ -74,7 +74,13 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
raws[name] = []byte(kept.JSON)
digests[name] = digest
}
refused, unnamed := judge.Shelf(raws, ix.predefinedList(ctx), 0)
predefined := ix.predefinedList(ctx)
// The old sections end when no module is left in them (issue 1966 D4): the count is the accepted set the last pass held.
var counts judge.OldSectionCounts
if ix.OldSections != nil {
counts = ix.heldOldSections(ctx)
}
refused, unnamed := judge.ShelfCounting(raws, predefined, 0, counts)
for _, p := range unnamed {
ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p)
}
@@ -83,6 +89,7 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
if id == "" {
id = judge.Identity()
}
var accepted []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(digests)) {
out.Judged++
v := delivery.ManifestVerdict{Module: name, State: delivery.VerdictAccepted, Judge: id, At: ix.now()}
@@ -94,6 +101,12 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
v.State, v.Problems = delivery.VerdictRefused, problems
}
}
if !v.Refused() {
if m, err := catalogue.Decode(raws[name]); err == nil {
m.Module = name
accepted = append(accepted, m)
}
}
wrote, err := ix.writeVerdict(ctx, digests[name], v)
if errors.Is(err, process.ErrNotYet) {
return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsBucket, err)
@@ -109,12 +122,20 @@ func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged,
if ix.SayRefused != nil {
ix.SayRefused(ctx, delivery.RefusedOf(digests[name], v))
}
} else if ix.SayAccepted != nil {
// A manifest asking for bus state, newly accepted, is said whole to the bus module (issue 1966 D3, ADR 0595 §4).
if m, problems := judge.Manifest(raws[name], predefined); len(problems) == 0 && len(m.BusState) > 0 {
ix.SayAccepted(ctx, delivery.AcceptedOf(digests[name], v, m))
}
}
}
if err := ix.carryVerdict(ctx, name, digests[name], v); err != nil {
ix.logf("judge: modules.%s does not carry its verdict: %v", name, err)
}
}
if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted)); err != nil {
ix.logf("judge: the count of the old sections is not kept: %v", err)
}
return out, nil
}
@@ -2,6 +2,8 @@ package index
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
@@ -118,9 +120,92 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if err := ix.writeOverlay(ctx, map[string]delivery.Module{"old": same}, nil, w); err != nil || !w.fresh[other] {
t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err)
}
// 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries
// whole; a verdict already held, and a manifest asking for none, say nothing.
var accepted []delivery.ManifestAccepted
ix.SayAccepted = func(_ context.Context, a delivery.ManifestAccepted) { accepted = append(accepted, a) }
moved := hold("moved", `{"module":"moved","version":"1","requires":[`+
`{"name":"cache","storage":"key-value","template":"state","options":{"history":8}},`+
`{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64}}]}`)
for pass := 0; pass < 2; pass++ {
if _, err := ix.JudgeAll(ctx, map[string]bool{moved: true}); err != nil {
t.Fatal(err)
}
}
if len(accepted) != 1 || accepted[0].Module != "moved" || accepted[0].Digest != moved || accepted[0].Judge != "judge-2" ||
len(accepted[0].BusState) != 2 || accepted[0].BusState[0].Name != "cache" || accepted[0].BusState[1].Template != "log" ||
string(accepted[0].BusState[0].Options["history"]) != "8" {
t.Fatalf("said accepted %+v", accepted)
}
// 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another
// module using logs is refused and the foundation's is accepted.
ix.OldSections = newMem()
hold("user", `{"module":"user","version":"1","state":["x"]}`)
hold("mesh-grants", `{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`)
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
counts := func() map[string]int {
value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey)
if err != nil || !found {
t.Fatalf("no count of the old sections: %v", err)
}
c, err := DecodeOldSections(value)
if err != nil {
t.Fatal(err)
}
return c.Counts
}
if c := counts(); c["state"] != 1 || c["logs"] != 0 || c["streams"] != 0 || c["objects"] != 0 {
t.Fatalf("the old sections count %v, want state 1 and the rest 0", c)
}
late := hold("late", `{"module":"late","version":"1","logs":[{"name":"l"}]}`)
grants := hold("mesh-grants", `{"module":"mesh-grants","version":"2","logs":[{"name":"l"},{"name":"m"}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil {
t.Fatal(err)
}
if v := verdict(late); !v.Refused() || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") {
t.Fatalf("late: %+v", v)
}
if v := verdict(grants); v.Refused() {
t.Fatalf("the foundation's manifest was refused: %+v", v)
}
// 1966/AC11: the sole user of an old section changes its manifest and keeps the section: accepted, since it was accepted
// with it (the count is the accepted set held from the last pass, not recounted from fresh digests).
hold("solo", `{"module":"solo","version":"1","objects":[{"name":"o","max-mib":1}]}`)
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
solo2 := hold("solo", `{"module":"solo","version":"2","objects":[{"name":"o","max-mib":2}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{solo2: true}); err != nil {
t.Fatal(err)
}
if v := verdict(solo2); v.Refused() {
t.Fatalf("the sole user of objects, changing its manifest and keeping objects, was refused: %+v", v)
}
// A count that cannot be read is never taken as 0: the zero rule is not applied in that pass and the log says so.
var logged []string
ix.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
held := ix.OldSections
ix.OldSections = unreadable{held}
retry := hold("late", `{"module":"late","version":"2","logs":[{"name":"l"}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{retry: true}); err != nil {
t.Fatal(err)
}
if v := verdict(retry); v.Refused() || !strings.Contains(strings.Join(logged, "\n"), "the zero rule is not applied") {
t.Fatalf("a count that could not be read: %+v, logged %q", v, logged)
}
ix.OldSections = held
// Without the verdicts state the indexer judges nothing.
ix.Verdicts = nil
if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 {
t.Fatalf("without the state: %+v %v", j, err)
}
}
// unreadable is a store whose reads fail.
type unreadable struct{ Store }
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
return nil, 0, false, errors.New("the bus did not answer")
}
@@ -0,0 +1,420 @@
package judge
import (
"bytes"
"encoding/json"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The rules for bus state asked for in `requires` (issue 1966 D1; novox/hq ADR 0595 §1 to §3, ADR 0596 §2, ADR 0597 §2,
// design 25 § 12.1). mesh-sdk keeps only the entry's shape (ADR 0578 §7); every bound below is this judge's. An entry is
// judged once, here, and a refusal names the module, the entry and the field.
// optionKind is what an option's value is.
type optionKind int
const (
optionInt optionKind = iota
optionFlag
optionText
optionEvents
)
// optionRule is one option a template takes: its kind, its bound where it is a number, and whether it must be given.
type optionRule struct {
kind optionKind
least, most int
required bool
}
// busTemplate is a template: the storage type it is on, the options it takes, and what it fixes in plain words.
type busTemplate struct {
storage string
options map[string]optionRule
fixed string
}
// busTemplates are the six templates of design 25 § 12.1 and the bounds of its table.
var busTemplates = map[string]busTemplate{
catalogue.TemplateState: {
storage: catalogue.StorageKeyValue,
fixed: "what happens when full, the size of a value and of the bucket, and the copies",
options: map[string]optionRule{
catalogue.OptionPastValues: {kind: optionInt, least: 1, most: catalogue.StateMostHistory},
catalogue.OptionTimeToLive: {kind: optionInt, least: 1, most: 315360000},
catalogue.OptionLifetimePerKey: {kind: optionFlag},
catalogue.OptionPerMachine: {kind: optionFlag},
},
},
catalogue.TemplateEvents: {
storage: catalogue.StorageStream,
fixed: "what happens when full (the oldest is dropped) and the module's own subjects",
options: map[string]optionRule{
"events": {kind: optionEvents, required: true},
catalogue.OptionAgeDays: {kind: optionInt, least: catalogue.StreamLeastDays, most: catalogue.StreamMostDays},
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
catalogue.OptionGrant: {kind: optionText},
},
},
catalogue.TemplateWorkQueue: {
storage: catalogue.StorageStream,
fixed: "what happens when full (a new message is refused), one taker, and no grant",
options: map[string]optionRule{
"events": {kind: optionEvents, required: true},
catalogue.OptionDeliveries: {kind: optionInt, least: catalogue.StreamLeastDeliveries, most: catalogue.StreamMostDeliveries},
catalogue.OptionDuplicates: {kind: optionInt, least: 1, most: catalogue.StreamMostDays * 86400},
catalogue.OptionAgeDays: {kind: optionInt, least: catalogue.StreamLeastDays, most: catalogue.StreamMostDays},
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
},
},
catalogue.TemplateLog: {
storage: catalogue.StorageStream,
fixed: "what happens when full (a new entry is refused), that an entry is never rewritten, and its size",
options: map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
catalogue.OptionAgeDays: {kind: optionInt, least: 1, most: 3650},
},
},
catalogue.TemplateObjects: {
storage: catalogue.StorageObjectStore,
fixed: "what happens when full (a new object is refused) and that objects are kept for good",
options: map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.ObjectsLeastMiB, most: catalogue.ObjectsMostMiB, required: true},
catalogue.OptionGrant: {kind: optionText},
},
},
catalogue.TemplateLedger: {
storage: catalogue.StorageLedger,
fixed: "what happens when full (a new entry is refused) and its state and index buckets",
options: map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB},
},
},
}
// bareStream is a stream of a module's own configuration, where no template fits: the three settings the judge
// refuses one for missing (design 25 § 12.1), and those it may also give.
var bareStream = map[string]optionRule{
catalogue.OptionCapMiB: {kind: optionInt, least: catalogue.StreamLeastMiB, most: catalogue.StreamMostMiB, required: true},
catalogue.OptionAgeDays: {kind: optionInt, least: 1, most: 3650, required: true},
catalogue.OptionGrant: {kind: optionText},
"events": {kind: optionEvents},
}
// ledgerConfiguration is what a ledger's own configuration may give: how long its history is kept (design 25 § 12.1).
var ledgerConfiguration = map[string]optionRule{
catalogue.OptionAgeDays: {kind: optionInt, least: 1, most: 3650},
}
// noAge is how a bare stream says it keeps its messages for good.
const noAge = "none"
// ledgerEffects are the four effects an operation may have (ADR 0597 §2).
var ledgerEffects = []string{catalogue.EffectSetFields, catalogue.EffectSetEntry, catalogue.EffectAppend, catalogue.EffectRemove}
// BusStateProblems is what is wrong with one manifest's bus state, judged on the manifest alone (issue 1966 D1).
func BusStateProblems(m catalogue.Manifest) []string {
if len(m.BusState) == 0 {
return nil
}
var problems []string
if !stateName.MatchString(m.Module) {
problems = append(problems, fmt.Sprintf(
"%s keeps bus state, and a module's name is part of its names on the bus, which take one plain name — "+
"no dot (novox/hq ADR 0201, ADR 0595 §1)", m.Module))
}
storageOf := map[string]string{}
oldSection := map[string]string{} // a local name an old section keeps, to that section (issue 1966 D2)
for _, st := range m.State {
oldSection[st.Name] = "state"
}
for _, l := range m.Logs {
oldSection[l.Name] = "logs"
}
for _, st := range m.Streams {
oldSection[st.Name] = "streams"
}
for _, o := range m.Objects {
oldSection[o.Name] = "objects"
}
terminal := catalogue.TerminalKeys(m)
for _, e := range m.BusState {
at := fmt.Sprintf("%s's bus state %q", m.Module, e.Name)
switch {
case strings.Contains(e.Name, "_"):
problems = append(problems, fmt.Sprintf(
"%s: its name holds an underscore, which separates the module from the local name on the bus; use a "+
"hyphen (novox/hq design 25 § 12.1)", at))
case !stateName.MatchString(e.Name):
problems = append(problems, fmt.Sprintf(
"%s: its name is a local name — lower-case letters, digits and hyphens, no dot and no underscore; the "+
"mesh derives the name on the bus (novox/hq ADR 0201)", at))
}
if first, taken := storageOf[e.Name]; taken {
problems = append(problems, fmt.Sprintf(
"%s: the name is used twice (as %s and as %s); each name is one, whatever its storage type (novox/hq "+
"ADR 0595 §1)", at, first, e.Storage))
}
storageOf[e.Name] = e.Storage
if section, kept := oldSection[e.Name]; kept {
problems = append(problems, fmt.Sprintf(
"%s: the name is also kept in the old section %s; one local name is one form, so a module that moves an "+
"entry removes it from the old section (novox/hq ADR 0595 §11)", at, section))
}
problems = append(problems, busEntryProblems(at, e, terminal)...)
}
return problems
}
// busEntryProblems judges one entry's storage type, template, options and configuration.
func busEntryProblems(at string, e catalogue.BusStateEntry, terminal []string) []string {
var problems []string
switch e.Storage {
case catalogue.StorageStream, catalogue.StorageKeyValue, catalogue.StorageObjectStore, catalogue.StorageLedger:
default:
return []string{fmt.Sprintf("%s: the storage type %q is unknown; it is %s", at, e.Storage,
strings.Join([]string{catalogue.StorageStream, catalogue.StorageKeyValue, catalogue.StorageObjectStore, catalogue.StorageLedger}, ", "))}
}
if e.Storage != catalogue.StorageLedger && len(e.Types) > 0 {
problems = append(problems, fmt.Sprintf("%s: types are a ledger's; storage type %s has none", at, e.Storage))
}
if e.Template == "" {
if e.Storage != catalogue.StorageStream {
return append(problems, fmt.Sprintf(
"%s: the storage type %s is asked for by its template; only a stream is asked for bare, with a "+
"configuration of its own (novox/hq design 25 § 12.1)", at, e.Storage))
}
if len(e.Options) > 0 {
problems = append(problems, fmt.Sprintf(
"%s: a bare stream gives its settings in its configuration, not in options", at))
}
return append(problems, settingsProblems(at, "configuration", e.Configuration, bareStream, "a bare stream", terminal, true)...)
}
tpl, known := busTemplates[e.Template]
switch {
case !known:
names := make([]string, 0, len(busTemplates))
for n := range busTemplates {
names = append(names, n)
}
slices.Sort(names)
return append(problems, fmt.Sprintf("%s: the template %q is unknown; it is %s", at, e.Template, strings.Join(names, ", ")))
case tpl.storage != e.Storage:
return append(problems, fmt.Sprintf(
"%s: the template %s is on the storage type %s, not %s", at, e.Template, tpl.storage, e.Storage))
}
problems = append(problems, settingsProblems(at, "options", e.Options, tpl.options, "the template "+e.Template+" (it fixes "+tpl.fixed+")", terminal, false)...)
if e.Template == catalogue.TemplateLedger {
problems = append(problems, settingsProblems(at, "configuration", e.Configuration, ledgerConfiguration, "a ledger's own configuration", terminal, false)...)
problems = append(problems, ledgerTypeProblems(at, e)...)
} else if len(e.Configuration) > 0 {
problems = append(problems, fmt.Sprintf(
"%s: the template %s is given a configuration; a configuration of its own is for a bare stream or a "+
"ledger whose history expires, and a template's settings are its options (novox/hq ADR 0595 §2)", at, e.Template))
}
if e.Template == catalogue.TemplateWorkQueue {
problems = append(problems, workQueueProblems(at, e)...)
}
return problems
}
// settingsProblems judges the given settings against the rules: an unknown one is refused (a fixed setting among them),
// a number outside its bound, a kind mismatch, and a required one missing. A bare stream's when-full and age are read
// here too.
func settingsProblems(at, field string, given map[string]json.RawMessage, rules map[string]optionRule, of string, terminal []string, bare bool) []string {
var problems []string
for _, k := range sortedKeys(given) {
raw := given[k]
if bare && k == catalogue.ConfigWhenFull {
var v string
if json.Unmarshal(raw, &v) != nil || (v != catalogue.WhenFullRefuse && v != catalogue.WhenFullDropOldest) {
problems = append(problems, fmt.Sprintf(
"%s: %s %q is %s; a bare stream says %q or %q", at, field, k, string(raw), catalogue.WhenFullRefuse, catalogue.WhenFullDropOldest))
}
continue
}
rule, ok := rules[k]
if !ok {
if k == catalogue.ConfigWhenFull {
problems = append(problems, fmt.Sprintf(
"%s: %s %q is given to %s; what happens when full is fixed and is never an option (novox/hq ADR 0595 §3)", at, field, k, of))
continue
}
problems = append(problems, fmt.Sprintf(
"%s: %s %q is not a setting of %s (novox/hq ADR 0595 §3)", at, field, k, of))
continue
}
problems = append(problems, valueProblems(at, field, k, raw, rule, terminal, bare)...)
}
for _, k := range sortedKeys(rules) {
if _, given := given[k]; !given && rules[k].required {
problems = append(problems, fmt.Sprintf("%s: %s %q is missing; %s gives it", at, field, k, of))
}
}
if bare {
if _, given := given[catalogue.ConfigWhenFull]; !given {
problems = append(problems, fmt.Sprintf(
"%s: %s %q is missing; a bare stream says %q or %q", at, field, catalogue.ConfigWhenFull, catalogue.WhenFullRefuse, catalogue.WhenFullDropOldest))
}
}
return problems
}
// valueProblems judges one setting's value against its rule.
func valueProblems(at, field, key string, raw json.RawMessage, rule optionRule, terminal []string, bare bool) []string {
switch rule.kind {
case optionInt:
if bare && key == catalogue.OptionAgeDays {
var word string
if json.Unmarshal(raw, &word) == nil && word == noAge {
return nil
}
}
n, ok := wholeNumber(raw)
if !ok {
return []string{fmt.Sprintf("%s: %s %q is %s, which is not a whole number", at, field, key, string(raw))}
}
if n < rule.least || n > rule.most {
extra := ""
if key == catalogue.OptionPastValues && n == 0 {
extra = "; 0 is refused here"
}
return []string{fmt.Sprintf("%s: %s %q is %d, outside its bound of %d to %d%s", at, field, key, n, rule.least, rule.most, extra)}
}
case optionFlag:
var b bool
if json.Unmarshal(raw, &b) != nil {
return []string{fmt.Sprintf("%s: %s %q is %s, which is not true or false", at, field, key, string(raw))}
}
case optionText:
var config string
if json.Unmarshal(raw, &config) != nil || !slices.Contains(terminal, config) {
return []string{fmt.Sprintf(
"%s: %s %q names %s, which is not a trusted config value of the module, set only at the controller's "+
"terminal or on the operator's Approve (novox/hq ADR 0348 §8)", at, field, key, string(raw))}
}
case optionEvents:
var events []string
if json.Unmarshal(raw, &events) != nil || len(events) == 0 {
return []string{fmt.Sprintf("%s: %s %q is %s; it is a list of at least one event", at, field, key, string(raw))}
}
var out []string
for _, ev := range events {
if err := eventPattern(ev, false); err != nil {
out = append(out, fmt.Sprintf("%s: %s %q carries %q: %v", at, field, key, ev, err))
}
}
return out
}
return nil
}
// wholeNumber reads a JSON number that is a whole number.
func wholeNumber(raw json.RawMessage) (int, bool) {
dec := json.NewDecoder(bytes.NewReader(raw))
dec.UseNumber()
var n json.Number
if dec.Decode(&n) != nil {
return 0, false
}
i, err := n.Int64()
return int(i), err == nil
}
// workQueueProblems holds a work queue's duplicate window to its age (design 25 § 12.1: up to its age).
func workQueueProblems(at string, e catalogue.BusStateEntry) []string {
raw, given := e.Options[catalogue.OptionDuplicates]
if !given {
return nil
}
window, ok := wholeNumber(raw)
if !ok {
return nil // valueProblems said so
}
days := catalogue.StreamDefaultDays
if age, ok := wholeNumber(e.Options[catalogue.OptionAgeDays]); ok && age > 0 {
days = age
}
if window > days*86400 {
return []string{fmt.Sprintf(
"%s: options %q is %d seconds, longer than the queue's age of %d days (novox/hq design 25 § 12.1)",
at, catalogue.OptionDuplicates, window, days)}
}
return nil
}
// ledgerTypeProblems judges a ledger's types: named locally and once, and every operation's effect one of the four.
func ledgerTypeProblems(at string, e catalogue.BusStateEntry) []string {
var problems []string
if len(e.Types) == 0 {
problems = append(problems, fmt.Sprintf("%s: a ledger names at least one type (novox/hq ADR 0597 §1)", at))
}
seen := map[string]bool{}
for _, t := range e.Types {
where := fmt.Sprintf("%s: type %q", at, t.Name)
switch {
case !stateName.MatchString(t.Name):
problems = append(problems, fmt.Sprintf("%s: its name is lower-case letters, digits and hyphens", where))
case seen[t.Name]:
problems = append(problems, fmt.Sprintf("%s: the type is named twice", where))
}
seen[t.Name] = true
operations := map[string]bool{}
for _, op := range t.Operations {
if operations[op.Name] {
problems = append(problems, fmt.Sprintf("%s: the operation %q is named twice", where, op.Name))
}
operations[op.Name] = true
if !slices.Contains(ledgerEffects, op.Effect) {
problems = append(problems, fmt.Sprintf(
"%s: the operation %q has the effect %q, outside the four (%s) (novox/hq ADR 0597 §2)",
where, op.Name, op.Effect, strings.Join(ledgerEffects, ", ")))
}
}
}
return problems
}
// BusStateCatalogueProblems are the rules of bus state no single manifest can be judged against: two modules' names on
// the bus alike, and a `reads` naming bus state in the new form, which has no readers (novox/hq ADR 0596 §2).
func BusStateCatalogueProblems(shelf catalogue.Shelf) []string {
var problems []string
owner := map[string]string{} // a name on the bus, upper-cased, to the module asking for it
for _, module := range shelfOrder(shelf) {
for _, o := range shelf[module].Objects {
owner[strings.ToUpper(o.Bucket(module))] = module
}
}
for _, module := range shelfOrder(shelf) {
for _, e := range shelf[module].BusState {
name := catalogue.ObjectsBucket(module, e.Name)
if first, taken := owner[strings.ToUpper(name)]; taken && first != module {
problems = append(problems, fmt.Sprintf(
"%s's bus state %q has the name %s on the bus, which %s's already has or is alike; a name on the bus "+
"has one owner (novox/hq ADR 0201 §4)", module, e.Name, name, first))
continue
}
owner[strings.ToUpper(name)] = module
}
}
for _, module := range shelfOrder(shelf) {
for _, r := range shelf[module].Reads {
from, local, err := catalogue.ReadState(r)
if err != nil || from == module {
continue // StateProblems says why
}
for _, e := range shelf[from].BusState {
if e.Name == local {
problems = append(problems, fmt.Sprintf(
"%s reads %q, which %s has moved to bus state in `requires`; the new form has no readers: ask %s's "+
"tools, or hear its events (novox/hq ADR 0596 §2)", module, r, from, from))
}
}
}
}
return problems
}
@@ -0,0 +1,95 @@
package judge
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// A manifest's bus state, asked for in `requires` by template, is judged once and each refusal names the module, the entry
// and the field (issue 1966 D1, D2; novox/hq design 25 § 12.1, ADR 0595, ADR 0596 §2, ADR 0597 §2).
func TestBusStateIsJudgedOnce(t *testing.T) {
const ledger = `{"name":"records","storage":"ledger","template":"ledger","options":{"max-mib":512},"types":[` +
`{"name":"decision","schema":{"title":{"type":"text"}},"operations":[{"name":"propose","effect":"set-fields","fields":["title"]}]}]}`
manifest := func(entries string) string {
return `{"module":"a","version":"1","requires":[` + entries + `]}`
}
for _, c := range []struct {
name string
raw string
refused []string // words the first problem holds, in order: absent is accepted
}{
// 1966/AC8: a state, a log, objects and a ledger within bounds
{"within bounds", manifest(`{"name":"cache","storage":"key-value","template":"state","options":{"history":8,"key-ttl":true}},` +
`{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64,"max-age-days":365}},` +
`{"name":"files","storage":"object-store","template":"objects","options":{"max-mib":2048}},` + ledger), nil},
// 1966/AC7: each refused case, naming the module, the entry and the field
{"unknown storage type", manifest(`{"name":"x","storage":"tape"}`), []string{"a's bus state \"x\"", "storage type \"tape\""}},
{"unknown template", manifest(`{"name":"x","storage":"stream","template":"pile"}`), []string{"\"x\"", "template \"pile\""}},
{"template on another storage type", manifest(`{"name":"x","storage":"stream","template":"state"}`), []string{"\"x\"", "template state is on the storage type key-value"}},
{"past values 0", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"history":0}}`), []string{"\"x\"", "options \"history\" is 0", "0 is refused"}},
{"past values 65", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"history":65}}`), []string{"\"x\"", "\"history\" is 65", "1 to 64"}},
{"a lifetime that is no number", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"ttl-seconds":"soon"}}`), []string{"\"x\"", "\"ttl-seconds\""}},
{"events age over 30 days", manifest(`{"name":"x","storage":"stream","template":"events","options":{"events":["seen"],"max-age-days":31}}`), []string{"\"x\"", "\"max-age-days\" is 31", "1 to 30"}},
{"events cap over 8192", manifest(`{"name":"x","storage":"stream","template":"events","options":{"events":["seen"],"max-mib":8193}}`), []string{"\"x\"", "\"max-mib\" is 8193"}},
{"events naming none", manifest(`{"name":"x","storage":"stream","template":"events","options":{}}`), []string{"\"x\"", "\"events\" is missing"}},
{"a grant that is no trusted value", manifest(`{"name":"x","storage":"stream","template":"events","options":{"events":["seen"],"grant":"nobody"}}`), []string{"\"x\"", "\"grant\"", "trusted config value"}},
{"deliveries 11", manifest(`{"name":"x","storage":"stream","template":"work-queue","options":{"events":["job"],"max-deliveries":11}}`), []string{"\"x\"", "\"max-deliveries\" is 11", "1 to 10"}},
{"duplicates past the age", manifest(`{"name":"x","storage":"stream","template":"work-queue","options":{"events":["job"],"max-age-days":1,"duplicates":86401}}`), []string{"\"x\"", "\"duplicates\"", "longer than the queue's age of 1 days"}},
{"log age 3651", manifest(`{"name":"x","storage":"stream","template":"log","options":{"max-age-days":3651}}`), []string{"\"x\"", "\"max-age-days\" is 3651"}},
{"objects without a cap", manifest(`{"name":"x","storage":"object-store","template":"objects"}`), []string{"\"x\"", "\"max-mib\" is missing"}},
{"when full given as an option", manifest(`{"name":"x","storage":"stream","template":"log","options":{"when-full":"drop-oldest"}}`), []string{"\"x\"", "\"when-full\"", "fixed"}},
{"another fixed setting given", manifest(`{"name":"x","storage":"key-value","template":"state","options":{"max-value-kib":1024}}`), []string{"\"x\"", "\"max-value-kib\" is not a setting of the template state"}},
{"a configuration beside a template", manifest(`{"name":"x","storage":"key-value","template":"state","configuration":{"max-mib":1}}`), []string{"\"x\"", "configuration"}},
{"a name used twice across storage types", manifest(
`{"name":"x","storage":"key-value","template":"state"},{"name":"x","storage":"stream","template":"log"}`), []string{"\"x\"", "used twice (as key-value and as stream)"}},
{"a local name holding an underscore", manifest(`{"name":"my_x","storage":"key-value","template":"state"}`), []string{"\"my_x\"", "underscore"}},
{"a bare stream without when full", manifest(`{"name":"x","storage":"stream","configuration":{"max-mib":10,"max-age-days":"none"}}`), []string{"\"x\"", "\"when-full\" is missing"}},
{"a bare stream without a cap", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"refuse","max-age-days":3}}`), []string{"\"x\"", "\"max-mib\" is missing"}},
{"a bare stream without an age", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"refuse","max-mib":10}}`), []string{"\"x\"", "\"max-age-days\" is missing"}},
{"a bare stream, complete", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"drop-oldest","max-mib":10,"max-age-days":"none"}}`), nil},
{"a bare stream with a wrong when full", manifest(`{"name":"x","storage":"stream","configuration":{"when-full":"crash","max-mib":10,"max-age-days":9}}`), []string{"\"x\"", "\"when-full\"", "\"refuse\" or \"drop-oldest\""}},
{"a ledger effect outside the four", manifest(strings.Replace(ledger, `"set-fields"`, `"upsert"`, 1)), []string{"\"records\"", "effect \"upsert\"", "outside the four"}},
{"a ledger whose history expires", manifest(strings.Replace(ledger, `"options"`, `"configuration":{"max-age-days":30},"options"`, 1)), nil},
} {
_, problems := Manifest([]byte(c.raw), nil)
switch {
case c.refused == nil && len(problems) > 0:
t.Errorf("%s: refused: %v", c.name, problems)
case c.refused != nil && len(problems) == 0:
t.Errorf("%s: accepted", c.name)
case c.refused != nil:
text := strings.Join(problems, "\n")
at := 0
for _, word := range c.refused {
i := strings.Index(text[at:], word)
if i < 0 {
t.Errorf("%s: problems %q lack %q", c.name, text, word)
break
}
at += i
}
}
}
// Between the manifests: two modules' names on the bus alike, and a read of bus state that moved (ADR 0596 §2).
entry := func(name string) []catalogue.BusStateEntry {
return []catalogue.BusStateEntry{{Name: name, Storage: catalogue.StorageKeyValue, Template: catalogue.TemplateState}}
}
for _, c := range []struct {
name string
shelf catalogue.Shelf
want string
}{
{"names alike", catalogue.Shelf{"a_b": {Module: "a_b", BusState: entry("c")}, "a": {Module: "a", BusState: entry("b_c")}}, "has one owner"},
{"a read of moved state", catalogue.Shelf{"owner": {Module: "owner", BusState: entry("cache")},
"reader": {Module: "reader", Reads: []string{"owner.cache"}}}, "reader reads \"owner.cache\", which owner has moved"},
{"a read of state still kept", catalogue.Shelf{"owner": {Module: "owner", State: []catalogue.StateDeclaration{{Name: "cache"}}},
"reader": {Module: "reader", Reads: []string{"owner.cache"}}}, ""},
} {
got := strings.Join(BusStateCatalogueProblems(c.shelf), "\n")
if (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
t.Errorf("%s: between the manifests: %q, want %q", c.name, got, c.want)
}
}
}
+45 -1
View File
@@ -69,6 +69,12 @@ func problemsOf(err error) []string {
// whose removal from the catalogue ends it (a rule two modules break together refuses both); one no removal ends is in
// unnamed, for the caller to say. A module not in the answer is accepted.
func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int) (refused map[string][]string, unnamed []string) {
return ShelfCounting(raws, predefined, longestMachine, nil)
}
// ShelfCounting is Shelf also refusing a manifest of a module outside the foundation that uses an old section whose count
// is 0 (OldSectionProblems; issue 1966 D4); nil counts refuse none.
func ShelfCounting(raws map[string][]byte, predefined catalogue.PredefinedList, longestMachine int, counts OldSectionCounts) (refused map[string][]string, unnamed []string) {
refused = map[string][]string{}
shelf := catalogue.Shelf{}
for name, raw := range raws {
@@ -81,6 +87,10 @@ func Shelf(raws map[string][]byte, predefined catalogue.PredefinedList, longestM
refused[name] = []string{fmt.Sprintf("the manifest is of module %q, not %q", m.Module, name)}
continue
}
if problems := OldSectionProblems(m, counts); len(problems) > 0 {
refused[name] = problems
continue
}
shelf[name] = m
}
between := Between(shelf, longestMachine)
@@ -179,6 +189,7 @@ const Unnamed = "(between manifests)"
func Request(r composer.Request) (composer.Verdict, error) {
v := composer.Verdict{V: composer.Version, SDK: Identity()}
seen := map[string]bool{}
counts := trunkOldSections(r)
for _, c := range r.Cases {
if seen[c.Name] {
return composer.Verdict{}, fmt.Errorf("the case %q is named twice", c.Name)
@@ -191,7 +202,7 @@ func Request(r composer.Request) (composer.Verdict, error) {
}
raws[name] = m.Raw
}
refused, unnamed := Shelf(raws, r.Predefined, 0)
refused, unnamed := ShelfCounting(raws, r.Predefined, 0, counts)
cv := composer.CaseVerdict{Name: c.Name, Manifests: map[string]composer.ManifestRead{}}
for name := range raws {
if problems := refused[name]; len(problems) > 0 {
@@ -208,6 +219,39 @@ func Request(r composer.Request) (composer.Verdict, error) {
return v, nil
}
// TrunkCase is the case of a request holding the catalogue as the trunk holds it (the checker's JudgeTrunk).
const TrunkCase = "trunk"
// trunkOldSections counts the old sections the modules accepted at the trunk use, as the indexer counts them from its
// verdicts (issue 1966 D4, ADR 0595 §11): a change is refused at the merge check exactly as at take-in (judged once, ADR
// 0578). Nil where the request holds no trunk case, which refuses no use of a section.
func trunkOldSections(r composer.Request) OldSectionCounts {
for _, c := range r.Cases {
if c.Name != TrunkCase {
continue
}
raws := map[string][]byte{}
for name, m := range c.Manifests {
if len(m.Raw) > 0 {
raws[name] = m.Raw
}
}
refused, _ := Shelf(raws, r.Predefined, 0)
var accepted []catalogue.Manifest
for name, raw := range raws {
if len(refused[name]) > 0 {
continue
}
if m, err := catalogue.Decode(raw); err == nil {
m.Module = name
accepted = append(accepted, m)
}
}
return CountOldSections(accepted)
}
return nil
}
var (
identityOnce sync.Once
identity string
@@ -86,6 +86,27 @@ func TestTheJudge(t *testing.T) {
v.Cases[0].Manifests["b"].Accepted || !strings.Contains(v.Cases[0].Manifests["b"].Refused, `requires "amqp"`) {
t.Fatalf("request: %+v %v", v, err)
}
// 1966/AC11: the checker's judge counts the old sections from the trunk case as the indexer does from its verdicts (the
// foundation left out): a change adding logs where no trunk module uses them is refused, the foundation's accepted, and
// state, which a trunk module uses, is accepted.
old := composer.Request{V: composer.Version, Cases: []composer.Case{
{Name: "change", Manifests: map[string]composer.Manifest{
"user": {Raw: json.RawMessage(`{"module":"user","version":"1","state":["x"]}`)},
"late": {Raw: json.RawMessage(`{"module":"late","version":"1","logs":[{"name":"l"}]}`)},
"newer": {Raw: json.RawMessage(`{"module":"newer","version":"1","state":["y"]}`)},
"mesh-grants": {Raw: json.RawMessage(`{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`)}}},
{Name: "trunk", Manifests: map[string]composer.Manifest{
"user": {Raw: json.RawMessage(`{"module":"user","version":"1","state":["x"]}`)},
"mesh-grants": {Raw: json.RawMessage(`{"module":"mesh-grants","version":"1","logs":[{"name":"l"}]}`)}}}}}
ov, err := Request(old)
if err != nil {
t.Fatal(err)
}
got := ov.Cases[0].Manifests
if got["late"].Accepted || !strings.Contains(got["late"].Refused, "late uses the old section logs") ||
!got["newer"].Accepted || !got["user"].Accepted || !got["mesh-grants"].Accepted {
t.Errorf("the change's old sections: %+v", got)
}
r.Cases[0].Manifests["c"] = composer.Manifest{Digest: strings.Repeat("a", 64)}
if _, err := Request(r); err == nil {
t.Fatal("a manifest left as a digest was judged")
@@ -0,0 +1,69 @@
package judge
import (
"fmt"
"slices"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The old sections end when no module is left in them (issue 1966 D4; novox/hq ADR 0595 §11): the indexer counts, per old
// section, the accepted modules using it, the foundation left out, and the judge refuses a manifest of another module
// that uses a section whose count is 0. The foundation (the bus module, the grants part and the indexer itself) is
// created before the bus module can create anything, so it keeps the old sections for good and is always accepted.
// OldSections are the sections of a manifest that keep bus state in the old forms.
var OldSections = []string{"state", "logs", "streams", "objects"}
// Foundation are the modules that keep using the old sections for good.
var Foundation = []string{"nats", "mesh-grants", "mesh-delivery-indexer"}
// OldSectionCounts is, per old section, how many accepted modules outside the foundation use it. Nil is no count: the
// judge then refuses no use of a section.
type OldSectionCounts map[string]int
// SectionsUsed are the old sections a manifest uses, in OldSections' order.
func SectionsUsed(m catalogue.Manifest) []string {
var used []string
for _, s := range OldSections {
if (s == "state" && len(m.State) > 0) || (s == "logs" && len(m.Logs) > 0) ||
(s == "streams" && len(m.Streams) > 0) || (s == "objects" && len(m.Objects) > 0) {
used = append(used, s)
}
}
return used
}
// CountOldSections counts, per old section, the manifests using it, the foundation's left out; every section is in the
// answer, with 0 where none uses it.
func CountOldSections(manifests []catalogue.Manifest) OldSectionCounts {
counts := OldSectionCounts{}
for _, s := range OldSections {
counts[s] = 0
}
for _, m := range manifests {
if slices.Contains(Foundation, m.Module) {
continue
}
for _, s := range SectionsUsed(m) {
counts[s]++
}
}
return counts
}
// OldSectionProblems are the old sections a manifest uses that no accepted module outside the foundation uses any more.
func OldSectionProblems(m catalogue.Manifest, counts OldSectionCounts) []string {
if counts == nil || slices.Contains(Foundation, m.Module) {
return nil
}
var problems []string
for _, s := range SectionsUsed(m) {
if counts[s] == 0 {
problems = append(problems, fmt.Sprintf(
"%s uses the old section %s, which no other module uses any more (0 modules count); ask for it in "+
"`requires` by template (novox/hq ADR 0595 §11)", m.Module, s))
}
}
return problems
}
+5 -1
View File
@@ -2626,6 +2626,8 @@ func parseManifest(raw []byte, predefined catalogue.PredefinedList) (catalogue.M
// And the object buckets it owns (novox/hq ADR 0592 §5, §9).
problems = append(problems, ObjectsProblems(m)...)
problems = append(problems, WorkQueueStreamProblems(m)...)
// And the bus state it asks for in `requires`, by template (busstate.go, novox/hq ADR 0595; issue 1966 D1).
problems = append(problems, BusStateProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262), or, written in the
// config form, the whole declaration of each value (config.go, novox/hq ADR 0348).
if m.DeclaresConfig() {
@@ -4554,6 +4556,8 @@ func CatalogueProblems(shelf catalogue.Shelf) []string {
problems = append(problems, StreamNameProblems(shelf)...)
// An object bucket's name has one owner, and another module reads it only through its owner's grant (ADR 0592 §5, §9).
problems = append(problems, ObjectsCatalogueProblems(shelf)...)
// Bus state in `requires`: a name on the bus has one owner, and the new form has no readers (ADR 0595, ADR 0596 §2).
problems = append(problems, BusStateCatalogueProblems(shelf)...)
// A module that hears a restricted stream runs as an account of its own (novox/hq ADR 0376 §2).
problems = append(problems, RestrictedHearingProblems(shelf)...)
// A work-queue stream is never asked for by another module's consumes (novox/hq ADR 0436 §6).
@@ -5358,7 +5362,7 @@ func StateProblems(m catalogue.Manifest) []string {
seen[s.Name] = true
if s.History < 0 || s.History > catalogue.StateMostHistory {
problems = append(problems, fmt.Sprintf(
"%s keeps %d values of %q; a key keeps between 1 and %d", m.Module, s.History, s.Name, catalogue.StateMostHistory))
"%s keeps %d values of %q; a key keeps 0 or 1 to %d (0 is taken as 1)", m.Module, s.History, s.Name, catalogue.StateMostHistory))
}
if s.TTLSeconds < 0 {
problems = append(problems, fmt.Sprintf("%s gives %q a negative lifetime", m.Module, s.Name))
@@ -54,12 +54,6 @@ func TestABinaryMustSayWhichSystemItIsFor(t *testing.T) {
}
}
func TestABinaryThatNamesASystemIsAccepted(t *testing.T) {
if got := buildProblemsOf(t, bundleFor("go", "arch")); got != "" {
t.Fatalf("a compiled bundle naming a system was refused:\n%s", got)
}
}
func TestABlockOutsideTheShapeIsRefused(t *testing.T) {
cases := map[string]string{
`{"seat":"node-bar","kind":"block","content":"[[block]]"}`: "has no data",
@@ -913,6 +907,12 @@ func TestALogIsNamedLocallyAndCapped(t *testing.T) {
{`{"module":"a.b","version":"1","logs":["c"]}`, `no dot`},
} {
_, err := ParseManifest([]byte(c.manifest))
if c.says == "" {
if err != nil {
t.Errorf("%s was refused: %v", c.manifest, err)
}
continue
}
if err == nil {
t.Errorf("%s was accepted", c.manifest)
continue
@@ -2356,18 +2356,19 @@ func TestAPlaceholderOutsideTheHolderIsRefused(t *testing.T) {
}
}
// **And a language that runs anywhere must not name one.** A system that decides nothing reads as
// though it did, which is the same fault as a restriction that restricts nothing (novox/hq ADR 0045).
func TestAPortableBundleMayNotNameASystem(t *testing.T) {
got := buildProblemsOf(t, bundleFor("typescript", "arch"))
if !strings.Contains(got, "runs anywhere") {
t.Fatalf("a portable bundle was allowed to name a system:\n%s", got)
}
}
func TestAPortableBundleNamingNoSystemIsAccepted(t *testing.T) {
if got := buildProblemsOf(t, bundleFor("typescript", "")); got != "" {
t.Fatalf("an ordinary bundle was refused:\n%s", got)
// **A language that runs anywhere must not name a system.** A system that decides nothing reads as
// though it did, which is the same fault as a restriction that restricts nothing (novox/hq ADR 0045); a compiled
// bundle names one, and a portable bundle naming none is an ordinary one.
func TestABundleNamesASystemOnlyWhereItCompiles(t *testing.T) {
for _, c := range []struct{ language, system, refused string }{
{"go", "arch", ""},
{"typescript", "", ""},
{"typescript", "arch", "runs anywhere"},
} {
got := buildProblemsOf(t, bundleFor(c.language, c.system))
if (c.refused == "") != (got == "") || !strings.Contains(got, c.refused) {
t.Errorf("a %s bundle naming %q: %q, want %q", c.language, c.system, got, c.refused)
}
}
}
@@ -2882,13 +2883,23 @@ func TestAStateNameIsLocalAndOneToken(t *testing.T) {
{`{"module":"a","version":"1","state":["mesh.servers"]}`, `keeps state "mesh.servers": a state is named locally`},
{`{"module":"a","version":"1","state":["my_servers"]}`, `keeps state "my_servers"`},
{`{"module":"a","version":"1","state":["s","s"]}`, `keeps state "s" twice`},
{`{"module":"a","version":"1","state":[{"name":"s","history":65}]}`, `a key keeps between 1 and 64`},
{`{"module":"a","version":"1","state":[{"name":"s","history":65}]}`, `a key keeps 0 or 1 to 64 (0 is taken as 1)`},
{`{"module":"a","version":"1","state":[{"name":"s","history":0}]}`, ``}, // 0 passes, as before (ADR 0579 §1)
// 1966/AC9: one name, one form
{`{"module":"a","version":"1","state":["x"],"requires":[{"name":"x","storage":"key-value","template":"state"}]}`,
`a's bus state "x": the name is also kept in the old section state`},
{`{"module":"a.b","version":"1","state":["s"]}`, `no dot`},
{`{"module":"a","version":"1","reads":["bindings"]}`, `a read is <module>.<name>`},
{`{"module":"a","version":"1","reads":["a.s"]}`, `which is its own state`},
{`{"module":"a","version":"1","state":[{"name":"s","shared":true}]}`, `{name, history, ttl-seconds, per-machine, key-ttl}`},
} {
_, err := ParseManifest([]byte(c.manifest))
if c.says == "" {
if err != nil {
t.Errorf("%s was refused: %v", c.manifest, err)
}
continue
}
if err == nil {
t.Errorf("%s was accepted", c.manifest)
continue
@@ -84,12 +84,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("module = %q", m.Module)
}
// 1126: placement is the seat mesh-placement's (ADR 0567): the indexer holds no placement state. ADR 0578 §1: it says
// a refused manifest as manifest-refused, its one event.
if !reflect.DeepEqual(m.Emits, []string{"manifest-refused"}) {
t.Errorf("emits = %v, want manifest-refused alone", m.Emits)
// a refused manifest as manifest-refused, and (issue 1966 D3, 1966/AC10) a newly accepted one asking for bus state as
// manifest-accepted.
if !reflect.DeepEqual(m.Emits, []string{"manifest-refused", "manifest-accepted"}) {
t.Errorf("emits = %v, want manifest-refused and manifest-accepted", m.Emits)
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
@@ -112,8 +113,9 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none.
if want := []string{"indexer_take_in"}; !reflect.DeepEqual(m.Tools, want) {
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11:
// indexer_old_sections reads the count of the old sections (state oldsections).
if want := []string{"indexer_take_in", "indexer_old_sections"}; !reflect.DeepEqual(m.Tools, want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// No streams, no bus, no seat (ADR 0437 §4); it says no event.
+5 -2
View File
@@ -3,7 +3,8 @@
"version": "1",
"slug": "indexer",
"tools": [
"indexer_take_in"
"indexer_take_in",
"indexer_old_sections"
],
"consumes": [
"mesh-build-go.toolchain.reported",
@@ -21,10 +22,12 @@
{ "name": "taken", "history": 1 },
{ "name": "takeins", "history": 1 },
{ "name": "verdicts", "history": 5 },
{ "name": "oldsections", "history": 1 },
{ "name": "packages", "history": 5 }
],
"emits": [
"manifest-refused"
"manifest-refused",
"manifest-accepted"
],
"reads": [
"mesh-build-go.toolchains",
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-grants
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1 h1:T66tg9T/1hHg6KN/bDScHPT41IHkxLAGGO8VJiAOlb8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103142843-f8bd12e4b7b1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+22
View File
@@ -2202,6 +2202,28 @@ func TestGrants(t *testing.T) {
if res := bare.Compose(); res.Accounts != nil || !strings.Contains(strings.Join(res.Errors, " "), "no bus address") {
t.Errorf("a part with no bus address wrote %+v: %v", res.Accounts, res.Errors)
}
// 1966/AC20 (issue 1966 D5, D5b; novox/hq ADR 0597 §9): a module asking for a ledger in `requires` is wanted a ledger
// account on each machine it is assigned to, minted by the vault like every kind and held in the tool runner's
// `ledgers`; the account is retired once the module leaves that machine.
l := fixture()
l.manifestOf("mod-x", strings.Replace(xManifest, `"state"`, `"requires":[{"name":"records","storage":"ledger","template":"ledger",`+
`"types":[{"name":"decision","schema":{},"operations":[{"name":"propose","effect":"set-fields"}]}]}],"state"`, 1))
l.assign("b", "mod-x")
l.loaded = shadowOf(t, l)
lp := partOn(l)
lp.Bus = grants.BusAddress{URL: "nats://a.mesh:4222"}
lp.Compose()
for _, user := range []string{"ledger.a.mod-x", "ledger.b.mod-x"} {
w, err := buscompose.ReadWantedAccount(user, l.own[grants.StateAccounts][user])
if err != nil || w.Kind != buscompose.KindLedger || w.OwnSecret() != "ledgers" || w.Module != "mod-x" {
t.Errorf("%s is wanted as %+v (%v)", user, w, err)
}
}
l.unassign("b", "mod-x")
if res := lp.Compose(); res.Accounts == nil || !slices.Equal(res.Accounts.Deleted, []string{"b.mod-x", "ledger.b.mod-x"}) ||
l.own[grants.StateAccounts]["ledger.a.mod-x"] == nil {
t.Errorf("after mod-x left b: %+v", res.Accounts)
}
})
// 774/AC5 (novox/hq ADR 0561 §1): the vault's account-issued wakes a pass, so the next list carries the new user's hash
@@ -50,7 +50,8 @@ type ClaimedSeat struct {
// WantedAccount is one key of the grants part's state `accounts`: an account the mesh needs, without its password.
type WantedAccount struct {
User string `json:"user"`
// Kind is KindModule (KindNodeTools for the runtime, named as a module is), KindProcess or KindProvisioner.
// Kind is KindModule (KindNodeTools for the runtime, named as a module is), KindProcess, KindProvisioner or
// KindLedger.
Kind Kind `json:"kind"`
Node string `json:"node"`
Module string `json:"module"`
@@ -65,8 +66,12 @@ type WantedAccount struct {
At time.Time `json:"at"`
}
// LedgersSecret is the tool runner's own secret holding its machine's ledger accounts, a map of module to credential
// (ADR 0597 §9).
const LedgersSecret = "ledgers"
// mintedKinds are the kinds of account the vault mints (ADR 0561 §1).
var mintedKinds = []Kind{KindModule, KindNodeTools, KindProcess, KindProvisioner}
var mintedKinds = []Kind{KindModule, KindNodeTools, KindProcess, KindProvisioner, KindLedger}
// principal is the account as the composer names it.
func (w WantedAccount) principal() Principal {
@@ -112,9 +117,12 @@ func (w WantedAccount) Same(o WantedAccount) bool {
}
// OwnSecret is the module's own secret the account is held in: `broker`, `broker-<process id>` or `provisioner-broker`
// (the controller's names, ADR 0436 §2, ADR 0349 §11).
// (the controller's names, ADR 0436 §2, ADR 0349 §11); a ledger account in the tool runner's `ledgers`, which keeps every
// ledger account of its machine, by module (ADR 0597 §9).
func (w WantedAccount) OwnSecret() string {
switch w.Kind {
case KindLedger:
return LedgersSecret
case KindProcess:
return "broker-" + w.Process
case KindProvisioner:
@@ -49,6 +49,7 @@ func TestTheCopiedComposerComposesAsTheControllers(t *testing.T) {
t.Run("only declared streams are created", theGrantsPartCreatesOnlyDeclaredStreams)
t.Run("a declared stream is created as the controller asserted it", declaredStreamsAreCreatedAsTheControllerAssertsThem)
t.Run("every node's compositions are read by their named readers alone and written by their node", everyNodesCompositions)
t.Run("a module's ledger is written by its ledger account alone", aLedgerIsWrittenByItsLedgerAccountAlone)
// ADR 0567 §3, §9: a node reads every key of the seat's placement and its candidates (it composes with the whole
// proposal laid over), and none of its open proposals; it writes neither.
reads := strings.Join(NodeComposeReads("laptop"), " ")
@@ -942,3 +943,75 @@ func everyNodesCompositions(t *testing.T) {
t.Errorf("the view's membership issues %+v", m.State)
}
}
// 1966/AC20, AC21 (issue 1966 D5, ADR 0597 §9): a module asking for a ledger, assigned on two machines, gets one ledger account
// on each, publishing only on that module's ledger histories and snapshots; the account is gone from a machine the module
// leaves; and the module's own account, its process and the machine's tool runner hold no publish on its ledger.
func aLedgerIsWrittenByItsLedgerAccountAlone(t *testing.T) {
assigned := func(nodes ...string) []Principal {
r := Records{Nodes: nodes, Assigned: map[string][]Declared{}}
for _, n := range nodes {
r.Assigned[n] = []Declared{
{Module: RuntimeModule},
{Module: "m", Ledgers: []string{"records", "audit"}, Processes: []ProcessPrincipal{{Process: "w"}}},
{Module: "plain"},
}
}
users, err := Users(r)
if err != nil {
t.Fatal(err)
}
return users
}
ledgers := func(users []Principal) []string {
var out []string
for _, p := range users {
if p.Kind == KindLedger {
out = append(out, p.Username())
}
}
return out
}
both := assigned("a", "b")
if got := ledgers(both); !slices.Equal(got, []string{"ledger.a.m", "ledger.b.m"}) {
t.Fatalf("ledger accounts %v, want ledger.a.m and ledger.b.m", got)
}
if got := ledgers(assigned("a")); !slices.Equal(got, []string{"ledger.a.m"}) {
t.Fatalf("after m left b: ledger accounts %v, want ledger.a.m alone", got)
}
want := []string{"mesh.ledger.m.audit.history.>", "mesh.ledger.m.audit.snapshot.>",
"mesh.ledger.m.records.history.>", "mesh.ledger.m.records.snapshot.>"}
raw := []string{"mesh.ledger.m.records.history.decision/0290", "mesh.ledger.m.records.snapshot.decision/0290"}
for _, p := range both {
perms, err := PermissionsFor(p)
if err != nil {
t.Fatalf("%s: %v", p.Username(), err)
}
if p.Kind == KindLedger {
if !slices.Equal(perms.Publish, want) || len(perms.PublishDeny) != 0 {
t.Errorf("%s may publish %v, want %v alone", p.Username(), perms.Publish, want)
}
continue
}
for _, pattern := range perms.Publish {
for _, subject := range raw {
if SubjectsOverlap(pattern, subject) {
t.Errorf("%s may publish %s, which reaches the ledger subject %s", p.Username(), pattern, subject)
}
}
}
if p.Kind == KindModule || p.Kind == KindNodeTools || p.Kind == KindProcess {
if err := CheckWriters(p, want); err == nil {
t.Errorf("%s was granted a raw publish on a ledger's history", p.Username())
}
}
}
if err := CheckWriters(Principal{Kind: KindLedger, Node: "a", Module: "other", Ledgers: []string{"records"}}, want); err == nil {
t.Error("another module's ledger account was granted m's ledger histories")
}
for _, bad := range []Principal{{Kind: KindLedger, Node: "a", Module: "m"}, {Kind: KindLedger, Node: "a", Module: "m", Ledgers: []string{"a.b"}}} {
if _, err := PermissionsFor(bad); err == nil {
t.Errorf("%+v was composed", bad)
}
}
}
@@ -0,0 +1,57 @@
package buscompose
import (
"fmt"
"slices"
"sort"
"strings"
sdk "git.novox.be/novox/mesh-sdk/go/buscompose"
)
// A module's ledger and the one account that writes it (issue 1966 D5; novox/hq ADR 0597 §9, design 25 § 12.4).
//
// **Only the tool runner's ledger credential publishes on a ledger's history** (ADR 0597 §9): one per module assignment,
// named ledger.<machine>.<module> (KindLedger), scoped to the subjects of that module's ledger histories and snapshots, and
// gone when the module is unassigned from the machine. The module's own account, its processes and the machine's tool
// runner hold no publish there, so a raw publish on a history is refused by the bus. The subjects are named in one place, mesh-sdk's
// buscompose (LedgerHistorySubject, LedgerSnapshotSubject), `mesh.ledger.<module>.<local name>.history|snapshot` and
// one subject per instance below it (design 25 § 12.4).
// LedgerPublishGrants are the publish patterns a module's ledger account is granted: the histories and the snapshots of the
// ledgers named, every instance of each, and nothing else. Sorted.
func LedgerPublishGrants(module string, locals []string) []string {
var out []string
for _, local := range locals {
out = append(out, sdk.LedgerHistorySubject(module, local)+".>", sdk.LedgerSnapshotSubject(module, local)+".>")
}
sort.Strings(out)
return out
}
// writesLedger is a module's ledger account publishing on that module's own ledgers' histories and snapshots, and no other
// principal (ADR 0597 §9).
func writesLedger(p Principal, pattern string) bool {
return p.Kind == KindLedger && slices.Contains(LedgerPublishGrants(p.Module, p.Ledgers), pattern)
}
// ledgerSubjects are the subjects no principal but a ledger account may publish on, for the writers table.
var ledgerSubjects = []string{"mesh.ledger.*.*.history.>", "mesh.ledger.*.*.snapshot.>"}
// ledgerPermissions are a ledger account's whole authority: publishing on its module's ledgers, and its own reply space for
// the stream's acknowledgements. Refused when it names no ledger or a name that is no subject token.
func ledgerPermissions(p Principal) (Permissions, error) {
pub := LedgerPublishGrants(p.Module, p.Ledgers)
if len(p.Ledgers) == 0 {
return Permissions{}, fmt.Errorf("%s asks for no ledger, so its account would publish nowhere", p.Username())
}
for _, l := range p.Ledgers {
if !safeSubject.MatchString(l) || strings.Contains(l, "_") {
return Permissions{}, fmt.Errorf("%s: %q cannot be part of a subject: a permission is a subject pattern, and this would widen it", p.Username(), l)
}
}
if err := CheckWriters(p, pub); err != nil {
return Permissions{}, err
}
return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil
}
@@ -52,6 +52,10 @@ const (
// exactly what its bus says — never root, never the operator account, and never able to answer a tool
// call or a seat's verb (§3). One per process and machine.
KindProcess Kind = "process"
// KindLedger is one module assignment's ledger credential (novox/hq ADR 0597 §9, design 25 § 12.4): held by that
// machine's tool runner, never by the module's own code, and the only account that publishes on that module's ledger
// histories and snapshots. One per module assignment, whose module asks for a ledger in `requires`.
KindLedger Kind = "ledger"
)
// RuntimeModule is the module that IS the node's tool runtime (novox/hq ADR 0175). Where it is
@@ -102,6 +106,9 @@ type Principal struct {
Kind Kind
Node string
Module string
// Ledgers are the local names of the ledgers its module asks for in `requires`, for a KindLedger principal alone
// (issue 1966 D5): its whole authority is publishing on those ledgers' histories and snapshots (LedgerPublishGrants).
Ledgers []string
// Process is the process resource's id, for a KindProcess principal alone (novox/hq ADR 0436 §2):
// Username and inbox are <node>.<module>.<process id>.
Process string
@@ -425,6 +432,10 @@ func (p Principal) Username() string {
return p.Login
}
return "node." + p.Node
case KindLedger:
// `ledger.<machine>.<module>`: never `<node>.<module>`, which is the module's own account or the tool runner's,
// so the module's code never holds it.
return "ledger." + p.Node + "." + p.Module
case KindProvisioner:
// Named for the machine, as a node is: one client per machine, whichever module runs it, and never
// `<node>.<module>`, which is the module's own account or the tool runner's.
@@ -567,6 +578,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
return Permissions{Publish: pub, Subscribe: []string{p.inbox()}}, nil
}
// A ledger account publishes on its module's ledger histories and snapshots and nothing else (issue 1966 D5).
if p.Kind == KindLedger {
return ledgerPermissions(p)
}
if p.Kind == KindProcess && len(p.Removes) > 0 {
pub, err := BusRemovalGrants(p.Removes, p.FilesFor)
if err != nil {
@@ -103,6 +103,9 @@ type Declared struct {
// composed whether or not the module itself has an account (NoAccount), because a process account is
// delivered in its own secret, broker-<process id>, never the module's `broker`.
Processes []ProcessPrincipal
// Ledgers are the local names of the ledgers it asks for in `requires` (catalogue.StorageLedger); each assignment of a
// module with any gets a ledger principal of its own (issue 1966 D5, ADR 0597 §9).
Ledgers []string
}
// ProcessPrincipal is one process of a module that declares bus (novox/hq ADR 0436 §1, §2), as the broker
@@ -229,6 +232,11 @@ func Users(r Records) ([]Principal, error) {
out[len(out)-1].FilesFor = sortedCopy(r.Nodes)
}
}
// **A ledger account per assignment of a module that asks for a ledger** (issue 1966 D5, ADR 0597 §9): held by the
// machine's tool runner, so it is composed whether or not the module has an account of its own.
if len(d.Ledgers) > 0 {
out = append(out, Principal{Kind: KindLedger, Node: node, Module: d.Module, Ledgers: sortedCopy(d.Ledgers)})
}
if runtimeHere && d.Module == RuntimeModule {
continue
}
@@ -236,6 +236,12 @@ var WritersTable = []WriterRow{
ProvisionerAppliedBucket,
Others: "the watchman reads it; a failure is that machine's failed apply",
Subjects: []string{"prov.applied.*.>"}, Writes: ownProvisioner},
// A ledger's history and its snapshots (novox/hq ADR 0597 §9, issue 1966 D5): the tool runner's ledger credential of
// the owning module alone, so the module's own code, its processes and the runner's own account cannot write one raw.
{State: "a ledger's history and snapshots", Writer: "the tool runner's ledger credential of the owning module",
KeptIn: "the bus, the ledger's history stream and its snapshot subject",
Others: "the owner reads them through the tool runner's calls; nobody else reads them (ADR 0596 §2)",
Subjects: ledgerSubjects, Writes: writesLedger},
}
// CheckWriters refuses a grant that lets a principal publish on a subject the writers table gives
@@ -46,8 +46,9 @@ type BusAddress struct {
Fingerprint string
}
// WantedAccounts is every account the vault mints that the records compose, keyed by user, with the bus's address. A
// node, the controller, an enrolment user or a person is never one (ADR 0561 §4, §5). Pure.
// WantedAccounts is every account the vault mints that the records compose, keyed by user, with the bus's address: a
// module's, a process's, a machine's provisioner client's and a module assignment's ledger account (issue 1966 D5b, ADR
// 0597 §9). A node, the controller, an enrolment user or a person is never one (ADR 0561 §4, §5). Pure.
func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[string]buscompose.WantedAccount, error) {
principals, err := buscompose.Users(recs)
if err != nil {
@@ -62,7 +63,7 @@ func WantedAccounts(in Inputs, recs buscompose.Records, bus BusAddress) (map[str
if m, ok := manifestOn(in, p.Node, p.Module); ok {
w.Claims = ClaimsOf(m)
}
case buscompose.KindProcess, buscompose.KindProvisioner:
case buscompose.KindProcess, buscompose.KindProvisioner, buscompose.KindLedger:
default:
continue
}
@@ -139,6 +139,8 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
Served: catalogue.ServedSettings(m),
// Its own object buckets (ADR 0592 §5).
Objects: objectsOf(m),
// The ledgers it asks for in `requires` (issue 1966 D5): each assignment gets a ledger account.
Ledgers: ledgersOf(m),
}
d.Loads = loadsABundle(m)
if _, reads := m.OwnSecrets["broker"]; !reads {
@@ -282,6 +284,17 @@ func readsOfObjects(module, node string, reads, refused []string, manifests map[
return states, objects, refusedOut
}
// ledgersOf are the local names of the ledgers a manifest asks for in `requires`.
func ledgersOf(m catalogue.Manifest) []string {
var out []string
for _, e := range m.BusState {
if e.Storage == catalogue.StorageLedger {
out = append(out, e.Name)
}
}
return out
}
func logsOf(m catalogue.Manifest) []buscompose.Log {
var out []buscompose.Log
for _, l := range m.Logs {