Compare commits

..
Author SHA1 Message Date
jschoubben 858b06a94e Take mesh-sdk at c8e3509, which counts an item's text as a placement of a config value (issue 1658)
mesh/repo-check head 858b06a94e69 merged into trunk 12752911b644 — fail: head 858b06a94e69 does not merge cleanly into trunk 12752911b644: in conflict go.m…
mesh/delivery-group group fix/1656-xterm-config rejected: a member's own check failed
mesh/merge-gate group fix/1656-xterm-config: composed again for the group's new heads
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 12:55:08 +02:00
86 changed files with 120 additions and 6544 deletions
+2 -21
View File
@@ -408,21 +408,12 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
return ""
}
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
_, reads := m.OwnSecrets["broker"]
if !reads && !runsTheProvisionerClient(m) {
// A module whose bus accounts are its processes' alone has none of its own: only the processes that
// are not issued yet are issued (issue 1226).
return processAccountsOnAssign(ctx, inv, m, node)
}
if !reads {
if _, reads := m.OwnSecrets["broker"]; !reads {
// A module running the provisioner client alone is minted the client's account (issue 917).
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
}
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
return ""
} else if minted {
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
return processAccountsOnAssign(ctx, inv, m, node)
}
busAddress, err := broker.BusAddress()
if err == nil {
@@ -435,16 +426,6 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
}
// processAccountsOnAssign issues the process accounts of an assigned module not issued yet, and says what it did.
func processAccountsOnAssign(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) string {
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
if err != nil {
return fmt.Sprintf("its process accounts are not issued (%v): `module issue %s --node %s` first — "+
"`push %s` refuses to send %s until it is", err, m.Module, node, node, m.Module)
}
return line
}
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
-2
View File
@@ -232,8 +232,6 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
fmt.Print(nothingReadsIt(repository, m))
return nil
}
// **A module the merge adds is asked for now** (novox/hq issue 1617), not when the batch is cut.
askNewModulesAtHearing(ctx, inv, m, entries, read, facts.from)
// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
own := ownPath(touches)
waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries))
-3
View File
@@ -768,9 +768,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
}
return manifest, kept, err
}
// A bus process the new definition gains gets its account now, on each machine assigned the module, so the
// next push composes a real account (issue 1226).
issueGainedProcessAccounts(ctx, inv, manifest)
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
return manifest, kept, err
-7
View File
@@ -32,13 +32,6 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
if err := broker.Raise(r, names); err != nil {
return nil, err
}
// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
// whenever the bus is up.
if files, ok := r.(interface{ EnsureDeclarationFiles(nodes []string) error }); ok {
if err := files.EnsureDeclarationFiles(names); err != nil {
return nil, err
}
}
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
// after something started asking for builds flushes the backlog instead of having lost it.
-6
View File
@@ -233,12 +233,6 @@ func prepareSend(ctx context.Context, open *stores, node string) ([]string, erro
if !slices.Contains(in.BusMinted, "provisioner."+node) {
continue
}
default:
// A bus process's account nobody issued still gets the stand-in, until the composition leaves
// the module off (issue 1753): the warning names the verb (issue 1226).
if line := processAccountWarning(m, node, name, in.BusMinted); line != "" {
sayWarning(ctx, line)
}
}
if _, err := inv.SecretForModule(ctx, node, m.Module, name); err == nil {
did = true
+1 -3
View File
@@ -200,7 +200,6 @@ func composeQuestion(ctx context.Context, open *stores, node string, mode compos
}
sayFound(ctx, d)
out := sendableOf(d)
out.fileReferences = readsFileReferences(in.Input, node)
if !withOrder {
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
}
@@ -282,7 +281,6 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
if len(left) == 0 && len(in.wouldMake) == 0 {
sayFound(ctx, d)
out := sendableOf(d)
out.fileReferences = readsFileReferences(in.Input, node)
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
here, _ := nodeOf(in.Input, node)
@@ -368,7 +366,7 @@ func sayFound(ctx context.Context, d compose.Declared) {
// sendableOf is a composed declaration as the controller's sends know it.
func sendableOf(d compose.Declared) sendable {
out := sendable{Resources: compactKept(d.Resources), Sequence: d.Sequence, Epoch: d.Epoch, Generation: d.Generation,
out := sendable{Resources: d.Resources, Sequence: d.Sequence, Epoch: d.Epoch, Generation: d.Generation,
ReadsModules: d.ReadsModules, moduleOf: d.ModuleOf, buildIDs: d.BuildIDs, modules: d.Modules,
localNetwork: d.LocalNetwork, Received: d.Received, Mesh: d.Mesh, BusUsers: d.BusUsers,
LeftOut: d.LeftOut, leftOutWhy: d.LeftOutWhy, withheld: d.Withheld, unbound: d.Unbound, foreseen: d.Foreseen,
@@ -1,65 +0,0 @@
package main
import (
"io"
"os"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Issue 1638 (novox/hq ADR 0480 clause 7), taken in with mesh-sdk's container `groups`: the controller sends a container's
// groups only to a node-engine that reports the capability container-groups, and holds the module on any other, naming it.
const n8nWithAGroup = `{"module":"n8n","version":"1",
"config":{"media-gid":{"type":"number","minimum":1,"required":true,"description":"the group that owns the media library"}},
"resources":[{"id":"server","type":"container","name":"n8n",
"image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0",
"groups":["${config:media-gid}"]}]}`
// printedBy runs reportUnhostable and returns what it printed.
func printedBy(t *testing.T, node string, plan catalogue.Resolution) string {
t.Helper()
stdout := os.Stdout
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
os.Stdout = w
reportUnhostable(node, plan)
w.Close()
os.Stdout = stdout
out, _ := io.ReadAll(r)
return string(out)
}
// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named.
func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) {
m, err := catalogue.ParseManifest([]byte(n8nWithAGroup))
if err != nil {
t.Fatal(err)
}
shelf := catalogue.Shelf{"n8n": m}
reads := catalogue.Node{Name: "ace", Capabilities: map[string]bool{"container-runtime": true, "container-groups": true}}
old := catalogue.Node{Name: "g14", Capabilities: map[string]bool{"container-runtime": true}}
got, err := catalogue.Resolve(shelf, []string{"n8n"}, reads, catalogue.World{})
if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 {
t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable)
}
if out := printedBy(t, "ace", got); out != "" {
t.Errorf("a node that reads groups was told: %q", out)
}
got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{})
if err != nil {
t.Fatal(err)
}
if len(got.Modules) != 0 || len(got.Unhostable) != 1 {
t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable)
}
if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") {
t.Errorf("the hold does not name the module and the capability: %q", out)
}
}
@@ -1,110 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/overlay"
"git.novox.be/novox/mesh-sdk/go/compose"
)
// 1696/AC2, ADR 0486 §5: a node is sent a file by digest only when its profile reports the capability. A node whose profile lacks
// it, and a node the mesh has no profile of, are sent every content inline; a node that reports it is sent the reference. The test
// goes through composeQuestion and composeSend, the two places readsFileReferences is read (compose_sdk.go).
func TestAFileTravelsByDigestOnlyToANodeThatReportsTheCapability(t *testing.T) {
open := aMesh(t) // anchor and laptop report no file-references
ctx := t.Context()
big := strings.Repeat("0123456789abcdef", 8<<10) // 128 KiB
register(t, open, catalogue.Manifest{Module: "bigfile", Version: "1",
Resources: []map[string]any{{"id": "blob", "type": "file", "path": "/etc/bigfile/blob", "mode": "0644", "content": big}}})
// reader reports the capability; bare has never reported a profile at all.
for i, n := range []struct {
name string
profile map[string]any
}{
{"reader", map[string]any{"capabilities": []any{
map[string]any{"name": "container-runtime", "present": true},
map[string]any{"name": "systemd", "present": true},
map[string]any{"name": sdkdecl.FileReferencesCapability, "present": true}}}},
{"bare", nil},
} {
record, err := open.inventory.AddNode(ctx, n.name)
if err != nil {
t.Fatal(err)
}
if err := open.inventory.SetPlace(ctx, n.name, n.name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+10)); err != nil {
t.Fatal(err)
}
if n.profile != nil {
if err := open.inventory.RecordProfile(ctx, record.ID, n.profile); err != nil {
t.Fatal(err)
}
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
if _, err := open.inventory.Assign(ctx, n.name, overlay.Name); err != nil {
t.Fatal(err)
}
}
for _, node := range []string{"laptop", "reader", "bare"} {
if _, err := assign(ctx, open, node, "bigfile"); err != nil {
t.Fatalf("assign %s: %v", node, err)
}
}
byDigest := map[string]bool{"laptop": false, "bare": false, "reader": true}
for node, want := range byDigest {
asked, _, err := composeQuestion(ctx, open, node, compose.Reading, false)
if err != nil {
t.Fatalf("%s: %v", node, err)
}
sent, err := composeSend(ctx, open, node)
if err != nil {
t.Fatalf("%s: %v", node, err)
}
for what, s := range map[string]sendable{"question": asked, "send": sent} {
if s.fileReferences != want {
t.Fatalf("%s, %s: fileReferences %v, want %v", node, what, s.fileReferences, want)
}
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
var got struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
var found bool
for _, r := range got.Resources {
if r["path"] != "/etc/bigfile/blob" {
continue
}
found = true
inline := r["content"] == big
if want && (inline || r["digest"] == nil || r["size"] == nil) {
t.Fatalf("%s, %s: a node that reads references is sent the content inline", node, what)
}
if !want && (!inline || r["digest"] != nil) {
t.Fatalf("%s, %s: a node that does not report the capability is sent a reference", node, what)
}
}
if !found {
t.Fatalf("%s, %s: the file is not in the declaration", node, what)
}
}
}
}
-253
View File
@@ -1,253 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"sync"
"git.novox.be/novox/mesh-sdk/go/compose"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/conditions"
)
// **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
//
// The declaration to a node that reports sdkdecl.FileReferencesCapability carries, in place of the content of every
// file resource of more than sdkdecl.FileReferenceThreshold bytes that may travel so (File.TravelsByDigest: plain content or
// bytes, not sealed, not filled with secrets, not written into another file), its sha256 digest and size. The content is put
// into the bucket of the bus under the hex of the digest before the declaration is sent. A node that does not report the
// capability is sent every content inline, as before: an older node-engine refuses a field it does not know, whole.
// readsFileReferences says the node reports the capability of reading a file given by digest (ADR 0486 §5). A node the mesh has
// no profile of reads none.
func readsFileReferences(in compose.Input, node string) bool {
n, ok := nodeOf(in, node)
return ok && n.Capabilities[sdkdecl.FileReferencesCapability]
}
// referencing is the resources as a node that reads references is sent them, and the content of each file that went by
// digest, by the name of its object (the hex of its digest). The resources it is given are not changed. A file that does not
// travel by digest is passed as it is.
func referencing(resources []map[string]any) ([]map[string]any, map[string][]byte, error) {
out := make([]map[string]any, len(resources))
var objects map[string][]byte
for i, r := range resources {
out[i] = r
if kind, _ := r["type"].(string); kind != string(sdkdecl.TypeFile) {
continue
}
// Cheap first: only a file with a content or bytes of the size is worth reading whole.
content, _ := r["content"].(string)
bytesField, _ := r["bytes"].(string)
if len(content) <= sdkdecl.FileReferenceThreshold && len(bytesField) <= sdkdecl.FileReferenceThreshold {
continue
}
raw, err := json.Marshal(r)
if err != nil {
return nil, nil, fmt.Errorf("resource %v: %w", r["id"], err)
}
var f sdkdecl.File
if err := json.Unmarshal(raw, &f); err != nil {
return nil, nil, fmt.Errorf("resource %v is not a file the controller can read: %w", r["id"], err)
}
if !f.TravelsByDigest() {
continue
}
ref, body, err := f.AsReference()
if err != nil {
return nil, nil, err
}
name, err := sdkdecl.ObjectName(ref.Digest)
if err != nil {
return nil, nil, err
}
with := make(map[string]any, len(r)+2)
for k, v := range r {
with[k] = v
}
delete(with, "content")
delete(with, "bytes")
with["digest"], with["size"] = ref.Digest, ref.Size
out[i] = with
if objects == nil {
objects = map[string][]byte{}
}
objects[name] = body
}
return out, objects, nil
}
// namedObjects are the objects of the bucket a body names: the object name of every file resource given by reference,
// with its size. It reads a declaration as the node does, so it also reads the latest one the bus holds.
func namedObjects(body []byte) (map[string]int64, error) {
var envelope struct {
Resources []struct {
Type string `json:"type"`
Digest string `json:"digest"`
Size int64 `json:"size"`
} `json:"resources"`
}
if err := json.Unmarshal(body, &envelope); err != nil {
return nil, err
}
var out map[string]int64
for _, r := range envelope.Resources {
if r.Type != string(sdkdecl.TypeFile) || r.Digest == "" {
continue
}
name, err := sdkdecl.ObjectName(r.Digest)
if err != nil {
return nil, err
}
if out == nil {
out = map[string]int64{}
}
out[name] = r.Size
}
return out, nil
}
// objectStore is what putting the objects needs of a node's bucket.
type objectStore interface {
// Has says whether an object of that name is there, complete, and whether it carries the sweep's mark of being named by no
// node's latest declaration.
Has(ctx context.Context, name string) (have, marked bool, err error)
// Put writes the object under that name, replacing one of the same name (and so erasing its mark).
Put(ctx context.Context, name string, content []byte) error
}
// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a node
// that reads the declaration finds each object. An object already there (the same content, the same name) is not written again,
// **unless the sweep marked it unnamed**: then it is written again, so the mark that would have it removed is gone with the
// old object (issue 1696, review: a mark read just before a send must not take an object the send relies on). Refuses on the
// first it cannot put, naming it, so the declaration is not sent.
func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
names := make([]string, 0, len(objects))
for name := range objects {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
have, marked, err := store.Has(ctx, name)
if err != nil {
return fmt.Errorf("whether the object %s is in the bucket: %w", name[:12], err)
}
if have && !marked {
continue
}
if err := store.Put(ctx, name, objects[name]); err != nil {
return fmt.Errorf("putting the object %s (%d bytes) in the bucket: %w", name[:12], len(objects[name]), err)
}
}
return nil
}
// busObjects is a node's bucket on the controller's own connection.
type busObjects struct{ store jetstream.ObjectStore }
// openBucket opens the bucket of a node's files, making it first if it is not there (it is asserted for every node at the
// controller's start; a node adopted since is covered here).
func openBucket(ctx context.Context, conn *nats.Conn, node string) (busObjects, error) {
js, err := jetstream.New(conn)
if err != nil {
return busObjects{}, err
}
bucket, err := broker.DeclarationFilesBucketOf(node)
if err != nil {
return busObjects{}, fmt.Errorf("the object bucket of node %q: %w", node, err)
}
store, err := js.ObjectStore(ctx, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
if err = broker.EnsureDeclarationFiles(ctx, js, node); err == nil {
store, err = js.ObjectStore(ctx, bucket)
}
}
if err != nil {
return busObjects{}, fmt.Errorf("the object bucket %s: %w", bucket, err)
}
return busObjects{store}, nil
}
func (b busObjects) Has(ctx context.Context, name string) (bool, bool, error) {
info, err := b.store.GetInfo(ctx, name)
switch {
case errors.Is(err, jetstream.ErrObjectNotFound):
return false, false, nil
case err != nil:
return false, false, err
}
_, marked := info.Metadata[declarationUnnamed]
return !info.Deleted, marked, nil
}
func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
_, err := b.store.PutBytes(ctx, name, content)
return err
}
// filesLocks hold a node's bucket for one act at a time in this process: a send puts its objects and publishes the declaration that
// names them under the node's lock, and the sweep judges the node's bucket under the same one, so the sweep never removes an
// object between a send's put and its declaration. Another process (a person's push) is covered by the put erasing a mark and by
// the sweep looking at an object again just before it removes it.
var filesLocks sync.Map
func lockFilesOf(node string) func() {
l, _ := filesLocks.LoadOrStore(node, &sync.Mutex{})
mu := l.(*sync.Mutex)
mu.Lock()
return mu.Unlock
}
// kindFilesNotPut is a node whose files that travel by digest could not be put in its bucket, send after send.
const kindFilesNotPut = "declaration-files-not-put"
// filesNotPutAfter is how many sends in a row may fail to put before the condition is raised: the first send to the machine holding
// the bus fails for want of the grant that very send delivers, and says so, and is not yet a fault.
const filesNotPutAfter = 2
var (
filesFailMu sync.Mutex
filesFailed = map[string]int{}
)
func filesNotPutObservation(node, summary string) conditions.Observation {
return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindFilesNotPut, Token: kindFilesNotPut,
Machine: node, Severity: conditions.Warning, Source: "sends", Summary: summary}
}
// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
// clears the count and the condition.
func notePut(ctx context.Context, node string, put error) {
filesFailMu.Lock()
if put == nil {
delete(filesFailed, node)
} else {
filesFailed[node]++
}
n := filesFailed[node]
filesFailMu.Unlock()
keeper := conditionsFrom
if keeper == nil {
return
}
key := filesNotPutObservation(node, "").Key()
switch {
case put == nil:
if _, err := keeper.Clear(ctx, key, node+" was sent its files by digest"); err != nil {
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
}
case n >= filesNotPutAfter:
summary := fmt.Sprintf("%s's files that travel by digest could not be put in its bucket %d sends in a row, so it is sent them "+
"inline: %v", node, n, put)
if _, err := keeper.Observe(ctx, filesNotPutObservation(node, summary)); err != nil {
fmt.Printf("the condition %s could not be raised: %v\n", key, err)
}
}
}
@@ -1,246 +0,0 @@
//go:build integration
package main
import (
"bytes"
"context"
"encoding/json"
"strings"
"testing"
"time"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
type sigOf struct{}
func (sigOf) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
// 1696/AC7 (ADR 0486 §1): on a real bus with its default largest message of 1 MiB, a declaration whose files add up to more than
// 3 MiB is sent as references, its objects are in the bucket, each fetched back whole and verified against the digest and size the
// declaration names; the same declaration inline is refused by the controller, naming size and limit. A changed byte in an object
// fails the verification.
func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) { // 1696/AC7
s := testbus.Start(t)
conn, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
nc, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
api, err := jetstream.New(conn)
if err != nil {
t.Fatal(err)
}
if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
t.Fatal(err)
}
// Run twice: an assertion of an existing bucket is no error and no change.
if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
t.Fatalf("asserting the bucket again: %v", err)
}
var resources []map[string]any
for i := 0; i < 3; i++ {
resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
}
resources = append(resources, aFileResource("small", "tiny\n"))
inline := sendable{Resources: resources}
byDigest := sendable{fileReferences: true, Resources: resources}
bus := link.OverNATS{Conn: conn, JS: nc}
body, err := inline.Body()
if err != nil {
t.Fatal(err)
}
if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
}
bucket, err := openBucket(ctx, conn, "novox")
if err != nil {
t.Fatal(err)
}
objects, err := byDigest.objects()
if err != nil || len(objects) != 3 {
t.Fatalf("objects: %d %v", len(objects), err)
}
if err := putObjects(ctx, bucket, objects); err != nil {
t.Fatal(err)
}
if err := putObjects(ctx, bucket, objects); err != nil {
t.Fatalf("putting them again: %v", err)
}
refBody, err := byDigest.Body()
if err != nil {
t.Fatal(err)
}
if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
t.Fatalf("the declaration by reference: %v", err)
}
// What the stream holds for the node is the reference form, and it names exactly the three objects.
latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
if err != nil || len(latest) != 1 {
t.Fatalf("the latest declarations: %v %v", len(latest), err)
}
named, err := namedByLatest("novox", latest["novox"])
if err != nil || len(named) != 3 {
t.Fatalf("what the latest declaration names: %v %v", named, err)
}
novoxBucket, err := broker.DeclarationFilesBucketOf("novox")
if err != nil {
t.Fatal(err)
}
store, err := api.ObjectStore(ctx, novoxBucket)
if err != nil {
t.Fatal(err)
}
var envelope struct {
Resources []sdkdecl.File `json:"resources"`
}
if err := json.Unmarshal(refBody, &envelope); err != nil {
t.Fatal(err)
}
for _, f := range envelope.Resources {
if !f.IsReference() {
continue
}
name, _ := sdkdecl.ObjectName(f.Digest)
got, err := store.GetBytes(ctx, name)
if err != nil {
t.Fatalf("the object of %s: %v", f.ID, err)
}
if err := f.Verify(got); err != nil {
t.Errorf("the object does not verify: %v", err)
}
got[len(got)/2] ^= 1
if err := f.Verify(got); err == nil {
t.Errorf("a changed byte in the object of %s passed the verification", f.ID)
}
}
// The sweep against the real bucket: all named, nothing marked; a fourth object named by none is marked, kept a day less a
// minute, and removed at the day.
extra := bytes.Repeat([]byte("x"), 100)
extraName := strings.Repeat("e", 64)
if err := putObjects(ctx, bucket, map[string][]byte{extraName: extra}); err != nil {
t.Fatal(err)
}
b := busSweptBucket{store}
t0 := time.Now()
if r, err := sweepDeclarationFiles(ctx, b, named, t0); err != nil || r.Marked != 1 || r.Removed != 0 || r.Kept != 3 {
t.Fatalf("first sweep: %+v %v", r, err)
}
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour-time.Minute)); err != nil || r.Removed != 0 {
t.Fatalf("a minute short of a day: %+v %v", r, err)
}
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour+time.Second)); err != nil || r.Removed != 1 {
t.Fatalf("a day after: %+v %v", r, err)
}
if infos, err := store.List(ctx); err != nil || len(infos) != 3 {
t.Fatalf("the bucket holds %d objects (%v), want the 3 the declaration names", len(infos), err)
}
}
// 1696/AC8 on a real bucket: the sweep removes an object only if it still carries the mark it judged it by. An object marked
// unnamed, then put again by a send (which erases the mark), is kept when the sweep deletes it by the old mark.
func TestTheSweepsDeleteLeavesAnObjectASendPutAgainAfterItWasMarked(t *testing.T) { // 1696/AC8
s := testbus.Start(t)
conn, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
bucket, err := openBucket(ctx, conn, "novox")
if err != nil {
t.Fatal(err)
}
name := strings.Repeat("a", 64)
content := map[string][]byte{name: bytes.Repeat([]byte("y"), 100)}
if err := putObjects(ctx, bucket, content); err != nil {
t.Fatal(err)
}
b := busSweptBucket{bucket.store}
judged := time.Now().Add(-25 * time.Hour).Truncate(time.Second)
if err := b.MarkUnnamed(ctx, name, judged); err != nil {
t.Fatal(err)
}
if _, marked, err := bucket.Has(ctx, name); err != nil || !marked {
t.Fatalf("the object was not marked: %v %v", marked, err)
}
// The sweep has read the mark; a send now puts the object again, which erases it.
if err := putObjects(ctx, bucket, content); err != nil {
t.Fatal(err)
}
if have, marked, err := bucket.Has(ctx, name); err != nil || !have || marked {
t.Fatalf("a send did not put the marked object again: have %v marked %v %v", have, marked, err)
}
gone, err := b.Delete(ctx, name, judged)
if err != nil || gone {
t.Fatalf("the sweep deleted by an old mark: gone %v, %v", gone, err)
}
if have, _, err := bucket.Has(ctx, name); err != nil || !have {
t.Fatalf("the object a send relies on is gone: %v %v", have, err)
}
// And the mark still on it is honoured: marked again and deleted by that mark, it goes.
if err := b.MarkUnnamed(ctx, name, judged); err != nil {
t.Fatal(err)
}
if gone, err := b.Delete(ctx, name, judged); err != nil || !gone {
t.Fatalf("an object still carrying the mark it was judged by is kept: gone %v, %v", gone, err)
}
}
// 1696/AC9 on a real bus: the sweep of a node's bucket takes the node's lock first, so it waits for a send that has put its objects
// and not yet published the declaration that names them.
func TestTheSweepOfANodesBucketHoldsTheNodesLock(t *testing.T) { // 1696/AC9
s := testbus.Start(t)
conn, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
const node = "sweep-lock-node"
if _, err := openBucket(ctx, conn, node); err != nil { // the bucket exists, or the sweep would return before the lock
t.Fatal(err)
}
release := lockFilesOf(node) // a send is between its put and its publication
done := make(chan struct{})
go func() {
_, _ = sweepNode(ctx, conn, node)
close(done)
}()
select {
case <-done:
t.Fatal("the sweep judged the node's bucket while a send held it")
case <-time.After(300 * time.Millisecond):
}
release()
select {
case <-done:
case <-time.After(20 * time.Second):
t.Fatal("the sweep did not go on once the send let go")
}
}
@@ -1,534 +0,0 @@
package main
import (
"context"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// contentOf is a deterministic content of n bytes that does not compress to nothing and holds no character JSON escapes.
func contentOf(seed string, n int) string {
var b strings.Builder
for i := 0; b.Len() < n; i++ {
fmt.Fprintf(&b, "%s-%07d-%x;", seed, i, i*2654435761%1000003)
}
return b.String()[:n]
}
func aFileResource(id, content string) map[string]any {
return map[string]any{"id": id, "type": "file", "path": "/etc/" + id, "mode": "0644", "content": content}
}
// controlNodeShape is a declaration shaped like the control node's on 2026-10-21 (issue 1689), the sizes measured then: a bus users
// file of 252,678 bytes, the vault's export of 138,475 bytes (compact), the tool runtime 21 KB, the packet filter 19 KB, the launcher
// 14 KB, and about 600 small resources for the rest (about 286 KB). `more` scales the count of small ones: 1 is the shape, 1.4 forty
// percent more of them.
func controlNodeShape(more float64) []map[string]any {
res := []map[string]any{
aFileResource("nats.bus-users", contentOf("users", 252678)),
aFileResource("mesh-vault.kept", contentOf("kept", 138475)),
aFileResource("node-tools.runtime", contentOf("runtime", 21<<10)),
aFileResource("nftables.filtering", contentOf("filter", 19<<10)),
aFileResource("mesh-host.launcher", contentOf("launcher", 14<<10)),
}
for i := 0; i < int(600*more); i++ {
res = append(res, aFileResource(fmt.Sprintf("module-%03d.config", i), contentOf(fmt.Sprintf("small%d", i), 380)))
}
return res
}
func wireSizeOf(t *testing.T, s sendable) int {
t.Helper()
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
// The signature the controller's key makes is 64 bytes.
msg, err := link.Envelope(body, make([]byte, 64))
if err != nil {
t.Fatal(err)
}
return len(msg)
}
// 1696/AC1: for a node that reports the capability, the two big files of a control-node-shaped declaration carry their digest and size
// and no content, and the declaration is far under the bus's 1 MiB.
func TestAFileOver64KiBTravelsByDigestToANodeThatReadsOne(t *testing.T) { // 1696/AC1
users, kept := contentOf("users", 252678), contentOf("kept", 138475)
s := sendable{fileReferences: true, Resources: []map[string]any{
aFileResource("nats.bus-users", users), aFileResource("mesh-vault.kept", kept),
aFileResource("small", "tiny\n"),
}}
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
var got struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
for i, want := range []string{users, kept} {
r := got.Resources[i]
digest := sdkdecl.DigestOf([]byte(want))
if _, has := r["content"]; has || r["digest"] != digest || r["size"] != float64(len(want)) {
t.Errorf("%v goes with content %v, digest %v, size %v; want no content, %s and %d", r["id"], r["content"] != nil,
r["digest"], r["size"], digest, len(want))
}
if r["path"] == nil || r["mode"] != "0644" {
t.Errorf("%v lost its path or mode: %v", r["id"], r)
}
}
if got.Resources[2]["content"] != "tiny\n" || got.Resources[2]["digest"] != nil {
t.Errorf("a small file is not inline: %v", got.Resources[2])
}
if strings.Contains(string(body), users[:200]) || strings.Contains(string(body), kept[:200]) {
t.Error("the content of a file that went by digest is in the body")
}
if n := wireSizeOf(t, s); n > 100<<10 {
t.Errorf("the declaration is %d bytes on the wire, want under 100 KB", n)
}
// The objects to put are the contents, each under the hex of its digest, and a reference verifies against its object.
objects, err := s.objects()
if err != nil || len(objects) != 2 {
t.Fatalf("objects: %v %v", objects, err)
}
for _, want := range []string{users, kept} {
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(want)))
if string(objects[name]) != want {
t.Errorf("the object %s does not hold the content", name[:12])
}
}
named, err := namedObjects(body)
if err != nil || len(named) != 2 {
t.Fatalf("the objects the body names: %v %v", named, err)
}
for name := range objects {
if _, ok := named[name]; !ok {
t.Errorf("the body does not name the object %s it was sent with", name[:12])
}
if _, err := hex.DecodeString(name); err != nil || len(name) != 64 {
t.Errorf("the object name %q is not 64 hex characters", name)
}
}
}
// 1696/AC2: a node that does not report the capability is sent every content inline, as before, and has no object put.
func TestANodeWithoutTheCapabilityIsSentEveryContentInline(t *testing.T) { // 1696/AC2
users := contentOf("users", 252678)
s := sendable{Resources: []map[string]any{aFileResource("nats.bus-users", users)}}
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
var got struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
if got.Resources[0]["content"] != users || got.Resources[0]["digest"] != nil || got.Resources[0]["size"] != nil {
t.Fatalf("a node that reads no references is not sent the content inline: %v", got.Resources[0]["digest"])
}
if objects, err := s.objects(); err != nil || len(objects) != 0 {
t.Fatalf("objects for a node that reads none: %v %v", len(objects), err)
}
}
// 1696/AC3: only a file that may travel by digest does: one of exactly 64 KiB, a sealed one, one filled with secrets, one written
// into another file, and a directory stay inline, whatever the node reads.
func TestOnlyAFileThatMayTravelByDigestDoes(t *testing.T) { // 1696/AC3
exact := contentOf("exact", 64<<10)
big := contentOf("big", 64<<10+1)
sealed := aFileResource("sealed", big)
sealed["sealed"], sealed["content"] = "box", ""
sealed["content"] = big // a sealed file's content is none; the body must pass it as it is, whatever it holds
secret := aFileResource("secret", big)
secret["secrets"] = map[string]any{"token": "x"}
into := aFileResource("into", big)
into["into"] = "/etc/other"
s := sendable{fileReferences: true, Resources: []map[string]any{
aFileResource("exact", exact), sealed, secret, into,
{"id": "dir", "type": "directory", "path": "/etc/d", "content": big},
aFileResource("big", big),
}}
body, err := s.Body()
if err != nil {
t.Fatal(err)
}
var got struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatal(err)
}
for i, id := range []string{"exact", "sealed", "secret", "into", "dir"} {
if got.Resources[i]["id"] != id || got.Resources[i]["digest"] != nil || got.Resources[i]["content"] == nil {
t.Errorf("%s went by digest or lost its content: %v", id, got.Resources[i]["digest"])
}
}
if big := got.Resources[5]; big["id"] != "big" || big["digest"] == nil || big["content"] != nil {
t.Errorf("a plain file of 64 KiB and a byte did not go by digest: %v", big["digest"])
}
}
// A file given as bytes goes by digest on its decoded length, with the decoded bytes in the object.
func TestAFileOfBytesTravelsByDigestOnItsDecodedLength(t *testing.T) {
raw := []byte(contentOf("png", 70<<10))
r := map[string]any{"id": "icon", "type": "file", "path": "/usr/share/icon.png", "bytes": stdBase64(raw)}
s := sendable{fileReferences: true, Resources: []map[string]any{r}}
objects, err := s.objects()
if err != nil || len(objects) != 1 {
t.Fatalf("%v %v", len(objects), err)
}
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf(raw))
if string(objects[name]) != string(raw) {
t.Fatal("the object is not the decoded bytes")
}
if r["bytes"] == nil {
t.Fatal("the composed resource was changed in place")
}
}
type memObjects struct {
have map[string][]byte
marked map[string]bool
puts []string
fail map[string]error
}
func (m *memObjects) Has(_ context.Context, name string) (bool, bool, error) {
_, ok := m.have[name]
return ok, m.marked[name], nil
}
func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
if err := m.fail[name]; err != nil {
return err
}
m.puts = append(m.puts, name)
m.have[name] = c
delete(m.marked, name)
return nil
}
// 1696/AC4: an object already in the bucket is not put again, one absent is, and a put that fails stops the send, naming the object.
func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696/AC4
objects := map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")}
b := &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one")}}
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("b", 64) {
t.Fatalf("put %v (%v), want the one absent", b.puts, err)
}
// An object the sweep marked unnamed is put again, which erases the mark: a send relies on it.
b = &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")},
marked: map[string]bool{strings.Repeat("a", 64): true}}
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("a", 64) || b.marked[strings.Repeat("a", 64)] {
t.Fatalf("a marked object: put %v (%v) marked %v, want it put again and the mark gone", b.puts, err, b.marked)
}
b = &memObjects{have: map[string][]byte{}, fail: map[string]error{strings.Repeat("a", 64): errors.New("permissions violation")}}
err := putObjects(t.Context(), b, objects)
if err == nil || !strings.Contains(err.Error(), "aaaaaaaaaaaa") || !strings.Contains(err.Error(), "permissions violation") {
t.Fatalf("a failed put: %v", err)
}
}
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
// controller's grant does, and takes the second.
type fallbackDelivery struct {
bodies [][]byte
first error
}
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
f.bodies = append(f.bodies, body)
if len(f.bodies) == 1 && f.first != nil {
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
}
return "digest", nil
}
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
// with every content inline and is made; a send that fails for another reason is not retried inline.
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
users := contentOf("users", 100<<10)
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
f := &fallbackDelivery{first: errors.New("timeout")}
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
}
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
t.Error("the first body should name the file by digest and the second carry it inline")
}
other := &fallbackDelivery{}
other.first = nil
failing := &failingDelivery{err: errors.New("the bus is down")}
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
}
}
type failingDelivery struct {
err error
calls int
}
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
f.calls++
return "", f.err
}
// A bucket for the sweep.
type sweepBucketFake struct {
objects map[string]time.Time // zero: named at the last look
deleted []string
}
func (b *sweepBucketFake) Objects(context.Context) ([]bucketObject, error) {
var out []bucketObject
for n, at := range b.objects {
out = append(out, bucketObject{Name: n, UnnamedSince: at})
}
return out, nil
}
func (b *sweepBucketFake) MarkUnnamed(_ context.Context, n string, at time.Time) error {
b.objects[n] = at
return nil
}
func (b *sweepBucketFake) ClearUnnamed(_ context.Context, n string) error {
b.objects[n] = time.Time{}
return nil
}
func (b *sweepBucketFake) Delete(_ context.Context, n string, markedAt time.Time) (bool, error) {
if at, ok := b.objects[n]; !ok || !at.Equal(markedAt) {
return false, nil // put again since it was judged: its mark is gone
}
delete(b.objects, n)
b.deleted = append(b.deleted, n)
return true, nil
}
// 1696/AC6: an object a latest declaration still names is not removed, however long; one unnamed is marked at the first look and removed
// only when the mark is 24 hours old; one named again loses its mark.
func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // 1696/AC6
named, unnamed, back := strings.Repeat("1", 64), strings.Repeat("2", 64), strings.Repeat("3", 64)
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
b := &sweepBucketFake{objects: map[string]time.Time{named: t0.Add(-100 * time.Hour), unnamed: {}, back: t0.Add(-5 * time.Hour)}}
named3 := map[string]bool{named: true, back: true}
r, err := sweepDeclarationFiles(t.Context(), b, named3, t0)
if err != nil || r.Marked != 1 || r.Cleared != 2 || r.Removed != 0 || len(b.deleted) != 0 {
t.Fatalf("first look: %+v %v %v", r, err, b.deleted)
}
if !b.objects[named].IsZero() || !b.objects[back].IsZero() || !b.objects[unnamed].Equal(t0) {
t.Fatalf("marks after the first look: %v", b.objects)
}
// Still unnamed 23 hours 59 minutes later: kept. A day after the mark: removed. The named ones stay for ever.
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour-time.Minute)); err != nil ||
r.Removed != 0 || len(b.deleted) != 0 {
t.Fatalf("a minute short of a day: %+v %v %v", r, err, b.deleted)
}
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour)); err != nil ||
r.Removed != 1 || len(b.deleted) != 1 || b.deleted[0] != unnamed {
t.Fatalf("a day after: %+v %v %v", r, err, b.deleted)
}
if _, ok := b.objects[named]; !ok {
t.Fatal("an object a latest declaration names was removed")
}
// An object named again before the day is up loses its mark and is not removed.
b = &sweepBucketFake{objects: map[string]time.Time{unnamed: t0}}
if _, err := sweepDeclarationFiles(t.Context(), b, map[string]bool{unnamed: true}, t0.Add(23*time.Hour)); err != nil || !b.objects[unnamed].IsZero() {
t.Fatalf("named again: %v %v", b.objects, err)
}
}
// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
// never "names none", which would mark an object a node needs.
func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
body, _ := s.Body()
msg, _ := link.Envelope(body, []byte("sig"))
named, err := namedByLatest("novox", msg)
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
if err != nil || len(named) != 1 || !named[name] {
t.Fatalf("%v %v", named, err)
}
if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 {
t.Fatalf("a declaration naming none: %v %v", none, err)
}
if _, err := namedByLatest("novox", []byte("not json")); err == nil {
t.Fatal("an unreadable latest declaration was read as naming none")
}
}
// 1698/AC2 (ADR 0486 §9): a declaration shaped like the control node's, for a node that reads references, stays under half of the
// bus's largest message on the wire; the inline form of the same does not fit at all; and the same shape with 40 percent more resources
// fails the test's own limit, so a change that eats the room fails in a pull request and not on the live mesh.
func TestTheControlNodesDeclarationInTheReferenceFormStaysUnderHalfTheBusLimit(t *testing.T) { // 1698/AC2
const limit = 1048576
shape := sendable{fileReferences: true, Resources: controlNodeShape(1)}
n := wireSizeOf(t, shape)
t.Logf("the control node's shape in the reference form: %d bytes on the wire, %.1f%% of %d", n, 100*float64(n)/limit, limit)
if n > limit/2 {
t.Errorf("the reference form is %d bytes on the wire, over half (%d) of the bus's largest message", n, limit/2)
}
if inline := wireSizeOf(t, sendable{Resources: controlNodeShape(1)}); inline < limit*3/4 {
t.Errorf("the inline form is %d bytes, which is not the shape that was over the limit", inline)
}
grown := wireSizeOf(t, sendable{fileReferences: true, Resources: controlNodeShape(1.4)})
if grown <= limit/2 {
t.Errorf("the shape with 40%% more resources is %d bytes, which the test would let pass", grown)
}
}
func stdBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
// A sweep that read an object's mark before a send put it again must not remove it: the object, now without the mark, is left.
func TestASweepDoesNotRemoveAnObjectASendPutAgainAfterItWasJudged(t *testing.T) { // 1696/AC8
n := strings.Repeat("4", 64)
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
b := &sweepBucketFake{objects: map[string]time.Time{n: t0}}
judged := bucketObject{Name: n, UnnamedSince: t0}
b.objects[n] = time.Time{} // the send put it again: no mark
if gone, err := b.Delete(t.Context(), judged.Name, judged.UnnamedSince); err != nil || gone || len(b.deleted) != 0 {
t.Fatalf("an object put again was removed: %v %v %v", gone, err, b.deleted)
}
}
// One node's bucket is held by one act at a time in this process: the sweep waits for a send that has put its objects and not yet
// published the declaration that names them (1696/AC9).
func TestTheSweepOfANodesBucketWaitsForASendThatHasPutItsObjects(t *testing.T) { // 1696/AC9
unlock := lockFilesOf("lock-test-node")
got := make(chan struct{})
go func() {
defer lockFilesOf("lock-test-node")()
close(got)
}()
select {
case <-got:
t.Fatal("the sweep took the node's bucket while a send held it")
case <-time.After(50 * time.Millisecond):
}
other := lockFilesOf("lock-test-other-node") // another node's bucket is not held
other()
unlock()
select {
case <-got:
case <-time.After(2 * time.Second):
t.Fatal("the sweep did not take the node's bucket once the send was done")
}
}
// A put that fails send after send raises a condition; the first failure does not (the grant it waits for travels in that very
// declaration), and a put that works clears it (1696/AC10).
func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testing.T) { // 1696/AC10
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
t.Cleanup(func() { k.Close(t.Context()) })
was := conditionsFrom
conditionsFrom = k
t.Cleanup(func() { conditionsFrom = was })
key := filesNotPutObservation("cond-node", "").Key()
open := func() bool {
_, found, err := conditions.ReadOne(t.Context(), store, key)
if err != nil {
t.Fatal(err)
}
return found
}
boom := errors.New("permissions violation")
notePut(t.Context(), "cond-node", boom)
if open() {
t.Fatal("the first failure raised the condition")
}
notePut(t.Context(), "cond-node", boom)
if !open() {
t.Fatal("the second failure in a row did not raise the condition")
}
notePut(t.Context(), "cond-node", nil)
if open() {
t.Fatal("a put that worked did not clear the condition")
}
notePut(t.Context(), "cond-node", boom)
if open() {
t.Fatal("the count did not start again after a put that worked")
}
}
// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
type aBusThatSays struct {
link.Bus
published chan string
heldAtPublish func() bool
}
func (b *aBusThatSays) PublishDeclaration(_ context.Context, node string, _ []byte) error {
if b.heldAtPublish != nil && !b.heldAtPublish() {
return errors.New("published while the node's bucket was not held")
}
b.published <- node
return errors.New("stop here: nothing is recorded")
}
type aSignerOfZeros struct{}
func (aSignerOfZeros) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
// whileHeld says whether the node's lock is held by someone right now, without keeping it.
func whileHeld(node string) bool {
got := make(chan struct{})
go func() {
defer lockFilesOf(node)()
close(got)
}()
select {
case <-got:
return false
case <-time.After(100 * time.Millisecond):
// held: the goroutine stays waiting and takes and releases the lock once it is let go.
return true
}
}
// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The
// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the
// lock is held.
func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9
const node = "lock-send-node"
bus := &aBusThatSays{published: make(chan string, 1)}
bus.heldAtPublish = func() bool { return whileHeld(node) }
send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}}
release := lockFilesOf(node) // a sweep is judging this node's bucket
done := make(chan error, 1)
go func() {
_, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`))
done <- err
}()
select {
case n := <-bus.published:
t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n)
case <-time.After(150 * time.Millisecond):
}
release()
select {
case <-bus.published:
case <-time.After(5 * time.Second):
t.Fatal("the send did not go on once the sweep let go")
}
if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") {
t.Fatalf("the send did not reach the publication (a lock held at it): %v", err)
}
}
-96
View File
@@ -1,96 +0,0 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
// the declaration is back under 75%. The limit is the connection's own, not a constant.
// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
const kindDeclarationNearLimit = "declaration-near-limit"
// declarationRoomSource is the condition's source: the sends.
const declarationRoomSource = "sends"
func declarationRoomObservation(node string, size, limit int64) conditions.Observation {
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindDeclarationNearLimit,
Token: kindDeclarationNearLimit, Machine: node, Severity: conditions.Warning, Source: declarationRoomSource}
if size > limit {
o.Severity = conditions.Urgent
}
o.Summary = fmt.Sprintf("%s's declaration is %d bytes on the wire, %d%% of the bus's largest message of %d bytes",
node, size, size*100/limit, limit)
if size > limit {
o.Summary += ": over it, so it cannot be sent"
}
return o
}
func init() {
link.DeclarationMeasured = func(ctx context.Context, node string, size, limit int64) {
settleDeclarationRoom(ctx, conditionsFrom, node, size, limit)
}
}
// settleDeclarationRoom raises or clears the condition of a declaration's size. A nil keeper (a tool, not the
// serving controller) says nothing.
func settleDeclarationRoom(ctx context.Context, keeper *conditions.Keeper, node string, size, limit int64) {
if keeper == nil || limit <= 0 {
return
}
obs := declarationRoomObservation(node, size, limit)
if float64(size) < link.NearTheLimit*float64(limit) {
if _, err := keeper.Clear(ctx, obs.Key(), fmt.Sprintf("%s's declaration is %d bytes, under the limit's margin", node, size)); err != nil {
fmt.Printf("the condition %s could not be cleared: %v\n", obs.Key(), err)
}
return
}
fmt.Printf("NEAR THE LIMIT: %s\n", obs.Summary)
if _, err := keeper.Observe(ctx, obs); err != nil {
fmt.Printf("the condition %s could not be raised: %v\n", obs.Key(), err)
}
}
// compactKept writes the vault's export compact. The composition (the SDK's keptFile) writes it indented, and its 334
// sealed secrets were 164,763 bytes of it against 138,474 compact: a fifth of the declaration's room, in whitespace,
// base64-encoded again on the wire. Every reader parses JSON, so the spacing is nobody's. A resource that is not
// the export, or whose content is not JSON, is left as it is. The resources are copied, never changed in place.
func compactKept(resources []map[string]any) []map[string]any {
out := make([]map[string]any, len(resources))
for i, r := range resources {
out[i] = r
id, _ := r["id"].(string)
path, _ := r["path"].(string)
content, ok := r["content"].(string)
if !ok || (id != "kept" && !strings.HasSuffix(id, ".kept")) || !strings.HasSuffix(path, "/export.json") {
continue
}
var dst bytes.Buffer
if json.Compact(&dst, []byte(content)) != nil {
continue
}
if strings.HasSuffix(content, "\n") {
dst.WriteByte('\n')
}
if dst.String() == content {
continue
}
c := make(map[string]any, len(r))
for k, v := range r {
c[k] = v
}
c["content"] = dst.String()
out[i] = c
}
return out
}
@@ -1,71 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
)
// The vault's export is sent compact (issue 1689): same document, no indentation, one rewrite.
func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
indented := "{\n \"export\": 1,\n \"kept\": [\n {\n \"name\": \"a\"\n }\n ]\n}\n"
other := "{\n \"x\": 1\n}\n"
in := []map[string]any{
{"id": "mesh-vault.kept", "type": "file", "path": "/srv/vault/export.json", "mode": "0600", "content": indented},
{"id": "mesh-vault.config", "type": "file", "path": "/srv/vault/config.json", "content": other},
{"id": "other.kept", "type": "file", "path": "/srv/other/notes.json", "content": other},
}
out := compactKept(in)
want := "{\"export\":1,\"kept\":[{\"name\":\"a\"}]}\n"
if out[0]["content"] != want || out[0]["mode"] != "0600" || out[0]["path"] != "/srv/vault/export.json" {
t.Fatalf("the export is %q (%v), expected %q with its mode and path kept", out[0]["content"], out[0], want)
}
if out[1]["content"] != other || out[2]["content"] != other {
t.Fatalf("a file that is not the export was rewritten: %v %v", out[1], out[2])
}
if in[0]["content"] != indented {
t.Fatalf("the composed resource was changed in place")
}
if again := compactKept(out); again[0]["content"] != want {
t.Fatalf("compacting twice changed it: %q", again[0]["content"])
}
}
// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
t.Cleanup(func() { k.Close(t.Context()) })
const limit = 1000
key := declarationRoomObservation("novox", 0, limit).Key()
read := func() (conditions.Condition, bool) {
t.Helper()
c, found, err := conditions.ReadOne(t.Context(), store, key)
if err != nil {
t.Fatal(err)
}
return c, found
}
settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
if _, found := read(); found {
t.Fatal("a declaration at 74.9% raised a condition")
}
settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
c, found := read()
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
t.Fatalf("at 75%% the condition is %v %+v", found, c)
}
settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
if c, _ := read(); c.Severity != conditions.Urgent {
t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
}
settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
if _, found := read(); found {
t.Fatal("the condition stayed after the declaration fell under 75%")
}
}
-273
View File
@@ -1,273 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
// node's latest declaration alone. Objects are immutable and regenerable from the
// controller's state, so a removal loses nothing.
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
const declarationUnnamed = "unnamed-since"
// sweepEvery is how often the bucket is judged: the removal is exact to within this.
const sweepEvery = time.Hour
// bucketObject is one object of the bucket as the sweep sees it.
type bucketObject struct {
Name string
// UnnamedSince is when it was first found named by none, zero when it was named at the last look.
UnnamedSince time.Time
}
// sweptBucket is what the sweep needs of the bucket.
type sweptBucket interface {
Objects(ctx context.Context) ([]bucketObject, error)
MarkUnnamed(ctx context.Context, name string, since time.Time) error
ClearUnnamed(ctx context.Context, name string) error
// Delete removes the object if it still carries the mark it was judged by, and says whether it did: a send that put it again
// since has erased the mark, and the object is left.
Delete(ctx context.Context, name string, markedAt time.Time) (bool, error)
}
// filesSweep says what a sweep did.
type filesSweep struct{ Kept, Marked, Cleared, Removed int }
// sweepDeclarationFiles judges every object against the names of the latest declarations: named ones are kept (and a
// mark of being unnamed is erased), unnamed ones are marked at the first look and removed once the mark is a day old.
func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[string]bool, now time.Time) (filesSweep, error) {
var r filesSweep
objects, err := bucket.Objects(ctx)
if err != nil {
return r, err
}
sort.Slice(objects, func(i, j int) bool { return objects[i].Name < objects[j].Name })
for _, o := range objects {
switch {
case named[o.Name]:
r.Kept++
if !o.UnnamedSince.IsZero() {
if err := bucket.ClearUnnamed(ctx, o.Name); err != nil {
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
}
r.Cleared++
}
case o.UnnamedSince.IsZero():
if err := bucket.MarkUnnamed(ctx, o.Name, now); err != nil {
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
}
r.Marked++
case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
gone, err := bucket.Delete(ctx, o.Name, o.UnnamedSince)
if err != nil {
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
}
if gone {
r.Removed++
} else {
r.Kept++
}
default:
r.Kept++
}
}
return r, nil
}
// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
// "names none": an object the node still needs would be marked for removal.
func namedByLatest(node string, signed []byte) (map[string]bool, error) {
var s link.Signed
if err := json.Unmarshal(signed, &s); err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
names, err := namedObjects(s.Declaration)
if err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
named := make(map[string]bool, len(names))
for name := range names {
named[name] = true
}
return named, nil
}
// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
// no entry.
func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
js, err := conn.JetStream(nats.Context(ctx))
if err != nil {
return nil, err
}
out := map[string][]byte{}
for _, node := range nodes {
msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
if errors.Is(err, nats.ErrMsgNotFound) {
continue
}
if err != nil {
return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
}
out[node] = msg.Data
}
return out, nil
}
// busSweptBucket is the bucket on the controller's connection.
type busSweptBucket struct{ store jetstream.ObjectStore }
func (b busSweptBucket) Objects(ctx context.Context) ([]bucketObject, error) {
infos, err := b.store.List(ctx)
if errors.Is(err, jetstream.ErrNoObjectsFound) {
return nil, nil
}
if err != nil {
return nil, err
}
out := make([]bucketObject, 0, len(infos))
for _, info := range infos {
o := bucketObject{Name: info.Name}
if at, err := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); err == nil {
o.UnnamedSince = at
}
out = append(out, o)
}
return out, nil
}
func (b busSweptBucket) meta(ctx context.Context, name string, change func(map[string]string)) error {
info, err := b.store.GetInfo(ctx, name)
if err != nil {
return err
}
meta := info.ObjectMeta
kept := map[string]string{}
for k, v := range meta.Metadata {
kept[k] = v
}
change(kept)
meta.Metadata = kept
return b.store.UpdateMeta(ctx, name, meta)
}
func (b busSweptBucket) MarkUnnamed(ctx context.Context, name string, since time.Time) error {
return b.meta(ctx, name, func(m map[string]string) { m[declarationUnnamed] = since.UTC().Format(time.RFC3339) })
}
func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
}
func (b busSweptBucket) Delete(ctx context.Context, name string, markedAt time.Time) (bool, error) {
info, err := b.store.GetInfo(ctx, name)
if errors.Is(err, jetstream.ErrObjectNotFound) {
return false, nil
}
if err != nil {
return false, err
}
if at, perr := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); perr != nil || !at.Equal(markedAt) {
return false, nil
}
return true, b.store.Delete(ctx, name)
}
// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
tick := time.NewTicker(sweepEvery)
defer tick.Stop()
failing := ""
for {
select {
case <-ctx.Done():
return
case <-tick.C:
}
if epoch, err := theLease.epoch(ctx); err != nil || epoch == 0 {
continue
}
why := sweepOnce(ctx, inv, conn)
if why != failing {
if why != "" {
fmt.Printf("the bucket of declaration files cannot be swept: %s\n", why)
} else {
fmt.Printf("the bucket of declaration files is swept again\n")
}
failing = why
}
}
}
// sweepOnce judges the bucket of every node against that node's latest declaration, one bucket at a time under the node's lock, with
// the declaration read again under it: a send that put an object and published its declaration is seen, and one that has not yet
// is waiting for the lock.
func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
nodes, err := inv.Nodes(ctx)
if err != nil {
return fmt.Sprintf("the nodes cannot be read: %v", err)
}
var problems []string
var total filesSweep
for _, n := range nodes {
r, err := sweepNode(ctx, conn, n.Name)
total.Kept, total.Marked, total.Cleared, total.Removed = total.Kept+r.Kept, total.Marked+r.Marked, total.Cleared+r.Cleared, total.Removed+r.Removed
if err != nil {
problems = append(problems, fmt.Sprintf("%s: %v", n.Name, err))
}
}
if total.Removed+total.Marked+total.Cleared > 0 {
fmt.Printf("the buckets of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
total.Kept, total.Marked, total.Cleared, total.Removed)
}
return strings.Join(problems, "; ")
}
func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, error) {
working, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
js, err := jetstream.New(conn)
if err != nil {
return filesSweep{}, err
}
bucket, err := broker.DeclarationFilesBucketOf(node)
if err != nil {
return filesSweep{}, fmt.Errorf("the bucket of node %q: %w", node, err)
}
store, err := js.ObjectStore(working, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
return filesSweep{}, nil // a node never sent a reference has no bucket yet
}
if err != nil {
return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
}
defer lockFilesOf(node)()
latest, err := latestDeclarations(working, conn, []string{node})
if err != nil {
return filesSweep{}, err
}
named := map[string]bool{}
if body, ok := latest[node]; ok {
if named, err = namedByLatest(node, body); err != nil {
return filesSweep{}, err
}
}
return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
}
@@ -1,37 +0,0 @@
package main
import (
"reflect"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// 1617/AC2 (unit): a new module whose first build a merge's hearing asked is not asked again when its batch is cut,
// and one whose ask failed, was never handed over or is long lost is.
func TestAModuleWhoseBuildIsComingIsNotAskedAgain(t *testing.T) {
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
source := buildSource{Repository: "http://forge/novox/mesh-catalog.git", Seat: "git"}
asked := func(path string, mod func(*inventory.RequestOutcome)) inventory.RequestOutcome {
r := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: "b-" + path, Repository: source.Repository,
Seat: "git", Path: path, For: "merge", At: now.Add(-3 * time.Minute)}}
if mod != nil {
mod(&r)
}
return r
}
requests := []inventory.RequestOutcome{
asked("modules/go", nil),
asked("modules/failed", func(r *inventory.RequestOutcome) { r.Heard, r.Failed = true, "no module.json" }),
asked("modules/notasked", func(r *inventory.RequestOutcome) { r.NotAsked = "the bus is silent" }),
asked("modules/lost", func(r *inventory.RequestOutcome) { r.At = now.Add(-2 * time.Hour) }),
asked("modules/byhand", func(r *inventory.RequestOutcome) { r.For = "build" }),
}
got := unaskedDirs([]string{"modules/go", "modules/failed", "modules/notasked", "modules/lost", "modules/byhand",
"modules/nodejs"}, requests, source, now)
want := []string{"modules/failed", "modules/notasked", "modules/lost", "modules/byhand", "modules/nodejs"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("asked again %v, want %v", got, want)
}
}
@@ -6,7 +6,6 @@ import (
"context"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
@@ -100,38 +99,3 @@ func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) {
t.Fatalf("a merge adding no module asked for %v", *asked)
}
}
// 1617/AC1 (integration): a merge adding a module to a repository the mesh builds modules from, heard while a batch of
// that repository is still assembling (its window running), asks the module's first build when it is heard, so the
// module is registered while the batch waits; cutting the batch asks it no second time (1617/AC2).
func TestAMergeAddingAModuleAsksItsBuildBeforeItsBatchIsCut(t *testing.T) {
open := windowed(t)
groupsWaiting(t, "pr/c1")
asked := asksWithPaths(t)
catalog := func(commit string, made time.Time, paths ...string) link.SourceMoved {
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
Head: "pr/" + commit, MergedAt: made.Format(time.RFC3339Nano), Paths: paths}
}
hear(t, open, catalog("c1", t0, "modules/app/index.ts"), t0)
if len(*asked) != 0 {
t.Fatalf("a merge adding no module asked %v", *asked)
}
hear(t, open, lone(catalog("c2", t0.Add(10*time.Second), "modules/go/packages.json", "modules/go/module.json")), t0.Add(10*time.Second))
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
t.Fatalf("the batch should still be assembling: %d walk(s), %d batch(es)", len(ws), len(bs))
}
want := [3]string{"novox/mesh-catalog", "modules/go", "main"}
if len(*asked) != 1 || (*asked)[0] != want {
t.Fatalf("heard with its batch assembling, the merge asked %v, not %v", *asked, want)
}
cutAt(t, open, t0.Add(10*time.Second+90*time.Second))
n := 0
for _, a := range *asked {
if a[1] == "modules/go" {
n++
}
}
if n != 1 {
t.Fatalf("modules/go was asked %d times: %v", n, *asked)
}
}
@@ -1,198 +0,0 @@
package main
import (
"context"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// heldElsewhere is the plans held by another act for longer than any test runs: waiting for them ends only with the
// context, and a look that does not wait is told at once that they are busy.
func heldElsewhere(ctx context.Context, wait bool) (func(), error) {
if !wait {
return nil, inventory.ErrPlansBusy
}
<-ctx.Done()
return nil, inventory.ErrPlansBusy
}
// 1708/AC1: given the plans held by another act, when the event loop takes a build's outcome, then it returns at
// once, without waiting the plans' two minutes (on 2026-10-21 at 14:20:45 nodered's outcome held the loop until
// 14:22:45, and the operator was told the controller stopped listening).
func TestAnOutcomeDoesNotWaitForThePlansAnotherActHolds(t *testing.T) {
ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
defer cancel()
start := time.Now()
release, err := holdPlansForOutcome(ctx, heldElsewhere)
if took := time.Since(start); took > time.Second {
t.Fatalf("taking an outcome waited %s for plans another act holds; the event loop takes nothing meanwhile", took)
}
if err == nil || release != nil {
t.Fatalf("plans another act holds were said to be held: release %v, err %v", release != nil, err)
}
}
// 1708/AC4: given the plans free, when the event loop takes a build's outcome, then it holds them and marks the plan
// from the outcome directly, as before.
func TestAnOutcomeHoldsThePlansWhenTheyAreFree(t *testing.T) {
let := false
release, err := holdPlansForOutcome(t.Context(), func(context.Context, bool) (func(), error) {
return func() { let = true }, nil
})
if err != nil || release == nil {
t.Fatalf("free plans were not held: %v", err)
}
release()
if !let {
t.Fatal("the hold given was not the one taken")
}
}
// withLooker installs a looker that is never run, and answers it, so a test can read whether a look was asked.
func withLooker(t *testing.T) *looker {
t.Helper()
l := newLooker(func(context.Context) bool { return false })
lookerMu.Lock()
before := theLooker
theLooker = l
lookerMu.Unlock()
t.Cleanup(func() {
lookerMu.Lock()
theLooker = before
lookerMu.Unlock()
})
return l
}
// 1708/AC2: given an outcome left to the build records because the plans were held, when it is left, then the look
// at the plans is asked for, so the records settle it once the holder lets go rather than at the next event.
func TestAnOutcomeLeftToTheRecordsAsksForTheLook(t *testing.T) {
l := withLooker(t)
leaveToRecords("nodered", "build-1708-ac2", "", "", inventory.ErrPlansBusy)
select {
case <-l.signal:
default:
t.Fatal("an outcome left to the build records asked for no look: the plan waits for whatever event comes next")
}
}
// 1708/AC3: given an outcome left to the records that said what the build did (ADR 0350 §4), when the look settles
// the plan from the build records by that build's id, then the module keeps those words, once.
func TestTheRecordsSettleAnOutcomeWithWhatTheBuildDid(t *testing.T) {
withLooker(t)
asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC)
plan := func() inventory.Plan {
return inventory.Plan{ID: "plan-1708", Tiers: [][]string{{"nodered"}},
Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac3"}}}
}
byID := map[string]inventory.Build{"build-1708-ac3": {ID: "build-1708-ac3", Module: "nodered", Commit: "4efac366",
At: asked.Add(5 * time.Minute)}}
leaveToRecords("nodered", "build-1708-ac3", "reused from build-1708-check", "", inventory.ErrPlansBusy)
p := plan()
if !settleFromRecords(&p, p.Tiers[0], nil, byID) || p.Modules["nodered"].State != "built" {
t.Fatalf("the records did not settle the outcome: %+v", p.Modules["nodered"])
}
if got := p.Modules["nodered"].Made; got != "reused from build-1708-check" {
t.Fatalf("settled from the records, the module says %q of what its build did, not the outcome's words", got)
}
again := plan()
settleFromRecords(&again, again.Tiers[0], nil, byID)
if got := again.Modules["nodered"].Made; got != "" {
t.Fatalf("an outcome's words were kept for a second settle: %q", got)
}
}
// 1708/AC5: given a serving controller (its look at the plans running), when an outcome was taken into the plans,
// then moving them (a release's sends, which may retry for minutes) is left to the look, off the event loop
// (2026-10-21 14:28:40 to 14:32:10: the loop sent okioki's release to g14 after mesh-delivery-builder's outcome).
func TestAnOutcomeLeavesMovingThePlansToTheLook(t *testing.T) {
l := withLooker(t)
moved := false
advanceAfterOutcome(func() { moved = true })
if moved {
t.Fatal("the plans were moved on the event loop after an outcome, where a send that retries holds every message")
}
select {
case <-l.signal:
default:
t.Fatal("the look was not asked to move the plans after an outcome")
}
}
// 1708/AC5: given no look running (a command, which has no event loop), when an outcome was taken, then the plans are
// moved at once, as before.
func TestAnOutcomeMovesThePlansWhereNoLookRuns(t *testing.T) {
lookerMu.Lock()
before := theLooker
theLooker = nil
lookerMu.Unlock()
t.Cleanup(func() {
lookerMu.Lock()
theLooker = before
lookerMu.Unlock()
})
moved := false
advanceAfterOutcome(func() { moved = true })
if !moved {
t.Fatal("with no look running, an outcome left the plans unmoved")
}
}
// 1708/AC6: given a registration the mesh refused (its build recorded as worked, the outcome failed), left to the
// records because the plans were held, when the look settles the plan from the records, then the module and the plan
// are failed, never built: settled as built, the tier would be sent the version from before (reviewer-1708@shanks).
func TestARefusedRegistrationLeftToTheRecordsFailsThePlan(t *testing.T) {
withLooker(t)
asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC)
p := inventory.Plan{ID: "plan-1708-ac6", State: inventory.PlanBuilding, Tiers: [][]string{{"nodered"}},
Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac6"}}}
byID := map[string]inventory.Build{"build-1708-ac6": {ID: "build-1708-ac6", Module: "nodered", Commit: "4efac366",
At: asked.Add(5 * time.Minute)}}
leaveToRecords("nodered", "build-1708-ac6", "", "the mesh does not register it: a port is taken", inventory.ErrPlansBusy)
overlayRefusals(nil, byID, refusalsKeptFor([]string{"build-1708-ac6"}))
settleFromRecords(&p, p.Tiers[0], nil, byID)
if s := p.Modules["nodered"]; s.State != "failed" || p.State != inventory.PlanFailed {
t.Fatalf("a refused registration left to the records settled as %q, the plan %q", s.State, p.State)
}
if !strings.Contains(p.Modules["nodered"].Why, "a port is taken") {
t.Fatalf("the refusal's words were lost: %q", p.Modules["nodered"].Why)
}
}
// 1708/AC6: a refusal kept in the store (a controller replaced before the settle) fails the record as one kept in
// the process does.
func TestARefusalFromTheStoreFailsTheRecord(t *testing.T) {
recorded := map[string][]inventory.Build{"nodered": {{ID: "b-store", Module: "nodered"}, {ID: "b-other", Module: "nodered"}}}
byID := map[string]inventory.Build{"b-store": {ID: "b-store", Module: "nodered"}}
overlayRefusals(recorded, byID, map[string]string{"b-store": "not registered"})
if recorded["nodered"][0].Worked() || byID["b-store"].Worked() {
t.Fatal("a build whose registration was refused still reads as worked")
}
if !recorded["nodered"][1].Worked() {
t.Fatal("a build nobody refused was failed")
}
}
// 1708/AC7: given the records settle a module as deleted at its source, or fail the plan, when the look settles, then
// it does what taking the outcome directly does: the deleted module is named to be forgotten, and the plan's failure
// is said (reviewer-1708@shanks).
func TestSettlingFromTheRecordsDoesWhatTakingTheOutcomeDoes(t *testing.T) {
p := inventory.Plan{State: inventory.PlanFailed, Modules: map[string]*inventory.PlanModule{
"gone": {State: planDeleted}, "x": {State: "failed"}, "y": {State: "built"}}}
before := map[string]string{"gone": "asked", "x": "asked", "y": "built"}
deleted, failed := settledEffects(before, &p, inventory.PlanBuilding)
if len(deleted) != 1 || deleted[0] != "gone" {
t.Fatalf("the module the records found deleted is not forgotten: %v", deleted)
}
if !failed {
t.Fatal("the plan the records failed is not said as failed")
}
if d, f := settledEffects(map[string]string{"gone": planDeleted}, &p, inventory.PlanFailed); len(d) != 0 || f {
t.Fatalf("what was deleted or failed before the settle is acted on again: %v %v", d, f)
}
}
-23
View File
@@ -195,29 +195,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
"there yet is sealed by the controller until the vault holds it.", m, m),
Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
}),
kindFilesNotPut: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "The big files for " + m + " cannot be put on the bus",
Needs: "look at the bus's permissions for the controller; the details give the reason.",
Explanation: fmt.Sprintf("The mesh could not put the large files of %s's instructions in their place on the bus, "+
"send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m),
Resolved: "Resolved: the big files for " + m + " reach the bus again"}
}),
kindDeclarationNearLimit: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
if o.Severity == conditions.Urgent {
return words{Headline: m + " cannot be sent its instructions",
Needs: "make the instructions for " + m + " smaller, for example by taking a module off it.",
Explanation: fmt.Sprintf("The instructions for %s have grown past the largest message the bus takes, so "+
"nothing new reaches %s until they are smaller.", m, m),
Resolved: "Resolved: the instructions for " + m + " fit the bus again"}
}
return words{Headline: "The instructions for " + m + " are nearly too big",
Needs: "do not add more to " + m + " until this is dealt with; the details give the size.",
Explanation: fmt.Sprintf("The instructions for %s are close to the largest message the bus takes. "+
"One more module or secret may stop anything new from reaching %s.", m, m),
Resolved: "Resolved: the instructions for " + m + " have room again"}
}),
kindCoreDowngrade: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
component := idPart(o, 0)
-47
View File
@@ -78,50 +78,3 @@ func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) {
// novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision,
// its last save and its `plan-moved` stay where its last change left them.
// novox/hq issue 920: a building plan whose current tier is all built waits on its note (a send refused at the
// gate), and its line, `status` and the stalled condition say that, not "building for ... LATE".
func TestAPlanLineSaysTheRefusalOnceItsTierIsBuilt(t *testing.T) {
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
built := now.Add(-time.Hour)
refusal := "tier 1: a build waits there for its gate: novox (mesh-delivery-indexer d68d7ef3→a5aa69a4) — tried again"
mk := func(second string) inventory.Plan {
return inventory.Plan{ID: "plan-r", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: refusal,
Tiers: [][]string{{"a"}, {"b", "c"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built},
"b": {State: "built", BuiltAt: &built}, "c": {State: second}}}
}
// 920/AC1
p := mk("built")
line := planLine(p, now)
if !strings.Contains(line, "a build waits there for its gate: novox") || strings.Contains(line, "building for") {
t.Errorf("920/AC1: a plan with its tier built and a refusal reads %q", line)
}
st := planStatuses([]inventory.Plan{p}, now, pauseView{}, nil)
if len(st) != 1 || !st[0].Late {
t.Errorf("920/AC1: status: %+v", st)
}
// 920/AC2
if line := planLine(mk("asked"), now); !strings.Contains(line, "building for 1h0m0s") {
t.Errorf("920/AC2: a plan with a tier module not built reads %q", line)
}
}
// 920/AC3: a tier move writes the previous tier's summary into the note; with the new tier all built, the line still
// says "building for", for that note is no refusal.
func TestAPlanLineDoesNotHoldOnATiersSummary(t *testing.T) {
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
built := now.Add(-time.Hour)
for _, note := range []string{"failed or refused its send (apply-failing says it): novox", "tier 0 built; sent mesh-sdk to novox first, in one send"} {
p := inventory.Plan{ID: "plan-s", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: note,
Tiers: [][]string{{"a"}, {"b"}},
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, "b": {State: "built", BuiltAt: &built}}}
if line := planLine(p, now); !strings.Contains(line, "building for 1h0m0s") || strings.Contains(line, "held") {
t.Errorf("920/AC3: a plan with the note %q reads %q", note, line)
}
}
}
-13
View File
@@ -136,18 +136,6 @@ func askToLook() {
}
}
// askedToLook asks the serving controller's look at the plans, and says whether there was one to ask.
func askedToLook() bool {
lookerMu.RLock()
l := theLooker
lookerMu.RUnlock()
if l == nil {
return false
}
l.ask()
return true
}
// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
// It answers a channel that is closed once the look has returned after the context ended, for a test that restores what
@@ -206,7 +194,6 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv
return func(_ *inventory.Inventory, p inventory.Plan) {
sayPlanMoved(ctx, bus, open.inventory, p)
planKept(p)
keepBuildingKey(ctx, p)
if p.WalkSaid == inventory.WalkEnded {
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
}
-238
View File
@@ -1,238 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The plans building (novox/hq ADR 0483 clause 2, issue 1687).
//
// **A key per delivery plan in its build phase, so that the build seat takes no new check while a delivery builds.** The phase
// runs from the ask of the plan's first build until every build of its last tier is built, or the plan fails, stops or waits
// on the operator. It is derived from the plan alone (buildPhaseOf), so a controller that restarts finds the same answer. The
// key is put when the first build is asked (sayBuilding), kept current at every save of the plan (keepBuildingKey) and by a
// look every link.PlansBuildingRefresh while this controller holds the lease (keepingPlansBuilding), and deleted at the phase's
// end; the bucket's age removes the key of a controller that stopped, so checks are held at most that long.
//
// The layout is documented in internal/link/holds.go.
// buildPhaseOf says whether a plan is in its build phase: open, not waiting for the operator's word, with a build asked and a
// build of its tiers still to be built. A tier built and sent leaves the plan rolling while later tiers are still to be asked;
// it is in the phase still, which is what keeps a check from taking a place between two tiers.
func buildPhaseOf(p inventory.Plan) bool {
if !p.Open() || p.Waiting() || p.Release != nil {
return false
}
asked, unbuilt := false, false
for _, tier := range p.Tiers {
for _, m := range tier {
st := p.Modules[m]
if st != nil && st.State != "" {
asked = true
}
if st == nil || (st.State != "built" && st.State != planDeleted) {
unbuilt = true
}
}
}
return asked && unbuilt
}
// buildingSince is when the plan's first build was asked, or fallback when no module says.
func buildingSince(p inventory.Plan, fallback time.Time) time.Time {
since := time.Time{}
for _, st := range p.Modules {
if st != nil && st.AskedAt != nil && (since.IsZero() || st.AskedAt.Before(since)) {
since = *st.AskedAt
}
}
if since.IsZero() {
return fallback
}
return since
}
// buildingStore is the bucket of the plans building.
type buildingStore interface {
Put(ctx context.Context, plan string, value []byte) error
Delete(ctx context.Context, plan string) error
Keys(ctx context.Context) ([]string, error)
}
// syncBuilding makes the bucket say what the plans say: a key for each plan in its build phase, put again, and none for any
// other plan, deleted. It returns the faults, each once.
func syncBuilding(ctx context.Context, store buildingStore, plans []inventory.Plan, now time.Time) []error {
var faults []error
live := map[string]bool{}
for _, p := range plans {
if !buildPhaseOf(p) {
continue
}
live[p.ID] = true
if err := putBuilding(ctx, store, p.ID, buildingSince(p, now), now); err != nil {
faults = append(faults, err)
}
}
keys, err := store.Keys(ctx)
if err != nil {
return append(faults, err)
}
for _, key := range keys {
if live[key] {
continue
}
if err := store.Delete(ctx, key); err != nil {
faults = append(faults, fmt.Errorf("the key of %s could not be taken away: %w", key, err))
}
}
return faults
}
func putBuilding(ctx context.Context, store buildingStore, plan string, since, now time.Time) error {
body, _ := json.Marshal(link.PlanBuilding{Plan: plan, Since: since.UTC(), Seen: now.UTC()})
if err := store.Put(ctx, plan, body); err != nil {
return fmt.Errorf("the key of %s could not be put: %w", plan, err)
}
return nil
}
// busBuilding is the store on the controller's bucket.
type busBuilding struct {
conn *nats.Conn
mu sync.Mutex
kv jetstream.KeyValue
}
func (b *busBuilding) bucket(ctx context.Context) (jetstream.KeyValue, error) {
b.mu.Lock()
defer b.mu.Unlock()
if b.kv != nil {
return b.kv, nil
}
api, err := jetstream.New(b.conn)
if err != nil {
return nil, err
}
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
if err != nil {
return nil, fmt.Errorf("the plans building: %w", err)
}
b.kv = kv
return kv, nil
}
func (b *busBuilding) Put(ctx context.Context, plan string, value []byte) error {
kv, err := b.bucket(ctx)
if err != nil {
return err
}
_, err = kv.Put(ctx, plan, value)
return err
}
func (b *busBuilding) Delete(ctx context.Context, plan string) error {
kv, err := b.bucket(ctx)
if err != nil {
return err
}
if err := kv.Delete(ctx, plan); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
func (b *busBuilding) Keys(ctx context.Context) ([]string, error) {
kv, err := b.bucket(ctx)
if err != nil {
return nil, err
}
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
// buildingKeys is the store of the serving controller; nil in a process that is not it, which says nothing.
var buildingKeys atomic.Pointer[buildingStore]
func setBuildingStore(s buildingStore) {
if s == nil {
buildingKeys.Store(nil)
return
}
buildingKeys.Store(&s)
}
// sayBuilding puts a plan's key: its first build is being asked (or another of its builds). A key that could not be put is
// said; the look that follows puts it again.
func sayBuilding(ctx context.Context, plan string, since time.Time) {
s := buildingKeys.Load()
if s == nil {
return
}
if err := putBuilding(ctx, *s, plan, since, time.Now()); err != nil {
fmt.Printf("plans: %v; checks may take a place of %s until the next look\n", err, plan)
}
}
// keepBuildingKey is told each plan this controller keeps: its key put again while it builds and taken away once it does not.
func keepBuildingKey(ctx context.Context, p inventory.Plan) {
s := buildingKeys.Load()
if s == nil {
return
}
if buildPhaseOf(p) {
sayBuilding(ctx, p.ID, buildingSince(p, time.Now()))
return
}
if err := (*s).Delete(ctx, p.ID); err != nil {
fmt.Printf("plans: the key of %s could not be taken away; it ages out: %v\n", p.ID, err)
}
}
// keepingPlansBuilding looks at the plans every link.PlansBuildingRefresh for as long as this controller serves and holds the
// lease, so a key is put again before its age and the key of a plan that left the phase unseen is taken away.
func keepingPlansBuilding(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
store := &busBuilding{conn: conn}
setBuildingStore(store)
defer setBuildingStore(nil)
tick := time.NewTicker(broker.PlansBuildingRefresh)
defer tick.Stop()
failing := ""
for {
if epoch, err := theLease.epoch(ctx); err == nil && epoch != 0 {
why := ""
if plans, err := inv.OpenPlans(ctx); err != nil {
why = fmt.Sprintf("the plans cannot be read: %v", err)
} else if faults := syncBuilding(ctx, store, plans, time.Now()); len(faults) > 0 {
why = fmt.Sprint(faults)
}
if why != failing {
if why != "" {
fmt.Printf("plans: the keys of the plans building cannot be kept: %s\n", why)
} else {
fmt.Printf("plans: the keys of the plans building are kept again\n")
}
failing = why
}
}
select {
case <-ctx.Done():
return
case <-tick.C:
}
}
}
-175
View File
@@ -1,175 +0,0 @@
package main
import (
"context"
"encoding/json"
"sort"
"sync"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// Issue 1687 (ADR 0483 clause 2): a delivery plan's key lives from the ask of its first build until every build of its last
// tier is built, or the plan fails, stops or waits on the operator.
type fakeBuildingStore struct {
mu sync.Mutex
keys map[string]link.PlanBuilding
puts map[string]int
}
func newFakeBuildingStore() *fakeBuildingStore {
return &fakeBuildingStore{keys: map[string]link.PlanBuilding{}, puts: map[string]int{}}
}
func (f *fakeBuildingStore) Put(_ context.Context, plan string, value []byte) error {
f.mu.Lock()
defer f.mu.Unlock()
var v link.PlanBuilding
if err := json.Unmarshal(value, &v); err != nil {
return err
}
f.keys[plan] = v
f.puts[plan]++
return nil
}
func (f *fakeBuildingStore) Delete(_ context.Context, plan string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.keys, plan)
return nil
}
func (f *fakeBuildingStore) Keys(context.Context) ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.keys {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
func aPlan(id, state string, tiers [][]string, states map[string]string) inventory.Plan {
p := inventory.Plan{ID: id, State: state, Tiers: tiers, Modules: map[string]*inventory.PlanModule{}}
now := time.Now().Add(-time.Minute)
for m, st := range states {
pm := &inventory.PlanModule{State: st}
if st != "" {
pm.AskedAt = &now
}
p.Modules[m] = pm
}
return p
}
func TestAPlanIsInItsBuildPhaseFromTheFirstAskToTheLastBuild(t *testing.T) {
tiers := [][]string{{"a", "b"}, {"c"}}
cases := []struct {
name string
plan inventory.Plan
want bool
}{
{"no build asked yet", aPlan("p", inventory.PlanBuilding, tiers, nil), false},
{"the first tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked", "b": "asked"}), true},
// Between two tiers: the first built and sent (rolling), the second not asked yet.
{"between two tiers", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"}), true},
{"the last tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "built", "b": "built", "c": "asked"}), true},
{"every build built", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": "built"}), false},
// A module the plan deletes is done as far as its tiers go (release_plan.go), so it holds no key through the rollout.
{"a deleted module is done", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": planDeleted}), false},
{"failed", aPlan("p", inventory.PlanFailed, tiers, map[string]string{"a": "failed", "b": "asked"}), false},
{"superseded", aPlan("p", inventory.PlanSuperseded, tiers, map[string]string{"a": "asked"}), false},
{"a batch whose window is open", aPlan("p", inventory.PlanAssembling, tiers, map[string]string{"a": "asked"}), false},
}
for _, c := range cases {
if got := buildPhaseOf(c.plan); got != c.want {
t.Errorf("%s: in the build phase = %v, want %v", c.name, got, c.want)
}
}
waiting := aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
waiting.Delivery = &inventory.PlanDelivery{Awaits: "the operator's word"}
if buildPhaseOf(waiting) {
t.Error("a plan that waits on the operator is in its build phase")
}
}
// The key's life: put, refreshed, deleted at the phase's end; a plan that left the phase unseen loses its key at the next look.
func TestAPlanKeyIsPutRefreshedAndDeletedAtThePhasesEnd(t *testing.T) {
ctx := context.Background()
store := newFakeBuildingStore()
tiers := [][]string{{"a"}, {"b"}}
building := aPlan("plan-1", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
if faults := syncBuilding(ctx, store, []inventory.Plan{building}, time.Now()); len(faults) > 0 {
t.Fatal(faults)
}
if _, ok := store.keys["plan-1"]; !ok {
t.Fatal("no key while the first build is asked")
}
first := store.keys["plan-1"]
// Refreshed: put again, the phase's start kept.
time.Sleep(2 * time.Millisecond)
syncBuilding(ctx, store, []inventory.Plan{building}, time.Now())
if store.puts["plan-1"] != 2 || !store.keys["plan-1"].Seen.After(first.Seen) || !store.keys["plan-1"].Since.Equal(first.Since) {
t.Errorf("the key was not refreshed with its start kept: %d puts, %+v then %+v", store.puts["plan-1"], first, store.keys["plan-1"])
}
// Between the tiers: still there.
between := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built"})
syncBuilding(ctx, store, []inventory.Plan{between}, time.Now())
if _, ok := store.keys["plan-1"]; !ok {
t.Error("the key was gone between two tiers")
}
// A key of another plan that is no longer in the phase.
store.keys["plan-0"] = link.PlanBuilding{Plan: "plan-0"}
done := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"})
syncBuilding(ctx, store, []inventory.Plan{done}, time.Now())
if len(store.keys) != 0 {
t.Errorf("keys remain after the last build was built: %v", store.keys)
}
}
// keepBuildingKey, told of each save, puts the key while the plan builds and deletes it when the plan fails.
func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
ctx := context.Background()
store := newFakeBuildingStore()
var s buildingStore = store
buildingKeys.Store(&s)
defer buildingKeys.Store(nil)
p := aPlan("plan-7", inventory.PlanBuilding, [][]string{{"a"}}, map[string]string{"a": "asked"})
keepBuildingKey(ctx, p)
if _, ok := store.keys["plan-7"]; !ok {
t.Fatal("the key was not put when the plan was saved building")
}
p.State = inventory.PlanFailed
keepBuildingKey(ctx, p)
if _, ok := store.keys["plan-7"]; ok {
t.Fatal("the key stayed after the plan failed")
}
}
// The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
store := newFakeBuildingStore()
var s buildingStore = store
buildingKeys.Store(&s)
defer buildingKeys.Store(nil)
p, byName := aTierOfThree()
keyThere := false
was := askABuild
askABuild = func(context.Context, buildSource, string, string) (string, error) {
_, keyThere = store.keys["plan-1"]
return "build-x", nil
}
t.Cleanup(func() { askABuild = was })
askModule(t.Context(), p, "a", byName)
if !keyThere {
t.Fatal("the plan's key was not there when its first build was asked")
}
}
-130
View File
@@ -1,130 +0,0 @@
package main
import (
"context"
"fmt"
"io"
"os"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
)
// processUser is the bus user of one process of a module on a node (novox/hq ADR 0436 §1).
func processUser(node, module, process string) string {
return broker.Principal{Kind: broker.KindProcess, Node: node, Module: module, Process: process}.Username()
}
// processAccountWarning is the warning line for the own secret broker-<process id> of a process that declares bus
// when nobody issued its account (novox/hq ADR 0436 §2, issue 1226): "" for any other secret and for an issued
// account. The send goes on with the stand-in until the composition leaves the module off (issue 1753), so the
// line is the only loud thing about it.
func processAccountWarning(m catalogue.Manifest, node, secret string, minted []string) string {
id, isProcess := strings.CutPrefix(secret, "broker-")
if !isProcess {
return ""
}
if _, declared := m.ProcessBuses()[id]; !declared {
return ""
}
user := processUser(node, m.Module, id)
for _, u := range minted {
if u == user {
return ""
}
}
return fmt.Sprintf("%s/%s process %s: its bus account is not issued — mesh-controller.issue %s %s",
node, m.Module, id, node, m.Module)
}
// sayWarning writes a line where the compose's other warnings go: the act's answer when it carries a writer,
// else the controller's journal.
func sayWarning(ctx context.Context, line string) {
var w io.Writer = os.Stderr
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
w = lw
}
fmt.Fprintln(w, line)
}
// processAccountsMissing is each process of m that declares bus and whose account is not minted yet on node.
func processAccountsMissing(m catalogue.Manifest, node string, minted []string) []broker.ProcessPrincipal {
have := map[string]bool{}
for _, u := range minted {
have[u] = true
}
var out []broker.ProcessPrincipal
for _, pr := range inventory.ProcessesOf(m) {
if !have[processUser(node, m.Module, pr.Process)] {
out = append(out, pr)
}
}
return out
}
// issueMissingProcessAccounts issues the accounts of m's bus processes on node that are not minted, and no
// other: an account already issued is rotated on purpose, never by a later delivery or assignment (issue 1226).
// It says what it issued, one line, and nothing when nothing was missing.
func issueMissingProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) (string, error) {
if len(m.ProcessBuses()) == 0 || mayIssue(m) != nil {
return "", nil
}
minted, err := inv.BusMinted(ctx)
if err != nil {
return "", err
}
missing := processAccountsMissing(m, node, minted)
if len(missing) == 0 {
return "", nil
}
known, err := broker.FromEnvironment()
if err != nil {
return "", fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
}
reachable, err := brokerReachableAt(ctx, inv, known, node)
if err != nil {
return "", err
}
busAddress, err := broker.BusAddress()
if err != nil {
busAddress = ""
}
var ids []string
for _, pr := range missing {
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
return "", err
}
ids = append(ids, pr.Process)
}
return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push",
strings.Join(ids, ", "), node), nil
}
// issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the
// accounts of its bus processes not yet minted are issued, so a process a delivery adds is not pushed with a
// random stand-in (issue 1226). Never fatal: the build worked and the module is registered, so a failure is said,
// and the push refuses the module until `mesh-controller.issue` is run.
func issueGainedProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) {
if len(m.ProcessBuses()) == 0 {
return
}
nodes, err := inv.Running(ctx, m.Module)
if err != nil {
fmt.Printf("%s: its machines are not read to issue its process accounts: %v\n", m.Module, err)
return
}
for _, node := range nodes {
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
if err != nil {
fmt.Printf("%s on %s: its process accounts are not issued (%v): `mesh-controller.issue %s %s` first\n",
m.Module, node, err, node, m.Module)
continue
}
if line != "" {
fmt.Printf("%s on %s: %s\n", m.Module, node, line)
}
}
}
@@ -1,135 +0,0 @@
//go:build integration
package main
import (
"bytes"
"encoding/json"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
func gainedAndPlain(t *testing.T) (plain, gained catalogue.Manifest) {
t.Helper()
plain, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
if err != nil {
t.Fatal(err)
}
return plain, mailSenderWithProcess()
}
// 1226/AC2: given an assigned module whose new definition adds a bus process, when the delivery records it
// (takeIn, the function that records a build result), the process's account is minted; and 1226/AC3: the same
// build heard again replaces nothing.
func TestAGainedProcessIsIssuedWhenTheDeliveryRecordsIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
dir := t.TempDir()
if err := busCertificate([]string{"--into", dir}); err != nil {
t.Fatal(err)
}
t.Setenv(broker.AddressVar, strings.TrimPrefix(testbus.URL(t), "nats://"))
t.Setenv(broker.CertificateVar, filepath.Join(dir, "tls.crt"))
plain, gained := gainedAndPlain(t)
register(t, open, plain)
if _, err := assign(ctx, open, "laptop", "mesh-mail"); err != nil {
t.Fatal(err)
}
if _, minted, _ := inv.BusUserHash(ctx, "laptop.mesh-mail.send"); minted {
t.Fatal("the account was minted before the module gained the process")
}
manifest, _ := json.Marshal(gained)
build := func(id, commit string) {
t.Helper()
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, link.BuildResult{
ID: id, Repository: "http://forge.internal:20000/novox/mesh-mail.git", Path: "modules/mesh-mail",
Ref: "main", On: "anchor", Commit: commit, Manifest: manifest,
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-mail"},
})); err != nil {
t.Fatal(err)
}
}
build("b-1", "aaaaaaa1111")
hash, held, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
if err != nil || !held {
t.Fatalf("1226/AC2: recording the gained definition did not mint the process's account: held %v, %v", held, err)
}
// And the next compose carries the account, with no warning (1226/AC2).
var said bytes.Buffer
if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
t.Fatalf("1226/AC2: the next compose failed: %v", err)
}
if strings.Contains(said.String(), "bus account is not issued") {
t.Fatalf("1226/AC2: the next compose warns of an account just minted: %q", said.String())
}
// The row the compose reads is the account issueProcess accepted (origin accepted), never a value the mesh
// made for it (a stand-in).
in, err := inputFor(ctx, open)
if err != nil {
t.Fatal(err)
}
var row *compose.OwnSecretRow
for i, r := range in.OwnSecrets {
if r.Node == "laptop" && r.Module == "mesh-mail" && r.Name == "broker-send" {
row = &in.OwnSecrets[i]
}
}
if row == nil {
t.Fatal("1226/AC2: the next compose carries no row for the process's account")
}
if row.Origin != inventory.OriginAccepted {
t.Fatalf("1226/AC2: the process's account is a stand-in the mesh made (origin %q), not the issued account", row.Origin)
}
build("b-2", "bbbbbbb2222")
again, _, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
if err != nil || again != hash {
t.Fatalf("1226/AC3: recording again replaced the issued account (%v)", err)
}
}
// 1226/AC1 and AC4 against the store: given two modules on a machine, one with an unissued process account, when
// the machine is composed, the rest of it is composed and sent, and only the warning in the push's answer names
// the first, with `mesh-controller.issue <node> <module>`.
func TestAnUnissuedProcessAccountIsWarnedOfAndTheRestIsComposed(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
_, gained := gainedAndPlain(t)
register(t, open, gained)
other := catalogue.Manifest{Module: "other", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/other/token"}}}
register(t, open, other)
for _, m := range []string{"mesh-mail", "other"} {
if _, err := assign(ctx, open, "laptop", m); err != nil {
t.Fatal(err)
}
}
var said bytes.Buffer
if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
t.Fatalf("1226/AC4: an unissued process account held the machine: %v", err)
}
want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
if !strings.Contains(said.String(), want) {
t.Fatalf("1226/AC1: the push says %q, want %q", said.String(), want)
}
if strings.Contains(said.String(), "other") {
t.Fatalf("1226/AC4: the other module is warned of: %q", said.String())
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "other", "token"); err != nil || !held {
t.Fatalf("1226/AC4: the other module's secret was not composed: held %v, %v", held, err)
}
}
@@ -1,36 +0,0 @@
package main
import "testing"
// 1226/AC1: given a module whose process declares bus and has no issued account, when a machine is composed, the
// push's warning names issue, and the machine's other modules are composed (AC4: nothing is refused).
func TestAProcessAccountNobodyIssuedIsWarnedOfNamingIssue(t *testing.T) {
m := mailSenderWithProcess()
got := processAccountWarning(m, "laptop", "broker-send", nil)
want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
if got != want {
t.Fatalf("1226/AC1: the warning is %q, want %q", got, want)
}
// Other names are not this rule's.
if processAccountWarning(m, "laptop", "broker", nil) != "" || processAccountWarning(m, "laptop", "broker-other", nil) != "" ||
processAccountWarning(m, "laptop", "token", nil) != "" {
t.Fatal("1226/AC1: a name that is no declared process's account was warned of")
}
if got := processAccountWarning(m, "laptop", "broker-send", []string{"laptop.mesh-mail.send"}); got != "" {
t.Fatalf("1226/AC3: an issued account was warned of: %q", got)
}
}
// 1226/AC3: an account already issued is not issued again by a later delivery or assignment.
func TestAnIssuedProcessAccountIsNotIssuedAgain(t *testing.T) {
m := mailSenderWithProcess()
if got := processAccountsMissing(m, "laptop", nil); len(got) != 1 || got[0].Process != "send" {
t.Fatalf("1226/AC3: an account not issued is not missing: %v", got)
}
if got := processAccountsMissing(m, "laptop", []string{"laptop.mesh-mail.send"}); len(got) != 0 {
t.Fatalf("1226/AC3: an issued account would be issued again: %v", got)
}
if got := processAccountsMissing(m, "ace", []string{"laptop.mesh-mail.send"}); len(got) != 1 {
t.Fatalf("1226/AC3: another machine's account stands in for this one's: %v", got)
}
}
+1 -57
View File
@@ -282,10 +282,6 @@ func serve(ctx context.Context) (err error) {
// assignment.
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
go mirroringRecords(ctx, inv, bus.Conn, bus)
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
// And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2).
go keepingPlansBuilding(ctx, inv, bus.Conn)
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
// to-be 45 §6).
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
@@ -983,18 +979,6 @@ func deliver(ctx context.Context, d delivery, holder string, sending []readyNode
return err
}
digest, err := d.declare(ctx, s, body)
if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
// references, and says so; the next one tries the bucket again.
fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
s.declared.fileReferences = false
if body, err = s.declared.Body(); err != nil {
return err
}
digest, err = d.declare(ctx, s, body)
}
if err != nil {
return err
}
@@ -1056,35 +1040,6 @@ type overTheBus struct {
indent string
}
// putFiles puts the contents s names by digest in the bucket.
func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
objects, err := s.declared.objects()
if err != nil {
return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
}
if len(objects) == 0 {
return nil
}
bus, ok := b.server.Bus().(link.OverNATS)
if !ok {
return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
}
putting, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
store, err := openBucket(putting, bus.Conn, s.node)
if err == nil {
err = putObjects(putting, store, objects)
}
notePut(ctx, s.node, err)
if err != nil {
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
}
return nil
}
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
// when they cannot be, and never what holds the send back (assertOnSend).
@@ -1097,20 +1052,9 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
}
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
// that applies it finds every one. A node reading no references has none.
// **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them.
unlock := lockFilesOf(s.node)
err := b.putFiles(ctx, s)
if err != nil {
unlock()
if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
return "", err
}
declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second)
unlock()
if declared != nil {
return "", declared
}
// After it is away, not before. A digest recorded for something that failed to send would make
// the machine look current for a declaration it never received.
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
+6 -40
View File
@@ -91,9 +91,7 @@ type queueAnswer struct {
Dead int `json:"dead"`
MaxDeliver int `json:"max-deliver"`
Asks []queuedAnswer `json:"asks"`
// HeldFor are the delivery plans building, while which the waiting checks are held (ADR 0483 clause 5); empty when none.
HeldFor []string `json:"held-for,omitempty"`
Text string `json:"text"`
Text string `json:"text"`
}
// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
@@ -122,9 +120,6 @@ type queuedAnswer struct {
// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
// (novox/hq issue 594).
Delivery int `json:"delivery,omitempty"`
// LetThrough is a check that waited past 30 minutes and is let through while a delivery builds: the next one, or the one
// that runs (ADR 0483 clause 5).
LetThrough bool `json:"let-through,omitempty"`
}
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
@@ -135,8 +130,7 @@ var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, HeldFor: q.Hold.Plans, Text: queueText(q, now)}
letNext, letRunning := q.LetThroughOf(now)
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
one := func(a link.QueuedAsk, position int) queuedAnswer {
kind := link.KindOf(a.Request)
since := a.AskedAt
@@ -149,7 +143,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
}
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
Position: position, Seq: a.Seq, Delivery: a.Delivery, LetThrough: a.ID != "" && (a.ID == letNext || a.ID == letRunning)}
Position: position, Seq: a.Seq, Delivery: a.Delivery}
}
for _, a := range running {
out.Asks = append(out.Asks, one(a, 0))
@@ -185,11 +179,6 @@ func queueText(q link.Queue, now time.Time) string {
}
return s
}
letNext, letRunning := q.LetThroughOf(now)
const passed = "waited past 30 minutes, let through"
if q.HoldFault != "" {
fmt.Fprintf(&b, "why checks wait cannot be read, so none is said held: %s\n", q.HoldFault)
}
if len(running) > 0 {
fmt.Fprintln(&b, "\nin flight:")
for _, a := range running {
@@ -204,27 +193,14 @@ func queueText(q link.Queue, now time.Time) string {
case a.Was != "":
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
}
note := ""
if a.ID != "" && a.ID == letRunning {
note = " — " + passed
}
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), where,
ago(a.AskedAt), a.Seq, note)
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
ago(a.AskedAt), a.Seq)
}
}
if len(waiting) > 0 {
fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
// **Why the checks wait** (ADR 0483 clause 5): above them, the plans they are held for.
if len(q.Hold.Plans) > 0 && hasCheck(waiting) {
fmt.Fprintf(&b, " checks are held while delivery %s builds; the oldest check that waited 30 minutes goes first, one at a time\n",
strings.Join(q.Hold.Plans, ", "))
}
for _, a := range waiting {
note := ""
if a.ID != "" && a.ID == letNext {
note = " — " + passed + " next"
}
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq, note)
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
}
}
if len(dead) > 0 {
@@ -242,16 +218,6 @@ func queueText(q link.Queue, now time.Time) string {
return b.String()
}
// hasCheck says a check is among the asks.
func hasCheck(asks []link.QueuedAsk) bool {
for _, a := range asks {
if a.Request.Check != nil {
return true
}
}
return false
}
// cancelCommand drops one waiting or dead ask.
func cancelCommand(ctx context.Context, args []string) error {
if len(args) != 1 {
-77
View File
@@ -1,77 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/link"
)
// Issue 1688 / 1652 AC5 (ADR 0483 clause 5): the queue names, above the waiting checks, the plan they are held for, and says of a
// check let through that it waited past 30 minutes.
func aHeldQueue(now time.Time) link.Queue {
check := func(id string, ago time.Duration, n int) link.QueuedAsk {
return link.QueuedAsk{Seq: uint64(n), ID: id, Repository: "ssh://forge/novox/r", AskedAt: now.Add(-ago), State: link.AskWaiting,
Request: link.BuildRequest{ID: id, Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: n}}}
}
return link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{
check("build-young", 5*time.Minute, 1),
check("build-old", 45*time.Minute, 2),
check("build-older", 50*time.Minute, 3),
}, Hold: link.CheckHold{Plans: []string{"plan-42"}}}
}
func TestTheQueueNamesThePlanTheChecksAreHeldFor(t *testing.T) {
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
text := queueText(aHeldQueue(now), now)
heldAt := strings.Index(text, "held while delivery plan-42 builds")
firstAt := strings.Index(text, "build-young")
if heldAt < 0 || firstAt < 0 || heldAt > firstAt {
t.Fatalf("the plan the checks are held for is not named above the waiting checks:\n%s", text)
}
// Nothing building: nothing held.
q := aHeldQueue(now)
q.Hold = link.CheckHold{}
if strings.Contains(queueText(q, now), "held while") {
t.Fatal("checks are said held with no delivery building")
}
}
func TestACheckLetThroughIsSaidToHaveWaitedPastThirtyMinutes(t *testing.T) {
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
q := aHeldQueue(now)
text := queueText(q, now)
var saidOf []string
for _, l := range strings.Split(text, "\n") {
if strings.Contains(l, "waited past 30 minutes, let through") {
saidOf = append(saidOf, strings.Fields(l)[0])
}
}
if len(saidOf) != 1 || saidOf[0] != "build-older" {
t.Fatalf("the check let through next (the oldest) is not the one said: %v\n%s", saidOf, text)
}
answer := queueAnswerOf(q, now)
if len(answer.HeldFor) != 1 || answer.HeldFor[0] != "plan-42" {
t.Errorf("held-for = %v", answer.HeldFor)
}
for _, a := range answer.Asks {
if a.LetThrough != (a.ID == "build-older") {
t.Errorf("%s let-through = %v", a.ID, a.LetThrough)
}
}
// A check in flight under the let-through key is said too.
q.Asks = append(q.Asks, link.QueuedAsk{Seq: 9, ID: "build-running", State: link.AskInFlight, On: "ace", AskedAt: now.Add(-time.Hour),
Started: now.Add(-time.Minute), Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: 9}}})
q.Hold.LetThrough = &link.LetThrough{Builder: "ace"}
if !strings.Contains(queueText(q, now), "build-running") || !strings.Contains(queueText(q, now), "waited past 30 minutes, let through\n") {
t.Errorf("the check in flight let through is not said:\n%s", queueText(q, now))
}
// Younger than the bound: none is said.
young := aHeldQueue(now)
young.Asks = young.Asks[:1]
if strings.Contains(queueText(young, now), "waited past") {
t.Error("a check of 5 minutes is said to have waited past 30")
}
}
+7 -254
View File
@@ -577,9 +577,6 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
}
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
// requests it answers, whose checks name the part of a differing key.
// **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save
// would hold a place for its whole run.
sayBuilding(ctx, p.ID, now)
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
e.Source.Path, ref)
if err != nil {
@@ -628,124 +625,16 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
}
// holdPlansForOutcome is how an outcome taken on the event loop holds the plans: **without waiting for them**
// (novox/hq issue 1708). The loop takes one message at a time, and the plans' holder may keep them for minutes while
// it sends machines their declarations; an outcome that waited its turn (up to inventory.HoldWaitFor, two minutes)
// held every report, build and merge behind it for as long, and the watchdog said the controller stopped listening
// (2026-10-21 14:20:45 to 14:22:45). Busy, the outcome is left to the build records, where it already is, and the
// look settles the plan from them (leaveToRecords).
func holdPlansForOutcome(ctx context.Context, hold func(context.Context, bool) (func(), error)) (func(), error) {
return hold(ctx, false)
}
// leaveToRecords says an outcome was not taken into the plans, which settle it from the build records (novox/hq issue
// 214). When another act holds the plans it asks for the look, which waits for them off the event loop and settles
// the plan from the records once they are let go (issue 1708); what the build did is kept for that settle by the
// build's id, so the walk's module says it as it would have (ADR 0350 §4).
func leaveToRecords(module, id, made, failed string, err error) {
fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
if !errors.Is(err, inventory.ErrPlansBusy) {
return
}
keepMadeFor(id, made, time.Now())
// A refused outcome (failed, its build recorded as worked) is kept as refused, or the records settle it built.
keepRefusalFor(id, failed, time.Now())
askToLook()
}
// refusalsKeptFor is the refusals kept in this process for these builds (keepRefusalFor), by id.
func refusalsKeptFor(ids []string) map[string]string {
out := map[string]string{}
refusedLeftMu.Lock()
defer refusedLeftMu.Unlock()
for _, id := range ids {
if k, ok := refusedLeft[id]; ok {
out[id] = k.made
}
}
return out
}
// refusedLeft is why each outcome left to the records was refused, by the build's id: kept in the process beside the
// store's record (RecordRefused), so a store that did not take it still fails the plan while this controller serves.
var (
refusedLeftMu sync.Mutex
refusedLeft = map[string]madeKept{}
)
// keepRefusalFor keeps why an outcome left to the records was refused, forgetting refusals older than madeLeftFor.
func keepRefusalFor(id, why string, now time.Time) {
if id == "" || why == "" {
return
}
refusedLeftMu.Lock()
defer refusedLeftMu.Unlock()
for k, v := range refusedLeft {
if now.Sub(v.at) > madeLeftFor {
delete(refusedLeft, k)
}
}
refusedLeft[id] = madeKept{made: why, at: now}
}
// madeLeft is what each build left to the records did, by the build's id, until a settle takes it: kept in the
// process, so a controller replaced before the settle loses only these words, never the outcome.
var (
madeLeftMu sync.Mutex
madeLeft = map[string]madeKept{}
)
type madeKept struct {
made string
at time.Time
}
// madeLeftFor is how long words nobody settled are kept: a plan that ended or moved on never asks for them.
const madeLeftFor = time.Hour
// keepMadeFor keeps what a build left to the records did for the settle that takes it, forgetting words older than
// madeLeftFor.
func keepMadeFor(id, made string, now time.Time) {
if id == "" || made == "" {
return
}
madeLeftMu.Lock()
defer madeLeftMu.Unlock()
for k, v := range madeLeft {
if now.Sub(v.at) > madeLeftFor {
delete(madeLeft, k)
}
}
madeLeft[id] = madeKept{made: made, at: now}
}
// takeMadeOf is what the build of this id did, if its outcome was left to the records with words; taken once.
func takeMadeOf(id string) string {
madeLeftMu.Lock()
defer madeLeftMu.Unlock()
k, ok := madeLeft[id]
if !ok {
return ""
}
delete(madeLeft, id)
return k.made
}
// planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
// ADR 0350 §4) — on the module of the walk it answers.
func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
inv := open.inventory
// One controller works the plans at a time (novox/hq issue 213). Not taken, the outcome is still in the build
// records, which the holder settles the plan from (issue 214).
release, err := holdPlansForOutcome(ctx, inv.HoldPlans)
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
// than write over what the holder is about to save. Not taken, it is still in the build records,
// which the holder settles the plan from (issue 214).
release, err := inv.HoldPlans(ctx, true)
if err != nil {
if failed != "" && id != "" && errors.Is(err, inventory.ErrPlansBusy) {
// In the store too, so a controller replaced before the look settles still fails the plan (issue 1708).
if err := inv.RecordRefused(ctx, id, module, failed); err != nil {
fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, err)
}
}
leaveToRecords(module, id, made, failed, err)
fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
return
}
defer release()
@@ -807,19 +696,7 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
}
}
advanceAfterOutcome(func() { advanceHeld(ctx, open) })
}
// advanceAfterOutcome moves the plans after an outcome was taken into them: **in the serving controller, by asking
// the look** (novox/hq issue 1708). The outcome is taken on the event loop, and moving the plans may send machines
// their declarations, a release's send retried for minutes among them; done here it held every report, build and
// merge behind it (2026-10-21 14:28:40 to 14:32:10). The look runs off the loop, holding the plans itself. Where no
// look runs (a command, which has no event loop) the plans are moved at once, as before.
func advanceAfterOutcome(advance func()) {
if askedToLook() {
return
}
advance()
advanceHeld(ctx, open)
}
// startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past
@@ -854,23 +731,6 @@ func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Pl
return behind, why, nil
}
// refusalNotePrefix and refusalNoteSuffix frame the note of a step the walk was refused: "tier N: <refusal> — tried
// again". Only a note so framed is a hold (novox/hq issue 920); a tier's summary left in the note is not.
const (
refusalNotePrefix = "tier "
refusalNoteSuffix = " — tried again"
)
// refusalNote is the note of a plan whose step at tier was refused with err.
func refusalNote(tier int, err error) string {
return refusalNotePrefix + fmt.Sprint(tier) + ": " + err.Error() + refusalNoteSuffix
}
// refused says a plan's note is a refusal the walk wrote.
func refused(p inventory.Plan) bool {
return strings.HasPrefix(p.Note, refusalNotePrefix) && strings.HasSuffix(p.Note, refusalNoteSuffix)
}
// deferredNote begins the note of a walk deferred behind another.
const deferredNote = "let go; starts once "
@@ -883,20 +743,6 @@ func deferred(p inventory.Plan) bool {
p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote)
}
// currentTierBuilt says every module of a plan's current tier is built (or deleted), so no build is what it waits on.
func currentTierBuilt(p inventory.Plan) bool {
if p.Tier < 0 || p.Tier >= len(p.Tiers) || len(p.Tiers[p.Tier]) == 0 {
return false
}
for _, m := range p.Tiers[p.Tier] {
s := p.Modules[m]
if s == nil || (s.State != "built" && s.State != planDeleted) {
return false
}
}
return true
}
// askedAny says a plan asked any module.
func askedAny(p inventory.Plan) bool {
for _, s := range p.Modules {
@@ -961,7 +807,7 @@ func advanceHeld(ctx context.Context, open *stores) (failed bool) {
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
// the state is left as it was and the step is tried again by the next look. Said and
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
p.Note = refusalNote(p.Tier, err)
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
if refusedForTheBus(err) {
busRefusedFirst.mark(p.ID, time.Now())
@@ -1075,17 +921,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
if err != nil {
return false, err
}
before, was := statesOf(p), p.State
if settleFromRecords(p, tier, recorded, byID) {
// As taking the outcome directly does (planBuiltSaying): a module deleted at its source is forgotten, and a
// failed plan says what it built and never sent (issue 1708).
deleted, failed := settledEffects(before, p, was)
for _, m := range deleted {
forgetDeleted(ctx, inv, m, p)
}
if failed {
sayUnsent(p, rollsOut)
}
return true, nil
}
// Asked: wait for every build.
@@ -2059,13 +1895,6 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
if p.State == inventory.PlanRolling {
what = p.Note
}
if p.State == inventory.PlanBuilding && refused(p) && currentTierBuilt(p) {
// Every build of the tier is in, so the plan waits on something else, and its note says what: a send
// refused ("a build waits there for its gate: <machine> (...)") is kept there. "building" with LATE
// hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920).
what = p.Note
return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late)
}
return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late)
}
@@ -2650,81 +2479,9 @@ func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.
}
}
}
refused, err := refusalsOf(ctx, inv, recorded, byID)
if err != nil {
return nil, nil, err
}
overlayRefusals(recorded, byID, refused)
return recorded, byID, nil
}
// overlayRefusals marks failed every build record whose registration the mesh refused (novox/hq issue 1708): the record
// keeps the builder's words, which worked, and settled from it as it is the refused build would be built and sent.
func overlayRefusals(recorded map[string][]inventory.Build, byID map[string]inventory.Build, refused map[string]string) {
if len(refused) == 0 {
return
}
fail := func(b *inventory.Build) {
if why, ok := refused[b.ID]; ok && b.Worked() {
b.Failed = why
}
}
for m := range recorded {
for i := range recorded[m] {
fail(&recorded[m][i])
}
}
for id, b := range byID {
fail(&b)
byID[id] = b
}
}
// refusalsOf is every refusal kept for the builds these records name, in the store and in this process.
func refusalsOf(ctx context.Context, inv *inventory.Inventory, recorded map[string][]inventory.Build,
byID map[string]inventory.Build) (map[string]string, error) {
var ids []string
for _, bs := range recorded {
for _, b := range bs {
ids = append(ids, b.ID)
}
}
for id := range byID {
ids = append(ids, id)
}
refused, err := inv.RefusedOf(ctx, ids)
if err != nil {
return nil, fmt.Errorf("the refused registrations cannot be read: %w", err)
}
for id, why := range refusalsKeptFor(ids) {
refused[id] = why
}
return refused, nil
}
// settledEffects is what settling from the records did that needs more than the plan: the modules it found deleted
// at their source, and whether it failed the plan.
func settledEffects(before map[string]string, p *inventory.Plan, was string) (deleted []string, failed bool) {
for m, s := range p.Modules {
if s != nil && s.State == planDeleted && before[m] != planDeleted {
deleted = append(deleted, m)
}
}
sort.Strings(deleted)
return deleted, p.State == inventory.PlanFailed && was != inventory.PlanFailed
}
// statesOf is each module's state in a plan, for settledEffects.
func statesOf(p *inventory.Plan) map[string]string {
out := map[string]string{}
for m, s := range p.Modules {
if s != nil {
out[m] = s.State
}
}
return out
}
// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
// Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
@@ -2774,10 +2531,6 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
s.State = "built"
s.BuiltAt = &at
s.Commit = outcome.Commit
// What the build did, when its outcome was left to the records with words (issue 1708).
if made := takeMadeOf(outcome.ID); made != "" {
s.Made = made
}
} else {
s.State = "failed"
s.Why = outcome.Failed
-24
View File
@@ -93,27 +93,3 @@ func TestTheSeatsListingShowsEachSeatsPaths(t *testing.T) {
}
}
}
// 1718/AC2: a manifest claiming node-files that serves upload and download passes the controller's module
// check, judged by the mesh-sdk it vendors; a verb outside the contract is still refused.
func TestAFilesHolderServingUploadAndDownloadCanHoldTheSeat(t *testing.T) {
seat, ok := catalogue.SeatNamed(catalogue.FilesSeat)
if !ok {
t.Fatal("node-files is not a seat")
}
serves := []string{"upload", "download"}
for _, v := range seat.Serves {
if v.Name != "upload" && v.Name != "download" {
serves = append(serves, v.Name)
}
}
holder := catalogue.Manifest{Module: "mesh-files", Version: "1", Claims: []catalogue.Claim{{
Name: catalogue.FilesSeat, Scope: catalogue.ScopeNode, Serves: serves}}}
if err := catalogue.CanHold(holder, seat); err != nil { // 1718/AC2
t.Fatalf("a holder serving upload and download is refused: %v", err)
}
holder.Claims[0].Serves = append(append([]string{}, serves...), "teleport")
if err := catalogue.CanHold(holder, seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
t.Fatalf("a verb outside the contract must be refused with 'does not promise': %v", err)
}
}
+3 -27
View File
@@ -39,10 +39,6 @@ type sendable struct {
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
ReadsModules bool
// fileReferences says the machine's node-engine reports the capability of reading a file given as a digest and a size
// (declaration.FileReferencesCapability, novox/hq ADR 0486 §5): then every file over 64 KiB that may travel so does (Body,
// objects). Not on the wire as a field; the bodies differ.
fileReferences bool
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
// kept beside the resources, and on the wire only to a machine that reads them.
moduleOf map[string]string
@@ -119,16 +115,12 @@ type adoptionEnvelope struct {
// Body is the declaration's bytes, as sent and as digested.
func (s sendable) Body() ([]byte, error) {
resources, _, err := s.byReference()
if err != nil {
return nil, err
}
envelope := map[string]any{"declaration": 1, "resources": resources}
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
if s.ReadsModules {
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
named := make([]map[string]any, len(resources))
for n, r := range resources {
named := make([]map[string]any, len(s.Resources))
for n, r := range s.Resources {
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
if !owned {
named[n] = r
@@ -173,19 +165,3 @@ func (s sendable) Body() ([]byte, error) {
}
return json.Marshal(envelope)
}
// byReference is the resources as the machine is sent them: with the contents of its large files replaced by their digests when
// it reports it reads them (issue 1696), and the contents it replaced, by the name of the object that holds each.
func (s sendable) byReference() ([]map[string]any, map[string][]byte, error) {
if !s.fileReferences {
return s.Resources, nil, nil
}
return referencing(s.Resources)
}
// objects are the contents the body names by digest, by the name of each one's object: what is put in the bucket before the
// body is sent. Empty for a machine that is sent every content inline.
func (s sendable) objects() (map[string][]byte, error) {
_, objects, err := s.byReference()
return objects, err
}
+1 -53
View File
@@ -415,59 +415,7 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
if len(added) == 0 || len(from) == 0 {
return nil
}
source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}
// Asked once (novox/hq issue 1617): a module is asked when its merge is heard, and again here when the batch is cut.
var requests []inventory.RequestOutcome
for _, dir := range added {
if got, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: source.Repository, Path: dir}.Name()); err == nil {
requests = append(requests, got...)
}
}
return askBuilds(ctx, inv, m, source, unaskedDirs(added, requests, source, time.Now()))
}
// askedInFlightFor is how long a merge's build request stands as the build coming for its module: past it, a lost one
// is asked again.
const askedInFlightFor = time.Hour
// unaskedDirs are the directories of a new module no build of a merge is on its way for: a kept request of the same
// repository and directory, asked for a merge and handed over, whose outcome has not come and is not given up on,
// is the build coming (novox/hq issue 1617).
func unaskedDirs(added []string, requests []inventory.RequestOutcome, source buildSource, now time.Time) []string {
var out []string
for _, dir := range added {
path := dir
if path == "." {
path = ""
}
coming := false
for _, r := range requests {
if r.For == "merge" && r.Repository == source.Repository && strings.Trim(r.Path, "/") == path &&
r.NotAsked == "" && r.OutcomeUnknown == "" && !r.Heard && now.Sub(r.At) < askedInFlightFor {
coming = true
break
}
}
if !coming {
out = append(out, dir)
}
}
return out
}
// askNewModulesAtHearing asks the first build of the modules a merge adds to a repository the mesh builds modules from,
// when the merge is heard, not when its batch is cut (novox/hq issue 1617): a batch waits out its window and the walk
// before it, and a module added by a merge delivered meanwhile could not be assigned. Outside the batch and its plan.
func askNewModulesAtHearing(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository, from []inventory.Entry) {
if len(from) == 0 {
return
}
_, added, _ := touchedBy(from, entries, m, read)
if asked := askNewModules(ctx, inv, m, from, added); len(asked) > 0 {
fmt.Printf("%s/%s merged into %s (%.8s) adds %s: the first build is asked now, before its batch is cut\n",
m.Owner, m.Repo, m.Base, m.Commit, strings.Join(asked, ", "))
}
return askBuilds(ctx, inv, m, buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}, added)
}
// askBuilds asks the first build of each directory of a merge's repository, as askNewModules does, from the source given.
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861
github.com/jackc/pgx/v5 v5.10.0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
+2 -2
View File
@@ -1,7 +1,7 @@
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75 h1:AcYfyJpYaUufhI93VG76+b001msXY4XX7kuRjnf33HY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861 h1:BsqolZMMsC5jLP8X88BwShqFssLgtREcgBIy6OfPSI8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-9
View File
@@ -46,7 +46,6 @@ func CheckStreams() []Stream {
Retention: RetentionWorkQueue,
MaxAge: 7 * 24 * 60 * 60,
MaxMsgsPerSubject: 1,
Direct: true,
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
"request still waiting",
@@ -57,7 +56,6 @@ func CheckStreams() []Stream {
Retention: RetentionWorkQueue,
MaxAge: 7 * 24 * 60 * 60,
MaxMsgsPerSubject: 1,
Direct: true,
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
},
@@ -100,13 +98,6 @@ func holderCheckGrants() []string {
"$KV."+CheckRunningBucket+".>",
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
// And, to hold checks while a delivery builds (ADR 0483, issues 1687 and 1688): the bucket of the plans building, **read and
// never written** (the controller's alone, so no `$KV.` subject of it is granted), and the oldest waiting ask of each check
// queue, read by a direct get that takes nothing from the queue and so counts against no delivery of the ask.
pub = append(pub, readGrants(PlansBuildingBucket, "")...)
for _, w := range CheckWorkers() {
pub = append(pub, "$JS.API.DIRECT.GET."+w.Stream, "$JS.API.DIRECT.GET."+w.Stream+".>")
}
return pub
}
-34
View File
@@ -40,37 +40,3 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
}
}
}
// Issue 1687 (ADR 0483 clause 2): a holder of the build seat reads the bucket of the plans building and writes none of it, and
// reads the oldest waiting ask of each check queue by a direct get, which takes nothing from the queue.
func TestTheBuildSeatsHoldersReadThePlansBuildingAndWriteNone(t *testing.T) {
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat}, PasswordHash: "x"}
perms, err := PermissionsFor(p)
if err != nil {
t.Fatal(err)
}
stream := "KV_" + PlansBuildingBucket
for _, s := range []string{
"$JS.API.STREAM.INFO." + stream,
"$JS.API.DIRECT.GET." + stream + ".>",
"$JS.API.CONSUMER.CREATE." + stream + ".>",
"$JS.API.DIRECT.GET.CHECKS_READY",
"$JS.API.DIRECT.GET.CHECKS_OTHER",
} {
has(t, perms.Publish, s)
}
for _, s := range perms.Publish {
if SubjectsOverlap(s, "$KV."+PlansBuildingBucket+".plan-1") {
t.Errorf("a holder of the build seat may write %s through %q", PlansBuildingBucket, s)
}
}
// And the controller alone writes it, as the writers table says.
for _, row := range WritersTable {
for _, s := range row.Subjects {
if s == "$KV."+PlansBuildingBucket+".>" && row.Writes(p, "$KV."+PlansBuildingBucket+".plan-1") {
t.Errorf("%q lets the build seat's holder write the plans building", row.State)
}
}
}
}
+2 -38
View File
@@ -63,26 +63,6 @@ var (
// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
// those heads. Written by the controller alone, from the announcement, when it holds the lease.
DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
// PlansBuildingBucket holds one key per delivery plan in its build phase (novox/hq ADR 0483 clause 2, issue 1687): the key
// is `<plan id>` (`plan-<digits>`), its value the plan, the time its build phase began and the time of the last refresh.
// The controller (lease holder) puts it when the plan's first build is asked, puts it again about every thirty seconds
// while the plan builds, and deletes it when the phase ends (every build of the last tier built, or the plan fails, stops
// or waits on the operator); the bucket's age, PlansBuildingKeptFor, is how long a key outlives a controller that
// stopped. **Read by the build seat's holders and the checker, written by the controller alone**: a holder takes no new
// check while any key is there, but for one that waited past ADR 0483's bound. A reader lists the keys and reads none of
// their values to decide that (`link.BuildingPlans`).
PlansBuildingBucket = BucketName(ControllerSeat, PlansBuilding)
)
// PlansBuilding is the local name of the bucket of the plans building, as a reader names it in `reads`
// (`mesh-controller.plans-building`).
const PlansBuilding = "plans-building"
// PlansBuildingKeptFor is the age of a plan-building key: the controller puts it again at PlansBuildingRefresh, so a
// controller that stopped leaves checks held at most this long (ADR 0483 clause 2).
const (
PlansBuildingKeptFor = 2 * time.Minute
PlansBuildingRefresh = 30 * time.Second
)
// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
@@ -224,14 +204,14 @@ func IsControllerBucket(bucket string) bool {
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
bucket == PlansBuildingBucket || bucket == delivery.HaveBucket
bucket == delivery.HaveBucket
}
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
func ControllerBuckets() []string {
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
}
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -374,22 +354,6 @@ func (j *JetStream) EnsureControllerBuckets() error {
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
}
// **The plans building: keys that age** (ADR 0483 clause 2, issue 1687). One message a key, each refresh a new one, so the
// age of the bucket is how long a key outlives a controller that stopped refreshing it.
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: PlansBuildingBucket,
Description: "the delivery plans in their build phase (novox/hq ADR 0483): `<plan id>` holds the plan and when its " +
"build phase began; put when the plan's first build is asked and about every thirty seconds, deleted when the " +
"phase ends, gone within two minutes of a controller that stopped; read by the build seat's holders, which " +
"take no new check while a key is there; written by the controller alone",
History: 1,
TTL: PlansBuildingKeptFor,
MaxValueSize: 4 << 10,
MaxBytes: 8 << 20,
Storage: jetstream.FileStorage,
}); err != nil {
return fmt.Errorf("asserting bucket %s: %w", PlansBuildingBucket, err)
}
// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: DeliveryMergesBucket,
-123
View File
@@ -1,123 +0,0 @@
package broker
import (
"context"
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/nats-io/nats.go/jetstream"
)
// The object buckets a declaration's large files travel in (novox/hq ADR 0486, issues 1696 and 1697): **one bucket for each
// node**, `declaration-files-<node>`.
//
// **The controller alone writes them, and a node-engine reads an object by its name from its own bucket** (ADR 0486 §2): its
// account may neither list the bucket nor write it, no account may touch another node's, and no other account has any access.
// Why a bucket each: a subject pattern cannot tell a name from a wildcard. A grant of `...M.*` matches a publish to the literal
// subject `...M.*` too (probed on nats-server 2.11.17 by the review of this issue: a node created a consumer filtered on
// `$O.<bucket>.C.*` and read every object of the bucket), so in one shared bucket a node could read every other node's files.
// With a bucket each, the same trick reads only the node's own files, which are the files of its own declaration.
//
// A bucket of the bus's object store is a stream, OBJ_<bucket>, whose messages are the objects' metadata
// (`$O.<bucket>.M.<base64 of the name>`) and chunks (`$O.<bucket>.C.<nuid>`).
// DeclarationFilesBucket is the prefix of the buckets' names, which mesh-sdk fixes for the node-engine and the controller alike.
const DeclarationFilesBucket = declaration.FileBucketPrefix
// DeclarationFilesBucketOf is the bucket a node's files are in: mesh-sdk's FileBucketFor, the one place the name is composed. A
// node name it refuses (one that could carry a subject wildcard or separator into a grant) is an error, never a bucket.
func DeclarationFilesBucketOf(node string) (string, error) { return declaration.FileBucketFor(node) }
// DeclarationFilesStreamOf is the stream under a node's bucket: mesh-sdk's FileStreamFor.
func DeclarationFilesStreamOf(node string) (string, error) { return declaration.FileStreamFor(node) }
// DeclarationFilesMaxBytes is a bucket's size: the objects are immutable and regenerable, and one node's hold well under a hundred
// megabytes.
const DeclarationFilesMaxBytes = 1 << 30
// DeclarationFilesKeptFor is how long an object stays after no node's latest declaration names it (ADR 0486 §3).
const DeclarationFilesKeptFor = 24 * time.Hour
// DeclarationFilesWrites is the one pattern the controller may publish to write a node's bucket: every object's metadata and
// chunks. The server's API for the stream is the controller's `$JS.API.>`.
func DeclarationFilesWrites(node string) (string, error) {
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return "", err
}
return "$O." + bucket + ".>", nil
}
// DeclarationFilesReads is what a node-engine's account may publish to read an object of its own bucket by its name, and nothing
// more. The set is the one node-engine's own call (mesh-host internal/link FileObject) makes, for B the node's bucket and S its
// stream (issue 1697, notes 4 and 5):
//
// - the object's metadata, by a direct get of the last message of the one subject that names it. **Only the suffixed
// form**: the bare `$JS.API.DIRECT.GET.S` takes a sequence number and would read any message of the stream.
// - the ordered consumer on that object's chunks: `$JS.API.CONSUMER.CREATE.S.<name>.$O.B.C.<nuid>`. **Not `S.>`**:
// a consumer filtered on `$O.B.M.>` would read every object's name.
// - the flow-control replies of that consumer, without which a large object stalls at the flow-control window.
// - the consumer's deletion, best effort: the consumer's inactivity threshold removes it if this is refused.
//
// **Never granted:** STREAM.INFO (a `subjects_filter` of `>` lists every name), STREAM.MSG.GET (any message by its sequence), a
// publish to `$O.B.>` (a write), and any subject of another node's stream. A literal `*` in a subject matches the grant's `*`: it
// reaches the objects of this node's own bucket and no other (above).
func DeclarationFilesReads(node string) ([]string, error) {
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return nil, err
}
stream, err := DeclarationFilesStreamOf(node)
if err != nil {
return nil, err
}
return []string{
"$JS.API.DIRECT.GET." + stream + ".$O." + bucket + ".M.*",
"$JS.API.CONSUMER.CREATE." + stream + ".*.$O." + bucket + ".C.*",
"$JS.API.CONSUMER.DELETE." + stream + ".*",
"$JS.FC." + stream + ".>",
}, nil
}
// EnsureDeclarationFiles creates the buckets of these nodes if absent and brings their options to match. A bucket exists whenever the
// bus does, so a node that is starting finds it (ADR 0486 §4). An update, never a delete.
func (j *JetStream) EnsureDeclarationFiles(nodes []string) error {
js, err := jetstream.New(j.conn)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
return EnsureDeclarationFiles(ctx, js, nodes...)
}
// EnsureDeclarationFiles is the same on a JetStream the caller holds.
func EnsureDeclarationFiles(ctx context.Context, js jetstream.JetStream, nodes ...string) error {
for _, node := range nodes {
cfg, err := DeclarationFilesConfig(node)
if err != nil {
return err
}
if _, err := js.CreateOrUpdateObjectStore(ctx, cfg); err != nil {
return fmt.Errorf("asserting the object bucket %s: %w", cfg.Bucket, err)
}
}
return nil
}
// DeclarationFilesConfig is a node's bucket as the controller asserts it.
func DeclarationFilesConfig(node string) (jetstream.ObjectStoreConfig, error) {
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return jetstream.ObjectStoreConfig{}, err
}
return jetstream.ObjectStoreConfig{
Bucket: bucket,
Description: "the contents of the files over 64 KiB of the declaration of " + node + ", each under the hex of its sha256 " +
"digest (novox/hq ADR 0486): written by the controller alone, read by the node-engine of that node by name, removed " +
"a day after the node's latest declaration no longer names it",
MaxBytes: DeclarationFilesMaxBytes,
Storage: jetstream.FileStorage,
}, nil
}
@@ -1,316 +0,0 @@
//go:build integration
package broker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/crypto/bcrypt"
)
// nodeEnginesFileObject is the node-engine's own call (mesh-host internal/link files.go, OverNATS.FileObject), as the
// engine makes it: the object's metadata by a direct get of one subject, then its chunks through an ordered consumer
// bound to the stream. It is copied here so the grants are judged by the call that has to work, and its subjects are the
// ones the grant names (issue 1697, notes 4 and 5); the node-engine's own integration test holds the other copy.
func nodeEnginesFileObject(ctx context.Context, nc *nats.Conn, node, name string) ([]byte, error) {
js, err := nc.JetStream(nats.MaxWait(10 * time.Second))
if err != nil {
return nil, err
}
stream, err := DeclarationFilesStreamOf(node)
if err != nil {
return nil, err
}
bucket, err := DeclarationFilesBucketOf(node)
if err != nil {
return nil, err
}
meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
nats.DirectGet(), nats.Context(ctx))
if err != nil {
return nil, err
}
var info struct {
NUID string `json:"nuid"`
Size uint64 `json:"size"`
}
if err := json.Unmarshal(meta.Data, &info); err != nil {
return nil, err
}
sub, err := js.SubscribeSync("$O."+bucket+".C."+info.NUID, nats.OrderedConsumer(), nats.BindStream(stream))
if err != nil {
return nil, err
}
defer sub.Unsubscribe()
body := make([]byte, 0, info.Size)
for uint64(len(body)) < info.Size {
msg, err := sub.NextMsgWithContext(ctx)
if err != nil {
return nil, err
}
body = append(body, msg.Data...)
}
return body, nil
}
// **The buckets' grants on a real server, judged by the node-engine's own call** (novox/hq ADR 0486 §2, issue 1697, 1697/AC2): the
// controller's account writes an object into each node's bucket, a node-engine's account reads one of its own bucket by its name,
// a large object included, and is refused a list of the bucket (a stream info asking for every subject), a message get by
// sequence, a direct get by sequence, a consumer on the names, and a write; it is refused every one of those and the read by name on
// another node's bucket, **the literal-`*` forms included** (a published subject `...M.*` or `...C.*` matches the grant's `*`: on
// its own bucket that reaches its own files, on another's it is refused); another module's account is refused all of them.
func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testing.T) { // 1697/AC2
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
users, err := Users(Records{Nodes: []string{"alpha", "beta"}, Assigned: map[string][]Declared{
"alpha": {{Module: RuntimeModule}, {Module: "other-module"}},
}})
if err != nil {
t.Fatal(err)
}
for i := range users {
users[i].PasswordHash = string(hash)
}
accounts, err := ComposeAccounts(users)
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(s.Shutdown)
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
connect := func(user string) (*nats.Conn, chan error) {
t.Helper()
refused := make(chan error, 16)
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
if err != nil {
t.Fatalf("%s cannot connect: %v", user, err)
}
t.Cleanup(nc.Close)
return nc, refused
}
// The controller makes the buckets (as it does at its start) and writes into each: for alpha a small object and one of
// several chunks, which needs the consumer's flow control; for beta one that holds a marker alpha must never read.
controller, _ := connect(ControllerName)
cjs, err := jetstream.New(controller)
if err != nil {
t.Fatal(err)
}
if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
t.Fatalf("the controller cannot make the buckets: %v", err)
}
if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
t.Fatalf("asserting them again: %v", err)
}
put := func(node string, content []byte) string {
t.Helper()
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(content)
name := hex.EncodeToString(sum[:])
if _, err := store.PutBytes(ctx, name, content); err != nil {
t.Fatalf("the controller cannot write the object %s of %s: %v", name[:8], node, err)
}
return name
}
small := bytes.Repeat([]byte("small file\n"), 100)
large := make([]byte, 900<<10)
for i := range large {
large[i] = byte(i*7 + i>>8)
}
secret := bytes.Repeat([]byte("BETA-SECRET "), 50)
smallName, largeName, betaName := put("alpha", small), put("alpha", large), put("beta", secret)
alpha, alphaRefused := connect("node.alpha")
for name, want := range map[string][]byte{smallName: small, largeName: large} {
got, err := nodeEnginesFileObject(ctx, alpha, "alpha", name)
if err != nil {
t.Fatalf("a node-engine cannot read the object %s of its own bucket by its name: %v", name[:8], err)
}
if !bytes.Equal(got, want) {
t.Fatalf("the node-engine read %d bytes of the object %s, want %d", len(got), name[:8], len(want))
}
}
beta, _ := connect("node.beta")
if got, err := nodeEnginesFileObject(ctx, beta, "beta", betaName); err != nil || !bytes.Equal(got, secret) {
t.Fatalf("beta's node-engine reads its own object: %v", err)
}
select {
case err := <-alphaRefused:
t.Fatalf("a node-engine's own read was refused something: %v", err)
default:
}
// What each account is refused, judged by the server.
alphaStream, alphaBucket := mustStream(t, "alpha"), mustBucket(t, "alpha")
betaStream, betaBucket := mustStream(t, "beta"), mustBucket(t, "beta")
listing := []byte(`{"subjects_filter":">"}`)
bySeq := []byte(`{"seq":1}`)
type refusal struct {
what string
do func(nc *nats.Conn)
}
onBucket := func(stream, bucket, which string) []refusal {
return []refusal{
{which + ": a list of the bucket (stream info asking for every subject)", func(nc *nats.Conn) {
_, _ = nc.Request("$JS.API.STREAM.INFO."+stream, listing, 300*time.Millisecond)
}},
{which + ": a message get by sequence", func(nc *nats.Conn) {
_, _ = nc.Request("$JS.API.STREAM.MSG.GET."+stream, bySeq, 300*time.Millisecond)
}},
{which + ": a direct get by sequence", func(nc *nats.Conn) {
_, _ = nc.Request("$JS.API.DIRECT.GET."+stream, bySeq, 300*time.Millisecond)
}},
{which + ": a consumer on every object's name", func(nc *nats.Conn) {
js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
if sub, err := js.SubscribeSync("$O."+bucket+".M.>", nats.OrderedConsumer(), nats.BindStream(stream)); err == nil {
sub.Unsubscribe()
}
}},
{which + ": a write of an object's metadata", func(nc *nats.Conn) {
_ = nc.Publish("$O."+bucket+".M.abc", []byte(`{}`))
_ = nc.Flush()
}},
{which + ": a write of a chunk", func(nc *nats.Conn) {
_ = nc.Publish("$O."+bucket+".C.abc", []byte("x"))
_ = nc.Flush()
}},
}
}
refusedWith := func(who string, nc *nats.Conn, ch chan error, r refusal) {
t.Helper()
for len(ch) > 0 {
<-ch
}
r.do(nc)
_ = nc.Flush()
deadline := time.After(3 * time.Second)
for {
select {
case err := <-ch:
if errors.Is(err, nats.ErrPermissionViolation) {
return
}
case <-deadline:
t.Errorf("the server did not refuse %s %s", who, r.what)
return
}
}
}
// Alpha's account on its own bucket (list, sequence reads, names, writes) and on beta's (all of that, and the read by name,
// and the literal-`*` probes of the review).
reachBeta := []refusal{
{"beta's bucket: a read of one object by its name", func(nc *nats.Conn) {
_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M."+
base64.URLEncoding.EncodeToString([]byte(betaName)), nil, 300*time.Millisecond)
}},
{"beta's bucket: a direct get with a literal `*` for the name", func(nc *nats.Conn) {
_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M.*", nil, 300*time.Millisecond)
}},
{"beta's bucket: a consumer with a literal `*` for the chunks", func(nc *nats.Conn) {
js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
if sub, err := js.SubscribeSync("$O."+betaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(betaStream)); err == nil {
sub.Unsubscribe()
}
}},
}
all := append(append(onBucket(alphaStream, alphaBucket, "its own bucket"), onBucket(betaStream, betaBucket, "beta's bucket")...), reachBeta...)
for _, r := range all {
nodeNC, nodeCh := connect("node.alpha")
refusedWith("a node-engine", nodeNC, nodeCh, r)
}
// Another module is refused all of that on both buckets, and the reads by name on both.
for _, r := range append(append(onBucket(alphaStream, alphaBucket, "alpha's bucket"), reachBeta...),
refusal{"alpha's bucket: a read of one object by its name", func(nc *nats.Conn) {
_, _ = nc.Request("$JS.API.DIRECT.GET."+alphaStream+".$O."+alphaBucket+".M."+
base64.URLEncoding.EncodeToString([]byte(smallName)), nil, 300*time.Millisecond)
}}) {
otherNC, otherCh := connect("alpha.other-module")
refusedWith("another module", otherNC, otherCh, r)
}
// The literal `*` on alpha's own bucket reaches alpha's own files and never beta's: the chunks it is given hold no marker of beta's.
js, err := alpha.JetStream(nats.MaxWait(2 * time.Second))
if err != nil {
t.Fatal(err)
}
if sub, err := js.SubscribeSync("$O."+alphaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(alphaStream)); err == nil {
for i := 0; i < 20; i++ {
msg, err := sub.NextMsg(300 * time.Millisecond)
if err != nil {
break
}
if bytes.Contains(msg.Data, []byte("BETA-SECRET")) {
t.Fatal("a literal `*` on alpha's bucket gave it a chunk of beta's file")
}
}
sub.Unsubscribe()
}
// Nothing was written by the refused: each bucket holds what the controller wrote.
for node, want := range map[string]int{"alpha": 2, "beta": 1} {
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
if err != nil {
t.Fatal(err)
}
infos, err := store.List(ctx)
if err != nil || len(infos) != want {
t.Errorf("the bucket of %s holds %d objects (%v), want the %d the controller wrote", node, len(infos), err, want)
}
}
}
func mustBucket(t *testing.T, node string) string {
t.Helper()
b, err := DeclarationFilesBucketOf(node)
if err != nil {
t.Fatal(err)
}
return b
}
func mustStream(t *testing.T, node string) string {
t.Helper()
b, err := DeclarationFilesStreamOf(node)
if err != nil {
t.Fatal(err)
}
return b
}
-131
View File
@@ -1,131 +0,0 @@
package broker
import (
"slices"
"strings"
"testing"
)
// 1697/AC1: the composed accounts give a node-engine exactly the four reads of one object by its name, on its own node's bucket and
// no other's; the controller the write of every node's bucket; and no other principal anything of any bucket.
func TestEachNodesBucketIsWrittenByTheControllerAndReadByNameByThatNodesEngine(t *testing.T) { // 1697/AC1
node, err := PermissionsFor(Principal{Kind: KindNode, Node: "alpha", PasswordHash: "x"})
if err != nil {
t.Fatal(err)
}
var mine []string
for _, s := range node.Publish {
if strings.Contains(s, "declaration-files") {
mine = append(mine, s)
}
}
want := []string{
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.*.$O.declaration-files-alpha.C.*",
"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-alpha.*",
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.*",
"$JS.FC.OBJ_declaration-files-alpha.>",
}
if !slices.Equal(mine, want) {
t.Fatalf("a node-engine may publish %q of the buckets, want exactly %q", mine, want)
}
// What would make a list, a write, or a read of another node's files: none of it, in any pattern a node holds. (The real
// server's judgement of the wildcard forms is the live test's.)
for _, refused := range []string{
"$JS.API.STREAM.INFO.OBJ_declaration-files-alpha",
"$JS.API.STREAM.MSG.GET.OBJ_declaration-files-alpha",
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha",
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.abc.def",
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj",
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj.more",
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha",
"$O.declaration-files-alpha.M.abc",
"$O.declaration-files-alpha.C.abc",
// another node's bucket, however it is named
"$JS.API.DIRECT.GET.OBJ_declaration-files-beta.$O.declaration-files-beta.M.YWJj",
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-beta.h4sh.$O.declaration-files-beta.C.NUID",
"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-beta.h4sh",
"$JS.FC.OBJ_declaration-files-beta.x",
// and the old shared bucket
"$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.YWJj",
} {
if MayPublish(node, refused) {
t.Errorf("a node-engine may publish %s, which lists or writes a bucket or reads another node's", refused)
}
}
for _, kept := range []string{
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.YWJj",
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.h4sh.$O.declaration-files-alpha.C.NUID",
} {
if !MayPublish(node, kept) {
t.Errorf("a node-engine may not publish %s, which reads one object of its own bucket by its name", kept)
}
}
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x", FilesFor: []string{"alpha", "beta"}})
if err != nil {
t.Fatal(err)
}
for _, s := range []string{"$O.declaration-files-alpha.M.abc", "$O.declaration-files-beta.C.nuid"} {
if !MayPublish(controller, s) {
t.Errorf("the controller may not write %s", s)
}
}
if MayPublish(controller, "$O.declaration-files-gamma.M.abc") {
t.Error("the controller may write the bucket of a node it was not given")
}
for _, p := range []Principal{
{Kind: KindModule, Module: "mesh-vault", PasswordHash: "x"},
{Kind: KindNodeTools, Node: "alpha", PasswordHash: "x"},
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
} {
perms, err := PermissionsFor(p)
if err != nil {
t.Fatal(err)
}
for _, s := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) {
if strings.Contains(s, "declaration-files") {
t.Errorf("%s holds %s, and no account but the controller's and the node-engines' has any access", p.Username(), s)
}
}
}
}
// The users the controller composes carry each node's bucket to the controller and to that node alone.
func TestTheComposedUsersGiveEachNodeItsOwnBucketAndTheControllerAll(t *testing.T) {
users, err := Users(Records{Nodes: []string{"alpha", "beta"}})
if err != nil {
t.Fatal(err)
}
var controller Principal
for _, u := range users {
if u.Kind == KindController {
controller = u
}
}
if !slices.Equal(controller.FilesFor, []string{"alpha", "beta"}) {
t.Fatalf("the controller writes the buckets of %v", controller.FilesFor)
}
}
// A node name that could carry a subject wildcard or separator into a grant is refused with an error, never composed into a bucket
// (ADR 0488): the grants of such a name would reach another node's bucket.
func TestANodeNameThatCarriesAWildcardGetsNoBucketAndNoGrant(t *testing.T) {
for _, node := range []string{"", "*", ">", "a.b", "a b", "a*"} {
if b, err := DeclarationFilesBucketOf(node); err == nil {
t.Errorf("node %q got the bucket %q", node, b)
}
if _, err := DeclarationFilesReads(node); err == nil {
t.Errorf("node %q got read grants", node)
}
if _, err := DeclarationFilesWrites(node); err == nil {
t.Errorf("node %q got write grants", node)
}
if _, err := DeclarationFilesConfig(node); err == nil {
t.Errorf("node %q got a bucket config", node)
}
}
if b, err := DeclarationFilesBucketOf("g14"); err != nil || b != "declaration-files-g14" {
t.Errorf("g14: %q %v", b, err)
}
}
-25
View File
@@ -104,10 +104,6 @@ type Principal struct {
// Username and inbox are <node>.<module>.<process id>.
Process string
// FilesFor are the nodes whose declaration-file buckets this principal writes, for the controller alone (novox/hq ADR 0486 §2):
// every node of the mesh, one bucket each.
FilesFor []string
Emits []string
Consumes []string
Serves []string
@@ -412,11 +408,6 @@ var DegradableGrants = []DegradableGrant{
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
"holders do (ADR 0406 §3, issue 974)",
},
{
Subject: "$KV." + PlansBuildingBucket + ".>",
Reason: "refused: the controller then says no plan builds, and no check is held while a delivery builds — a check run " +
"wrongly costs a delivery a place, a check held wrongly costs a pull request its turn (ADR 0483 clause 2, issue 1687)",
},
{
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
@@ -745,15 +736,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
}
pub = append(pub, "$KV."+bucket+".>")
}
// **And the object bucket a declaration's large files travel in** (novox/hq ADR 0486 §2, issue 1697): the
// controller alone writes it, as a publish to the bucket's own subjects, which `$JS.API.>` does not cover.
for _, node := range p.FilesFor {
writes, err := DeclarationFilesWrites(node)
if err != nil {
return Permissions{}, err
}
pub = append(pub, writes)
}
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
// consumer that gave up on a message, or one that was deleted. The server already publishes
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
@@ -853,13 +835,6 @@ func PermissionsFor(p Principal) (Permissions, error) {
return Permissions{}, err
}
pub = append(pub, haveKeys...)
// And an object of the declaration bucket by its name, and nothing of the bucket's list or writing
// (novox/hq ADR 0486 §2, issue 1697).
fileReads, err := DeclarationFilesReads(p.Node)
if err != nil {
return Permissions{}, err
}
pub = append(pub, fileReads...)
// And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the
// values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call.
if p.MovesIntoTheVault {
-134
View File
@@ -1,134 +0,0 @@
//go:build integration
package broker
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/crypto/bcrypt"
)
// **A holder of the build seat reads the plans building and the server refuses its write** (novox/hq ADR 0483 clause 2, issue
// 1687): on a real server whose users are composed as the controller composes them, the controller makes the bucket and puts a
// plan's key; the build agent's credential lists the keys and gets one, and its put of a key is refused with a permissions
// violation. It also reads the first waiting ask of a check queue by a direct get, which takes nothing from the queue.
func TestNatsTheBuildSeatsHoldersReadThePlansBuildingAndTheServerRefusesTheirWrite(t *testing.T) { // 1687
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
users, err := Users(Records{Nodes: []string{"alpha"}, Assigned: map[string][]Declared{
"alpha": {{Module: RuntimeModule}, {Module: "build-agent", Holds: []Seat{seat}}},
}})
if err != nil {
t.Fatal(err)
}
for i := range users {
users[i].PasswordHash = string(hash)
}
accounts, err := ComposeAccounts(users)
if err != nil {
t.Fatal(err)
}
conf := filepath.Join(t.TempDir(), "bus.conf")
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatalf("the composed accounts do not parse: %v", err)
}
opts.NoLog, opts.NoSigs = true, true
s, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go s.Start()
if !s.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(s.Shutdown)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
connect := func(user string) (*nats.Conn, chan error) {
t.Helper()
refused := make(chan error, 8)
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
if err != nil {
t.Fatalf("%s cannot connect: %v", user, err)
}
t.Cleanup(nc.Close)
return nc, refused
}
controller, _ := connect(ControllerName)
cj := OnConn(controller)
if err := AssertMeshStreams(cj); err != nil {
t.Fatalf("the controller cannot make the streams: %v", err)
}
if err := cj.EnsureControllerBuckets(); err != nil {
t.Fatalf("the controller cannot make its buckets: %v", err)
}
cjs, _ := jetstream.New(controller)
ckv, err := cjs.KeyValue(ctx, PlansBuildingBucket)
if err != nil {
t.Fatal(err)
}
if _, err := ckv.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
t.Fatalf("the controller cannot put a plan's key: %v", err)
}
holder, refused := connect("alpha.build-agent")
hjs, err := jetstream.New(holder)
if err != nil {
t.Fatal(err)
}
kv, err := hjs.KeyValue(ctx, PlansBuildingBucket)
if err != nil {
t.Fatalf("a holder of the build seat cannot bind %s: %v", PlansBuildingBucket, err)
}
keys, err := kv.Keys(ctx)
if err != nil || len(keys) != 1 || keys[0] != "plan-1" {
t.Fatalf("the holder lists %v (%v), want plan-1", keys, err)
}
if e, err := kv.Get(ctx, "plan-1"); err != nil || e == nil {
t.Fatalf("the holder cannot get the key: %v", err)
}
// Its write is refused by the server.
if err := holder.Publish("$KV."+PlansBuildingBucket+".plan-2", []byte(`{}`)); err != nil {
t.Fatal(err)
}
_ = holder.Flush()
select {
case err := <-refused:
if !errors.Is(err, nats.ErrPermissionViolation) {
t.Errorf("the server said %v, want a permissions violation", err)
}
case <-time.After(3 * time.Second):
t.Error("the server did not refuse the holder's put of a plan's key")
}
if _, err := ckv.Get(ctx, "plan-2"); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Errorf("the refused put left a key: %v", err)
}
// And the direct get of a check queue's first waiting ask, which is granted.
reply, err := holder.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+ChecksOtherStream, []byte(`{"seq":1,"next_by_subj":"`+
CheckSubjectPrefix(TheBuildSeat, false)+`>"}`))
if err != nil {
t.Fatalf("the holder's direct get of %s was not answered: %v", ChecksOtherStream, err)
}
if reply.Header.Get("Status") != "404" {
t.Errorf("an empty queue answered %q, want 404", reply.Header.Get("Status"))
}
}
+2 -2
View File
@@ -29,7 +29,7 @@ accounts {
]
users = [
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_plans-building.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
allow_responses: { max: 1, ttl: "1m" }
} }
@@ -38,7 +38,7 @@ accounts {
subscribe: { allow: ["_INBOX.enrol.one.>"] }
} }
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-one.*.$O.declaration-files-one.C.*", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-one.*", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-one.$O.declaration-files-one.M.*", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.FC.OBJ_declaration-files-one.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
allow_responses: { max: 1, ttl: "1m" }
} }
+1 -1
View File
@@ -135,7 +135,7 @@ type Records struct {
// correct.
func Users(r Records) ([]Principal, error) {
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r),
AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig), FilesFor: sortedCopy(r.Nodes)}}
AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig)}}
for _, node := range sortedCopy(r.Nodes) {
witness := false
-11
View File
@@ -177,10 +177,6 @@ var WritersTable = []WriterRow{
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
{State: "a delivery plan in its build phase", Writer: "controller (lease holder), at the first build's ask and at each refresh",
KeptIn: "key-value " + PlansBuildingBucket + ", aged out when not refreshed",
Others: "the build seat's holders read it and take no new check while a key is there (ADR 0483)",
Subjects: kvOf(PlansBuildingBucket), Writes: isController},
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
@@ -223,13 +219,6 @@ var WritersTable = []WriterRow{
{State: "a machine's have", Writer: "the node-engine of the machine, under its own keys `<node>.>`",
KeptIn: "key-value " + delivery.HaveBucket + ", twenty revisions of each key",
Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys},
// The contents of a declaration's files over 64 KiB (novox/hq ADR 0486 §2): the controller alone writes the bucket, each
// object under the digest of its content; a node-engine reads one by its name, with no list and no write.
{State: "a declaration's large files", Writer: "controller, each object under the sha256 digest of its content",
KeptIn: "the object bucket " + DeclarationFilesBucket + "-<node> of each node, an object removed a day after the " +
"node's latest declaration no longer names it",
Others: "a node-engine reads an object of its own bucket by its name and may neither list the bucket nor write it",
Subjects: []string{"$O.*.>"}, Writes: isController},
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
-4
View File
@@ -29,8 +29,6 @@ var designRows = []string{
"builds and their outcomes",
"a pull request's check asked",
"a pull request's check running",
// The plans in their build phase, a bucket the build seat's holders read (ADR 0483 clause 2, issue 1687).
"a delivery plan in its build phase",
"a merge announced",
"a pull request's head announced",
"a pull request's merge check",
@@ -45,8 +43,6 @@ var designRows = []string{
"the rollouts of the chain",
// What each machine has of each module, written by its node-engine under its own keys (ADR 0429 §1, issue 1136).
"a machine's have",
// The contents of a declaration's large files, in the bus's object bucket (ADR 0486 §2, issue 1697).
"a declaration's large files",
"the declared-config mirror",
"the predefined-keys mirror",
// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).
-15
View File
@@ -1773,24 +1773,9 @@ func whatFailed(s string) string {
}
}
}
// The controller's module check refuses with a count line and one line per problem, then prints its
// warning counts: the refusal is what failed, not the count printed last (novox/hq issue 1310).
for i, line := range lines {
if line = strings.TrimSpace(line); problemsLine.MatchString(line) {
if i+1 < len(lines) {
if first := strings.TrimSpace(lines[i+1]); first != "" {
return line + ": " + first
}
}
return line
}
}
return lastLine(s)
}
// problemsLine is the module check's refusal: "mesh-controller: 3 problem(s) in 139 manifest(s)".
var problemsLine = regexp.MustCompile(`^\S+: [1-9]\d* problem\(s\) in \d+ manifest\(s\)$`)
func lastLine(s string) string {
lines := strings.Split(strings.TrimSpace(s), "\n")
return strings.TrimSpace(lines[len(lines)-1])
-26
View File
@@ -199,29 +199,3 @@ func TestOnlyATrysSelectionReachesTheScript(t *testing.T) {
t.Fatalf("a try's selection: %v", env)
}
}
// **Issue 1310**: the controller's module check prints its warning counts after its refusal; a failed check
// is said by the refusal, never by the count printed last.
func TestAFailedModuleCheckIsSaidByItsRefusalNotItsWarningCount(t *testing.T) {
log := "mesh-controller: 3 problem(s) in 139 manifest(s)\n" +
"mesh-files claims node-files and says it serves upload, download, which that seat's protocol does not promise\n" +
"minio claims the seat \"object-store\", which no module declares and the mesh does not define\n" +
"only-office claims the seat \"document-converter\", which no module declares and the mesh does not define\n" +
"adwaita: ok, 4 tool(s)\nlong-running resources without health: 59\n" +
"files asking for a setting without saying whether it is trusted: 36\n"
// 1310/AC1
want := "mesh-controller: 3 problem(s) in 139 manifest(s): mesh-files claims node-files and says it serves upload, " +
"download, which that seat's protocol does not promise"
if got := whatFailed(log); got != want {
t.Errorf("1310/AC1: the captured log is said as %q, not %q", got, want)
}
// 1310/AC2
if got, want := whatFailed("ok \tx/a\t1s\n--- FAIL: TestX (0.01s)\n x_test.go:12: got 1\nFAIL\nlong-running resources without health: 59\n"),
"--- FAIL: TestX (0.01s)"; got != want {
t.Errorf("1310/AC2: a failing test is said as %q, not %q", got, want)
}
// 1310/AC3
if got, want := whatFailed("starting\nsomething odd happened\n"), "something odd happened"; got != want {
t.Errorf("1310/AC3: a log with no failure line is said as %q, not %q", got, want)
}
}
+2 -3
View File
@@ -100,9 +100,8 @@ var ErrPlansBusy = errors.New("another controller is working the plans")
// returned function and by the session ending, so a controller that dies holding it holds nothing.
//
// wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is
// moving the plans already; and a build outcome's on the event loop, which the build records keep
// for the look to settle (novox/hq issue 1708). wait true looks again every HoldPoll for up to
// HoldWaitFor — a merge's way, which must be written.
// moving the plans already. wait true looks again every HoldPoll for up to HoldWaitFor — an
// outcome's or a merge's way, which must be written.
func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) {
deadline := time.Now().Add(HoldWaitFor)
for {
@@ -1,10 +0,0 @@
-- A build's outcome whose registration the mesh refused, by the build's id (novox/hq issue 1708). The build record says
-- what the builder said, which worked; an outcome the event loop could not take into the plans (another act held them)
-- is settled from the records by the look, and without this it would read the refused build as built. Kept in the
-- store so a controller replaced before the settle still fails the plan.
create table build_refused (
id text primary key,
module text not null,
why text not null,
at timestamptz not null default now()
);
-33
View File
@@ -1,33 +0,0 @@
package inventory
import "context"
// RecordRefused keeps that the mesh refused to register a build's outcome, and why (novox/hq issue 1708): the build
// record says the builder's words, which worked, and the look that settles a plan from the records reads this to fail
// it instead. The first refusal of an id stands.
func (i *Inventory) RecordRefused(ctx context.Context, id, module, why string) error {
_, err := i.store.Pool().Exec(ctx,
`insert into build_refused (id, module, why) values ($1, $2, $3) on conflict (id) do nothing`, id, module, why)
return err
}
// RefusedOf is why each of these builds' registration was refused, by id; a build not refused is absent.
func (i *Inventory) RefusedOf(ctx context.Context, ids []string) (map[string]string, error) {
out := map[string]string{}
if len(ids) == 0 {
return out, nil
}
rows, err := i.store.Pool().Query(ctx, `select id, why from build_refused where id = any($1)`, ids)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var id, why string
if err := rows.Scan(&id, &why); err != nil {
return nil, err
}
out[id] = why
}
return out, rows.Err()
}
-28
View File
@@ -1,28 +0,0 @@
//go:build integration
package inventory
import "testing"
// novox/hq issue 1708 (1708/AC6): a refused registration is kept by its build's id, the first refusal standing, and
// read back for the builds asked about only.
func TestARefusedRegistrationIsKeptByItsBuild(t *testing.T) {
inv := ForTest(t)
ctx := t.Context()
if err := inv.RecordRefused(ctx, "build-1708-a", "nodered", "a port is taken"); err != nil {
t.Fatal(err)
}
if err := inv.RecordRefused(ctx, "build-1708-a", "nodered", "said again"); err != nil {
t.Fatalf("a refusal said twice was refused: %v", err)
}
got, err := inv.RefusedOf(ctx, []string{"build-1708-a", "build-1708-b"})
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got["build-1708-a"] != "a port is taken" {
t.Fatalf("the refusals read back are %v", got)
}
if none, err := inv.RefusedOf(ctx, nil); err != nil || len(none) != 0 {
t.Fatalf("no builds asked about answered %v %v", none, err)
}
}
+54 -203
View File
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"os"
"sync"
"time"
"github.com/nats-io/nats.go"
@@ -163,21 +162,11 @@ type natsMachine struct {
// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
subs []*takeQueue
opts MachineOptions
// gate is what the loop asks of the bus to hold checks while a delivery builds; nil is the bus's (a test sets a fake).
gate checkGate
// said is the faults of the hold already said, letKey whether this holder created the let-through key for the check it
// took last (ADR 0483 clause 3).
said map[string]bool
letKey bool
// mu guards the subscriptions of subs, bound and let go of by the taking loop and closed by Close.
mu sync.Mutex
}
// takeQueue is one queue a machine pulls from.
type takeQueue struct {
sub *nats.Subscription
// pull is what the queue is fetched through: the subscription once bound (a test sets a fake).
pull puller
// kind is what its asks are: a build, a ready check, or another check.
kind string
// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
@@ -185,11 +174,6 @@ type takeQueue struct {
bound bool
}
// puller is the fetch of a queue.
type puller interface {
Fetch(batch int, opts ...nats.PullOpt) ([]*nats.Msg, error)
}
// The kinds of queue a machine takes from, in the order it takes from them.
const (
queueBuilds = "builds"
@@ -211,9 +195,6 @@ type MachineOptions struct {
// the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or
// the other, never neither. Nil claims every one.
Claim func() bool
// LetThroughAfter is how long a check waits while a delivery builds before it is let through; zero is ADR 0483's thirty
// minutes. Set by a test.
LetThroughAfter time.Duration
}
// MachineOverNATS takes build work from the current build role.
@@ -241,8 +222,6 @@ const (
)
func (m *natsMachine) Close() {
m.mu.Lock()
defer m.mu.Unlock()
for _, q := range m.subs {
if q.sub != nil {
_ = q.sub.Unsubscribe()
@@ -265,9 +244,6 @@ func (m *natsMachine) Close() {
// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
//
// **Unless a delivery builds** (novox/hq ADR 0483): while the controller's bucket of the plans building holds a key, no check queue
// is fetched from, but for the one check that waited LetThroughAfter, which goes first (see next).
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
@@ -312,19 +288,6 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
}
continue
}
// The seat's own queue is bound again when a running ask made the machine let go of it (quiesce): a machine that cannot
// is not taking, which is said and tried again.
if !m.subs[0].bound {
if err := m.bind(m.subs[0]); err != nil {
fmt.Fprintf(os.Stderr, "this machine cannot bind %s again after a build, and tries again: %v\n", m.seat, err)
select {
case <-ctx.Done():
return nil
case <-time.After(pausedPoll):
}
continue
}
}
if time.Now().After(bindAgain) {
for _, q := range m.subs[1:] {
if !q.bound {
@@ -349,7 +312,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
// say why (2026-10-03, the first build agents). Asked again.
continue
default:
if m.subs[0].sub != nil && m.subs[0].sub.IsValid() {
if m.subs[0].sub.IsValid() {
// A transient fault in asking — a reconnect, a slow server — is asked past rather
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
time.Sleep(time.Second)
@@ -357,138 +320,62 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
}
return fmt.Errorf("the bus stopped delivering build work: %w", err)
}
m.handOver(ctx, msg, q, do)
// And the queues it let go of are bound again at once, not after the half minute.
bindAgain = time.Time{}
}
}
// handOver settles one fetched ask: dropped, handed back or built. **A check let through keeps its key for as long as it
// stays here** (ADR 0483 clause 3): refreshed while it runs and deleted when it ends, however it ends.
func (m *natsMachine) handOver(ctx context.Context, msg *nats.Msg, q *takeQueue, do func(context.Context, Build)) {
if m.letKey {
defer m.keepLetThrough(ctx, heartbeatWhileWorking)()
}
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
// the one that arrived in that moment. Handed back at once for another holder — counted as
// a delivery, which a pause landing exactly then costs and nothing else does.
if m.opts.Paused != nil && m.opts.Paused() {
_ = msg.Nak()
return
}
var request BuildRequest
if err := json.Unmarshal(msg.Data, &request); err != nil {
// Unreadable: terminated rather than retried, because the next attempt reads the same
// bytes. Nobody waiting hears an answer, which is right — there was no request.
_ = msg.Term()
return
}
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
// id into the seat's cancelled set before deleting the ask, so one taken in between is
// ended here, terminated rather than redelivered, and its outcome said as failed — never
// built. A set that cannot be read is said and the ask built: a cancel is a person's
// exception, and a holder that refused every build while its grant was missing would
// stop the mesh building for it.
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
} else if cancelled {
endCancelled(ctx, build)
return
}
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
if q.kind == queueOther {
if why := m.judgedAlready(ctx, request); why != "" {
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
_ = msg.Ack()
return
}
}
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
// landing then is.
if m.opts.Claim != nil && !m.opts.Claim() {
_ = msg.Nak()
return
}
// **A duplicate of a check another builder is running is dropped** (issue 1649): an ask redelivered to this machine
// because its first holder did not say, in time, that it had it. The running key names that builder and this same
// ask; running it a second time would spend a second place on one check.
if why := m.runningElsewhere(ctx, request); why != "" {
fmt.Fprintf(os.Stderr, "%s is handed out again while another builder runs it and is dropped: %s\n", request.ID, why)
_ = msg.Ack()
return
}
// A build outlives the acknowledgement window many times over; said while it runs,
// as the controller says it for its own long handlers, so the server neither hands
// the ask to a second machine nor counts the wait against its deliveries.
working := make(chan struct{})
go stillWorking(msg, working, heartbeatWhileWorking)
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
// deleted when it ends, whatever the outcome.
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
// **And holds nothing else while it runs** (issue 1747): see quiesce.
m.quiesce(q)
do(ctx, build)
stopRunning()
close(working)
// **The queue it came from too, once the ask is settled**: a fetch that returned a late message leaves its pull request
// open, and the server can deliver one more into the subscription (issue 1747). Take binds every queue again first.
m.quiesce(nil)
}
// quiesce lets go of every queue but the one the running ask came from, for as long as it runs. **A builder never holds a
// message it does not start** (issue 1747). A message the server sends after a fetch gave up, within the few milliseconds its
// pull request still lived, lands in that queue's subscription and stays there until the next fetch of it: held, never
// acknowledged as in progress, for as long as the builder runs its current ask, five or ten minutes for a check, while the
// holders that could build it are all busy and a delivery waits behind it. The legacy pull subscription cannot be read
// without sending a new pull request, which could take one more, so it cannot be searched for such a message. It is dropped
// instead: the subscription is closed, a message in it or on its way is lost to this builder and the server hands it out again
// when its worker's wait for an acknowledgement ends (a minute), to any holder that is free, and the queue is bound again
// before the next fetch (Take). The queue the running ask came from keeps its subscription, which its acknowledgements use.
func (m *natsMachine) quiesce(keep *takeQueue) {
m.mu.Lock()
defer m.mu.Unlock()
for _, q := range m.subs {
if q == keep || !q.bound {
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
// the one that arrived in that moment. Handed back at once for another holder — counted as
// a delivery, which a pause landing exactly then costs and nothing else does.
if m.opts.Paused != nil && m.opts.Paused() {
_ = msg.Nak()
continue
}
if q.sub != nil {
_ = q.sub.Unsubscribe()
var request BuildRequest
if err := json.Unmarshal(msg.Data, &request); err != nil {
// Unreadable: terminated rather than retried, because the next attempt reads the same
// bytes. Nobody waiting hears an answer, which is right — there was no request.
_ = msg.Term()
continue
}
q.sub, q.pull, q.bound = nil, nil, false
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
// id into the seat's cancelled set before deleting the ask, so one taken in between is
// ended here, terminated rather than redelivered, and its outcome said as failed — never
// built. A set that cannot be read is said and the ask built: a cancel is a person's
// exception, and a holder that refused every build while its grant was missing would
// stop the mesh building for it.
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
} else if cancelled {
endCancelled(ctx, build)
continue
}
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
if q.kind == queueOther {
if why := m.judgedAlready(ctx, request); why != "" {
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
_ = msg.Ack()
continue
}
}
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
// landing then is.
if m.opts.Claim != nil && !m.opts.Claim() {
_ = msg.Nak()
continue
}
// A build outlives the acknowledgement window many times over; said while it runs,
// as the controller says it for its own long handlers, so the server neither hands
// the ask to a second machine nor counts the wait against its deliveries.
working := make(chan struct{})
go stillWorking(msg, working, heartbeatWhileWorking)
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
// deleted when it ends, whatever the outcome.
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
do(ctx, build)
stopRunning()
close(working)
}
}
// runningElsewhere says why an ask need not run here, or nothing: the bucket of the checks running says another builder
// runs this very ask (its id, its head and its trunk commit). A key that cannot be read runs the check.
func (m *natsMachine) runningElsewhere(ctx context.Context, r BuildRequest) string {
if r.Check == nil {
return ""
}
running, ok, err := ReadRunning(ctx, m.js.Conn(), r)
if err != nil {
fmt.Fprintf(os.Stderr, "could not read whether %s is running elsewhere, so it runs: %v\n", r.ID, err)
return ""
}
return duplicateOf(running, ok, r, m.on)
}
// runningFreshFor is how recent a running key's last heartbeat must be for its builder to be taken as alive: two missed
// heartbeats of ten seconds are not yet a dead builder, and a redelivery after a crash comes a minute after the last one.
const runningFreshFor = 25 * time.Second
// duplicateOf says why an ask is a second hand-out of one another builder is running, or nothing.
func duplicateOf(running CheckRunning, found bool, r BuildRequest, me string) string {
// Seen is the server's time and runningFreshFor is judged against this builder's clock: a skew over about 35 s would matter.
// A key not renewed for runningFreshFor is a builder that has gone: its ask is redelivered after the ack wait, and
// that is the recovery, not a duplicate (the key itself lives two minutes).
if found && running.Build == r.ID && running.Builder != me && running.Same(r) && time.Since(running.Seen) < runningFreshFor {
return fmt.Sprintf("%s is running it as %s since %s", running.Builder, running.Build, running.Started.Local().Format("15:04:05"))
}
return ""
}
// bind binds a queue's worker, which exists and is not created here.
//
// **The consumer's own filter, not the one subject this machine cares about.** The client checks what is asked for
@@ -496,39 +383,25 @@ func duplicateOf(running CheckRunning, found bool, r BuildRequest, me string) st
// subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned
// twice now, on two different consumers, which is why it is written down here.
func (m *natsMachine) bind(q *takeQueue) error {
m.mu.Lock()
defer m.mu.Unlock()
sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name,
nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck())
if err != nil {
return err
}
q.sub, q.pull, q.bound = sub, sub, true
q.sub, q.bound = sub, true
return nil
}
// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
// on for a second. Nothing, with no error, when there was nothing.
func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
// **While a delivery builds, only delivery builds are taken, but for one check that waited** (novox/hq ADR 0483): the
// bucket of the plans building is read before either check queue is fetched from, and a key there holds both. The check
// that waited LetThroughAfter goes first, ahead of the builds that wait; the rest of the holders take builds only.
if m.seat == broker.TheBuildSeat && len(m.subs) > 1 {
if plans := m.building(ctx); len(plans) > 0 {
if msg, q := m.letOneThrough(ctx); msg != nil {
fmt.Fprintf(os.Stderr, "a check that waited %s is let through while %s builds\n", m.letThroughAfter(), whyHeld(plans))
return msg, q, nil
}
return m.nextBuild(ctx)
}
}
var last error
for _, q := range m.subs {
if !q.bound {
continue
}
trying, end := context.WithTimeout(ctx, checkTakeWait)
fetched, err := q.pull.Fetch(1, nats.Context(trying))
fetched, err := q.sub.Fetch(1, nats.Context(trying))
end()
if len(fetched) > 0 {
return fetched[0], q, nil
@@ -544,18 +417,9 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
if last != nil {
return nil, nil, last
}
return m.nextBuild(ctx)
}
// nextBuild waits for a second at most for a delivery build, the seat's own queue.
func (m *natsMachine) nextBuild(ctx context.Context) (*nats.Msg, *takeQueue, error) {
if !m.subs[0].bound {
// Let go of while an ask ran and not bound again yet (Take binds it before it asks): nothing to take from.
return nil, nil, nil
}
waiting, end := context.WithTimeout(ctx, takeIdleWait)
defer end()
fetched, err := m.subs[0].pull.Fetch(1, nats.Context(waiting))
fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
if len(fetched) > 0 {
return fetched[0], m.subs[0], nil
}
@@ -718,16 +582,3 @@ func endCancelled(ctx context.Context, b *natsBuild) {
}
_ = b.msg.Term()
}
// boundKinds are the kinds of queue this machine holds a subscription to now.
func (m *natsMachine) boundKinds() []string {
m.mu.Lock()
defer m.mu.Unlock()
var out []string
for _, q := range m.subs {
if q.bound {
out = append(out, q.kind)
}
}
return out
}
-8
View File
@@ -184,14 +184,6 @@ func (b OverNATS) PublishMembershipAfter(ctx context.Context, node, module strin
return nil
}
// MaxPayload is the largest message the connected server takes, as the server told this connection.
func (b OverNATS) MaxPayload() int64 {
if b.Conn == nil {
return 0
}
return b.Conn.MaxPayload()
}
func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
if err != nil {
-3
View File
@@ -24,6 +24,3 @@ func ConnectNats(js *broker.JetStream, enroller Enroller, listener Listener) *Se
// Bus is the controller's outbound, whichever transport it connected over. Callers that send a
// declaration or ask a tool use this rather than the channel, which one transport does not have.
func (s *Server) Bus() Bus { return s.bus }
// SendingOn is a server that only sends, over the bus given: for a test of a sender with a bus it holds in its hand.
func SendingOn(bus Bus) *Server { return &Server{bus: bus} }
+3 -24
View File
@@ -253,25 +253,11 @@ func watchPullIn(ctx context.Context, buckets []string, open func(ctx context.Co
// checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it
// waits on the first when all three are empty.
//
// **A fetch's pull request must end before the fetch does** (issue 1649). nats.go gives the server-side request an expiry a tenth
// shorter than the fetch's wait (fetchExpiryOf), and the server then answers with a status message at the expiry; a wait so
// short that the answer can land after the fetch gave up left a status message in the pull's buffer, and anything a request
// could still deliver would be held by a machine that is busy. Each wait leaves at least minFetchMargin between the two.
const (
checkTakeWait = 100 * time.Millisecond
checkTakeWait = 30 * time.Millisecond
takeIdleWait = time.Second
)
// minFetchMargin is the least time between a fetch's server-side expiry and the end of its wait.
const minFetchMargin = 10 * time.Millisecond
// fetchExpiryOf is the expiry nats.go gives the pull request of a fetch that waits this long: the wait less a tenth of it, or
// less five seconds.
func fetchExpiryOf(wait time.Duration) time.Duration {
return wait - min(wait/10, 5*time.Second)
}
// CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406
// §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf).
type CheckRunning struct {
@@ -282,21 +268,15 @@ type CheckRunning struct {
// Started is when the check started, kept on every heartbeat: the entry's own time is the last heartbeat's, and a
// reader that asks how long a check has run reads this (novox/hq ADR 0433 §6).
Started time.Time `json:"started"`
// Seen is when the key was last put, which is the last heartbeat: read from the entry, never written (issue 1649).
Seen time.Time `json:"-"`
}
// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
// composed check; `try.<id>` for a try, which no other request replaces.
// composed check; none for a try, which no other request replaces.
func RunningKeyOf(r BuildRequest) (string, bool) {
c := r.Check
switch {
case c == nil:
case c == nil, c.Try:
return "", false
case c.Try:
// A try says it runs under its own id, which no other request shares: so a reader of the bucket can tell a try that runs
// from a check left waiting for a redelivery (ADR 0483 clause 3).
return "try." + tokenOf(r.ID), true
case c.Group != "":
return "group." + tokenOf(c.Group), true
case c.Number <= 0:
@@ -339,7 +319,6 @@ func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRun
if err := json.Unmarshal(e.Value(), &running); err != nil {
return CheckRunning{}, false, nil // a key of a builder from before this means no match
}
running.Seen = e.Created()
return running, true, nil
}
@@ -510,113 +510,3 @@ func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
stop()
}
}
// An ask handed out a second time to another builder while the first still runs it is acknowledged and not run: the
// running key names the other builder and this same ask (issue 1649).
func TestAnAskHandedOutAgainWhileAnotherBuilderRunsItIsNotRunTwice(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
running := make(chan string, 4)
release := make(chan struct{})
a := MachineOverNATS(js, "builder-a")
defer a.Close()
go func() {
_ = a.Take(ctx, func(ctx context.Context, work Build) {
running <- work.Request().ID
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
check := aPullCheck("build-twice", 7, "head-7-000000000", "trunk-1-000000", false)
if err := asks.Ask(ctx, check); err != nil {
t.Fatal(err)
}
select {
case <-running:
case <-time.After(10 * time.Second):
t.Fatal("builder a took nothing")
}
// The bus's redelivery, as it reaches the other machine: the same ask, the same id.
if err := asks.Ask(ctx, check); err != nil {
t.Fatal(err)
}
b := MachineOverNATS(js, "builder-b")
defer b.Close()
took := make(chan string, 4)
go func() {
_ = b.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Done()
})
}()
select {
case id := <-took:
t.Fatalf("builder b ran %s, which builder a is running", id)
case <-time.After(3 * time.Second):
}
if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
t.Errorf("%d request(s) still wait; builder b was to acknowledge the duplicate", n)
}
close(release)
}
// A busy machine fetches nothing, so no ask waiting in the queue is taken by it, whatever status message sits in its pulls
// (issue 1649). A pull request of our own that expires at once leaves such a message in the other checks' pull.
func TestABusyMachineTakesNoWaitingAskWhateverItsPullsHold(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
busy := make(chan string, 4)
release := make(chan struct{})
a := MachineOverNATS(js, "builder-a").(*natsMachine)
defer a.Close()
go func() {
_ = a.Take(ctx, func(ctx context.Context, work Build) {
busy <- work.Request().ID
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
if err := asks.Ask(ctx, BuildRequest{ID: "busy", Repository: "/d"}); err != nil {
t.Fatal(err)
}
select {
case <-busy:
case <-time.After(10 * time.Second):
t.Fatal("builder a took nothing")
}
// **A busy machine holds no pull at all** (issues 1649 and 1747): the queues the running ask did not come from are let go of,
// so no message, a status message or a late delivery, can sit in one while it runs.
if kinds := a.boundKinds(); len(kinds) != 1 || kinds[0] != queueBuilds {
t.Fatalf("builder a holds %v while it builds; want only the queue its build came from", kinds)
}
other := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
time.Sleep(500 * time.Millisecond)
for i, id := range []string{"wait-1", "wait-2"} {
if err := asks.Ask(ctx, aPullCheck(id, 20+i, "head-x-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
}
time.Sleep(5 * time.Second)
info, err := js.Context().ConsumerInfo(other.worker.Stream, other.worker.Name)
if err != nil {
t.Fatal(err)
}
if info.NumAckPending != 0 || info.NumRedelivered != 0 || info.Delivered.Consumer != 0 {
t.Fatalf("a busy machine took from the queue: %d delivered, %d pending, %d redelivered", info.Delivered.Consumer,
info.NumAckPending, info.NumRedelivered)
}
if n := len(queueOf(t, js).Of(AskWaiting)); n != 2 {
t.Fatalf("%d asks wait; want the 2 nobody took", n)
}
close(release)
}
@@ -1,479 +0,0 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"fmt"
"github.com/nats-io/nats.go"
"reflect"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687, 1688), against a real bus.
// aBusWithTheBuckets is the bus with the build role and the controller's buckets, the bucket of the checks running with an age
// of two seconds so a key a holder stopped refreshing is seen to expire.
func aBusWithTheBuckets(t *testing.T) (*broker.JetStream, jetstream.KeyValue) {
t.Helper()
js := aBusWithTheBuildRole(t)
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
ctx := t.Context()
if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: 2 * time.Second}); err != nil {
t.Fatal(err)
}
plans, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.PlansBuildingBucket, History: 1, TTL: broker.PlansBuildingKeptFor})
if err != nil {
t.Fatal(err)
}
return js, plans
}
func aCheckAskedAgo(ago time.Duration, number int, ready bool) BuildRequest {
r := aPullCheck(fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano()), number, fmt.Sprintf("head-%d-0000000", number), "trunk-0-000000", ready)
return r
}
// 1652/AC6: a plan's builds asked in two tiers with a check waiting, the check is handed out only after the last tier's builds,
// and not in the gap between the tiers.
func TestNatsACheckIsHandedOutOnlyAfterTheLastTierOfAPlan(t *testing.T) {
js, plans := aBusWithTheBuckets(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
t.Fatal(err)
}
if err := asks.Ask(ctx, aCheckAskedAgo(time.Minute, 7, true)); err != nil {
t.Fatal(err)
}
if err := asks.Ask(ctx, BuildRequest{ID: "tier-1-a", Repository: "/a"}); err != nil {
t.Fatal(err)
}
took := make(chan string, 16)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
expect := func(want string) {
t.Helper()
select {
case got := <-took:
if got != want {
t.Fatalf("taken %s, want %s", got, want)
}
case <-time.After(15 * time.Second):
t.Fatalf("%s was not taken", want)
}
}
expect("tier-1-a")
// The gap between two tiers: the delivery's queue is empty, the plan's key is there, and the check is not taken.
select {
case got := <-took:
t.Fatalf("%s was taken in the gap between two tiers", got)
case <-time.After(3 * time.Second):
}
if err := asks.Ask(ctx, BuildRequest{ID: "tier-2-a", Repository: "/b"}); err != nil {
t.Fatal(err)
}
expect("tier-2-a")
select {
case got := <-took:
t.Fatalf("%s was taken before the plan's last build was built and its key deleted", got)
case <-time.After(2 * time.Second):
}
// The phase ends: the key is deleted, and the check is handed out.
if err := plans.Delete(ctx, "plan-1"); err != nil {
t.Fatal(err)
}
select {
case got := <-took:
if got == "tier-1-a" || got == "tier-2-a" {
t.Fatalf("a build was taken twice: %s", got)
}
case <-time.After(15 * time.Second):
t.Fatal("the check was not handed out once the plan's key was deleted")
}
}
// 1688: two holders racing for checks that waited past the bound: exactly one is let through at a time, by a create that fails
// while the key is there, and the key is deleted when its check ends, after which the next oldest goes.
func TestNatsTwoHoldersRacingForOldChecksLetExactlyOneThrough(t *testing.T) {
js, plans := aBusWithTheBuckets(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
// The buckets' age is two seconds here, so the keys are refreshed well within it.
was := heartbeatWhileWorking
heartbeatWhileWorking = 400 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = was })
asks := BuildsOn(js, TheBuildMachine)
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
t.Fatal(err)
}
for i, ago := range []time.Duration{50 * time.Minute, 40 * time.Minute, 35 * time.Minute} {
if err := asks.Ask(ctx, aCheckAskedAgo(ago, i+1, false)); err != nil {
t.Fatal(err)
}
}
var mu sync.Mutex
running, most := 0, 0
started := make(chan string, 8)
release := make(chan struct{})
do := func(ctx context.Context, work Build) {
mu.Lock()
running++
most = max(most, running)
mu.Unlock()
started <- work.Request().Check.Repo + fmt.Sprint(work.Request().Check.Number)
<-release
mu.Lock()
running--
mu.Unlock()
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "x", Failed: "no"})
_ = work.Done()
}
for _, on := range []string{"ace", "shanks"} {
machine := MachineOverNATS(js, on)
defer machine.Close()
go func() { _ = machine.Take(ctx, do) }()
}
var order []string
for range 3 {
select {
case id := <-started:
order = append(order, id)
case <-time.After(20 * time.Second):
t.Fatalf("started so far: %v", order)
}
// While it runs, no second check starts and the key is there.
select {
case id := <-started:
t.Fatalf("%s started beside a check let through", id)
case <-time.After(2500 * time.Millisecond):
}
kv := runningKV(t, js)
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
t.Fatalf("the let-through key is not there while the check runs: %v", err)
}
release <- struct{}{}
// It is deleted when the check ends: the next check let through creates the key again at once, and a create fails
// while the key is there, so its start (the next round) is the proof; after the last one the key stays gone.
if len(order) == 3 {
deadline := time.Now().Add(10 * time.Second)
for {
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
break
}
if time.Now().After(deadline) {
t.Fatal("the let-through key was not deleted when the last check ended")
}
time.Sleep(50 * time.Millisecond)
}
}
}
if want := []string{"a1", "a2", "a3"}; !reflect.DeepEqual(order, want) {
t.Fatalf("checks went %v, want the oldest first %v", order, want)
}
mu.Lock()
defer mu.Unlock()
if most != 1 {
t.Fatalf("%d checks ran at once while a delivery built", most)
}
}
func runningKV(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
t.Helper()
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
kv, err := api.KeyValue(t.Context(), broker.CheckRunningBucket)
if err != nil {
t.Fatal(err)
}
return kv
}
// 1688: the create of the let-through key succeeds for exactly one of holders asking at once, and a key its holder stopped
// refreshing expires, after which the next create succeeds.
func TestNatsOneCreateOfTheLetThroughKeySucceedsAndAStoppedHoldersKeyExpires(t *testing.T) {
js, _ := aBusWithTheBuckets(t)
gate := busGate{conn: js.Conn()}
ctx := t.Context()
var wg sync.WaitGroup
var mu sync.Mutex
won := 0
for i := range 8 {
wg.Add(1)
go func() {
defer wg.Done()
ok, err := gate.TakeLetThrough(ctx, fmt.Sprint("holder-", i))
if err != nil {
t.Error(err)
return
}
if ok {
mu.Lock()
won++
mu.Unlock()
}
}()
}
wg.Wait()
if won != 1 {
t.Fatalf("%d of 8 holders created the let-through key, want exactly one", won)
}
// The holder stopped: nothing refreshes the key, and the bucket's age (two seconds here) removes it.
deadline := time.Now().Add(15 * time.Second)
for {
ok, err := gate.TakeLetThrough(ctx, "next")
if err != nil {
t.Fatal(err)
}
if ok {
break
}
if time.Now().After(deadline) {
t.Fatal("the key of a stopped holder did not expire")
}
time.Sleep(250 * time.Millisecond)
}
// And a refreshed key does not expire.
for range 6 {
if err := gate.KeepLetThrough(ctx, "next"); err != nil {
t.Fatal(err)
}
time.Sleep(700 * time.Millisecond)
}
if ok, _ := gate.TakeLetThrough(ctx, "other"); ok {
t.Fatal("a refreshed key expired")
}
var v LetThrough
e, err := runningKV(t, js).Get(ctx, LetThroughKey)
if err != nil || json.Unmarshal(e.Value(), &v) != nil || v.Builder != "next" {
t.Fatalf("the key says %+v (%v)", v, err)
}
}
// 1687: the oldest waiting ask is read without being taken: no delivery is counted against it.
func TestNatsTheOldestWaitingAskIsReadWithoutTakingIt(t *testing.T) {
js, _ := aBusWithTheBuckets(t)
ctx := t.Context()
asks := BuildsOn(js, TheBuildMachine)
old := aCheckAskedAgo(45*time.Minute, 1, false)
if err := asks.Ask(ctx, old); err != nil {
t.Fatal(err)
}
gate := busGate{conn: js.Conn()}
q := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
at, found, err := gate.OldestWaiting(ctx, q)
if err != nil || !found {
t.Fatalf("the first waiting ask was not read: %v %v", found, err)
}
if want, _ := BuildAskedAt(old.ID); !at.Equal(want) {
t.Fatalf("read %v, asked %v", at, want)
}
for range 3 {
if _, _, err := gate.OldestWaiting(ctx, q); err != nil {
t.Fatal(err)
}
}
info, err := js.Context().ConsumerInfo(q.worker.Stream, q.worker.Name)
if err != nil {
t.Fatal(err)
}
if info.Delivered.Consumer != 0 || info.NumAckPending != 0 || info.NumPending != 1 {
t.Fatalf("reading the age took the ask: delivered %d, pending acks %d, waiting %d", info.Delivered.Consumer, info.NumAckPending, info.NumPending)
}
// And an empty queue reads as none.
if _, found, err := gate.OldestWaiting(ctx, &takeQueue{kind: queueReady, worker: broker.CheckWorkers()[0]}); err != nil || found {
t.Fatalf("an empty queue read as %v (%v)", found, err)
}
}
// Issue 1747: a message that reaches a builder's subscription outside a fetch is not held while the builder runs another ask. A
// stray check lands in the other queue's subscription of builder A, which then runs a long check; the server hands the stray
// out again when its wait for an acknowledgement ends (two seconds here), builder B runs it, and A, finished, does not run it a
// second time from the subscription it held.
func TestNatsAMessageDeliveredOutsideAFetchIsNotHeldWhileTheBuilderRunsAnotherAsk(t *testing.T) { // 1747
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
for _, w := range broker.CheckWorkers() {
w.AckWaitSeconds = 2
if err := js.EnsureConsumer(w); err != nil {
t.Fatal(err)
}
}
// The long check says it is alive well within the ack wait of two seconds.
was := heartbeatWhileWorking
heartbeatWhileWorking = 400 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = was })
asks := BuildsOn(js, TheBuildMachine)
long := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()), 1, "head-1-0000000", "trunk-0-000000", true)
stray := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()+1), 2, "head-2-0000000", "trunk-0-000000", false)
for _, r := range []BuildRequest{long, stray} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
var mu sync.Mutex
runs := map[string][]string{} // ask id -> the machines that ran it
release := make(chan struct{})
started := make(chan string, 8)
do := func(on string) func(context.Context, Build) {
return func(ctx context.Context, work Build) {
id := work.Request().ID
mu.Lock()
runs[id] = append(runs[id], on)
mu.Unlock()
started <- on + ":" + id
if id == long.ID {
<-release
}
_ = work.Announce(ctx, BuildResult{ID: id, On: on, Failed: "no"})
_ = work.Done()
}
}
var a *natsMachine
strayLanded := make(chan error, 1)
a = MachineOverNATSWith(js, "ace", TheBuildMachine, MachineOptions{Claim: func() bool {
// The moment the long check is claimed: the server hands the other queue's waiting check to A's subscription there, as
// one that arrives after a fetch gave up does. Done once.
select {
case <-strayLanded:
return true
default:
}
other := a.subs[2]
inbox := strings.TrimSuffix(other.sub.Subject, "*") + "stray"
body, _ := json.Marshal(map[string]any{"batch": 1, "expires": int64(30 * time.Second)})
if err := js.Conn().PublishRequest("$JS.API.CONSUMER.MSG.NEXT."+other.worker.Stream+"."+other.worker.Name, inbox, body); err != nil {
strayLanded <- err
return true
}
_ = js.Conn().Flush()
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
if n, _, _ := other.sub.Pending(); n > 0 {
break
}
}
strayLanded <- nil
return true
}}).(*natsMachine)
defer a.Close()
go func() { _ = a.Take(ctx, do("ace")) }()
select {
case got := <-started:
if got != "ace:"+long.ID {
t.Fatalf("started %s, want the long check on ace first", got)
}
case <-time.After(15 * time.Second):
t.Fatal("the long check did not start")
}
if err := <-strayLanded; err != nil {
t.Fatal(err)
}
// B, free, runs the stray once the server hands it out again.
b := MachineOverNATS(js, "shanks")
defer b.Close()
go func() { _ = b.Take(ctx, do("shanks")) }()
select {
case got := <-started:
if got != "shanks:"+stray.ID {
t.Fatalf("started %s, want the stray on shanks", got)
}
case <-time.After(20 * time.Second):
t.Fatal("the stray was held by the builder that ran the long check and nobody ran it")
}
// A finishes, and does not run the stray from the subscription it held.
release <- struct{}{}
select {
case got := <-started:
t.Fatalf("%s started after the long check: the stray ran twice", got)
case <-time.After(6 * time.Second):
}
mu.Lock()
defer mu.Unlock()
if len(runs[stray.ID]) != 1 {
t.Fatalf("the stray ran on %v, want once", runs[stray.ID])
}
}
// Review of 1688: a young check pending redelivery (handed back by a Nak) stands ahead of a check older than 30 minutes in the
// same queue while a plan builds. The let-through fetch would return the young one first, so nothing is fetched: the young
// check's delivery count stays 1, it is not handed back every second until it is dead-lettered, and it is not run.
func TestNatsAYoungRedeliveryAheadOfAnOldCheckIsNeverFetchedByTheLetThrough(t *testing.T) {
js, plans := aBusWithTheBuckets(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
young := aCheckAskedAgo(time.Minute, 1, false)
if err := asks.Ask(ctx, young); err != nil {
t.Fatal(err)
}
// Someone took it and handed it back: it is pending redelivery.
worker := broker.CheckWorkers()[1]
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
if err != nil || len(got) != 1 {
t.Fatalf("the young check was not fetched to be handed back: %v", err)
}
if err := got[0].Nak(); err != nil {
t.Fatal(err)
}
_ = sub.Unsubscribe()
old := aCheckAskedAgo(45*time.Minute, 2, false)
if err := asks.Ask(ctx, old); err != nil {
t.Fatal(err)
}
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
t.Fatal(err)
}
started := make(chan string, 4)
machine := MachineOverNATS(js, "ace")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
started <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "ace", Failed: "no"})
_ = work.Done()
})
}()
select {
case id := <-started:
t.Fatalf("%s was run while the young redelivery stood ahead of it", id)
case <-time.After(6 * time.Second):
}
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
if err != nil {
t.Fatal(err)
}
if info.Delivered.Consumer != 1 || info.NumRedelivered != 0 {
t.Fatalf("the young check was handed out %d time(s), %d redelivered: the let-through fetched it", info.Delivered.Consumer, info.NumRedelivered)
}
}
+1 -42
View File
@@ -41,54 +41,13 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
return fmt.Errorf("cannot sign a declaration for %s: %w", node, err)
}
body, err := Envelope(declaration, signature)
body, err := json.Marshal(Signed{Declaration: declaration, Signature: signature})
if err != nil {
return err
}
// **Measured before it is sent** (novox/hq issue 1689): the bare bus error said only "maximum payload
// exceeded", on 2026-10-21 for the control node's declaration, 2 KB over. The limit is the connection's own.
if limit := payloadLimitOf(bus); limit > 0 {
size := int64(len(body))
if DeclarationMeasured != nil {
DeclarationMeasured(ctx, node, size, limit)
}
if size > limit {
return fmt.Errorf("%s's declaration is %d bytes on the wire (%d bytes of declaration, signed and encoded), "+
"over the bus's largest message of %d bytes by %d: it cannot be sent (novox/hq issue 1689)",
node, size, len(declaration), limit, size-limit)
}
}
publish, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
return bus.PublishDeclaration(publish, node, body)
}
// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON
// document. Its length is what the bus's largest message bounds.
func Envelope(declaration, signature []byte) ([]byte, error) {
return json.Marshal(Signed{Declaration: declaration, Signature: signature})
}
// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that
// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486
// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline).
const NearTheLimit = 0.75
// DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message,
// so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689).
// Set by the controller; nil in a test or a tool.
var DeclarationMeasured func(ctx context.Context, node string, size, limit int64)
// payloadLimiter is a bus that knows its largest message.
type payloadLimiter interface{ MaxPayload() int64 }
// payloadLimitOf is the bus's largest message in bytes, or 0 when the bus does not say.
func payloadLimitOf(bus Bus) int64 {
if l, ok := bus.(payloadLimiter); ok {
return l.MaxPayload()
}
return 0
}
@@ -1,52 +0,0 @@
//go:build integration
package link
import (
"context"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// Against a real bus whose largest message is small: the limit is read from the connection, the one that fits goes
// through, the one over is refused by us with its size and the limit, not by the client's bare error (issue 1689).
func TestADeclarationIsMeasuredAgainstTheRealBusLimit(t *testing.T) {
s := testbus.StartWithMaxPayload(t, 2048)
conn, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
js, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := js.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}}); err != nil {
t.Fatal(err)
}
bus := OverNATS{Conn: conn, JS: js}
if bus.MaxPayload() != 2048 {
t.Fatalf("the limit read from the connection is %d, expected 2048", bus.MaxPayload())
}
var told []int64
was := DeclarationMeasured
DeclarationMeasured = func(_ context.Context, _ string, size, limit int64) { told = append(told, size, limit) }
t.Cleanup(func() { DeclarationMeasured = was })
fits := []byte(`{"a":"` + strings.Repeat("x", 600) + `"}`)
if err := Declare(t.Context(), bus, fixedSigner{}, "novox", fits, 5*time.Second); err != nil {
t.Fatalf("a declaration under the limit: %v", err)
}
over := []byte(`{"a":"` + strings.Repeat("x", 1600) + `"}`)
err = Declare(t.Context(), bus, fixedSigner{}, "novox", over, 5*time.Second)
if err == nil || !strings.Contains(err.Error(), "2048 bytes") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("a declaration over the limit: %v", err)
}
if len(told) != 4 || told[1] != 2048 || told[2] <= 2048 {
t.Fatalf("measured %v", told)
}
}
-57
View File
@@ -1,57 +0,0 @@
package link
import (
"context"
"strings"
"testing"
"time"
)
type roomBus struct {
Bus
limit int64
sent [][]byte
}
func (b *roomBus) MaxPayload() int64 { return b.limit }
func (b *roomBus) PublishDeclaration(_ context.Context, _ string, body []byte) error {
b.sent = append(b.sent, body)
return nil
}
type fixedSigner struct{}
func (fixedSigner) Sign(context.Context, []byte) ([]byte, error) { return []byte("sig"), nil }
// A declaration is measured as it goes on the wire against the bus's own limit; one over it is refused saying size and
// limit, and what the measure sees is told to whoever raises a condition (issue 1689).
func TestADeclarationOverTheBusLimitIsRefusedSayingSizeAndLimit(t *testing.T) {
var seen []int64
was := DeclarationMeasured
DeclarationMeasured = func(_ context.Context, node string, size, limit int64) {
if node != "novox" || limit != 400 {
t.Errorf("measured %s against %d", node, limit)
}
seen = append(seen, size)
}
t.Cleanup(func() { DeclarationMeasured = was })
bus := &roomBus{limit: 400}
small := []byte(`{"a":"` + strings.Repeat("x", 100) + `"}`)
if err := Declare(t.Context(), bus, fixedSigner{}, "novox", small, time.Second); err != nil || len(bus.sent) != 1 {
t.Fatalf("a small declaration: %v, %d sent", err, len(bus.sent))
}
if int64(len(bus.sent[0])) != seen[0] {
t.Fatalf("measured %d bytes, sent %d", seen[0], len(bus.sent[0]))
}
big := []byte(`{"a":"` + strings.Repeat("x", 300) + `"}`)
err := Declare(t.Context(), bus, fixedSigner{}, "novox", big, time.Second)
if err == nil || !strings.Contains(err.Error(), "400 bytes") || !strings.Contains(err.Error(), "novox") ||
!strings.Contains(err.Error(), "over") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("an oversize declaration: %v", err)
}
if len(bus.sent) != 1 || len(seen) != 2 || seen[1] <= 400 {
t.Fatalf("it was sent (%d) or not measured (%v)", len(bus.sent), seen)
}
}
-43
View File
@@ -1,43 +0,0 @@
package link
import (
"testing"
"time"
)
// An ask a machine holds but does not start gets no heartbeat, and the bus hands it to another machine after the ack
// wait, so two machines run one check (issue 1649).
// A fetch's server-side pull request ends clearly before the fetch does, so the server's answer and any message it could still
// deliver come inside the fetch and never into a buffer nobody reads while the machine builds.
func TestAFetchsPullRequestEndsClearlyBeforeTheFetchDoes(t *testing.T) {
for name, wait := range map[string]time.Duration{"a check queue": checkTakeWait, "the idle wait": takeIdleWait} {
if margin := wait - fetchExpiryOf(wait); margin < minFetchMargin {
t.Errorf("%s: the pull request ends %v before a fetch of %v does; want at least %v", name, margin, wait, minFetchMargin)
}
}
}
func TestAnAskAnotherBuilderIsRunningIsADuplicateAndYoursIsNot(t *testing.T) {
r := aPullCheck("build-1", 423, "head-4771ee53aa", "trunk-0123456789", false)
run := CheckRunning{Head: r.Ref, Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-1", Started: time.Now().Add(-time.Minute), Seen: time.Now().Add(-4 * time.Second)}
dead := run
dead.Seen = time.Now().Add(-60 * time.Second)
for name, c := range map[string]struct {
running CheckRunning
found bool
me string
dup bool
}{
"another builder runs this very ask": {run, true, "ace", true},
"its builder stopped renewing the key a minute ago": {dead, true, "ace", false},
"this builder runs it itself": {run, true, "shanks", false},
"nobody runs it": {CheckRunning{}, false, "ace", false},
"another ask of the same head": {CheckRunning{Head: r.Ref, Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-0"}, true, "ace", false},
"the same ask at another head": {CheckRunning{Head: "head-other-0000", Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-1"}, true, "ace", false},
} {
if got := duplicateOf(c.running, c.found, r, c.me) != ""; got != c.dup {
t.Errorf("%s: duplicate = %v; want %v", name, got, c.dup)
}
}
}
-384
View File
@@ -1,384 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687 and 1688).
//
// **The layout other parts read** (ADR 0484 clause 12 has the checker read the same key; a layout shared between modules
// belongs in mesh-sdk, and this one is not there yet):
//
// - bucket `mesh-controller_plans-building` (broker.PlansBuildingBucket): one key per delivery plan in its build phase, named
// by the plan's id (`plan-<digits>`), value a PlanBuilding as JSON. Written by the controller (lease holder) alone: put when
// the plan's first build is asked, put again about every thirty seconds, deleted when the phase ends; the bucket's age
// (two minutes) removes the key of a controller that stopped. A reader lists the keys and needs no value to decide.
// - key `let-through` in bucket `mesh-controller_checks-running` (broker.CheckRunningBucket), value a LetThrough as JSON:
// created by the holder that takes a check which waited LetThroughAfter, refreshed at each heartbeat of that check, deleted
// when it ends. At most one such check runs at a time on the build seat, because the create fails while the key is there.
// LetThroughAfter is how long a check waits, while a delivery builds, before it is let through (ADR 0483 clause 3).
const LetThroughAfter = 30 * time.Minute
// LetThroughKey is the key of the one check let through, in the bucket of the checks running. It has no dot, so it is no
// pull request's key (`<owner>.<repo>.<number>`) and no group's (`group.<name>`).
const LetThroughKey = "let-through"
// PlanBuilding is the value of a plan's key in the bucket of the plans building.
type PlanBuilding struct {
Plan string `json:"plan"`
Since time.Time `json:"since"`
Seen time.Time `json:"seen"`
}
// LetThrough is the value of the let-through key: the builder that runs the check and when it took it.
type LetThrough struct {
Builder string `json:"builder"`
Since time.Time `json:"since"`
}
// checkGate is what a taking loop asks of the bus to hold checks while a delivery builds; a variable of the machine so a test
// hands out against fakes.
type checkGate interface {
// Building is the ids of the plans in their build phase: the keys of the bucket. An error is a bucket that cannot be read.
Building(ctx context.Context) ([]string, error)
// OldestWaiting is when the first waiting ask of a check queue was asked, read without taking it.
OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error)
// TakeLetThrough creates the let-through key for a builder: false when it is there (another builder runs a check let through).
TakeLetThrough(ctx context.Context, builder string) (bool, error)
// KeepLetThrough puts the key again, and DropLetThrough deletes it.
KeepLetThrough(ctx context.Context, builder string) error
DropLetThrough(ctx context.Context) error
}
// busGate is the gate over a connection.
type busGate struct{ conn *nats.Conn }
func (g busGate) Building(ctx context.Context) ([]string, error) {
return BuildingPlans(ctx, g.conn)
}
// BuildingPlans is the ids of the delivery plans in their build phase, from the controller's bucket. A reader that is not the
// build seat's (the checker, ADR 0484 clause 12) calls this. A bucket that cannot be read is an error; none is none building.
func BuildingPlans(ctx context.Context, conn *nats.Conn) ([]string, error) {
api, err := jetstream.New(conn)
if err != nil {
return nil, err
}
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
if err != nil {
return nil, fmt.Errorf("the plans building: %w", err)
}
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("the plans building: %w", err)
}
return keys, nil
}
// OldestWaiting is when the check a fetch of this queue would return was asked, read without taking anything.
//
// **It sees redeliveries** (review of issue 1688). A work queue keeps only what is not acknowledged: what the worker handed out
// (sequence up to its delivered mark) is running or waiting to be handed out again, and a fetch returns those before anything
// new. An ask there that is not said running (a key of the checks running names its build) is such a redelivery. If one is
// younger than the bound, the queue's next fetch would return it, and a young check is never let through: the queue is not a
// candidate, and nothing is fetched, so nothing is handed back and no delivery is spent. Otherwise the candidate is the oldest
// of the redeliveries and the first ask after the delivered mark.
func (g busGate) OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error) {
info, err := jsOf(g.conn).ConsumerInfo(q.worker.Stream, q.worker.Name, nats.Context(ctx))
if err != nil {
return time.Time{}, false, fmt.Errorf("the %s worker: %w", q.kind, err)
}
var oldest time.Time
note := func(at time.Time) {
if oldest.IsZero() || at.Before(oldest) {
oldest = at
}
}
for seq := uint64(1); seq <= info.Delivered.Stream; {
data, stored, got, err := g.directGet(ctx, q, seq)
if err != nil {
return time.Time{}, false, err
}
if got == 0 || got > info.Delivered.Stream {
break
}
seq = got + 1
var r BuildRequest
if json.Unmarshal(data, &r) != nil {
continue
}
running, ok, err := ReadRunning(ctx, g.conn, r)
if err != nil {
return time.Time{}, false, fmt.Errorf("the checks running: %w", err)
}
if ok && running.Build == r.ID {
continue // running now
}
at := askedAtOf(data, stored)
if time.Since(at) < LetThroughAfter {
return time.Time{}, false, nil // a young redelivery is next in line
}
note(at)
}
data, stored, got, err := g.directGet(ctx, q, info.Delivered.Stream+1)
if err != nil {
return time.Time{}, false, err
}
if got != 0 {
note(askedAtOf(data, stored))
}
return oldest, !oldest.IsZero(), nil
}
// directGet reads the first ask at or after a sequence of a check queue's stream, without taking it: its body, the time the
// stream stored it, and its sequence (0 when there is none).
func (g busGate) directGet(ctx context.Context, q *takeQueue, seq uint64) ([]byte, string, uint64, error) {
body, _ := json.Marshal(map[string]any{"seq": seq, "next_by_subj": q.worker.Filters[0]})
reply, err := g.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+q.worker.Stream, body)
if err != nil {
return nil, "", 0, fmt.Errorf("the asks of the %s queue: %w", q.kind, err)
}
if st := reply.Header.Get("Status"); st == "404" || st == "408" {
return nil, "", 0, nil
}
got, _ := strconv.ParseUint(reply.Header.Get("Nats-Sequence"), 10, 64)
return reply.Data, reply.Header.Get("Nats-Time-Stamp"), got, nil
}
// askedAtOf is when a request was asked: its id says (NewBuildID), the stream's own time otherwise, and a request that says
// neither is taken as asked now, so it is never let through on a guess.
func askedAtOf(data []byte, stored string) time.Time {
var r BuildRequest
if json.Unmarshal(data, &r) == nil {
if at, ok := BuildAskedAt(r.ID); ok {
return at
}
}
if at, err := time.Parse(time.RFC3339Nano, stored); err == nil {
return at
}
return time.Now()
}
func jsOf(conn *nats.Conn) nats.JetStreamContext {
js, _ := conn.JetStream()
return js
}
func (g busGate) TakeLetThrough(ctx context.Context, builder string) (bool, error) {
kv, err := runningBucket(ctx, g.conn)
if err != nil {
return false, err
}
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
if _, err := kv.Create(ctx, LetThroughKey, body); err != nil {
if errors.Is(err, jetstream.ErrKeyExists) {
return false, nil
}
return false, err
}
return true, nil
}
func (g busGate) KeepLetThrough(ctx context.Context, builder string) error {
kv, err := runningBucket(ctx, g.conn)
if err != nil {
return err
}
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
_, err = kv.Put(ctx, LetThroughKey, body)
return err
}
func (g busGate) DropLetThrough(ctx context.Context) error {
kv, err := runningBucket(ctx, g.conn)
if err != nil {
return err
}
if err := kv.Delete(ctx, LetThroughKey); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
return err
}
return nil
}
// CheckHold is what a reader of the queue (the controller's own verb) says of why checks wait: the plans building and the check
// let through now, if one.
type CheckHold struct {
Plans []string
LetThrough *LetThrough
}
// ReadCheckHold reads the plans building and the let-through key. A bucket that cannot be read is an error the caller says.
func ReadCheckHold(ctx context.Context, conn *nats.Conn) (CheckHold, error) {
var h CheckHold
plans, err := BuildingPlans(ctx, conn)
if err != nil {
return h, err
}
h.Plans = plans
kv, err := runningBucket(ctx, conn)
if err != nil {
return h, err
}
e, err := kv.Get(ctx, LetThroughKey)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return h, nil
}
if err != nil {
return h, err
}
var l LetThrough
if json.Unmarshal(e.Value(), &l) == nil {
h.LetThrough = &l
}
return h, nil
}
// gateOf is the machine's gate: the one a test set, or the bus's.
func (m *natsMachine) gateOf() checkGate {
if m.gate != nil {
return m.gate
}
return busGate{conn: m.js.Conn()}
}
// sayOnce says a fault of the hold the first time it happens and again once it has recovered and happened anew, so a bucket
// that cannot be read is in the log once and not every second.
func (m *natsMachine) sayOnce(kind string, err error) {
if m.said == nil {
m.said = map[string]bool{}
}
if err == nil {
delete(m.said, kind)
return
}
if !m.said[kind] {
m.said[kind] = true
fmt.Fprintf(os.Stderr, "%s; read as empty, so no check is held for it: %v\n", kind, err)
}
}
// building says whether a delivery builds: any key in the bucket of the plans building. **A bucket that cannot be read is said
// and read as empty** (ADR 0483 clause 2): a check wrongly held costs a pull request its turn, a fault that held every check
// would stop the mesh checking.
func (m *natsMachine) building(ctx context.Context) []string {
plans, err := m.gateOf().Building(ctx)
m.sayOnce("the bucket of the plans building cannot be read", err)
if err != nil {
// empty-on-error: ADR 0483 clause 2 — a bucket that cannot be read is said (sayOnce) and read as empty, because a check wrongly held costs a pull request its turn and a fault that held every check would stop the mesh checking.
return nil
}
return plans
}
// letOneThrough takes the oldest waiting check of the two check queues once it waited LetThroughAfter, by the one let-through
// key: nil when none waited that long, when another holder runs the check let through, or when the check was taken by
// another holder in the moment. The caller owns the key while it returns a message and releases it with dropLetThrough.
func (m *natsMachine) letOneThrough(ctx context.Context) (*nats.Msg, *takeQueue) {
gate := m.gateOf()
var oldest *takeQueue
var oldestAt time.Time
for _, q := range m.subs[1:] {
if !q.bound {
continue
}
at, found, err := gate.OldestWaiting(ctx, q)
m.sayOnce("the first waiting check of the "+q.kind+" queue cannot be read", err)
if err != nil || !found {
continue
}
if oldest == nil || at.Before(oldestAt) {
oldest, oldestAt = q, at
}
}
if oldest == nil || time.Since(oldestAt) < m.letThroughAfter() {
return nil, nil
}
created, err := gate.TakeLetThrough(ctx, m.on)
m.sayOnce("the let-through key cannot be taken", err)
if err != nil || !created {
return nil, nil
}
trying, end := context.WithTimeout(ctx, checkTakeWait)
defer end()
fetched, err := oldest.pull.Fetch(1, nats.Context(trying))
if len(fetched) == 0 {
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.Canceled) {
m.sayOnce("the "+oldest.kind+" check queue cannot be taken from", err)
}
m.dropLetThrough(ctx)
return nil, nil
}
// Another holder took the one that was oldest in the moment between the read and the fetch: what came is a younger check,
// handed back and the key released, never run during a delivery.
if askedAtOf(fetched[0].Data, "").Add(m.letThroughAfter()).After(time.Now()) {
_ = fetched[0].NakWithDelay(time.Minute) // a delay keeps it off the queue of redeliveries a fetch reads first
m.dropLetThrough(ctx)
return nil, nil
}
m.letKey = true
return fetched[0], oldest
}
func (m *natsMachine) letThroughAfter() time.Duration {
if m.opts.LetThroughAfter > 0 {
return m.opts.LetThroughAfter
}
return LetThroughAfter
}
// dropLetThrough deletes the let-through key this holder created.
func (m *natsMachine) dropLetThrough(ctx context.Context) {
m.letKey = false
if err := m.gateOf().DropLetThrough(context.WithoutCancel(ctx)); err != nil {
fmt.Fprintf(os.Stderr, "could not take the let-through key away; it ages out: %v\n", err)
}
}
// keepLetThrough refreshes the key at every heartbeat while the check it was created for runs, and returns the end of that: it
// stops the refreshing and deletes the key.
func (m *natsMachine) keepLetThrough(ctx context.Context, every time.Duration) (end func()) {
done := make(chan struct{})
finished := make(chan struct{})
go func() {
defer close(finished)
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
case <-done:
return
case <-tick.C:
putting, stop := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
if err := m.gateOf().KeepLetThrough(putting, m.on); err != nil {
fmt.Fprintf(os.Stderr, "could not keep the let-through key: %v\n", err)
}
stop()
}
}
}()
return func() {
close(done)
<-finished
m.dropLetThrough(ctx)
}
}
// whyHeld words the plans a check is held for, for the log.
func whyHeld(plans []string) string { return strings.Join(plans, ", ") }
-307
View File
@@ -1,307 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1687 and 1688): the order the taking loop hands out in, against
// fake queues and a fake bus.
// fakeQueue is a queue the loop fetches from: the asks waiting, handed out in order, and how often it was fetched from.
type fakeQueue struct {
mu sync.Mutex
waiting []*nats.Msg
fetched int
}
func (f *fakeQueue) Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.fetched++
if len(f.waiting) == 0 {
return nil, nats.ErrTimeout
}
m := f.waiting[0]
f.waiting = f.waiting[1:]
return []*nats.Msg{m}, nil
}
func (f *fakeQueue) fetches() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.fetched
}
// askedAgo is a request asked this long ago, as the id says, for a check of this pull request.
func askedAgo(ago time.Duration, number int, ready bool) *nats.Msg {
id := fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano())
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "r", Check: &CheckRequest{Owner: "o", Repo: "r", Number: number, Ready: ready}})
return &nats.Msg{Data: body}
}
func buildAsk() *nats.Msg {
body, _ := json.Marshal(BuildRequest{ID: fmt.Sprintf("build-%d", time.Now().UnixNano()), Repository: "r"})
return &nats.Msg{Data: body}
}
// fakeGate is the bus as the loop sees it for the hold.
type fakeGate struct {
mu sync.Mutex
plans []string
plansErr error
letKey bool
taken int
kept int
dropped int
oldestFrom map[*takeQueue]*fakeQueue
}
func (g *fakeGate) Building(context.Context) ([]string, error) {
g.mu.Lock()
defer g.mu.Unlock()
return g.plans, g.plansErr
}
// OldestWaiting reads the first waiting ask of the fake queue without taking it.
func (g *fakeGate) OldestWaiting(_ context.Context, q *takeQueue) (time.Time, bool, error) {
f := g.oldestFrom[q]
f.mu.Lock()
defer f.mu.Unlock()
if len(f.waiting) == 0 {
return time.Time{}, false, nil
}
return askedAtOf(f.waiting[0].Data, ""), true, nil
}
func (g *fakeGate) TakeLetThrough(context.Context, string) (bool, error) {
g.mu.Lock()
defer g.mu.Unlock()
if g.letKey {
return false, nil
}
g.letKey = true
g.taken++
return true, nil
}
func (g *fakeGate) KeepLetThrough(context.Context, string) error {
g.mu.Lock()
defer g.mu.Unlock()
g.kept++
return nil
}
func (g *fakeGate) DropLetThrough(context.Context) error {
g.mu.Lock()
defer g.mu.Unlock()
g.letKey = false
g.dropped++
return nil
}
func (g *fakeGate) setPlans(plans ...string) {
g.mu.Lock()
defer g.mu.Unlock()
g.plans = plans
}
// aMachine is a loop on the build seat with a delivery queue and the two check queues, all fake.
func aMachine() (m *natsMachine, builds, ready, other *fakeQueue, gate *fakeGate) {
builds, ready, other = &fakeQueue{}, &fakeQueue{}, &fakeQueue{}
subs := []*takeQueue{
{kind: queueBuilds, pull: builds, bound: true},
{kind: queueReady, pull: ready, bound: true},
{kind: queueOther, pull: other, bound: true},
}
gate = &fakeGate{oldestFrom: map[*takeQueue]*fakeQueue{subs[1]: ready, subs[2]: other}}
m = &natsMachine{on: "ace", seat: broker.TheBuildSeat, subs: subs, gate: gate}
return m, builds, ready, other, gate
}
func nextAsk(t *testing.T, m *natsMachine) (*nats.Msg, *takeQueue) {
t.Helper()
msg, q, err := m.next(context.Background())
if err != nil {
t.Fatal(err)
}
return msg, q
}
// 1652/AC6, 1687: a waiting check is not taken while a plan key is there, also when the delivery's queue is empty for the moment
// (between two tiers).
func TestNoCheckIsTakenWhileAPlanBuilds(t *testing.T) {
m, builds, ready, other, gate := aMachine()
gate.setPlans("plan-1")
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
other.waiting = []*nats.Msg{askedAgo(5*time.Minute, 2, false)}
// The delivery's queue is empty: between two tiers.
if msg, _ := nextAsk(t, m); msg != nil {
t.Fatalf("a check was taken while plan-1 builds: %s", msg.Data)
}
if ready.fetches() != 0 || other.fetches() != 0 {
t.Fatalf("the check queues were fetched from while plan-1 builds: ready %d, other %d", ready.fetches(), other.fetches())
}
if builds.fetches() == 0 {
t.Fatal("the delivery's queue was not looked at")
}
}
// A delivery build is still taken first, and the check waits behind it.
func TestADeliveryBuildIsTakenWhileAPlanBuilds(t *testing.T) {
m, builds, ready, _, gate := aMachine()
gate.setPlans("plan-1")
builds.waiting = []*nats.Msg{buildAsk()}
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
msg, q := nextAsk(t, m)
if msg == nil || q.kind != queueBuilds {
t.Fatalf("the delivery build was not taken first: %v %v", msg, q)
}
if ready.fetches() != 0 {
t.Fatal("a check queue was fetched from")
}
}
// It is taken once the key is gone (deleted at the phase's end, or expired).
func TestACheckIsTakenOnceTheKeyIsGone(t *testing.T) {
m, _, ready, _, gate := aMachine()
gate.setPlans("plan-1")
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
if msg, _ := nextAsk(t, m); msg != nil {
t.Fatal("taken while the key is there")
}
gate.setPlans()
msg, q := nextAsk(t, m)
if msg == nil || q.kind != queueReady {
t.Fatalf("the check was not taken once the key was gone: %v", q)
}
if m.letKey {
t.Fatal("a check taken without a delivery building took the let-through key")
}
}
// 1687: a bucket that cannot be read is said and read as empty, so the checks are taken.
func TestAnUnreadableBucketHoldsNoCheck(t *testing.T) {
m, _, ready, _, gate := aMachine()
gate.plansErr = errors.New("permissions violation")
gate.plans = []string{"plan-1"}
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
msg, q := nextAsk(t, m)
if msg == nil || q.kind != queueReady {
t.Fatal("a check was held because the bucket could not be read")
}
if !m.said["the bucket of the plans building cannot be read"] {
t.Fatalf("the fault was not said: %v", m.said)
}
}
// 1688: a check asked 30 minutes ago is let through while a key is there; the oldest across both queues by ask time, ahead of a
// waiting delivery build; a younger one is never taken before it.
func TestTheOldestCheckPastTheBoundGoesFirstAcrossBothQueues(t *testing.T) {
m, builds, ready, other, gate := aMachine()
gate.setPlans("plan-1")
builds.waiting = []*nats.Msg{buildAsk()}
ready.waiting = []*nats.Msg{askedAgo(31*time.Minute, 1, true)}
other.waiting = []*nats.Msg{askedAgo(50*time.Minute, 2, false)}
msg, q := nextAsk(t, m)
if msg == nil || q.kind != queueOther || !strings.Contains(string(msg.Data), `"number":2`) {
t.Fatalf("the oldest check (50 minutes, other queue) was not let through first: %v", q)
}
if !m.letKey || gate.taken != 1 {
t.Fatalf("the let-through key was not created for it: held %v, created %d", m.letKey, gate.taken)
}
}
// 1688: several checks past the bound go one at a time, oldest first: while the key is there every other holder takes builds.
func TestChecksPastTheBoundGoOneAtATimeOldestFirst(t *testing.T) {
m, builds, ready, other, gate := aMachine()
gate.setPlans("plan-1")
ready.waiting = []*nats.Msg{askedAgo(33*time.Minute, 1, true), askedAgo(31*time.Minute, 3, true)}
other.waiting = []*nats.Msg{askedAgo(40*time.Minute, 2, false)}
var order []string
for range 3 {
msg, q := nextAsk(t, m)
if msg == nil {
t.Fatalf("no check let through after %v", order)
}
var r BuildRequest
_ = json.Unmarshal(msg.Data, &r)
order = append(order, fmt.Sprint(r.Check.Number, q.kind))
// While it runs another holder (the same fake gate, key held) takes builds only.
builds.waiting = []*nats.Msg{buildAsk()}
other2 := &natsMachine{on: "shanks", seat: broker.TheBuildSeat, subs: m.subs, gate: gate}
if next, nq := nextAsk(t, other2); next == nil || nq.kind != queueBuilds {
t.Fatalf("a second holder did not take a delivery build while the let-through key is held: %v", nq)
}
m.dropLetThrough(context.Background())
}
if want := []string{"2other", "1ready", "3ready"}; strings.Join(order, " ") != strings.Join(want, " ") {
t.Fatalf("checks went %v, want %v (oldest first, one at a time)", order, want)
}
}
// 1688: a check younger than the bound is never let through, and no key is created for it.
func TestACheckYoungerThanTheBoundIsHeld(t *testing.T) {
m, _, ready, _, gate := aMachine()
gate.setPlans("plan-1")
ready.waiting = []*nats.Msg{askedAgo(29*time.Minute, 1, true)}
if msg, _ := nextAsk(t, m); msg != nil {
t.Fatal("a check of 29 minutes was let through")
}
if gate.taken != 0 {
t.Fatal("a let-through key was created for a young check")
}
if ready.fetches() != 0 {
t.Fatal("the age was read by a fetch; it must be read without taking")
}
}
// 1688: the holder that lost the race for the key, or the check, takes builds only; and a younger check that arrives in the
// moment between the read and the fetch is handed back with the key released.
func TestAYoungerCheckFetchedInTheRaceIsHandedBackAndTheKeyReleased(t *testing.T) {
m, _, ready, _, gate := aMachine()
gate.setPlans("plan-1")
// The read sees the old check; before the fetch another holder takes it and a young one is first.
gate.oldestFrom[m.subs[1]] = &fakeQueue{waiting: []*nats.Msg{askedAgo(40*time.Minute, 1, true)}}
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
if msg, _ := nextAsk(t, m); msg != nil {
t.Fatalf("a young check was run during a delivery: %s", msg.Data)
}
if gate.letKey || m.letKey {
t.Fatal("the let-through key was left held")
}
}
// The key of a check let through is refreshed while it runs and deleted when it ends.
func TestTheLetThroughKeyLivesAsLongAsTheCheckRuns(t *testing.T) {
m, _, ready, _, gate := aMachine()
gate.setPlans("plan-1")
ready.waiting = []*nats.Msg{askedAgo(40*time.Minute, 1, true)}
if msg, _ := nextAsk(t, m); msg == nil {
t.Fatal("not let through")
}
end := m.keepLetThrough(context.Background(), 20*time.Millisecond)
time.Sleep(120 * time.Millisecond)
gate.mu.Lock()
held, kept := gate.letKey, gate.kept
gate.mu.Unlock()
if !held || kept == 0 {
t.Errorf("while the check ran the key was held %v and refreshed %d times", held, kept)
}
end()
gate.mu.Lock()
defer gate.mu.Unlock()
if gate.letKey || m.letKey {
t.Error("the key was not deleted when the check ended")
}
}
-44
View File
@@ -88,40 +88,6 @@ type Queue struct {
AckPending int `json:"ack-pending"`
MaxDeliver int `json:"max-deliver"`
Asks []QueuedAsk `json:"asks"`
// Hold is why the checks wait: the delivery plans building and the check let through now (ADR 0483, issues 1687 and 1688);
// HoldFault says the bucket could not be read, in which case Hold is empty.
Hold CheckHold `json:"hold"`
HoldFault string `json:"hold-fault,omitempty"`
}
// LetThroughOf says which ask is let through under ADR 0483 clause 3: the check that waited LetThroughAfter or longer and is next
// (the oldest waiting check across the two check queues, while a delivery builds), and the check in flight that was let through
// (the let-through key names the machine that runs it). Empty when none.
func (q Queue) LetThroughOf(now time.Time) (next, running string) {
if len(q.Hold.Plans) == 0 && q.Hold.LetThrough == nil {
return "", ""
}
var oldest *QueuedAsk
for i := range q.Asks {
a := &q.Asks[i]
if a.State != AskWaiting || a.Request.Check == nil || a.AskedAt.IsZero() {
continue
}
if oldest == nil || a.AskedAt.Before(oldest.AskedAt) {
oldest = a
}
}
if oldest != nil && len(q.Hold.Plans) > 0 && now.Sub(oldest.AskedAt) >= LetThroughAfter {
next = oldest.ID
}
if q.Hold.LetThrough != nil {
for _, a := range q.Asks {
if a.State == AskInFlight && a.Request.Check != nil && a.On == q.Hold.LetThrough.Builder {
running = a.ID
}
}
}
return next, running
}
// Of is the asks in one state, in queue order.
@@ -353,16 +319,6 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
for _, r := range reads {
q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
}
// And why the checks wait, for the build seat that has them (ADR 0483 clause 5). A bucket that cannot be read is said by the
// queue, which then names no plan.
if seat == broker.TheBuildSeat {
hold, err := ReadCheckHold(ctx, js.Conn())
if err != nil {
q.HoldFault = err.Error()
} else {
q.Hold = hold
}
}
return q, nil
}
-54
View File
@@ -1,54 +0,0 @@
package link
import (
"testing"
"time"
"github.com/nats-io/nats.go"
)
// Issue 1747: a builder never holds a message it does not start. A message the server sends after a fetch gave up lands in that
// queue's subscription; while an ask runs, the queues it does not come from are let go of, so such a message is not held
// unheartbeated for the whole run, and the queues are bound again before the next fetch.
func TestARunningAskLetsGoOfTheQueuesItDidNotComeFrom(t *testing.T) {
m, builds, _, other, _ := aMachine()
// A delivery build reached the builds queue's subscription after its fetch gave up: it sits in the buffer.
builds.waiting = []*nats.Msg{buildAsk()}
took := m.subs[1] // the running ask came from the ready queue
m.quiesce(took)
if !took.bound || took.pull == nil {
t.Error("the queue the running ask came from was let go of; its acknowledgements need it")
}
for _, q := range []*takeQueue{m.subs[0], m.subs[2]} {
if q.bound || q.pull != nil || q.sub != nil {
t.Errorf("the %s queue is still held while an ask runs", q.kind)
}
}
// Nothing is fetched from a queue that was let go of: a message in it cannot be taken unheartbeated.
if msg, _, _ := m.next(t.Context()); msg != nil {
t.Fatalf("a message was taken from a queue that was let go of: %s", msg.Data)
}
if builds.fetches() != 0 || other.fetches() != 0 {
t.Errorf("fetched from a queue that was let go of: builds %d, other %d", builds.fetches(), other.fetches())
}
}
// Once the ask is settled the queue it came from is let go of too: a second message the server delivered into its subscription
// on a pull request still open is not run after the ask ends (review of 1747).
func TestTheQueueTheAskCameFromIsLetGoOfWhenTheAskEnds(t *testing.T) {
m, _, ready, _, _ := aMachine()
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
m.quiesce(nil)
for _, q := range m.subs {
if q.bound || q.pull != nil {
t.Errorf("the %s queue is still held after the ask ended", q.kind)
}
}
m.subs[0].bound, m.subs[0].pull = true, &fakeQueue{}
if msg, _, _ := m.next(t.Context()); msg != nil || ready.fetches() != 0 {
t.Fatal("a message held in the kept queue's subscription was taken after the ask ended")
}
}
+2 -6
View File
@@ -56,14 +56,10 @@ var buses sync.Map
// Start is a server with JetStream, for this test alone: on loopback, on a port the system chooses, its
// store in the test's own directory.
func Start(t testing.TB) *server.Server { return StartWithMaxPayload(t, 0) }
// StartWithMaxPayload is Start with the largest message the server takes set to maxPayload bytes; zero keeps the
// server's default (1 MiB, as the mesh's bus has).
func StartWithMaxPayload(t testing.TB, maxPayload int32) *server.Server {
func Start(t testing.TB) *server.Server {
t.Helper()
opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(),
NoLog: true, NoSigs: true, MaxPayload: maxPayload}
NoLog: true, NoSigs: true}
s, err := server.NewServer(opts)
if err != nil {
t.Fatalf("a bus for this test could not be made: %v", err)
+2 -2
View File
@@ -1,5 +1,5 @@
module route-proxy
go 1.26.0
go 1.25.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6
+4 -2
View File
@@ -1,2 +1,4 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0 h1:EK84so3Y8JdFT3BHrzj9cPQQrDUPa/d+IdOGnerlmEI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261016174426-a4e7f480b3b0/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6 h1:Sv1Qzix+U1RpaUkw5IbB0IJEyZ+FqgzHmbZrCZkcWf0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
@@ -17,5 +17,3 @@ at the same version as the TypeScript package.
What it holds, under ADR 0039's test: the stdio loop and the tool type, and `cli`, the local protocol
`mesh-cli` speaks to the node-engine on its machine (novox/hq ADR 0272): the socket's path and one request and one
answer; and `seal`, the mesh's sealed box (novox/hq ADR 0206: X25519, HKDF-SHA256, AES-256-GCM, `{v, eph, iv, tag, ct}`), byte for byte the box the licence manager made before it lived here and the TypeScript `@novox/mesh-sdk/seal` makes. Nothing else — no bus, no client, nothing of any module's. `go build ./example` and pipe a `tools/list` request into it.
From release 4a of design 47 (novox/hq ADR 0453 clause 3, ADR 0446 clause 4) it also holds the composition of a machine's declaration, over a plain input and with nothing behind it: `declaration` and `validate`, the node-engine's parser and its validator, copied from mesh-host; `catalogue`, the manifests, their resolution and rendering, and `overlay`, the private network's generator, copied from mesh-controller; `compose`, the controller's planFor, renderingFor and declarationWith as functions of an `Input` (the composition never allocates, makes, keeps or writes: what a send would do it names as foreseen or returns); `sdkversion`, which mesh-sdk version a program was built with; and, in `delivery`, the layouts of the keys the builder, the checker and the indexer write (the source fingerprint's new form, the builds, the verdicts and kept verdicts, the manifests by digest). None of them holds a bus, a store or a vault client, which `go list -deps` of `compose` shows.
@@ -5,127 +5,16 @@
# seat in the mesh's Go toolchain, its own container, because the SDK's merge-check.sh declares it
# (`# mesh-check-also: go go/merge-check.sh`) — and by hand, from anywhere: `sh go/merge-check.sh`.
#
# Formatted, vetted (with and without the build tag `integration`), and run at two levels (novox/hq ADR 0391):
#
# 1. the unit level: `go test ./...` untagged, no bus or other process, timed and held to a bound
# (unit_bound below); over it the check fails and names the slowest tests (issue 895);
# 2. the integration level: the whole suite with `-tags integration`, each test against a bus of its
# own (internal/testbus, which issue 1171 brings with the first test that needs a bus) so nothing is
# serialised. With no tagged test yet, the level runs the unit tests again and says their count.
#
# **A test that starts a bus or another process carries `//go:build integration`** as the first line of
# its file; a file that held both kinds is two files (`<name>_integration_test.go` beside `<name>_test.go`).
# Each level says its test count and its time.
#
# No dependency is committed (no vendor directory: .gitignore; the operator, 2026-10-19; issue 1304). Every dependency
# is resolved by the version and checksum go.sum names (novox/hq ADR 0405 §5), from its public source: this binding
# requires nothing of the mesh's own forge. `go mod verify` fails by module name when a downloaded dependency does not
# match go.sum, and -mod=readonly fails a build whose go.sum lacks an entry instead of writing it. gofmt reads only
# the binding's own files.
#
# Exit 0 is a pass; anything else fails `mesh/repo-check` with the last line printed.
# Formatted, vetted, and its whole suite run: the stdio loop and the local protocol mesh-cli speaks to the
# node-engine. Standard library only, so nothing is fetched.
set -eu
cd "$(dirname "$0")"
# The toolchain the check runs in sets -mod=vendor; there is no vendor directory, so it is replaced.
GOFLAGS="$(printf '%s' "${GOFLAGS:-}" | sed 's/-mod=[a-z]*//g') -mod=readonly"
export GOFLAGS
export GOPRIVATE=git.novox.be
# **The unit level's bound, in seconds** (issue 895): the slowest package's own test time of the untagged run,
# the compile not counted. A run over it fails the check and names the slowest tests. The SDK's unit suite is
# small (its slowest package measured well under a second on a quiet machine), so 2 s leaves room for a busier
# build node without hiding a test that should carry `integration` instead.
unit_bound=2
# judge_unit_run says the unit level as `go test -v` reported it and fails when it was over the bound: $1 the
# file holding the output of the untagged `go test -v` of every package (packages one after another, each
# ending in its `ok <package> <time>` line), $2 the bound in seconds. Packages run side by side, so the
# level's time is that of its slowest package; the five slowest tests of the run are named when it is over,
# so that the bound is kept by making them cheap or by tagging them `integration`, and never by raising it
# (issue 895).
judge_unit_run() {
out=$1 bound=$2
awk '
/^--- (PASS|FAIL|SKIP): / {
name = $3; took = $4; gsub(/[()s]/, "", took)
pending[++n] = took "\t" name
if ($2 == "SKIP:") skipped++
next
}
/^(ok|FAIL)[ \t]/ && NF >= 3 && $3 ~ /^[0-9.]+s$/ {
t = $3; sub(/s$/, "", t)
for (i = 1; i <= n; i++) print "T\t" pending[i] "\t" $2
n = 0
print "P\t" t "\t" $2
}
END { print "S\t" skipped + 0 }
' "$out" >"$out.judged"
tests=$(grep -c '^T' "$out.judged" || true)
skipped=$(awk -F'\t' '$1 == "S" {print $2}' "$out.judged")
level=$(awk -F'\t' '$1 == "P" && $2 + 0 > m {m = $2 + 0} END {printf "%.2f", m}' "$out.judged")
echo "unit level: $tests tests ($skipped skipped), $level s (bound $bound s)"
if awk -v t="$level" -v b="$bound" 'BEGIN {exit !(t + 0 > b + 0)}'; then
echo "the unit level took $level s, over its bound of $bound s; the slowest tests:"
awk -F'\t' '$1 == "T" {print $2 "\t" $4 "\t" $3}' "$out.judged" | sort -t"$(printf '\t')" -k1,1 -nr | head -n 5 |
awk -F'\t' '{printf " %s s %s %s\n", $1, $3, $2}'
echo "make them cheap (no wait, no process), or give their files //go:build integration if they start a bus or another process"
return 1
fi
}
# `sh go/merge-check.sh judge-unit-run <output> <bound>` runs only the judging of the unit level, on a
# recorded output.
if [ "${1:-}" = judge-unit-run ]; then
shift
judge_unit_run "$@"
exit
fi
if ! fetched=$(go mod download 2>&1); then
echo "FAILED: the dependencies cannot be fetched: $(printf '%s\n' "$fetched" | tail -n 1)"
exit 1
fi
if ! verified=$(go mod verify 2>&1); then
echo "FAILED: a dependency does not match go.sum: $verified"
exit 1
fi
unformatted=$(find . -name '*.go' -not -path './vendor/*' | xargs gofmt -l)
unformatted=$(gofmt -l .)
if [ -n "$unformatted" ]; then
echo "not gofmt'd:"
echo "$unformatted"
exit 1
fi
go vet ./...
# The files tagged `integration` are vetted too, or nothing would read them.
go vet -tags integration ./...
# NOT RACE-CHECKED here: the SDK's suite is small and carries no store or other shared state between
# packages, so the race detector is not required of this check (novox/hq issue 1175); a module that needs
# it asks for it in its own merge-check.sh.
echo "NOT RACE-CHECKED: the Go binding's check runs without the race detector"
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
# The unit level first and alone: every package untagged, no bus or other process (judged above).
status=0
go test -count=1 -v ./... >"$work/unit.out" 2>&1 || status=$?
if [ "$status" -ne 0 ]; then
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/unit.out"
echo "the unit level failed (go test exited $status)"
exit 1
fi
judge_unit_run "$work/unit.out" "$unit_bound"
# The integration level: the whole suite with the tag, each test against its own bus (internal/testbus).
integration_start=$(date +%s)
status=0
go test -tags integration -count=1 -v ./... >"$work/integration.out" 2>&1 || status=$?
grep -vE '^(=== |[[:space:]]*--- PASS)' "$work/integration.out" || true
tests=$(grep -cE '^--- (PASS|FAIL|SKIP): ' "$work/integration.out" || true)
skipped=$(grep -cE '^--- SKIP: ' "$work/integration.out" || true)
echo "integration level: $tests tests ($skipped skipped), $(($(date +%s) - integration_start)) s"
if [ "$status" -ne 0 ]; then
echo "the integration level failed (go test exited $status)"
exit 1
fi
go test -count=1 ./...
@@ -57,11 +57,7 @@ type message struct {
// channel is the one conversation with the runtime: what this bundle writes, serialised, and what
// it asked and waits for.
type channel struct {
mu sync.Mutex
// wmu orders the writes to the runtime and nothing else: a write waits on the pipe for as long as
// the runtime does not read, and mu must never wait with it, or the loop that reads the runtime's
// answers stops (issue 1286).
wmu sync.Mutex
mu sync.Mutex
enc *json.Encoder
next int
waiting map[string]chan message
@@ -94,8 +90,8 @@ type Envelope struct {
func (c *channel) say(m map[string]any) {
m["jsonrpc"] = "2.0"
c.wmu.Lock()
defer c.wmu.Unlock()
c.mu.Lock()
defer c.mu.Unlock()
_ = c.enc.Encode(m)
}
@@ -476,15 +472,10 @@ func matchFrom(p []string, pi int, k []string, ki int) bool {
return matchFrom(p, pi+1, k, ki+1)
}
// EmitOptions is the provenance an emitter may attach (ADR 0042), and the message id it may choose.
// EmitOptions is the provenance an emitter may attach (ADR 0042).
type EmitOptions struct {
CausationID string
Schema string
// MessageID is the id this emit is sent under, as x-event-id (and, by the tool runner, as the bus's
// own Nats-Msg-Id): chosen by the emitter so a repeated publish inside the stream's duplicate window
// is kept to one, never generated once an id is given (novox/hq ADR 0436 clause 4). Empty is a random
// one, as on main.
MessageID string
}
// Emit asks the runtime to publish an event as this bundle's module, and returns once the bus has
@@ -504,17 +495,8 @@ func Emit(eventType string, body any, opts ...EmitOptions) error {
if node == "" {
node = "unknown"
}
id := ""
for _, o := range opts {
if o.MessageID != "" {
id = o.MessageID
}
}
if id == "" {
id = eventID()
}
headers := map[string]string{
"x-event-id": id, "x-source": source, "x-node": node,
"x-event-id": eventID(), "x-source": source, "x-node": node,
"x-time": time.Now().UTC().Format(time.RFC3339Nano), "content-type": "application/json",
}
for _, o := range opts {
+2 -2
View File
@@ -1,4 +1,4 @@
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2
## explicit; go 1.26.0
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018003858-f185aedbf5d6
## explicit; go 1.22
git.novox.be/novox/mesh-sdk/go
git.novox.be/novox/mesh-sdk/go/provider
+1 -1
View File
@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile
mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock
mesh-sdk 725590e06e75c67b96ce58d015b3056c2d261249 conformance/events/module-event.json
mesh-sdk c8e35097e8611c174c7937d9ade86ed292314e9a conformance/events/module-event.json
To move them, from this repository's root, with the two repositories checked out beside it:
-1
View File
@@ -167,4 +167,3 @@
./cmd/mesh-controller TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule
./cmd/mesh-controller TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued
./cmd/mesh-controller TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent
./internal/inventory TestARefusedRegistrationIsKeptByItsBuild
@@ -74,22 +74,6 @@ func filesVerbs() []Verb {
map[string]string{"path": "an absolute path", "owner": "an account's name or number",
"group": "a group's name or number (optional)", "recursive": "\"true\": also everything under a directory"},
[]string{"path", "owner"}, []string{"group", "recursive"}, "chown", "chgrp"),
Verb{Name: "upload", Description: "Send one file of this node into a bucket of the object store, over a short signed link " +
"and never over the bus; answers the key (the file's SHA-256 when none is given), the size and the SHA-256. " +
"At most 100 MiB. A place a secret is kept, a known credential file, or a link to one is refused by name, " +
"before anything is sent. Read " + filesAsAccount + ".",
Input: schema(map[string]string{"path": "an absolute path to a file", "bucket": "the bucket the file goes into",
"key": "the object's key (optional; the file's SHA-256 when not given)"}, []string{"path", "bucket"})},
// download is a plain Verb, not through write(): write() types its optional arguments as true/false
// switches, which a hash and a reason are not (issue 1719).
Verb{Name: "download", Description: "Write one object of a bucket to a file of this node, over a short signed link and " +
"never over the bus; the old file goes to the node's trash. At most 100 MiB. The bytes must hash to sha256, or to " +
"the key when the key is a SHA-256; otherwise no file is written. Refuses a path a module owns " +
"unless override names a reason; done as the node's operator account, through sudo -n where its rights do not reach.",
Input: schema(map[string]string{"bucket": "the bucket", "key": "the object's key", "path": "an absolute path",
"sha256": "the SHA-256 the bytes must have (optional; the key when that is a SHA-256)",
"reason": "why the file is changed, for the audit trail",
"override": overrideWord}, []string{"bucket", "key", "path", "reason"})},
Verb{Name: "trash", Description: "What the node's trash holds: each deleted or replaced item with its id, " +
"original path and when it went.",
Input: schema(map[string]string{}, nil), Replaces: []string{"ls ~/.local/share/Trash"}},
@@ -2214,7 +2214,6 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
problems = append(problems, replacesProblems(m)...)
problems = append(problems, UnitSettingProblems(m)...)
problems = append(problems, OwnerSettingProblems(m)...)
problems = append(problems, GroupSettingProblems(m)...)
problems = append(problems, endpointNameProblems(m)...)
problems = append(problems, RouteProblems(m)...)
for _, port := range m.Guards {
+1 -18
View File
@@ -904,7 +904,7 @@ func isModule(catalogue map[string]Manifest, want string) bool {
// machine can host the module as far as its capabilities go.
func missingCapabilities(m Manifest, node Node) []string {
var missing []string
for _, c := range append(append([]string(nil), m.Capabilities...), impliedCapabilities(m)...) {
for _, c := range m.Capabilities {
if !node.Capabilities[c] {
missing = append(missing, c)
}
@@ -1457,20 +1457,3 @@ func packageNames(m map[string][]string) []string {
sort.Strings(names)
return names
}
// ContainerGroups is the capability a node-engine reports when it reads a container's `groups` and passes each as a
// supplementary group (novox/hq ADR 0480 clause 7, issue 1638). An older engine refuses the key (strict decoding).
const ContainerGroups = "container-groups"
// impliedCapabilities are the capabilities a module needs for what its resources say, beyond those it declares: a
// container that names groups needs an engine that reads them, so the module is left out of a machine whose report
// lacks the capability, naming it, and the field is sent to no engine that would refuse it. None for a module whose
// containers name no groups.
func impliedCapabilities(m Manifest) []string {
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) == "container" && len(groupsOf(r)) > 0 {
return []string{ContainerGroups}
}
}
return nil
}
@@ -66,8 +66,7 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
// names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its
// contribution out on every machine, silently, so it is refused. A detector added there is added here.
var KnownCapabilities = []string{
"battery", "container-groups", "container-runtime", "file-references", "firewall", "graphical-session", "overlay",
"package-manager",
"battery", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager",
"power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager",
"uplink-systemd-networkd", "user-namespaces", "virtualisation",
}
@@ -85,9 +85,6 @@ func settingInto(resource map[string]any, layers []Layer, module string) error {
if fmt.Sprint(resource["type"]) == "directory" {
return settingIntoOwner(resource, layers, module)
}
if fmt.Sprint(resource["type"]) == "container" {
return settingIntoGroups(resource, layers, module)
}
if fmt.Sprint(resource["type"]) != "file" {
return nil
}
@@ -327,7 +324,7 @@ func settingKeysUsedBy(m Manifest) map[string]bool {
return used
}
// settingKeysAskedOutsideOwners is every key a module's files, units, contributions and served facts
// settingKeysAskedOutsideOwners is every key a module's files, units, items, contributions and served facts
// ask for: what a directory's owner may follow (OwnerSettingProblems).
func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
used := map[string]bool{}
@@ -346,14 +343,12 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
if unit, ok := r["unit"].(string); ok {
note(unit)
}
case "container":
for _, g := range groupsOf(r) {
if s, ok := g.(string); ok {
note(s)
}
}
}
}
// An item's text, filled from its template (WithTemplatesFrom), places a key as a file's does (issue 1658).
for _, it := range m.Items {
note(it.Content)
}
inValues := func(values map[string]any) {
for _, v := range values {
if s, ok := v.(string); ok {
@@ -374,100 +369,3 @@ func settingKeysAskedOutsideOwners(m Manifest) map[string]bool {
}
return used
}
// A container's supplementary groups may be an operator's value (novox/hq ADR 0480 clause 7, ADR 0348 §11, issue 1636):
// n8n joins the media library's group, whose number is the machine's, never the definition's (ADR 0112 §1). Each
// element of `groups` is a whole placeholder or a whole number; a placeholder is filled with the number the
// config value holds, and the list reaches the node-engine as numbers.
// groupSide is one element of a container's groups: a whole placeholder or a whole number.
var groupSide = regexp.MustCompile(`^(?:\$\{setting:[a-z0-9][a-z0-9_.-]*\}|[0-9]+)$`)
// groupsOf is a container resource's groups as the manifest wrote them (text or numbers), none when it names none.
func groupsOf(r map[string]any) []any {
groups, _ := r["groups"].([]any)
return groups
}
// GroupSettingProblems refuses, where a manifest is read, a container whose groups hold anything but whole
// placeholders and whole numbers, or a placeholder whose key the module neither declares nor asks for elsewhere.
func GroupSettingProblems(m Manifest) []string {
var problems []string
for _, r := range m.Resources {
if fmt.Sprint(r["type"]) != "container" {
continue
}
for _, g := range groupsOf(r) {
s, isText := g.(string)
if !isText {
continue // a number: the declaration judges its value
}
if !groupSide.MatchString(s) {
problems = append(problems, fmt.Sprintf("%s: the container %v's group %q is not a whole "+
"${config:key} or a whole number: a supplementary group is a number, and nothing else is "+
"written into it (novox/hq ADR 0480 clause 7)", m.Module, r["id"], s))
continue
}
if m.Templated() {
continue
}
for _, key := range settingsUsed(s) {
if _, declared := m.Settings[key]; !declared && !m.DeclaresConfig() {
continue // the settings form judges its own keys
}
if _, declared := m.Settings[key]; !declared {
problems = append(problems, fmt.Sprintf("%s: the container %v's group says ${config:%s}, and the module "+
"declares no such value (novox/hq ADR 0348 §4)", m.Module, r["id"], key))
}
}
}
}
return problems
}
// settingIntoGroups fills ${setting:…} in a container's groups, refusing — with the key and why, the resource left as
// it was — a key nothing sets and a value that is not a whole number above zero. A number is kept; the list that
// results is numbers alone.
func settingIntoGroups(resource map[string]any, layers []Layer, module string) error {
groups := groupsOf(resource)
if len(groups) == 0 {
return nil
}
filled := make([]any, 0, len(groups))
for _, g := range groups {
s, isText := g.(string)
if !isText {
if f, isNumber := g.(float64); isNumber && f == float64(int(f)) {
g = int(f) // a number the manifest wrote, as the numbers a placeholder gives
}
filled = append(filled, g)
continue
}
keys := settingsUsed(s)
if len(keys) != 1 || settingRef.ReplaceAllString(s, "") != "" {
filled = append(filled, g) // not a whole placeholder: the manifest check refuses it
continue
}
key := keys[0]
value, set := settingValue(layers, key)
if !set {
return &UnsetSettingError{Module: module, Setting: key,
where: fmt.Sprintf("a container %v whose groups", resource["id"]), setToday: orNoSettings(layers),
said: fmt.Sprintf(
"%s has a container %v whose groups say ${setting:%s}, and nothing sets %q for it — an operator's "+
"value is the assignment's, never the definition's (novox/hq ADR 0112): "+
"`settings set %s <file>` with {%q: …}%s",
module, resource["id"], key, key, module, key, orNoSettings(layers))}
}
v := plainly(value)
id, whole := wholeID(v)
if !whole || id == 0 {
return fmt.Errorf("%s: the setting %q is %q, which cannot be a group of the container %v: a supplementary "+
"group is a whole number above zero, never a name or other text (novox/hq ADR 0480 clause 7, ADR 0328 §15)",
module, key, v, resource["id"])
}
filled = append(filled, int(id))
}
resource["groups"] = filled
return nil
}
@@ -56,7 +56,7 @@ type filler struct {
// seatInto, machineInto and contributionsInto. A pass that comes to read another field adds it here,
// or the sweep refuses the placeholder it would have filled.
var fillers = []filler{
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}, "container": {"groups"}}},
{namespace: "setting", patterns: []*regexp.Regexp{settingRef}, fields: map[string][]string{"file": {"content"}, "service": {"unit"}, "directory": {"owner"}}},
{namespace: "dir", patterns: []*regexp.Regexp{dirRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
{namespace: "access", patterns: []*regexp.Regexp{accessRef}, fields: map[string][]string{"*": {"path", "content", "volumes", "env", "env-file"}}},
{namespace: "secret", patterns: []*regexp.Regexp{placeholder}, fields: map[string][]string{"file": {"content"}},
@@ -324,15 +324,6 @@ type File struct {
// depend on a field somewhere else.
Bytes string `json:"bytes,omitempty"`
// Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as
// the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes
// (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucketFor(node), checks what it
// received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes,
// VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine
// refuses a field it does not know, whole.
Digest string `json:"digest,omitempty"`
Size int64 `json:"size,omitempty"`
// Owner is the user this belongs to, by name. Absent means root.
Owner string `json:"owner,omitempty"`
@@ -474,10 +465,6 @@ func (f *File) validate(where string, _ bool) []string {
problems = append(problems, where+": a file the vault sealed whole has no holes for secrets")
}
}
problems = append(problems, f.checkReference(where)...)
if f.Digest != "" || f.Size != 0 {
said = append(said, "digest")
}
if len(said) > 1 {
sort.Strings(said)
problems = append(problems, where+
@@ -1467,14 +1454,6 @@ type Container struct {
// container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because
// a resource's Identity is its id.
RunAs *ContainerIdentity `json:"identity,omitempty"`
// Groups are the supplementary groups the container's process joins, as numbers, each passed to the runtime
// as a group to add (novox/hq ADR 0480 clause 7, issue 1636): n8n writes the media library through the
// library's group, which a stock image knows nothing of. Each is above zero and below the kernel's "no group"
// (ADR 0328 §15: never root). Part of the spec, so a change recreates the container. Absent: none are added.
// An older node-engine refuses the key (strict decoding), so the controller sends it only to a node that
// reports it reads it.
Groups []int64 `json:"groups,omitempty"`
}
// The ways an image is told who to run as (novox/hq ADR 0325 §2).
@@ -1542,30 +1521,6 @@ func (i *ContainerIdentity) problems(where string, env map[string]string) []stri
return problems
}
// groupProblems refuses a supplementary group that is root's or below it, the kernel's "no group" number, or a
// number named twice (novox/hq ADR 0480 clause 7, ADR 0328 §15).
func (c *Container) groupProblems(where string) []string {
var problems []string
seen := map[int64]bool{}
for _, g := range c.Groups {
switch {
case g == 0:
problems = append(problems, fmt.Sprintf("%s: container %s adds group 0, and the mesh never makes or keeps anything "+
"as root: a supplementary group is never root's (novox/hq ADR 0328 §15)", where, c.Name))
case g < 0:
problems = append(problems, fmt.Sprintf("%s: container %s adds group %d, below zero: a group is a whole number "+
"above zero", where, c.Name, g))
case g >= 1<<32-1:
problems = append(problems, fmt.Sprintf("%s: container %s adds group %d, which is the kernel's \"no group\" "+
"number or beyond it", where, c.Name, g))
case seen[g]:
problems = append(problems, fmt.Sprintf("%s: container %s adds group %d twice", where, c.Name, g))
}
seen[g] = true
}
return problems
}
func (c *Container) Identity() string { return c.ID }
func (c *Container) Kind() Type { return TypeContainer }
func (c *Container) Target() string { return c.Name }
@@ -1648,7 +1603,6 @@ func (c *Container) validate(where string, _ bool) []string {
"runs and exits, and joins nothing afterwards")
}
problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...)
problems = append(problems, c.groupProblems(where)...)
return append(problems, checkImage(where, c.Image)...)
}
@@ -1,150 +0,0 @@
package declaration
import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"regexp"
)
// The three names of a file travelling by digest (novox/hq ADR 0486 §1, §2, §5), shared by the node-engine that
// reads them and the controller that writes them.
const (
// FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer
// stays in the declaration as it is.
FileReferenceThreshold = 64 * 1024
// FileBucketPrefix starts the name of a node's object bucket, FileBucketFor(node), which holds the contents of the
// files of that node's declaration, each under the hex of its sha256 digest (ObjectName). One bucket per node,
// so a grant names it and a literal `*` is never needed (ADR 0486 §2, changed by ADR 0488 §1 and §2). The controller alone writes it; the
// node's engine reads an object by name.
FileBucketPrefix = "declaration-files-"
// FileStreamPrefix starts the name of the stream behind such a bucket, which the bus's grants name.
FileStreamPrefix = "OBJ_"
// FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size.
FileReferencesCapability = "file-references"
)
var nodeToken = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
// FileBucketFor is the name of the object bucket of a node's files, "declaration-files-<node>" (novox/hq ADR 0488
// §4, which replaces the single bucket constant of issue 1694). The node name must be a name the bus accepts in a
// bucket and in a subject token: letters, digits, '_' and '-', so no '.', '*', '>' or space.
func FileBucketFor(node string) (string, error) {
if !nodeToken.MatchString(node) {
return "", fmt.Errorf("node %q; a node's bucket is named by letters, digits, '_' and '-' only", node)
}
return FileBucketPrefix + node, nil
}
// FileStreamFor is the name of the stream behind a node's bucket, "OBJ_" and the bucket's name.
func FileStreamFor(node string) (string, error) {
b, err := FileBucketFor(node)
if err != nil {
return "", err
}
return FileStreamPrefix + b, nil
}
var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>".
func DigestOf(content []byte) string {
sum := sha256.Sum256(content)
return "sha256:" + hex.EncodeToString(sum[:])
}
// ObjectName is the name of the object holding the content a digest names: the hex without its "sha256:" prefix.
func ObjectName(digest string) (string, error) {
if !digestForm.MatchString(digest) {
return "", fmt.Errorf("digest %q is not sha256:<64 lower-case hex characters>", digest)
}
return digest[len("sha256:"):], nil
}
// IsReference reports whether the file gives its content as a digest and size instead of carrying it.
func (f *File) IsReference() bool { return f.Digest != "" }
// checkReference judges the digest and size, and what a reference may not be given with.
func (f *File) checkReference(where string) []string {
if f.Digest == "" && f.Size == 0 {
return nil
}
var problems []string
if !digestForm.MatchString(f.Digest) {
problems = append(problems, fmt.Sprintf("%s: digest %q; a file by reference names its content as "+
"sha256:<64 lower-case hex characters>", where, f.Digest))
}
if f.Size <= 0 {
problems = append(problems, fmt.Sprintf("%s: size %d; a file by reference states its size in bytes, "+
"more than zero", where, f.Size))
}
if f.Content != "" || f.Bytes != "" || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 {
problems = append(problems, where+": a file by reference carries its content in the object its digest "+
"names, so it says no content, bytes, sealed, vault-sealed or secrets")
}
if f.Into != "" {
problems = append(problems, where+": a file by reference is written whole; a file written into says its "+
"part in content")
}
return problems
}
// Body is the bytes the file's content stands for: its content, or its bytes decoded.
func (f *File) Body() ([]byte, error) {
if f.Bytes == "" {
return []byte(f.Content), nil
}
b, err := base64.StdEncoding.DecodeString(f.Bytes)
if err != nil {
return nil, fmt.Errorf("file %s: bytes are not base64: %w", f.ID, err)
}
return b, nil
}
// TravelsByDigest reports whether the controller may put this file's content in the bucket: it is plain content
// or bytes (not sealed, not filled with secrets, not written into another file) of more than FileReferenceThreshold
// bytes.
func (f *File) TravelsByDigest() bool {
if f.IsReference() || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 || f.Into != "" {
return false
}
if f.Bytes != "" {
// DecodedLen counts padding, so the exact length is taken off the decoded bytes.
body, err := f.Body()
return err == nil && len(body) > FileReferenceThreshold
}
return len(f.Content) > FileReferenceThreshold
}
// AsReference is the file as it is sent to an engine that reports FileReferencesCapability: a copy with the digest
// and size in place of the content, and the content the controller puts into the bucket under ObjectName(digest).
// A file that does not travel by digest is refused, not passed through, so a caller cannot send a small file empty.
func (f *File) AsReference() (File, []byte, error) {
if !f.TravelsByDigest() {
return File{}, nil, fmt.Errorf("file %s does not travel by digest", f.ID)
}
body, err := f.Body()
if err != nil {
return File{}, nil, err
}
ref := *f
ref.Content, ref.Bytes = "", ""
ref.Digest, ref.Size = DigestOf(body), int64(len(body))
return ref, body, nil
}
// Verify checks what was fetched against the digest and size the file names; the node-engine writes the file only
// after it returns nil. A mismatch is the apply's failure, never a partial apply (ADR 0486 §1).
func (f *File) Verify(body []byte) error {
if !f.IsReference() {
return fmt.Errorf("file %s is not given by reference", f.ID)
}
if int64(len(body)) != f.Size {
return fmt.Errorf("file %s: the object is %d bytes and the declaration says %d", f.ID, len(body), f.Size)
}
if got := DigestOf(body); got != f.Digest {
return fmt.Errorf("file %s: the object hashes to %s and the declaration names %s", f.ID, got, f.Digest)
}
return nil
}
@@ -25,7 +25,7 @@ type BuildRecord struct {
}
// Artifact is one thing a build made: mesh-controller internal/chain/records.go Artifact, json keys name, kind and
// reference, nothing else (novox/hq ADR 0478 clause 8).
// reference, nothing else (novox/hq ADR 0478 clause 7).
type Artifact struct {
Name string `json:"name"`
Kind string `json:"kind"`
+1 -1
View File
@@ -390,7 +390,7 @@ type Module struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
// Path is the module's directory in its repository, trimmed of slashes, and RootPath (".") when its manifest is the
// repository's root (novox/hq ADR 0478 clause 9, issue 1615): the build tools need it. Empty means not known: a value
// repository's root (novox/hq ADR 0478 clause 8, issue 1615): the build tools need it. Empty means not known: a value
// written before the indexer learned paths has none.
Path string `json:"path,omitempty"`
// Toolchains are the toolchains FP2 was computed with, by language (go, typescript): the version installed and the
+1 -1
View File
@@ -1,4 +1,4 @@
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861
## explicit; go 1.26.0
git.novox.be/novox/mesh-sdk/go/asks
git.novox.be/novox/mesh-sdk/go/catalogue