Compare commits
33
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7cc315fd0 | ||
|
|
60253c82b7 | ||
|
|
bb8f076cc0 | ||
|
|
bb4d2e1da4 | ||
|
|
44033f4b6d | ||
|
|
22c9815f65 | ||
|
|
a14dc04b7b | ||
|
|
181b579ac6 | ||
|
|
b08c6372e1 | ||
|
|
4e11965a2a | ||
|
|
faf8beee84 | ||
|
|
3827c3eb60 | ||
|
|
6bcfbeb5b2 | ||
|
|
deac3e232c | ||
|
|
612f493c49 | ||
|
|
2377d75ed8 | ||
|
|
5ba2ffe8e6 | ||
|
|
8fc0d135bb | ||
|
|
d568d76b97 | ||
|
|
610d88e6bf | ||
|
|
0b11b6c98d | ||
|
|
92a5193e31 | ||
|
|
3217f097d3 | ||
|
|
650274d700 | ||
|
|
678a517843 | ||
|
|
516af253c6 | ||
|
|
5157ccc3a8 | ||
|
|
087e9266d1 | ||
|
|
a3b4e0601d | ||
|
|
da364c2730 | ||
|
|
4771ee53d9 | ||
|
|
ceaa616d01 | ||
|
|
fa5b5e1c11 |
@@ -408,12 +408,21 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
_, reads := m.OwnSecrets["broker"]
|
||||
if !reads && !runsTheProvisionerClient(m) {
|
||||
// A module whose bus accounts are its processes' alone has none of its own: only the processes that
|
||||
// are not issued yet are issued (issue 1226).
|
||||
return processAccountsOnAssign(ctx, inv, m, node)
|
||||
}
|
||||
if !reads {
|
||||
// A module running the provisioner client alone is minted the client's account (issue 917).
|
||||
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
||||
}
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
||||
return ""
|
||||
} else if minted {
|
||||
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
|
||||
return processAccountsOnAssign(ctx, inv, m, node)
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err == nil {
|
||||
@@ -426,6 +435,16 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
|
||||
// processAccountsOnAssign issues the process accounts of an assigned module not issued yet, and says what it did.
|
||||
func processAccountsOnAssign(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) string {
|
||||
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("its process accounts are not issued (%v): `module issue %s --node %s` first — "+
|
||||
"`push %s` refuses to send %s until it is", err, m.Module, node, node, m.Module)
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
|
||||
// novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
|
||||
// the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every
|
||||
|
||||
@@ -768,6 +768,9 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// A bus process the new definition gains gets its account now, on each machine assigned the module, so the
|
||||
// next push composes a real account (issue 1226).
|
||||
issueGainedProcessAccounts(ctx, inv, manifest)
|
||||
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
|
||||
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
|
||||
return manifest, kept, err
|
||||
|
||||
@@ -32,6 +32,13 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
|
||||
if err := broker.Raise(r, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
|
||||
// whenever the bus is up.
|
||||
if files, ok := r.(interface{ EnsureDeclarationFiles(nodes []string) error }); ok {
|
||||
if err := files.EnsureDeclarationFiles(names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
|
||||
// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
|
||||
// after something started asking for builds flushes the backlog instead of having lost it.
|
||||
|
||||
@@ -233,6 +233,12 @@ func prepareSend(ctx context.Context, open *stores, node string) ([]string, erro
|
||||
if !slices.Contains(in.BusMinted, "provisioner."+node) {
|
||||
continue
|
||||
}
|
||||
default:
|
||||
// A bus process's account nobody issued still gets the stand-in, until the composition leaves
|
||||
// the module off (issue 1753): the warning names the verb (issue 1226).
|
||||
if line := processAccountWarning(m, node, name, in.BusMinted); line != "" {
|
||||
sayWarning(ctx, line)
|
||||
}
|
||||
}
|
||||
if _, err := inv.SecretForModule(ctx, node, m.Module, name); err == nil {
|
||||
did = true
|
||||
|
||||
@@ -200,6 +200,7 @@ func composeQuestion(ctx context.Context, open *stores, node string, mode compos
|
||||
}
|
||||
sayFound(ctx, d)
|
||||
out := sendableOf(d)
|
||||
out.fileReferences = readsFileReferences(in.Input, node)
|
||||
if !withOrder {
|
||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||
}
|
||||
@@ -281,6 +282,7 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
|
||||
if len(left) == 0 && len(in.wouldMake) == 0 {
|
||||
sayFound(ctx, d)
|
||||
out := sendableOf(d)
|
||||
out.fileReferences = readsFileReferences(in.Input, node)
|
||||
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
|
||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||
here, _ := nodeOf(in.Input, node)
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
)
|
||||
|
||||
// 1696/AC2, ADR 0486 §5: a node is sent a file by digest only when its profile reports the capability. A node whose profile lacks
|
||||
// it, and a node the mesh has no profile of, are sent every content inline; a node that reports it is sent the reference. The test
|
||||
// goes through composeQuestion and composeSend, the two places readsFileReferences is read (compose_sdk.go).
|
||||
func TestAFileTravelsByDigestOnlyToANodeThatReportsTheCapability(t *testing.T) {
|
||||
open := aMesh(t) // anchor and laptop report no file-references
|
||||
ctx := t.Context()
|
||||
big := strings.Repeat("0123456789abcdef", 8<<10) // 128 KiB
|
||||
register(t, open, catalogue.Manifest{Module: "bigfile", Version: "1",
|
||||
Resources: []map[string]any{{"id": "blob", "type": "file", "path": "/etc/bigfile/blob", "mode": "0644", "content": big}}})
|
||||
|
||||
// reader reports the capability; bare has never reported a profile at all.
|
||||
for i, n := range []struct {
|
||||
name string
|
||||
profile map[string]any
|
||||
}{
|
||||
{"reader", map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "container-runtime", "present": true},
|
||||
map[string]any{"name": "systemd", "present": true},
|
||||
map[string]any{"name": sdkdecl.FileReferencesCapability, "present": true}}}},
|
||||
{"bare", nil},
|
||||
} {
|
||||
record, err := open.inventory.AddNode(ctx, n.name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetPlace(ctx, n.name, n.name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n.profile != nil {
|
||||
if err := open.inventory.RecordProfile(ctx, record.ID, n.profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, n.name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, node := range []string{"laptop", "reader", "bare"} {
|
||||
if _, err := assign(ctx, open, node, "bigfile"); err != nil {
|
||||
t.Fatalf("assign %s: %v", node, err)
|
||||
}
|
||||
}
|
||||
|
||||
byDigest := map[string]bool{"laptop": false, "bare": false, "reader": true}
|
||||
for node, want := range byDigest {
|
||||
asked, _, err := composeQuestion(ctx, open, node, compose.Reading, false)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
sent, err := composeSend(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", node, err)
|
||||
}
|
||||
for what, s := range map[string]sendable{"question": asked, "send": sent} {
|
||||
if s.fileReferences != want {
|
||||
t.Fatalf("%s, %s: fileReferences %v, want %v", node, what, s.fileReferences, want)
|
||||
}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, r := range got.Resources {
|
||||
if r["path"] != "/etc/bigfile/blob" {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
inline := r["content"] == big
|
||||
if want && (inline || r["digest"] == nil || r["size"] == nil) {
|
||||
t.Fatalf("%s, %s: a node that reads references is sent the content inline", node, what)
|
||||
}
|
||||
if !want && (!inline || r["digest"] != nil) {
|
||||
t.Fatalf("%s, %s: a node that does not report the capability is sent a reference", node, what)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("%s, %s: the file is not in the declaration", node, what)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
|
||||
//
|
||||
// The declaration to a node that reports sdkdecl.FileReferencesCapability carries, in place of the content of every
|
||||
// file resource of more than sdkdecl.FileReferenceThreshold bytes that may travel so (File.TravelsByDigest: plain content or
|
||||
// bytes, not sealed, not filled with secrets, not written into another file), its sha256 digest and size. The content is put
|
||||
// into the bucket of the bus under the hex of the digest before the declaration is sent. A node that does not report the
|
||||
// capability is sent every content inline, as before: an older node-engine refuses a field it does not know, whole.
|
||||
|
||||
// readsFileReferences says the node reports the capability of reading a file given by digest (ADR 0486 §5). A node the mesh has
|
||||
// no profile of reads none.
|
||||
func readsFileReferences(in compose.Input, node string) bool {
|
||||
n, ok := nodeOf(in, node)
|
||||
return ok && n.Capabilities[sdkdecl.FileReferencesCapability]
|
||||
}
|
||||
|
||||
// referencing is the resources as a node that reads references is sent them, and the content of each file that went by
|
||||
// digest, by the name of its object (the hex of its digest). The resources it is given are not changed. A file that does not
|
||||
// travel by digest is passed as it is.
|
||||
func referencing(resources []map[string]any) ([]map[string]any, map[string][]byte, error) {
|
||||
out := make([]map[string]any, len(resources))
|
||||
var objects map[string][]byte
|
||||
for i, r := range resources {
|
||||
out[i] = r
|
||||
if kind, _ := r["type"].(string); kind != string(sdkdecl.TypeFile) {
|
||||
continue
|
||||
}
|
||||
// Cheap first: only a file with a content or bytes of the size is worth reading whole.
|
||||
content, _ := r["content"].(string)
|
||||
bytesField, _ := r["bytes"].(string)
|
||||
if len(content) <= sdkdecl.FileReferenceThreshold && len(bytesField) <= sdkdecl.FileReferenceThreshold {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v: %w", r["id"], err)
|
||||
}
|
||||
var f sdkdecl.File
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return nil, nil, fmt.Errorf("resource %v is not a file the controller can read: %w", r["id"], err)
|
||||
}
|
||||
if !f.TravelsByDigest() {
|
||||
continue
|
||||
}
|
||||
ref, body, err := f.AsReference()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(ref.Digest)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
with := make(map[string]any, len(r)+2)
|
||||
for k, v := range r {
|
||||
with[k] = v
|
||||
}
|
||||
delete(with, "content")
|
||||
delete(with, "bytes")
|
||||
with["digest"], with["size"] = ref.Digest, ref.Size
|
||||
out[i] = with
|
||||
if objects == nil {
|
||||
objects = map[string][]byte{}
|
||||
}
|
||||
objects[name] = body
|
||||
}
|
||||
return out, objects, nil
|
||||
}
|
||||
|
||||
// namedObjects are the objects of the bucket a body names: the object name of every file resource given by reference,
|
||||
// with its size. It reads a declaration as the node does, so it also reads the latest one the bus holds.
|
||||
func namedObjects(body []byte) (map[string]int64, error) {
|
||||
var envelope struct {
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out map[string]int64
|
||||
for _, r := range envelope.Resources {
|
||||
if r.Type != string(sdkdecl.TypeFile) || r.Digest == "" {
|
||||
continue
|
||||
}
|
||||
name, err := sdkdecl.ObjectName(r.Digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]int64{}
|
||||
}
|
||||
out[name] = r.Size
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// objectStore is what putting the objects needs of a node's bucket.
|
||||
type objectStore interface {
|
||||
// Has says whether an object of that name is there, complete, and whether it carries the sweep's mark of being named by no
|
||||
// node's latest declaration.
|
||||
Has(ctx context.Context, name string) (have, marked bool, err error)
|
||||
// Put writes the object under that name, replacing one of the same name (and so erasing its mark).
|
||||
Put(ctx context.Context, name string, content []byte) error
|
||||
}
|
||||
|
||||
// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a node
|
||||
// that reads the declaration finds each object. An object already there (the same content, the same name) is not written again,
|
||||
// **unless the sweep marked it unnamed**: then it is written again, so the mark that would have it removed is gone with the
|
||||
// old object (issue 1696, review: a mark read just before a send must not take an object the send relies on). Refuses on the
|
||||
// first it cannot put, naming it, so the declaration is not sent.
|
||||
func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
|
||||
names := make([]string, 0, len(objects))
|
||||
for name := range objects {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
have, marked, err := store.Has(ctx, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("whether the object %s is in the bucket: %w", name[:12], err)
|
||||
}
|
||||
if have && !marked {
|
||||
continue
|
||||
}
|
||||
if err := store.Put(ctx, name, objects[name]); err != nil {
|
||||
return fmt.Errorf("putting the object %s (%d bytes) in the bucket: %w", name[:12], len(objects[name]), err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busObjects is a node's bucket on the controller's own connection.
|
||||
type busObjects struct{ store jetstream.ObjectStore }
|
||||
|
||||
// openBucket opens the bucket of a node's files, making it first if it is not there (it is asserted for every node at the
|
||||
// controller's start; a node adopted since is covered here).
|
||||
func openBucket(ctx context.Context, conn *nats.Conn, node string) (busObjects, error) {
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return busObjects{}, err
|
||||
}
|
||||
bucket, err := broker.DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return busObjects{}, fmt.Errorf("the object bucket of node %q: %w", node, err)
|
||||
}
|
||||
store, err := js.ObjectStore(ctx, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
if err = broker.EnsureDeclarationFiles(ctx, js, node); err == nil {
|
||||
store, err = js.ObjectStore(ctx, bucket)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return busObjects{}, fmt.Errorf("the object bucket %s: %w", bucket, err)
|
||||
}
|
||||
return busObjects{store}, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Has(ctx context.Context, name string) (bool, bool, error) {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrObjectNotFound):
|
||||
return false, false, nil
|
||||
case err != nil:
|
||||
return false, false, err
|
||||
}
|
||||
_, marked := info.Metadata[declarationUnnamed]
|
||||
return !info.Deleted, marked, nil
|
||||
}
|
||||
|
||||
func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
|
||||
_, err := b.store.PutBytes(ctx, name, content)
|
||||
return err
|
||||
}
|
||||
|
||||
// filesLocks hold a node's bucket for one act at a time in this process: a send puts its objects and publishes the declaration that
|
||||
// names them under the node's lock, and the sweep judges the node's bucket under the same one, so the sweep never removes an
|
||||
// object between a send's put and its declaration. Another process (a person's push) is covered by the put erasing a mark and by
|
||||
// the sweep looking at an object again just before it removes it.
|
||||
var filesLocks sync.Map
|
||||
|
||||
func lockFilesOf(node string) func() {
|
||||
l, _ := filesLocks.LoadOrStore(node, &sync.Mutex{})
|
||||
mu := l.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
}
|
||||
|
||||
// kindFilesNotPut is a node whose files that travel by digest could not be put in its bucket, send after send.
|
||||
const kindFilesNotPut = "declaration-files-not-put"
|
||||
|
||||
// filesNotPutAfter is how many sends in a row may fail to put before the condition is raised: the first send to the machine holding
|
||||
// the bus fails for want of the grant that very send delivers, and says so, and is not yet a fault.
|
||||
const filesNotPutAfter = 2
|
||||
|
||||
var (
|
||||
filesFailMu sync.Mutex
|
||||
filesFailed = map[string]int{}
|
||||
)
|
||||
|
||||
func filesNotPutObservation(node, summary string) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindFilesNotPut, Token: kindFilesNotPut,
|
||||
Machine: node, Severity: conditions.Warning, Source: "sends", Summary: summary}
|
||||
}
|
||||
|
||||
// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
|
||||
// clears the count and the condition.
|
||||
func notePut(ctx context.Context, node string, put error) {
|
||||
filesFailMu.Lock()
|
||||
if put == nil {
|
||||
delete(filesFailed, node)
|
||||
} else {
|
||||
filesFailed[node]++
|
||||
}
|
||||
n := filesFailed[node]
|
||||
filesFailMu.Unlock()
|
||||
keeper := conditionsFrom
|
||||
if keeper == nil {
|
||||
return
|
||||
}
|
||||
key := filesNotPutObservation(node, "").Key()
|
||||
switch {
|
||||
case put == nil:
|
||||
if _, err := keeper.Clear(ctx, key, node+" was sent its files by digest"); err != nil {
|
||||
fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
|
||||
}
|
||||
case n >= filesNotPutAfter:
|
||||
summary := fmt.Sprintf("%s's files that travel by digest could not be put in its bucket %d sends in a row, so it is sent them "+
|
||||
"inline: %v", node, n, put)
|
||||
if _, err := keeper.Observe(ctx, filesNotPutObservation(node, summary)); err != nil {
|
||||
fmt.Printf("the condition %s could not be raised: %v\n", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
type sigOf struct{}
|
||||
|
||||
func (sigOf) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
|
||||
|
||||
// 1696/AC7 (ADR 0486 §1): on a real bus with its default largest message of 1 MiB, a declaration whose files add up to more than
|
||||
// 3 MiB is sent as references, its objects are in the bucket, each fetched back whole and verified against the digest and size the
|
||||
// declaration names; the same declaration inline is refused by the controller, naming size and limit. A changed byte in an object
|
||||
// fails the verification.
|
||||
func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) { // 1696/AC7
|
||||
s := testbus.Start(t)
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
nc, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
|
||||
MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Run twice: an assertion of an existing bucket is no error and no change.
|
||||
if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
|
||||
t.Fatalf("asserting the bucket again: %v", err)
|
||||
}
|
||||
|
||||
var resources []map[string]any
|
||||
for i := 0; i < 3; i++ {
|
||||
resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
|
||||
}
|
||||
resources = append(resources, aFileResource("small", "tiny\n"))
|
||||
inline := sendable{Resources: resources}
|
||||
byDigest := sendable{fileReferences: true, Resources: resources}
|
||||
|
||||
bus := link.OverNATS{Conn: conn, JS: nc}
|
||||
body, err := inline.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
|
||||
!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
|
||||
t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
|
||||
}
|
||||
|
||||
bucket, err := openBucket(ctx, conn, "novox")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objects, err := byDigest.objects()
|
||||
if err != nil || len(objects) != 3 {
|
||||
t.Fatalf("objects: %d %v", len(objects), err)
|
||||
}
|
||||
if err := putObjects(ctx, bucket, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := putObjects(ctx, bucket, objects); err != nil {
|
||||
t.Fatalf("putting them again: %v", err)
|
||||
}
|
||||
refBody, err := byDigest.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
|
||||
t.Fatalf("the declaration by reference: %v", err)
|
||||
}
|
||||
|
||||
// What the stream holds for the node is the reference form, and it names exactly the three objects.
|
||||
latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
|
||||
if err != nil || len(latest) != 1 {
|
||||
t.Fatalf("the latest declarations: %v %v", len(latest), err)
|
||||
}
|
||||
named, err := namedByLatest("novox", latest["novox"])
|
||||
if err != nil || len(named) != 3 {
|
||||
t.Fatalf("what the latest declaration names: %v %v", named, err)
|
||||
}
|
||||
novoxBucket, err := broker.DeclarationFilesBucketOf("novox")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err := api.ObjectStore(ctx, novoxBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope struct {
|
||||
Resources []sdkdecl.File `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(refBody, &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range envelope.Resources {
|
||||
if !f.IsReference() {
|
||||
continue
|
||||
}
|
||||
name, _ := sdkdecl.ObjectName(f.Digest)
|
||||
got, err := store.GetBytes(ctx, name)
|
||||
if err != nil {
|
||||
t.Fatalf("the object of %s: %v", f.ID, err)
|
||||
}
|
||||
if err := f.Verify(got); err != nil {
|
||||
t.Errorf("the object does not verify: %v", err)
|
||||
}
|
||||
got[len(got)/2] ^= 1
|
||||
if err := f.Verify(got); err == nil {
|
||||
t.Errorf("a changed byte in the object of %s passed the verification", f.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// The sweep against the real bucket: all named, nothing marked; a fourth object named by none is marked, kept a day less a
|
||||
// minute, and removed at the day.
|
||||
extra := bytes.Repeat([]byte("x"), 100)
|
||||
extraName := strings.Repeat("e", 64)
|
||||
if err := putObjects(ctx, bucket, map[string][]byte{extraName: extra}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := busSweptBucket{store}
|
||||
t0 := time.Now()
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0); err != nil || r.Marked != 1 || r.Removed != 0 || r.Kept != 3 {
|
||||
t.Fatalf("first sweep: %+v %v", r, err)
|
||||
}
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour-time.Minute)); err != nil || r.Removed != 0 {
|
||||
t.Fatalf("a minute short of a day: %+v %v", r, err)
|
||||
}
|
||||
if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour+time.Second)); err != nil || r.Removed != 1 {
|
||||
t.Fatalf("a day after: %+v %v", r, err)
|
||||
}
|
||||
if infos, err := store.List(ctx); err != nil || len(infos) != 3 {
|
||||
t.Fatalf("the bucket holds %d objects (%v), want the 3 the declaration names", len(infos), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC8 on a real bucket: the sweep removes an object only if it still carries the mark it judged it by. An object marked
|
||||
// unnamed, then put again by a send (which erases the mark), is kept when the sweep deletes it by the old mark.
|
||||
func TestTheSweepsDeleteLeavesAnObjectASendPutAgainAfterItWasMarked(t *testing.T) { // 1696/AC8
|
||||
s := testbus.Start(t)
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
bucket, err := openBucket(ctx, conn, "novox")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
name := strings.Repeat("a", 64)
|
||||
content := map[string][]byte{name: bytes.Repeat([]byte("y"), 100)}
|
||||
if err := putObjects(ctx, bucket, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := busSweptBucket{bucket.store}
|
||||
judged := time.Now().Add(-25 * time.Hour).Truncate(time.Second)
|
||||
if err := b.MarkUnnamed(ctx, name, judged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, marked, err := bucket.Has(ctx, name); err != nil || !marked {
|
||||
t.Fatalf("the object was not marked: %v %v", marked, err)
|
||||
}
|
||||
// The sweep has read the mark; a send now puts the object again, which erases it.
|
||||
if err := putObjects(ctx, bucket, content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have, marked, err := bucket.Has(ctx, name); err != nil || !have || marked {
|
||||
t.Fatalf("a send did not put the marked object again: have %v marked %v %v", have, marked, err)
|
||||
}
|
||||
gone, err := b.Delete(ctx, name, judged)
|
||||
if err != nil || gone {
|
||||
t.Fatalf("the sweep deleted by an old mark: gone %v, %v", gone, err)
|
||||
}
|
||||
if have, _, err := bucket.Has(ctx, name); err != nil || !have {
|
||||
t.Fatalf("the object a send relies on is gone: %v %v", have, err)
|
||||
}
|
||||
// And the mark still on it is honoured: marked again and deleted by that mark, it goes.
|
||||
if err := b.MarkUnnamed(ctx, name, judged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gone, err := b.Delete(ctx, name, judged); err != nil || !gone {
|
||||
t.Fatalf("an object still carrying the mark it was judged by is kept: gone %v, %v", gone, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC9 on a real bus: the sweep of a node's bucket takes the node's lock first, so it waits for a send that has put its objects
|
||||
// and not yet published the declaration that names them.
|
||||
func TestTheSweepOfANodesBucketHoldsTheNodesLock(t *testing.T) { // 1696/AC9
|
||||
s := testbus.Start(t)
|
||||
conn, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
const node = "sweep-lock-node"
|
||||
if _, err := openBucket(ctx, conn, node); err != nil { // the bucket exists, or the sweep would return before the lock
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := lockFilesOf(node) // a send is between its put and its publication
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
_, _ = sweepNode(ctx, conn, node)
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("the sweep judged the node's bucket while a send held it")
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("the sweep did not go on once the send let go")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// contentOf is a deterministic content of n bytes that does not compress to nothing and holds no character JSON escapes.
|
||||
func contentOf(seed string, n int) string {
|
||||
var b strings.Builder
|
||||
for i := 0; b.Len() < n; i++ {
|
||||
fmt.Fprintf(&b, "%s-%07d-%x;", seed, i, i*2654435761%1000003)
|
||||
}
|
||||
return b.String()[:n]
|
||||
}
|
||||
|
||||
func aFileResource(id, content string) map[string]any {
|
||||
return map[string]any{"id": id, "type": "file", "path": "/etc/" + id, "mode": "0644", "content": content}
|
||||
}
|
||||
|
||||
// controlNodeShape is a declaration shaped like the control node's on 2026-10-21 (issue 1689), the sizes measured then: a bus users
|
||||
// file of 252,678 bytes, the vault's export of 138,475 bytes (compact), the tool runtime 21 KB, the packet filter 19 KB, the launcher
|
||||
// 14 KB, and about 600 small resources for the rest (about 286 KB). `more` scales the count of small ones: 1 is the shape, 1.4 forty
|
||||
// percent more of them.
|
||||
func controlNodeShape(more float64) []map[string]any {
|
||||
res := []map[string]any{
|
||||
aFileResource("nats.bus-users", contentOf("users", 252678)),
|
||||
aFileResource("mesh-vault.kept", contentOf("kept", 138475)),
|
||||
aFileResource("node-tools.runtime", contentOf("runtime", 21<<10)),
|
||||
aFileResource("nftables.filtering", contentOf("filter", 19<<10)),
|
||||
aFileResource("mesh-host.launcher", contentOf("launcher", 14<<10)),
|
||||
}
|
||||
for i := 0; i < int(600*more); i++ {
|
||||
res = append(res, aFileResource(fmt.Sprintf("module-%03d.config", i), contentOf(fmt.Sprintf("small%d", i), 380)))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func wireSizeOf(t *testing.T, s sendable) int {
|
||||
t.Helper()
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The signature the controller's key makes is 64 bytes.
|
||||
msg, err := link.Envelope(body, make([]byte, 64))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(msg)
|
||||
}
|
||||
|
||||
// 1696/AC1: for a node that reports the capability, the two big files of a control-node-shaped declaration carry their digest and size
|
||||
// and no content, and the declaration is far under the bus's 1 MiB.
|
||||
func TestAFileOver64KiBTravelsByDigestToANodeThatReadsOne(t *testing.T) { // 1696/AC1
|
||||
users, kept := contentOf("users", 252678), contentOf("kept", 138475)
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{
|
||||
aFileResource("nats.bus-users", users), aFileResource("mesh-vault.kept", kept),
|
||||
aFileResource("small", "tiny\n"),
|
||||
}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, want := range []string{users, kept} {
|
||||
r := got.Resources[i]
|
||||
digest := sdkdecl.DigestOf([]byte(want))
|
||||
if _, has := r["content"]; has || r["digest"] != digest || r["size"] != float64(len(want)) {
|
||||
t.Errorf("%v goes with content %v, digest %v, size %v; want no content, %s and %d", r["id"], r["content"] != nil,
|
||||
r["digest"], r["size"], digest, len(want))
|
||||
}
|
||||
if r["path"] == nil || r["mode"] != "0644" {
|
||||
t.Errorf("%v lost its path or mode: %v", r["id"], r)
|
||||
}
|
||||
}
|
||||
if got.Resources[2]["content"] != "tiny\n" || got.Resources[2]["digest"] != nil {
|
||||
t.Errorf("a small file is not inline: %v", got.Resources[2])
|
||||
}
|
||||
if strings.Contains(string(body), users[:200]) || strings.Contains(string(body), kept[:200]) {
|
||||
t.Error("the content of a file that went by digest is in the body")
|
||||
}
|
||||
if n := wireSizeOf(t, s); n > 100<<10 {
|
||||
t.Errorf("the declaration is %d bytes on the wire, want under 100 KB", n)
|
||||
}
|
||||
// The objects to put are the contents, each under the hex of its digest, and a reference verifies against its object.
|
||||
objects, err := s.objects()
|
||||
if err != nil || len(objects) != 2 {
|
||||
t.Fatalf("objects: %v %v", objects, err)
|
||||
}
|
||||
for _, want := range []string{users, kept} {
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(want)))
|
||||
if string(objects[name]) != want {
|
||||
t.Errorf("the object %s does not hold the content", name[:12])
|
||||
}
|
||||
}
|
||||
named, err := namedObjects(body)
|
||||
if err != nil || len(named) != 2 {
|
||||
t.Fatalf("the objects the body names: %v %v", named, err)
|
||||
}
|
||||
for name := range objects {
|
||||
if _, ok := named[name]; !ok {
|
||||
t.Errorf("the body does not name the object %s it was sent with", name[:12])
|
||||
}
|
||||
if _, err := hex.DecodeString(name); err != nil || len(name) != 64 {
|
||||
t.Errorf("the object name %q is not 64 hex characters", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC2: a node that does not report the capability is sent every content inline, as before, and has no object put.
|
||||
func TestANodeWithoutTheCapabilityIsSentEveryContentInline(t *testing.T) { // 1696/AC2
|
||||
users := contentOf("users", 252678)
|
||||
s := sendable{Resources: []map[string]any{aFileResource("nats.bus-users", users)}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Resources[0]["content"] != users || got.Resources[0]["digest"] != nil || got.Resources[0]["size"] != nil {
|
||||
t.Fatalf("a node that reads no references is not sent the content inline: %v", got.Resources[0]["digest"])
|
||||
}
|
||||
if objects, err := s.objects(); err != nil || len(objects) != 0 {
|
||||
t.Fatalf("objects for a node that reads none: %v %v", len(objects), err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC3: only a file that may travel by digest does: one of exactly 64 KiB, a sealed one, one filled with secrets, one written
|
||||
// into another file, and a directory stay inline, whatever the node reads.
|
||||
func TestOnlyAFileThatMayTravelByDigestDoes(t *testing.T) { // 1696/AC3
|
||||
exact := contentOf("exact", 64<<10)
|
||||
big := contentOf("big", 64<<10+1)
|
||||
sealed := aFileResource("sealed", big)
|
||||
sealed["sealed"], sealed["content"] = "box", ""
|
||||
sealed["content"] = big // a sealed file's content is none; the body must pass it as it is, whatever it holds
|
||||
secret := aFileResource("secret", big)
|
||||
secret["secrets"] = map[string]any{"token": "x"}
|
||||
into := aFileResource("into", big)
|
||||
into["into"] = "/etc/other"
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{
|
||||
aFileResource("exact", exact), sealed, secret, into,
|
||||
{"id": "dir", "type": "directory", "path": "/etc/d", "content": big},
|
||||
aFileResource("big", big),
|
||||
}}
|
||||
body, err := s.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, id := range []string{"exact", "sealed", "secret", "into", "dir"} {
|
||||
if got.Resources[i]["id"] != id || got.Resources[i]["digest"] != nil || got.Resources[i]["content"] == nil {
|
||||
t.Errorf("%s went by digest or lost its content: %v", id, got.Resources[i]["digest"])
|
||||
}
|
||||
}
|
||||
if big := got.Resources[5]; big["id"] != "big" || big["digest"] == nil || big["content"] != nil {
|
||||
t.Errorf("a plain file of 64 KiB and a byte did not go by digest: %v", big["digest"])
|
||||
}
|
||||
}
|
||||
|
||||
// A file given as bytes goes by digest on its decoded length, with the decoded bytes in the object.
|
||||
func TestAFileOfBytesTravelsByDigestOnItsDecodedLength(t *testing.T) {
|
||||
raw := []byte(contentOf("png", 70<<10))
|
||||
r := map[string]any{"id": "icon", "type": "file", "path": "/usr/share/icon.png", "bytes": stdBase64(raw)}
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{r}}
|
||||
objects, err := s.objects()
|
||||
if err != nil || len(objects) != 1 {
|
||||
t.Fatalf("%v %v", len(objects), err)
|
||||
}
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf(raw))
|
||||
if string(objects[name]) != string(raw) {
|
||||
t.Fatal("the object is not the decoded bytes")
|
||||
}
|
||||
if r["bytes"] == nil {
|
||||
t.Fatal("the composed resource was changed in place")
|
||||
}
|
||||
}
|
||||
|
||||
type memObjects struct {
|
||||
have map[string][]byte
|
||||
marked map[string]bool
|
||||
puts []string
|
||||
fail map[string]error
|
||||
}
|
||||
|
||||
func (m *memObjects) Has(_ context.Context, name string) (bool, bool, error) {
|
||||
_, ok := m.have[name]
|
||||
return ok, m.marked[name], nil
|
||||
}
|
||||
func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
|
||||
if err := m.fail[name]; err != nil {
|
||||
return err
|
||||
}
|
||||
m.puts = append(m.puts, name)
|
||||
m.have[name] = c
|
||||
delete(m.marked, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1696/AC4: an object already in the bucket is not put again, one absent is, and a put that fails stops the send, naming the object.
|
||||
func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696/AC4
|
||||
objects := map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")}
|
||||
b := &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one")}}
|
||||
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("b", 64) {
|
||||
t.Fatalf("put %v (%v), want the one absent", b.puts, err)
|
||||
}
|
||||
// An object the sweep marked unnamed is put again, which erases the mark: a send relies on it.
|
||||
b = &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")},
|
||||
marked: map[string]bool{strings.Repeat("a", 64): true}}
|
||||
if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("a", 64) || b.marked[strings.Repeat("a", 64)] {
|
||||
t.Fatalf("a marked object: put %v (%v) marked %v, want it put again and the mark gone", b.puts, err, b.marked)
|
||||
}
|
||||
b = &memObjects{have: map[string][]byte{}, fail: map[string]error{strings.Repeat("a", 64): errors.New("permissions violation")}}
|
||||
err := putObjects(t.Context(), b, objects)
|
||||
if err == nil || !strings.Contains(err.Error(), "aaaaaaaaaaaa") || !strings.Contains(err.Error(), "permissions violation") {
|
||||
t.Fatalf("a failed put: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
|
||||
// controller's grant does, and takes the second.
|
||||
type fallbackDelivery struct {
|
||||
bodies [][]byte
|
||||
first error
|
||||
}
|
||||
|
||||
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
|
||||
f.bodies = append(f.bodies, body)
|
||||
if len(f.bodies) == 1 && f.first != nil {
|
||||
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
|
||||
}
|
||||
return "digest", nil
|
||||
}
|
||||
|
||||
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
|
||||
// with every content inline and is made; a send that fails for another reason is not retried inline.
|
||||
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
|
||||
users := contentOf("users", 100<<10)
|
||||
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
|
||||
f := &fallbackDelivery{first: errors.New("timeout")}
|
||||
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
|
||||
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
|
||||
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
|
||||
}
|
||||
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
|
||||
t.Error("the first body should name the file by digest and the second carry it inline")
|
||||
}
|
||||
other := &fallbackDelivery{}
|
||||
other.first = nil
|
||||
failing := &failingDelivery{err: errors.New("the bus is down")}
|
||||
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
|
||||
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type failingDelivery struct {
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
|
||||
f.calls++
|
||||
return "", f.err
|
||||
}
|
||||
|
||||
// A bucket for the sweep.
|
||||
type sweepBucketFake struct {
|
||||
objects map[string]time.Time // zero: named at the last look
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (b *sweepBucketFake) Objects(context.Context) ([]bucketObject, error) {
|
||||
var out []bucketObject
|
||||
for n, at := range b.objects {
|
||||
out = append(out, bucketObject{Name: n, UnnamedSince: at})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (b *sweepBucketFake) MarkUnnamed(_ context.Context, n string, at time.Time) error {
|
||||
b.objects[n] = at
|
||||
return nil
|
||||
}
|
||||
func (b *sweepBucketFake) ClearUnnamed(_ context.Context, n string) error {
|
||||
b.objects[n] = time.Time{}
|
||||
return nil
|
||||
}
|
||||
func (b *sweepBucketFake) Delete(_ context.Context, n string, markedAt time.Time) (bool, error) {
|
||||
if at, ok := b.objects[n]; !ok || !at.Equal(markedAt) {
|
||||
return false, nil // put again since it was judged: its mark is gone
|
||||
}
|
||||
delete(b.objects, n)
|
||||
b.deleted = append(b.deleted, n)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// 1696/AC6: an object a latest declaration still names is not removed, however long; one unnamed is marked at the first look and removed
|
||||
// only when the mark is 24 hours old; one named again loses its mark.
|
||||
func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // 1696/AC6
|
||||
named, unnamed, back := strings.Repeat("1", 64), strings.Repeat("2", 64), strings.Repeat("3", 64)
|
||||
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
b := &sweepBucketFake{objects: map[string]time.Time{named: t0.Add(-100 * time.Hour), unnamed: {}, back: t0.Add(-5 * time.Hour)}}
|
||||
named3 := map[string]bool{named: true, back: true}
|
||||
r, err := sweepDeclarationFiles(t.Context(), b, named3, t0)
|
||||
if err != nil || r.Marked != 1 || r.Cleared != 2 || r.Removed != 0 || len(b.deleted) != 0 {
|
||||
t.Fatalf("first look: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if !b.objects[named].IsZero() || !b.objects[back].IsZero() || !b.objects[unnamed].Equal(t0) {
|
||||
t.Fatalf("marks after the first look: %v", b.objects)
|
||||
}
|
||||
// Still unnamed 23 hours 59 minutes later: kept. A day after the mark: removed. The named ones stay for ever.
|
||||
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour-time.Minute)); err != nil ||
|
||||
r.Removed != 0 || len(b.deleted) != 0 {
|
||||
t.Fatalf("a minute short of a day: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour)); err != nil ||
|
||||
r.Removed != 1 || len(b.deleted) != 1 || b.deleted[0] != unnamed {
|
||||
t.Fatalf("a day after: %+v %v %v", r, err, b.deleted)
|
||||
}
|
||||
if _, ok := b.objects[named]; !ok {
|
||||
t.Fatal("an object a latest declaration names was removed")
|
||||
}
|
||||
// An object named again before the day is up loses its mark and is not removed.
|
||||
b = &sweepBucketFake{objects: map[string]time.Time{unnamed: t0}}
|
||||
if _, err := sweepDeclarationFiles(t.Context(), b, map[string]bool{unnamed: true}, t0.Add(23*time.Hour)); err != nil || !b.objects[unnamed].IsZero() {
|
||||
t.Fatalf("named again: %v %v", b.objects, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
|
||||
// never "names none", which would mark an object a node needs.
|
||||
func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
|
||||
s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
|
||||
body, _ := s.Body()
|
||||
msg, _ := link.Envelope(body, []byte("sig"))
|
||||
named, err := namedByLatest("novox", msg)
|
||||
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
|
||||
if err != nil || len(named) != 1 || !named[name] {
|
||||
t.Fatalf("%v %v", named, err)
|
||||
}
|
||||
if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 {
|
||||
t.Fatalf("a declaration naming none: %v %v", none, err)
|
||||
}
|
||||
if _, err := namedByLatest("novox", []byte("not json")); err == nil {
|
||||
t.Fatal("an unreadable latest declaration was read as naming none")
|
||||
}
|
||||
}
|
||||
|
||||
// 1698/AC2 (ADR 0486 §9): a declaration shaped like the control node's, for a node that reads references, stays under half of the
|
||||
// bus's largest message on the wire; the inline form of the same does not fit at all; and the same shape with 40 percent more resources
|
||||
// fails the test's own limit, so a change that eats the room fails in a pull request and not on the live mesh.
|
||||
func TestTheControlNodesDeclarationInTheReferenceFormStaysUnderHalfTheBusLimit(t *testing.T) { // 1698/AC2
|
||||
const limit = 1048576
|
||||
shape := sendable{fileReferences: true, Resources: controlNodeShape(1)}
|
||||
n := wireSizeOf(t, shape)
|
||||
t.Logf("the control node's shape in the reference form: %d bytes on the wire, %.1f%% of %d", n, 100*float64(n)/limit, limit)
|
||||
if n > limit/2 {
|
||||
t.Errorf("the reference form is %d bytes on the wire, over half (%d) of the bus's largest message", n, limit/2)
|
||||
}
|
||||
if inline := wireSizeOf(t, sendable{Resources: controlNodeShape(1)}); inline < limit*3/4 {
|
||||
t.Errorf("the inline form is %d bytes, which is not the shape that was over the limit", inline)
|
||||
}
|
||||
grown := wireSizeOf(t, sendable{fileReferences: true, Resources: controlNodeShape(1.4)})
|
||||
if grown <= limit/2 {
|
||||
t.Errorf("the shape with 40%% more resources is %d bytes, which the test would let pass", grown)
|
||||
}
|
||||
}
|
||||
|
||||
func stdBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
|
||||
|
||||
// A sweep that read an object's mark before a send put it again must not remove it: the object, now without the mark, is left.
|
||||
func TestASweepDoesNotRemoveAnObjectASendPutAgainAfterItWasJudged(t *testing.T) { // 1696/AC8
|
||||
n := strings.Repeat("4", 64)
|
||||
t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
b := &sweepBucketFake{objects: map[string]time.Time{n: t0}}
|
||||
judged := bucketObject{Name: n, UnnamedSince: t0}
|
||||
b.objects[n] = time.Time{} // the send put it again: no mark
|
||||
if gone, err := b.Delete(t.Context(), judged.Name, judged.UnnamedSince); err != nil || gone || len(b.deleted) != 0 {
|
||||
t.Fatalf("an object put again was removed: %v %v %v", gone, err, b.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// One node's bucket is held by one act at a time in this process: the sweep waits for a send that has put its objects and not yet
|
||||
// published the declaration that names them (1696/AC9).
|
||||
func TestTheSweepOfANodesBucketWaitsForASendThatHasPutItsObjects(t *testing.T) { // 1696/AC9
|
||||
unlock := lockFilesOf("lock-test-node")
|
||||
got := make(chan struct{})
|
||||
go func() {
|
||||
defer lockFilesOf("lock-test-node")()
|
||||
close(got)
|
||||
}()
|
||||
select {
|
||||
case <-got:
|
||||
t.Fatal("the sweep took the node's bucket while a send held it")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
other := lockFilesOf("lock-test-other-node") // another node's bucket is not held
|
||||
other()
|
||||
unlock()
|
||||
select {
|
||||
case <-got:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the sweep did not take the node's bucket once the send was done")
|
||||
}
|
||||
}
|
||||
|
||||
// A put that fails send after send raises a condition; the first failure does not (the grant it waits for travels in that very
|
||||
// declaration), and a put that works clears it (1696/AC10).
|
||||
func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testing.T) { // 1696/AC10
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
|
||||
t.Cleanup(func() { k.Close(t.Context()) })
|
||||
was := conditionsFrom
|
||||
conditionsFrom = k
|
||||
t.Cleanup(func() { conditionsFrom = was })
|
||||
key := filesNotPutObservation("cond-node", "").Key()
|
||||
open := func() bool {
|
||||
_, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return found
|
||||
}
|
||||
boom := errors.New("permissions violation")
|
||||
notePut(t.Context(), "cond-node", boom)
|
||||
if open() {
|
||||
t.Fatal("the first failure raised the condition")
|
||||
}
|
||||
notePut(t.Context(), "cond-node", boom)
|
||||
if !open() {
|
||||
t.Fatal("the second failure in a row did not raise the condition")
|
||||
}
|
||||
notePut(t.Context(), "cond-node", nil)
|
||||
if open() {
|
||||
t.Fatal("a put that worked did not clear the condition")
|
||||
}
|
||||
notePut(t.Context(), "cond-node", boom)
|
||||
if open() {
|
||||
t.Fatal("the count did not start again after a put that worked")
|
||||
}
|
||||
}
|
||||
|
||||
// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
|
||||
type aBusThatSays struct {
|
||||
link.Bus
|
||||
published chan string
|
||||
heldAtPublish func() bool
|
||||
}
|
||||
|
||||
func (b *aBusThatSays) PublishDeclaration(_ context.Context, node string, _ []byte) error {
|
||||
if b.heldAtPublish != nil && !b.heldAtPublish() {
|
||||
return errors.New("published while the node's bucket was not held")
|
||||
}
|
||||
b.published <- node
|
||||
return errors.New("stop here: nothing is recorded")
|
||||
}
|
||||
|
||||
type aSignerOfZeros struct{}
|
||||
|
||||
func (aSignerOfZeros) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
|
||||
|
||||
// whileHeld says whether the node's lock is held by someone right now, without keeping it.
|
||||
func whileHeld(node string) bool {
|
||||
got := make(chan struct{})
|
||||
go func() {
|
||||
defer lockFilesOf(node)()
|
||||
close(got)
|
||||
}()
|
||||
select {
|
||||
case <-got:
|
||||
return false
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// held: the goroutine stays waiting and takes and releases the lock once it is let go.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The
|
||||
// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the
|
||||
// lock is held.
|
||||
func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9
|
||||
const node = "lock-send-node"
|
||||
bus := &aBusThatSays{published: make(chan string, 1)}
|
||||
bus.heldAtPublish = func() bool { return whileHeld(node) }
|
||||
send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}}
|
||||
release := lockFilesOf(node) // a sweep is judging this node's bucket
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case n := <-bus.published:
|
||||
t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-bus.published:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the send did not go on once the sweep let go")
|
||||
}
|
||||
if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") {
|
||||
t.Fatalf("the send did not reach the publication (a lock held at it): %v", err)
|
||||
}
|
||||
}
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
|
||||
// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
|
||||
// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
|
||||
// So the size is measured at every send: a warning condition from 90% of the limit, urgent over it, cleared when
|
||||
// the declaration is back under 90%. The limit is the connection's own, not a constant.
|
||||
// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
|
||||
// the declaration is back under 75%. The limit is the connection's own, not a constant.
|
||||
|
||||
// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
|
||||
const kindDeclarationNearLimit = "declaration-near-limit"
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// From 90% of the bus's largest message the declaration is a warning; over it, urgent; back under 90%, cleared.
|
||||
// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
|
||||
func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
|
||||
store := conditions.NewInMemory()
|
||||
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
|
||||
@@ -50,22 +50,22 @@ func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *test
|
||||
return c, found
|
||||
}
|
||||
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 899, limit)
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("a declaration at 89.9% raised a condition")
|
||||
t.Fatal("a declaration at 74.9% raised a condition")
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 900, limit)
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
|
||||
c, found := read()
|
||||
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "900 bytes") ||
|
||||
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
|
||||
!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
|
||||
t.Fatalf("at 90%% the condition is %v %+v", found, c)
|
||||
t.Fatalf("at 75%% the condition is %v %+v", found, c)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
|
||||
if c, _ := read(); c.Severity != conditions.Urgent {
|
||||
t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
|
||||
}
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 850, limit)
|
||||
settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
|
||||
if _, found := read(); found {
|
||||
t.Fatal("the condition stayed after the declaration fell under 90%")
|
||||
t.Fatal("the condition stayed after the declaration fell under 75%")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
|
||||
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
|
||||
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
|
||||
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
|
||||
// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
|
||||
// node's latest declaration alone. Objects are immutable and regenerable from the
|
||||
// controller's state, so a removal loses nothing.
|
||||
|
||||
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
|
||||
const declarationUnnamed = "unnamed-since"
|
||||
|
||||
// sweepEvery is how often the bucket is judged: the removal is exact to within this.
|
||||
const sweepEvery = time.Hour
|
||||
|
||||
// bucketObject is one object of the bucket as the sweep sees it.
|
||||
type bucketObject struct {
|
||||
Name string
|
||||
// UnnamedSince is when it was first found named by none, zero when it was named at the last look.
|
||||
UnnamedSince time.Time
|
||||
}
|
||||
|
||||
// sweptBucket is what the sweep needs of the bucket.
|
||||
type sweptBucket interface {
|
||||
Objects(ctx context.Context) ([]bucketObject, error)
|
||||
MarkUnnamed(ctx context.Context, name string, since time.Time) error
|
||||
ClearUnnamed(ctx context.Context, name string) error
|
||||
// Delete removes the object if it still carries the mark it was judged by, and says whether it did: a send that put it again
|
||||
// since has erased the mark, and the object is left.
|
||||
Delete(ctx context.Context, name string, markedAt time.Time) (bool, error)
|
||||
}
|
||||
|
||||
// filesSweep says what a sweep did.
|
||||
type filesSweep struct{ Kept, Marked, Cleared, Removed int }
|
||||
|
||||
// sweepDeclarationFiles judges every object against the names of the latest declarations: named ones are kept (and a
|
||||
// mark of being unnamed is erased), unnamed ones are marked at the first look and removed once the mark is a day old.
|
||||
func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[string]bool, now time.Time) (filesSweep, error) {
|
||||
var r filesSweep
|
||||
objects, err := bucket.Objects(ctx)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
sort.Slice(objects, func(i, j int) bool { return objects[i].Name < objects[j].Name })
|
||||
for _, o := range objects {
|
||||
switch {
|
||||
case named[o.Name]:
|
||||
r.Kept++
|
||||
if !o.UnnamedSince.IsZero() {
|
||||
if err := bucket.ClearUnnamed(ctx, o.Name); err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
r.Cleared++
|
||||
}
|
||||
case o.UnnamedSince.IsZero():
|
||||
if err := bucket.MarkUnnamed(ctx, o.Name, now); err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
r.Marked++
|
||||
case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
|
||||
gone, err := bucket.Delete(ctx, o.Name, o.UnnamedSince)
|
||||
if err != nil {
|
||||
return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
|
||||
}
|
||||
if gone {
|
||||
r.Removed++
|
||||
} else {
|
||||
r.Kept++
|
||||
}
|
||||
default:
|
||||
r.Kept++
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
|
||||
// "names none": an object the node still needs would be marked for removal.
|
||||
func namedByLatest(node string, signed []byte) (map[string]bool, error) {
|
||||
var s link.Signed
|
||||
if err := json.Unmarshal(signed, &s); err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
names, err := namedObjects(s.Declaration)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
|
||||
}
|
||||
named := make(map[string]bool, len(names))
|
||||
for name := range names {
|
||||
named[name] = true
|
||||
}
|
||||
return named, nil
|
||||
}
|
||||
|
||||
// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
|
||||
// no entry.
|
||||
func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, node := range nodes {
|
||||
msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
|
||||
if errors.Is(err, nats.ErrMsgNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
|
||||
}
|
||||
out[node] = msg.Data
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// busSweptBucket is the bucket on the controller's connection.
|
||||
type busSweptBucket struct{ store jetstream.ObjectStore }
|
||||
|
||||
func (b busSweptBucket) Objects(ctx context.Context) ([]bucketObject, error) {
|
||||
infos, err := b.store.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]bucketObject, 0, len(infos))
|
||||
for _, info := range infos {
|
||||
o := bucketObject{Name: info.Name}
|
||||
if at, err := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); err == nil {
|
||||
o.UnnamedSince = at
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (b busSweptBucket) meta(ctx context.Context, name string, change func(map[string]string)) error {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta := info.ObjectMeta
|
||||
kept := map[string]string{}
|
||||
for k, v := range meta.Metadata {
|
||||
kept[k] = v
|
||||
}
|
||||
change(kept)
|
||||
meta.Metadata = kept
|
||||
return b.store.UpdateMeta(ctx, name, meta)
|
||||
}
|
||||
|
||||
func (b busSweptBucket) MarkUnnamed(ctx context.Context, name string, since time.Time) error {
|
||||
return b.meta(ctx, name, func(m map[string]string) { m[declarationUnnamed] = since.UTC().Format(time.RFC3339) })
|
||||
}
|
||||
|
||||
func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
|
||||
return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
|
||||
}
|
||||
|
||||
func (b busSweptBucket) Delete(ctx context.Context, name string, markedAt time.Time) (bool, error) {
|
||||
info, err := b.store.GetInfo(ctx, name)
|
||||
if errors.Is(err, jetstream.ErrObjectNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if at, perr := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); perr != nil || !at.Equal(markedAt) {
|
||||
return false, nil
|
||||
}
|
||||
return true, b.store.Delete(ctx, name)
|
||||
}
|
||||
|
||||
// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
|
||||
func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
|
||||
tick := time.NewTicker(sweepEvery)
|
||||
defer tick.Stop()
|
||||
failing := ""
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
if epoch, err := theLease.epoch(ctx); err != nil || epoch == 0 {
|
||||
continue
|
||||
}
|
||||
why := sweepOnce(ctx, inv, conn)
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("the bucket of declaration files cannot be swept: %s\n", why)
|
||||
} else {
|
||||
fmt.Printf("the bucket of declaration files is swept again\n")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sweepOnce judges the bucket of every node against that node's latest declaration, one bucket at a time under the node's lock, with
|
||||
// the declaration read again under it: a send that put an object and published its declaration is seen, and one that has not yet
|
||||
// is waiting for the lock.
|
||||
func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("the nodes cannot be read: %v", err)
|
||||
}
|
||||
var problems []string
|
||||
var total filesSweep
|
||||
for _, n := range nodes {
|
||||
r, err := sweepNode(ctx, conn, n.Name)
|
||||
total.Kept, total.Marked, total.Cleared, total.Removed = total.Kept+r.Kept, total.Marked+r.Marked, total.Cleared+r.Cleared, total.Removed+r.Removed
|
||||
if err != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s: %v", n.Name, err))
|
||||
}
|
||||
}
|
||||
if total.Removed+total.Marked+total.Cleared > 0 {
|
||||
fmt.Printf("the buckets of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
|
||||
total.Kept, total.Marked, total.Cleared, total.Removed)
|
||||
}
|
||||
return strings.Join(problems, "; ")
|
||||
}
|
||||
|
||||
func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, error) {
|
||||
working, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
bucket, err := broker.DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return filesSweep{}, fmt.Errorf("the bucket of node %q: %w", node, err)
|
||||
}
|
||||
store, err := js.ObjectStore(working, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return filesSweep{}, nil // a node never sent a reference has no bucket yet
|
||||
}
|
||||
if err != nil {
|
||||
return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
|
||||
}
|
||||
defer lockFilesOf(node)()
|
||||
latest, err := latestDeclarations(working, conn, []string{node})
|
||||
if err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
named := map[string]bool{}
|
||||
if body, ok := latest[node]; ok {
|
||||
if named, err = namedByLatest(node, body); err != nil {
|
||||
return filesSweep{}, err
|
||||
}
|
||||
}
|
||||
return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
|
||||
}
|
||||
@@ -195,6 +195,14 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"there yet is sealed by the controller until the vault holds it.", m, m),
|
||||
Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
|
||||
}),
|
||||
kindFilesNotPut: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
return words{Headline: "The big files for " + m + " cannot be put on the bus",
|
||||
Needs: "look at the bus's permissions for the controller; the details give the reason.",
|
||||
Explanation: fmt.Sprintf("The mesh could not put the large files of %s's instructions in their place on the bus, "+
|
||||
"send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m),
|
||||
Resolved: "Resolved: the big files for " + m + " reach the bus again"}
|
||||
}),
|
||||
kindDeclarationNearLimit: worded(func(o conditions.Observation) words {
|
||||
m := machineOr(o, "a machine")
|
||||
if o.Severity == conditions.Urgent {
|
||||
|
||||
@@ -78,3 +78,50 @@ func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) {
|
||||
|
||||
// novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision,
|
||||
// its last save and its `plan-moved` stay where its last change left them.
|
||||
|
||||
// novox/hq issue 920: a building plan whose current tier is all built waits on its note (a send refused at the
|
||||
// gate), and its line, `status` and the stalled condition say that, not "building for ... LATE".
|
||||
func TestAPlanLineSaysTheRefusalOnceItsTierIsBuilt(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
built := now.Add(-time.Hour)
|
||||
refusal := "tier 1: a build waits there for its gate: novox (mesh-delivery-indexer d68d7ef3→a5aa69a4) — tried again"
|
||||
mk := func(second string) inventory.Plan {
|
||||
return inventory.Plan{ID: "plan-r", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
|
||||
Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: refusal,
|
||||
Tiers: [][]string{{"a"}, {"b", "c"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built},
|
||||
"b": {State: "built", BuiltAt: &built}, "c": {State: second}}}
|
||||
}
|
||||
|
||||
// 920/AC1
|
||||
p := mk("built")
|
||||
line := planLine(p, now)
|
||||
if !strings.Contains(line, "a build waits there for its gate: novox") || strings.Contains(line, "building for") {
|
||||
t.Errorf("920/AC1: a plan with its tier built and a refusal reads %q", line)
|
||||
}
|
||||
st := planStatuses([]inventory.Plan{p}, now, pauseView{}, nil)
|
||||
if len(st) != 1 || !st[0].Late {
|
||||
t.Errorf("920/AC1: status: %+v", st)
|
||||
}
|
||||
|
||||
// 920/AC2
|
||||
if line := planLine(mk("asked"), now); !strings.Contains(line, "building for 1h0m0s") {
|
||||
t.Errorf("920/AC2: a plan with a tier module not built reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// 920/AC3: a tier move writes the previous tier's summary into the note; with the new tier all built, the line still
|
||||
// says "building for", for that note is no refusal.
|
||||
func TestAPlanLineDoesNotHoldOnATiersSummary(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||
built := now.Add(-time.Hour)
|
||||
for _, note := range []string{"failed or refused its send (apply-failing says it): novox", "tier 0 built; sent mesh-sdk to novox first, in one send"} {
|
||||
p := inventory.Plan{ID: "plan-s", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
|
||||
Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: note,
|
||||
Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, "b": {State: "built", BuiltAt: &built}}}
|
||||
if line := planLine(p, now); !strings.Contains(line, "building for 1h0m0s") || strings.Contains(line, "held") {
|
||||
t.Errorf("920/AC3: a plan with the note %q reads %q", note, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -206,6 +206,7 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv
|
||||
return func(_ *inventory.Inventory, p inventory.Plan) {
|
||||
sayPlanMoved(ctx, bus, open.inventory, p)
|
||||
planKept(p)
|
||||
keepBuildingKey(ctx, p)
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The plans building (novox/hq ADR 0483 clause 2, issue 1687).
|
||||
//
|
||||
// **A key per delivery plan in its build phase, so that the build seat takes no new check while a delivery builds.** The phase
|
||||
// runs from the ask of the plan's first build until every build of its last tier is built, or the plan fails, stops or waits
|
||||
// on the operator. It is derived from the plan alone (buildPhaseOf), so a controller that restarts finds the same answer. The
|
||||
// key is put when the first build is asked (sayBuilding), kept current at every save of the plan (keepBuildingKey) and by a
|
||||
// look every link.PlansBuildingRefresh while this controller holds the lease (keepingPlansBuilding), and deleted at the phase's
|
||||
// end; the bucket's age removes the key of a controller that stopped, so checks are held at most that long.
|
||||
//
|
||||
// The layout is documented in internal/link/holds.go.
|
||||
|
||||
// buildPhaseOf says whether a plan is in its build phase: open, not waiting for the operator's word, with a build asked and a
|
||||
// build of its tiers still to be built. A tier built and sent leaves the plan rolling while later tiers are still to be asked;
|
||||
// it is in the phase still, which is what keeps a check from taking a place between two tiers.
|
||||
func buildPhaseOf(p inventory.Plan) bool {
|
||||
if !p.Open() || p.Waiting() || p.Release != nil {
|
||||
return false
|
||||
}
|
||||
asked, unbuilt := false, false
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
st := p.Modules[m]
|
||||
if st != nil && st.State != "" {
|
||||
asked = true
|
||||
}
|
||||
if st == nil || (st.State != "built" && st.State != planDeleted) {
|
||||
unbuilt = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return asked && unbuilt
|
||||
}
|
||||
|
||||
// buildingSince is when the plan's first build was asked, or fallback when no module says.
|
||||
func buildingSince(p inventory.Plan, fallback time.Time) time.Time {
|
||||
since := time.Time{}
|
||||
for _, st := range p.Modules {
|
||||
if st != nil && st.AskedAt != nil && (since.IsZero() || st.AskedAt.Before(since)) {
|
||||
since = *st.AskedAt
|
||||
}
|
||||
}
|
||||
if since.IsZero() {
|
||||
return fallback
|
||||
}
|
||||
return since
|
||||
}
|
||||
|
||||
// buildingStore is the bucket of the plans building.
|
||||
type buildingStore interface {
|
||||
Put(ctx context.Context, plan string, value []byte) error
|
||||
Delete(ctx context.Context, plan string) error
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// syncBuilding makes the bucket say what the plans say: a key for each plan in its build phase, put again, and none for any
|
||||
// other plan, deleted. It returns the faults, each once.
|
||||
func syncBuilding(ctx context.Context, store buildingStore, plans []inventory.Plan, now time.Time) []error {
|
||||
var faults []error
|
||||
live := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
if !buildPhaseOf(p) {
|
||||
continue
|
||||
}
|
||||
live[p.ID] = true
|
||||
if err := putBuilding(ctx, store, p.ID, buildingSince(p, now), now); err != nil {
|
||||
faults = append(faults, err)
|
||||
}
|
||||
}
|
||||
keys, err := store.Keys(ctx)
|
||||
if err != nil {
|
||||
return append(faults, err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if live[key] {
|
||||
continue
|
||||
}
|
||||
if err := store.Delete(ctx, key); err != nil {
|
||||
faults = append(faults, fmt.Errorf("the key of %s could not be taken away: %w", key, err))
|
||||
}
|
||||
}
|
||||
return faults
|
||||
}
|
||||
|
||||
func putBuilding(ctx context.Context, store buildingStore, plan string, since, now time.Time) error {
|
||||
body, _ := json.Marshal(link.PlanBuilding{Plan: plan, Since: since.UTC(), Seen: now.UTC()})
|
||||
if err := store.Put(ctx, plan, body); err != nil {
|
||||
return fmt.Errorf("the key of %s could not be put: %w", plan, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// busBuilding is the store on the controller's bucket.
|
||||
type busBuilding struct {
|
||||
conn *nats.Conn
|
||||
mu sync.Mutex
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (b *busBuilding) bucket(ctx context.Context) (jetstream.KeyValue, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.kv != nil {
|
||||
return b.kv, nil
|
||||
}
|
||||
api, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
b.kv = kv
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
func (b *busBuilding) Put(ctx context.Context, plan string, value []byte) error {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = kv.Put(ctx, plan, value)
|
||||
return err
|
||||
}
|
||||
|
||||
func (b *busBuilding) Delete(ctx context.Context, plan string) error {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kv.Delete(ctx, plan); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *busBuilding) Keys(ctx context.Context) ([]string, error) {
|
||||
kv, err := b.bucket(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
// buildingKeys is the store of the serving controller; nil in a process that is not it, which says nothing.
|
||||
var buildingKeys atomic.Pointer[buildingStore]
|
||||
|
||||
func setBuildingStore(s buildingStore) {
|
||||
if s == nil {
|
||||
buildingKeys.Store(nil)
|
||||
return
|
||||
}
|
||||
buildingKeys.Store(&s)
|
||||
}
|
||||
|
||||
// sayBuilding puts a plan's key: its first build is being asked (or another of its builds). A key that could not be put is
|
||||
// said; the look that follows puts it again.
|
||||
func sayBuilding(ctx context.Context, plan string, since time.Time) {
|
||||
s := buildingKeys.Load()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if err := putBuilding(ctx, *s, plan, since, time.Now()); err != nil {
|
||||
fmt.Printf("plans: %v; checks may take a place of %s until the next look\n", err, plan)
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuildingKey is told each plan this controller keeps: its key put again while it builds and taken away once it does not.
|
||||
func keepBuildingKey(ctx context.Context, p inventory.Plan) {
|
||||
s := buildingKeys.Load()
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if buildPhaseOf(p) {
|
||||
sayBuilding(ctx, p.ID, buildingSince(p, time.Now()))
|
||||
return
|
||||
}
|
||||
if err := (*s).Delete(ctx, p.ID); err != nil {
|
||||
fmt.Printf("plans: the key of %s could not be taken away; it ages out: %v\n", p.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// keepingPlansBuilding looks at the plans every link.PlansBuildingRefresh for as long as this controller serves and holds the
|
||||
// lease, so a key is put again before its age and the key of a plan that left the phase unseen is taken away.
|
||||
func keepingPlansBuilding(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
|
||||
store := &busBuilding{conn: conn}
|
||||
setBuildingStore(store)
|
||||
defer setBuildingStore(nil)
|
||||
tick := time.NewTicker(broker.PlansBuildingRefresh)
|
||||
defer tick.Stop()
|
||||
failing := ""
|
||||
for {
|
||||
if epoch, err := theLease.epoch(ctx); err == nil && epoch != 0 {
|
||||
why := ""
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil {
|
||||
why = fmt.Sprintf("the plans cannot be read: %v", err)
|
||||
} else if faults := syncBuilding(ctx, store, plans, time.Now()); len(faults) > 0 {
|
||||
why = fmt.Sprint(faults)
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("plans: the keys of the plans building cannot be kept: %s\n", why)
|
||||
} else {
|
||||
fmt.Printf("plans: the keys of the plans building are kept again\n")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Issue 1687 (ADR 0483 clause 2): a delivery plan's key lives from the ask of its first build until every build of its last
|
||||
// tier is built, or the plan fails, stops or waits on the operator.
|
||||
|
||||
type fakeBuildingStore struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]link.PlanBuilding
|
||||
puts map[string]int
|
||||
}
|
||||
|
||||
func newFakeBuildingStore() *fakeBuildingStore {
|
||||
return &fakeBuildingStore{keys: map[string]link.PlanBuilding{}, puts: map[string]int{}}
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Put(_ context.Context, plan string, value []byte) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var v link.PlanBuilding
|
||||
if err := json.Unmarshal(value, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
f.keys[plan] = v
|
||||
f.puts[plan]++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Delete(_ context.Context, plan string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.keys, plan)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeBuildingStore) Keys(context.Context) ([]string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []string
|
||||
for k := range f.keys {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func aPlan(id, state string, tiers [][]string, states map[string]string) inventory.Plan {
|
||||
p := inventory.Plan{ID: id, State: state, Tiers: tiers, Modules: map[string]*inventory.PlanModule{}}
|
||||
now := time.Now().Add(-time.Minute)
|
||||
for m, st := range states {
|
||||
pm := &inventory.PlanModule{State: st}
|
||||
if st != "" {
|
||||
pm.AskedAt = &now
|
||||
}
|
||||
p.Modules[m] = pm
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestAPlanIsInItsBuildPhaseFromTheFirstAskToTheLastBuild(t *testing.T) {
|
||||
tiers := [][]string{{"a", "b"}, {"c"}}
|
||||
cases := []struct {
|
||||
name string
|
||||
plan inventory.Plan
|
||||
want bool
|
||||
}{
|
||||
{"no build asked yet", aPlan("p", inventory.PlanBuilding, tiers, nil), false},
|
||||
{"the first tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked", "b": "asked"}), true},
|
||||
// Between two tiers: the first built and sent (rolling), the second not asked yet.
|
||||
{"between two tiers", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"}), true},
|
||||
{"the last tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "built", "b": "built", "c": "asked"}), true},
|
||||
{"every build built", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": "built"}), false},
|
||||
// A module the plan deletes is done as far as its tiers go (release_plan.go), so it holds no key through the rollout.
|
||||
{"a deleted module is done", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": planDeleted}), false},
|
||||
{"failed", aPlan("p", inventory.PlanFailed, tiers, map[string]string{"a": "failed", "b": "asked"}), false},
|
||||
{"superseded", aPlan("p", inventory.PlanSuperseded, tiers, map[string]string{"a": "asked"}), false},
|
||||
{"a batch whose window is open", aPlan("p", inventory.PlanAssembling, tiers, map[string]string{"a": "asked"}), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := buildPhaseOf(c.plan); got != c.want {
|
||||
t.Errorf("%s: in the build phase = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
waiting := aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
|
||||
waiting.Delivery = &inventory.PlanDelivery{Awaits: "the operator's word"}
|
||||
if buildPhaseOf(waiting) {
|
||||
t.Error("a plan that waits on the operator is in its build phase")
|
||||
}
|
||||
}
|
||||
|
||||
// The key's life: put, refreshed, deleted at the phase's end; a plan that left the phase unseen loses its key at the next look.
|
||||
func TestAPlanKeyIsPutRefreshedAndDeletedAtThePhasesEnd(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newFakeBuildingStore()
|
||||
tiers := [][]string{{"a"}, {"b"}}
|
||||
building := aPlan("plan-1", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
|
||||
|
||||
if faults := syncBuilding(ctx, store, []inventory.Plan{building}, time.Now()); len(faults) > 0 {
|
||||
t.Fatal(faults)
|
||||
}
|
||||
if _, ok := store.keys["plan-1"]; !ok {
|
||||
t.Fatal("no key while the first build is asked")
|
||||
}
|
||||
first := store.keys["plan-1"]
|
||||
// Refreshed: put again, the phase's start kept.
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
syncBuilding(ctx, store, []inventory.Plan{building}, time.Now())
|
||||
if store.puts["plan-1"] != 2 || !store.keys["plan-1"].Seen.After(first.Seen) || !store.keys["plan-1"].Since.Equal(first.Since) {
|
||||
t.Errorf("the key was not refreshed with its start kept: %d puts, %+v then %+v", store.puts["plan-1"], first, store.keys["plan-1"])
|
||||
}
|
||||
// Between the tiers: still there.
|
||||
between := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built"})
|
||||
syncBuilding(ctx, store, []inventory.Plan{between}, time.Now())
|
||||
if _, ok := store.keys["plan-1"]; !ok {
|
||||
t.Error("the key was gone between two tiers")
|
||||
}
|
||||
// A key of another plan that is no longer in the phase.
|
||||
store.keys["plan-0"] = link.PlanBuilding{Plan: "plan-0"}
|
||||
done := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"})
|
||||
syncBuilding(ctx, store, []inventory.Plan{done}, time.Now())
|
||||
if len(store.keys) != 0 {
|
||||
t.Errorf("keys remain after the last build was built: %v", store.keys)
|
||||
}
|
||||
}
|
||||
|
||||
// keepBuildingKey, told of each save, puts the key while the plan builds and deletes it when the plan fails.
|
||||
func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := newFakeBuildingStore()
|
||||
var s buildingStore = store
|
||||
buildingKeys.Store(&s)
|
||||
defer buildingKeys.Store(nil)
|
||||
|
||||
p := aPlan("plan-7", inventory.PlanBuilding, [][]string{{"a"}}, map[string]string{"a": "asked"})
|
||||
keepBuildingKey(ctx, p)
|
||||
if _, ok := store.keys["plan-7"]; !ok {
|
||||
t.Fatal("the key was not put when the plan was saved building")
|
||||
}
|
||||
p.State = inventory.PlanFailed
|
||||
keepBuildingKey(ctx, p)
|
||||
if _, ok := store.keys["plan-7"]; ok {
|
||||
t.Fatal("the key stayed after the plan failed")
|
||||
}
|
||||
}
|
||||
|
||||
// The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
|
||||
func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
|
||||
store := newFakeBuildingStore()
|
||||
var s buildingStore = store
|
||||
buildingKeys.Store(&s)
|
||||
defer buildingKeys.Store(nil)
|
||||
p, byName := aTierOfThree()
|
||||
keyThere := false
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
_, keyThere = store.keys["plan-1"]
|
||||
return "build-x", nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
askModule(t.Context(), p, "a", byName)
|
||||
if !keyThere {
|
||||
t.Fatal("the plan's key was not there when its first build was asked")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// processUser is the bus user of one process of a module on a node (novox/hq ADR 0436 §1).
|
||||
func processUser(node, module, process string) string {
|
||||
return broker.Principal{Kind: broker.KindProcess, Node: node, Module: module, Process: process}.Username()
|
||||
}
|
||||
|
||||
// processAccountWarning is the warning line for the own secret broker-<process id> of a process that declares bus
|
||||
// when nobody issued its account (novox/hq ADR 0436 §2, issue 1226): "" for any other secret and for an issued
|
||||
// account. The send goes on with the stand-in until the composition leaves the module off (issue 1753), so the
|
||||
// line is the only loud thing about it.
|
||||
func processAccountWarning(m catalogue.Manifest, node, secret string, minted []string) string {
|
||||
id, isProcess := strings.CutPrefix(secret, "broker-")
|
||||
if !isProcess {
|
||||
return ""
|
||||
}
|
||||
if _, declared := m.ProcessBuses()[id]; !declared {
|
||||
return ""
|
||||
}
|
||||
user := processUser(node, m.Module, id)
|
||||
for _, u := range minted {
|
||||
if u == user {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s/%s process %s: its bus account is not issued — mesh-controller.issue %s %s",
|
||||
node, m.Module, id, node, m.Module)
|
||||
}
|
||||
|
||||
// sayWarning writes a line where the compose's other warnings go: the act's answer when it carries a writer,
|
||||
// else the controller's journal.
|
||||
func sayWarning(ctx context.Context, line string) {
|
||||
var w io.Writer = os.Stderr
|
||||
if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
|
||||
w = lw
|
||||
}
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
|
||||
// processAccountsMissing is each process of m that declares bus and whose account is not minted yet on node.
|
||||
func processAccountsMissing(m catalogue.Manifest, node string, minted []string) []broker.ProcessPrincipal {
|
||||
have := map[string]bool{}
|
||||
for _, u := range minted {
|
||||
have[u] = true
|
||||
}
|
||||
var out []broker.ProcessPrincipal
|
||||
for _, pr := range inventory.ProcessesOf(m) {
|
||||
if !have[processUser(node, m.Module, pr.Process)] {
|
||||
out = append(out, pr)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// issueMissingProcessAccounts issues the accounts of m's bus processes on node that are not minted, and no
|
||||
// other: an account already issued is rotated on purpose, never by a later delivery or assignment (issue 1226).
|
||||
// It says what it issued, one line, and nothing when nothing was missing.
|
||||
func issueMissingProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) (string, error) {
|
||||
if len(m.ProcessBuses()) == 0 || mayIssue(m) != nil {
|
||||
return "", nil
|
||||
}
|
||||
minted, err := inv.BusMinted(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
missing := processAccountsMissing(m, node, minted)
|
||||
if len(missing) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
|
||||
}
|
||||
reachable, err := brokerReachableAt(ctx, inv, known, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
busAddress = ""
|
||||
}
|
||||
var ids []string
|
||||
for _, pr := range missing {
|
||||
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ids = append(ids, pr.Process)
|
||||
}
|
||||
return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push",
|
||||
strings.Join(ids, ", "), node), nil
|
||||
}
|
||||
|
||||
// issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the
|
||||
// accounts of its bus processes not yet minted are issued, so a process a delivery adds is not pushed with a
|
||||
// random stand-in (issue 1226). Never fatal: the build worked and the module is registered, so a failure is said,
|
||||
// and the push refuses the module until `mesh-controller.issue` is run.
|
||||
func issueGainedProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) {
|
||||
if len(m.ProcessBuses()) == 0 {
|
||||
return
|
||||
}
|
||||
nodes, err := inv.Running(ctx, m.Module)
|
||||
if err != nil {
|
||||
fmt.Printf("%s: its machines are not read to issue its process accounts: %v\n", m.Module, err)
|
||||
return
|
||||
}
|
||||
for _, node := range nodes {
|
||||
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
||||
if err != nil {
|
||||
fmt.Printf("%s on %s: its process accounts are not issued (%v): `mesh-controller.issue %s %s` first\n",
|
||||
m.Module, node, err, node, m.Module)
|
||||
continue
|
||||
}
|
||||
if line != "" {
|
||||
fmt.Printf("%s on %s: %s\n", m.Module, node, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
func gainedAndPlain(t *testing.T) (plain, gained catalogue.Manifest) {
|
||||
t.Helper()
|
||||
plain, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
|
||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return plain, mailSenderWithProcess()
|
||||
}
|
||||
|
||||
// 1226/AC2: given an assigned module whose new definition adds a bus process, when the delivery records it
|
||||
// (takeIn, the function that records a build result), the process's account is minted; and 1226/AC3: the same
|
||||
// build heard again replaces nothing.
|
||||
func TestAGainedProcessIsIssuedWhenTheDeliveryRecordsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
dir := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", dir}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(broker.AddressVar, strings.TrimPrefix(testbus.URL(t), "nats://"))
|
||||
t.Setenv(broker.CertificateVar, filepath.Join(dir, "tls.crt"))
|
||||
|
||||
plain, gained := gainedAndPlain(t)
|
||||
register(t, open, plain)
|
||||
if _, err := assign(ctx, open, "laptop", "mesh-mail"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, minted, _ := inv.BusUserHash(ctx, "laptop.mesh-mail.send"); minted {
|
||||
t.Fatal("the account was minted before the module gained the process")
|
||||
}
|
||||
|
||||
manifest, _ := json.Marshal(gained)
|
||||
build := func(id, commit string) {
|
||||
t.Helper()
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, link.BuildResult{
|
||||
ID: id, Repository: "http://forge.internal:20000/novox/mesh-mail.git", Path: "modules/mesh-mail",
|
||||
Ref: "main", On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-mail"},
|
||||
})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
build("b-1", "aaaaaaa1111")
|
||||
hash, held, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
|
||||
if err != nil || !held {
|
||||
t.Fatalf("1226/AC2: recording the gained definition did not mint the process's account: held %v, %v", held, err)
|
||||
}
|
||||
// And the next compose carries the account, with no warning (1226/AC2).
|
||||
var said bytes.Buffer
|
||||
if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
|
||||
t.Fatalf("1226/AC2: the next compose failed: %v", err)
|
||||
}
|
||||
if strings.Contains(said.String(), "bus account is not issued") {
|
||||
t.Fatalf("1226/AC2: the next compose warns of an account just minted: %q", said.String())
|
||||
}
|
||||
// The row the compose reads is the account issueProcess accepted (origin accepted), never a value the mesh
|
||||
// made for it (a stand-in).
|
||||
in, err := inputFor(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var row *compose.OwnSecretRow
|
||||
for i, r := range in.OwnSecrets {
|
||||
if r.Node == "laptop" && r.Module == "mesh-mail" && r.Name == "broker-send" {
|
||||
row = &in.OwnSecrets[i]
|
||||
}
|
||||
}
|
||||
if row == nil {
|
||||
t.Fatal("1226/AC2: the next compose carries no row for the process's account")
|
||||
}
|
||||
if row.Origin != inventory.OriginAccepted {
|
||||
t.Fatalf("1226/AC2: the process's account is a stand-in the mesh made (origin %q), not the issued account", row.Origin)
|
||||
}
|
||||
build("b-2", "bbbbbbb2222")
|
||||
again, _, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
|
||||
if err != nil || again != hash {
|
||||
t.Fatalf("1226/AC3: recording again replaced the issued account (%v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1226/AC1 and AC4 against the store: given two modules on a machine, one with an unissued process account, when
|
||||
// the machine is composed, the rest of it is composed and sent, and only the warning in the push's answer names
|
||||
// the first, with `mesh-controller.issue <node> <module>`.
|
||||
func TestAnUnissuedProcessAccountIsWarnedOfAndTheRestIsComposed(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
_, gained := gainedAndPlain(t)
|
||||
register(t, open, gained)
|
||||
other := catalogue.Manifest{Module: "other", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/other/token"}}}
|
||||
register(t, open, other)
|
||||
for _, m := range []string{"mesh-mail", "other"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var said bytes.Buffer
|
||||
if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
|
||||
t.Fatalf("1226/AC4: an unissued process account held the machine: %v", err)
|
||||
}
|
||||
want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Fatalf("1226/AC1: the push says %q, want %q", said.String(), want)
|
||||
}
|
||||
if strings.Contains(said.String(), "other") {
|
||||
t.Fatalf("1226/AC4: the other module is warned of: %q", said.String())
|
||||
}
|
||||
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "other", "token"); err != nil || !held {
|
||||
t.Fatalf("1226/AC4: the other module's secret was not composed: held %v, %v", held, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1226/AC1: given a module whose process declares bus and has no issued account, when a machine is composed, the
|
||||
// push's warning names issue, and the machine's other modules are composed (AC4: nothing is refused).
|
||||
func TestAProcessAccountNobodyIssuedIsWarnedOfNamingIssue(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
got := processAccountWarning(m, "laptop", "broker-send", nil)
|
||||
want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
|
||||
if got != want {
|
||||
t.Fatalf("1226/AC1: the warning is %q, want %q", got, want)
|
||||
}
|
||||
// Other names are not this rule's.
|
||||
if processAccountWarning(m, "laptop", "broker", nil) != "" || processAccountWarning(m, "laptop", "broker-other", nil) != "" ||
|
||||
processAccountWarning(m, "laptop", "token", nil) != "" {
|
||||
t.Fatal("1226/AC1: a name that is no declared process's account was warned of")
|
||||
}
|
||||
if got := processAccountWarning(m, "laptop", "broker-send", []string{"laptop.mesh-mail.send"}); got != "" {
|
||||
t.Fatalf("1226/AC3: an issued account was warned of: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1226/AC3: an account already issued is not issued again by a later delivery or assignment.
|
||||
func TestAnIssuedProcessAccountIsNotIssuedAgain(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
if got := processAccountsMissing(m, "laptop", nil); len(got) != 1 || got[0].Process != "send" {
|
||||
t.Fatalf("1226/AC3: an account not issued is not missing: %v", got)
|
||||
}
|
||||
if got := processAccountsMissing(m, "laptop", []string{"laptop.mesh-mail.send"}); len(got) != 0 {
|
||||
t.Fatalf("1226/AC3: an issued account would be issued again: %v", got)
|
||||
}
|
||||
if got := processAccountsMissing(m, "ace", []string{"laptop.mesh-mail.send"}); len(got) != 1 {
|
||||
t.Fatalf("1226/AC3: another machine's account stands in for this one's: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -282,6 +282,10 @@ func serve(ctx context.Context) (err error) {
|
||||
// assignment.
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
go mirroringRecords(ctx, inv, bus.Conn, bus)
|
||||
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
|
||||
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
|
||||
// And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2).
|
||||
go keepingPlansBuilding(ctx, inv, bus.Conn)
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
@@ -979,6 +983,18 @@ func deliver(ctx context.Context, d delivery, holder string, sending []readyNode
|
||||
return err
|
||||
}
|
||||
digest, err := d.declare(ctx, s, body)
|
||||
if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
|
||||
// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
|
||||
// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
|
||||
// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
|
||||
// references, and says so; the next one tries the bucket again.
|
||||
fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
|
||||
s.declared.fileReferences = false
|
||||
if body, err = s.declared.Body(); err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err = d.declare(ctx, s, body)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1040,6 +1056,35 @@ type overTheBus struct {
|
||||
indent string
|
||||
}
|
||||
|
||||
// putFiles puts the contents s names by digest in the bucket.
|
||||
func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
|
||||
objects, err := s.declared.objects()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
return nil
|
||||
}
|
||||
bus, ok := b.server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
|
||||
}
|
||||
putting, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
store, err := openBucket(putting, bus.Conn, s.node)
|
||||
if err == nil {
|
||||
err = putObjects(putting, store, objects)
|
||||
}
|
||||
notePut(ctx, s.node, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
|
||||
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||
@@ -1052,9 +1097,20 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
}
|
||||
|
||||
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
|
||||
// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
|
||||
// that applies it finds every one. A node reading no references has none.
|
||||
// **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them.
|
||||
unlock := lockFilesOf(s.node)
|
||||
err := b.putFiles(ctx, s)
|
||||
if err != nil {
|
||||
unlock()
|
||||
return "", err
|
||||
}
|
||||
declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second)
|
||||
unlock()
|
||||
if declared != nil {
|
||||
return "", declared
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
||||
|
||||
@@ -91,7 +91,9 @@ type queueAnswer struct {
|
||||
Dead int `json:"dead"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []queuedAnswer `json:"asks"`
|
||||
Text string `json:"text"`
|
||||
// HeldFor are the delivery plans building, while which the waiting checks are held (ADR 0483 clause 5); empty when none.
|
||||
HeldFor []string `json:"held-for,omitempty"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
|
||||
@@ -120,6 +122,9 @@ type queuedAnswer struct {
|
||||
// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
|
||||
// (novox/hq issue 594).
|
||||
Delivery int `json:"delivery,omitempty"`
|
||||
// LetThrough is a check that waited past 30 minutes and is let through while a delivery builds: the next one, or the one
|
||||
// that runs (ADR 0483 clause 5).
|
||||
LetThrough bool `json:"let-through,omitempty"`
|
||||
}
|
||||
|
||||
// aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
|
||||
@@ -130,7 +135,8 @@ var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
|
||||
func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||
out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
|
||||
MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, HeldFor: q.Hold.Plans, Text: queueText(q, now)}
|
||||
letNext, letRunning := q.LetThroughOf(now)
|
||||
one := func(a link.QueuedAsk, position int) queuedAnswer {
|
||||
kind := link.KindOf(a.Request)
|
||||
since := a.AskedAt
|
||||
@@ -143,7 +149,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
|
||||
}
|
||||
return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
|
||||
Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery}
|
||||
Position: position, Seq: a.Seq, Delivery: a.Delivery, LetThrough: a.ID != "" && (a.ID == letNext || a.ID == letRunning)}
|
||||
}
|
||||
for _, a := range running {
|
||||
out.Asks = append(out.Asks, one(a, 0))
|
||||
@@ -179,6 +185,11 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
letNext, letRunning := q.LetThroughOf(now)
|
||||
const passed = "waited past 30 minutes, let through"
|
||||
if q.HoldFault != "" {
|
||||
fmt.Fprintf(&b, "why checks wait cannot be read, so none is said held: %s\n", q.HoldFault)
|
||||
}
|
||||
if len(running) > 0 {
|
||||
fmt.Fprintln(&b, "\nin flight:")
|
||||
for _, a := range running {
|
||||
@@ -193,14 +204,27 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq)
|
||||
note := ""
|
||||
if a.ID != "" && a.ID == letRunning {
|
||||
note = " — " + passed
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s, %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), where,
|
||||
ago(a.AskedAt), a.Seq, note)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
|
||||
// **Why the checks wait** (ADR 0483 clause 5): above them, the plans they are held for.
|
||||
if len(q.Hold.Plans) > 0 && hasCheck(waiting) {
|
||||
fmt.Fprintf(&b, " checks are held while delivery %s builds; the oldest check that waited 30 minutes goes first, one at a time\n",
|
||||
strings.Join(q.Hold.Plans, ", "))
|
||||
}
|
||||
for _, a := range waiting {
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
|
||||
note := ""
|
||||
if a.ID != "" && a.ID == letNext {
|
||||
note = " — " + passed + " next"
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %-8s %s — %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq, note)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
@@ -218,6 +242,16 @@ func queueText(q link.Queue, now time.Time) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// hasCheck says a check is among the asks.
|
||||
func hasCheck(asks []link.QueuedAsk) bool {
|
||||
for _, a := range asks {
|
||||
if a.Request.Check != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// cancelCommand drops one waiting or dead ask.
|
||||
func cancelCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Issue 1688 / 1652 AC5 (ADR 0483 clause 5): the queue names, above the waiting checks, the plan they are held for, and says of a
|
||||
// check let through that it waited past 30 minutes.
|
||||
|
||||
func aHeldQueue(now time.Time) link.Queue {
|
||||
check := func(id string, ago time.Duration, n int) link.QueuedAsk {
|
||||
return link.QueuedAsk{Seq: uint64(n), ID: id, Repository: "ssh://forge/novox/r", AskedAt: now.Add(-ago), State: link.AskWaiting,
|
||||
Request: link.BuildRequest{ID: id, Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: n}}}
|
||||
}
|
||||
return link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{
|
||||
check("build-young", 5*time.Minute, 1),
|
||||
check("build-old", 45*time.Minute, 2),
|
||||
check("build-older", 50*time.Minute, 3),
|
||||
}, Hold: link.CheckHold{Plans: []string{"plan-42"}}}
|
||||
}
|
||||
|
||||
func TestTheQueueNamesThePlanTheChecksAreHeldFor(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
text := queueText(aHeldQueue(now), now)
|
||||
heldAt := strings.Index(text, "held while delivery plan-42 builds")
|
||||
firstAt := strings.Index(text, "build-young")
|
||||
if heldAt < 0 || firstAt < 0 || heldAt > firstAt {
|
||||
t.Fatalf("the plan the checks are held for is not named above the waiting checks:\n%s", text)
|
||||
}
|
||||
// Nothing building: nothing held.
|
||||
q := aHeldQueue(now)
|
||||
q.Hold = link.CheckHold{}
|
||||
if strings.Contains(queueText(q, now), "held while") {
|
||||
t.Fatal("checks are said held with no delivery building")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACheckLetThroughIsSaidToHaveWaitedPastThirtyMinutes(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
q := aHeldQueue(now)
|
||||
text := queueText(q, now)
|
||||
var saidOf []string
|
||||
for _, l := range strings.Split(text, "\n") {
|
||||
if strings.Contains(l, "waited past 30 minutes, let through") {
|
||||
saidOf = append(saidOf, strings.Fields(l)[0])
|
||||
}
|
||||
}
|
||||
if len(saidOf) != 1 || saidOf[0] != "build-older" {
|
||||
t.Fatalf("the check let through next (the oldest) is not the one said: %v\n%s", saidOf, text)
|
||||
}
|
||||
answer := queueAnswerOf(q, now)
|
||||
if len(answer.HeldFor) != 1 || answer.HeldFor[0] != "plan-42" {
|
||||
t.Errorf("held-for = %v", answer.HeldFor)
|
||||
}
|
||||
for _, a := range answer.Asks {
|
||||
if a.LetThrough != (a.ID == "build-older") {
|
||||
t.Errorf("%s let-through = %v", a.ID, a.LetThrough)
|
||||
}
|
||||
}
|
||||
// A check in flight under the let-through key is said too.
|
||||
q.Asks = append(q.Asks, link.QueuedAsk{Seq: 9, ID: "build-running", State: link.AskInFlight, On: "ace", AskedAt: now.Add(-time.Hour),
|
||||
Started: now.Add(-time.Minute), Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: 9}}})
|
||||
q.Hold.LetThrough = &link.LetThrough{Builder: "ace"}
|
||||
if !strings.Contains(queueText(q, now), "build-running") || !strings.Contains(queueText(q, now), "waited past 30 minutes, let through\n") {
|
||||
t.Errorf("the check in flight let through is not said:\n%s", queueText(q, now))
|
||||
}
|
||||
// Younger than the bound: none is said.
|
||||
young := aHeldQueue(now)
|
||||
young.Asks = young.Asks[:1]
|
||||
if strings.Contains(queueText(young, now), "waited past") {
|
||||
t.Error("a check of 5 minutes is said to have waited past 30")
|
||||
}
|
||||
}
|
||||
@@ -577,6 +577,9 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
}
|
||||
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||
// requests it answers, whose checks name the part of a differing key.
|
||||
// **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save
|
||||
// would hold a place for its whole run.
|
||||
sayBuilding(ctx, p.ID, now)
|
||||
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
|
||||
e.Source.Path, ref)
|
||||
if err != nil {
|
||||
@@ -851,6 +854,23 @@ func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Pl
|
||||
return behind, why, nil
|
||||
}
|
||||
|
||||
// refusalNotePrefix and refusalNoteSuffix frame the note of a step the walk was refused: "tier N: <refusal> — tried
|
||||
// again". Only a note so framed is a hold (novox/hq issue 920); a tier's summary left in the note is not.
|
||||
const (
|
||||
refusalNotePrefix = "tier "
|
||||
refusalNoteSuffix = " — tried again"
|
||||
)
|
||||
|
||||
// refusalNote is the note of a plan whose step at tier was refused with err.
|
||||
func refusalNote(tier int, err error) string {
|
||||
return refusalNotePrefix + fmt.Sprint(tier) + ": " + err.Error() + refusalNoteSuffix
|
||||
}
|
||||
|
||||
// refused says a plan's note is a refusal the walk wrote.
|
||||
func refused(p inventory.Plan) bool {
|
||||
return strings.HasPrefix(p.Note, refusalNotePrefix) && strings.HasSuffix(p.Note, refusalNoteSuffix)
|
||||
}
|
||||
|
||||
// deferredNote begins the note of a walk deferred behind another.
|
||||
const deferredNote = "let go; starts once "
|
||||
|
||||
@@ -863,6 +883,20 @@ func deferred(p inventory.Plan) bool {
|
||||
p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote)
|
||||
}
|
||||
|
||||
// currentTierBuilt says every module of a plan's current tier is built (or deleted), so no build is what it waits on.
|
||||
func currentTierBuilt(p inventory.Plan) bool {
|
||||
if p.Tier < 0 || p.Tier >= len(p.Tiers) || len(p.Tiers[p.Tier]) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
if s == nil || (s.State != "built" && s.State != planDeleted) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// askedAny says a plan asked any module.
|
||||
func askedAny(p inventory.Plan) bool {
|
||||
for _, s := range p.Modules {
|
||||
@@ -927,7 +961,7 @@ func advanceHeld(ctx context.Context, open *stores) (failed bool) {
|
||||
// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
|
||||
// the state is left as it was and the step is tried again by the next look. Said and
|
||||
// kept when it is new: the same refusal on every tick is one fact, not one per tick.
|
||||
p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
|
||||
p.Note = refusalNote(p.Tier, err)
|
||||
// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
|
||||
if refusedForTheBus(err) {
|
||||
busRefusedFirst.mark(p.ID, time.Now())
|
||||
@@ -2025,6 +2059,13 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
|
||||
if p.State == inventory.PlanRolling {
|
||||
what = p.Note
|
||||
}
|
||||
if p.State == inventory.PlanBuilding && refused(p) && currentTierBuilt(p) {
|
||||
// Every build of the tier is in, so the plan waits on something else, and its note says what: a send
|
||||
// refused ("a build waits there for its gate: <machine> (...)") is kept there. "building" with LATE
|
||||
// hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920).
|
||||
what = p.Note
|
||||
return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late)
|
||||
}
|
||||
return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late)
|
||||
}
|
||||
|
||||
|
||||
@@ -39,6 +39,10 @@ type sendable struct {
|
||||
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
|
||||
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
|
||||
ReadsModules bool
|
||||
// fileReferences says the machine's node-engine reports the capability of reading a file given as a digest and a size
|
||||
// (declaration.FileReferencesCapability, novox/hq ADR 0486 §5): then every file over 64 KiB that may travel so does (Body,
|
||||
// objects). Not on the wire as a field; the bodies differ.
|
||||
fileReferences bool
|
||||
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
|
||||
// kept beside the resources, and on the wire only to a machine that reads them.
|
||||
moduleOf map[string]string
|
||||
@@ -115,12 +119,16 @@ type adoptionEnvelope struct {
|
||||
|
||||
// Body is the declaration's bytes, as sent and as digested.
|
||||
func (s sendable) Body() ([]byte, error) {
|
||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||
resources, _, err := s.byReference()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
envelope := map[string]any{"declaration": 1, "resources": resources}
|
||||
if s.ReadsModules {
|
||||
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
|
||||
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
|
||||
named := make([]map[string]any, len(s.Resources))
|
||||
for n, r := range s.Resources {
|
||||
named := make([]map[string]any, len(resources))
|
||||
for n, r := range resources {
|
||||
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
|
||||
if !owned {
|
||||
named[n] = r
|
||||
@@ -165,3 +173,19 @@ func (s sendable) Body() ([]byte, error) {
|
||||
}
|
||||
return json.Marshal(envelope)
|
||||
}
|
||||
|
||||
// byReference is the resources as the machine is sent them: with the contents of its large files replaced by their digests when
|
||||
// it reports it reads them (issue 1696), and the contents it replaced, by the name of the object that holds each.
|
||||
func (s sendable) byReference() ([]map[string]any, map[string][]byte, error) {
|
||||
if !s.fileReferences {
|
||||
return s.Resources, nil, nil
|
||||
}
|
||||
return referencing(s.Resources)
|
||||
}
|
||||
|
||||
// objects are the contents the body names by digest, by the name of each one's object: what is put in the bucket before the
|
||||
// body is sent. Empty for a machine that is sent every content inline.
|
||||
func (s sendable) objects() (map[string][]byte, error) {
|
||||
_, objects, err := s.byReference()
|
||||
return objects, err
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021144048-d4deecdb380b
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021144048-d4deecdb380b h1:5UlQ8ytlHNgqlm5AbwJEnZwuf1EWUiNyXX2hcOV9Yro=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021144048-d4deecdb380b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75 h1:AcYfyJpYaUufhI93VG76+b001msXY4XX7kuRjnf33HY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -46,6 +46,7 @@ func CheckStreams() []Stream {
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Direct: true,
|
||||
Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
|
||||
"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
|
||||
"request still waiting",
|
||||
@@ -56,6 +57,7 @@ func CheckStreams() []Stream {
|
||||
Retention: RetentionWorkQueue,
|
||||
MaxAge: 7 * 24 * 60 * 60,
|
||||
MaxMsgsPerSubject: 1,
|
||||
Direct: true,
|
||||
Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
|
||||
"request, the newest request kept; one left behind for a head already judged is dropped when taken",
|
||||
},
|
||||
@@ -98,6 +100,13 @@ func holderCheckGrants() []string {
|
||||
"$KV."+CheckRunningBucket+".>",
|
||||
"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
|
||||
"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
|
||||
// And, to hold checks while a delivery builds (ADR 0483, issues 1687 and 1688): the bucket of the plans building, **read and
|
||||
// never written** (the controller's alone, so no `$KV.` subject of it is granted), and the oldest waiting ask of each check
|
||||
// queue, read by a direct get that takes nothing from the queue and so counts against no delivery of the ask.
|
||||
pub = append(pub, readGrants(PlansBuildingBucket, "")...)
|
||||
for _, w := range CheckWorkers() {
|
||||
pub = append(pub, "$JS.API.DIRECT.GET."+w.Stream, "$JS.API.DIRECT.GET."+w.Stream+".>")
|
||||
}
|
||||
return pub
|
||||
}
|
||||
|
||||
|
||||
@@ -40,3 +40,37 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1687 (ADR 0483 clause 2): a holder of the build seat reads the bucket of the plans building and writes none of it, and
|
||||
// reads the oldest waiting ask of each check queue by a direct get, which takes nothing from the queue.
|
||||
func TestTheBuildSeatsHoldersReadThePlansBuildingAndWriteNone(t *testing.T) {
|
||||
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat}, PasswordHash: "x"}
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + PlansBuildingBucket
|
||||
for _, s := range []string{
|
||||
"$JS.API.STREAM.INFO." + stream,
|
||||
"$JS.API.DIRECT.GET." + stream + ".>",
|
||||
"$JS.API.CONSUMER.CREATE." + stream + ".>",
|
||||
"$JS.API.DIRECT.GET.CHECKS_READY",
|
||||
"$JS.API.DIRECT.GET.CHECKS_OTHER",
|
||||
} {
|
||||
has(t, perms.Publish, s)
|
||||
}
|
||||
for _, s := range perms.Publish {
|
||||
if SubjectsOverlap(s, "$KV."+PlansBuildingBucket+".plan-1") {
|
||||
t.Errorf("a holder of the build seat may write %s through %q", PlansBuildingBucket, s)
|
||||
}
|
||||
}
|
||||
// And the controller alone writes it, as the writers table says.
|
||||
for _, row := range WritersTable {
|
||||
for _, s := range row.Subjects {
|
||||
if s == "$KV."+PlansBuildingBucket+".>" && row.Writes(p, "$KV."+PlansBuildingBucket+".plan-1") {
|
||||
t.Errorf("%q lets the build seat's holder write the plans building", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,6 +63,26 @@ var (
|
||||
// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
|
||||
// those heads. Written by the controller alone, from the announcement, when it holds the lease.
|
||||
DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
|
||||
// PlansBuildingBucket holds one key per delivery plan in its build phase (novox/hq ADR 0483 clause 2, issue 1687): the key
|
||||
// is `<plan id>` (`plan-<digits>`), its value the plan, the time its build phase began and the time of the last refresh.
|
||||
// The controller (lease holder) puts it when the plan's first build is asked, puts it again about every thirty seconds
|
||||
// while the plan builds, and deletes it when the phase ends (every build of the last tier built, or the plan fails, stops
|
||||
// or waits on the operator); the bucket's age, PlansBuildingKeptFor, is how long a key outlives a controller that
|
||||
// stopped. **Read by the build seat's holders and the checker, written by the controller alone**: a holder takes no new
|
||||
// check while any key is there, but for one that waited past ADR 0483's bound. A reader lists the keys and reads none of
|
||||
// their values to decide that (`link.BuildingPlans`).
|
||||
PlansBuildingBucket = BucketName(ControllerSeat, PlansBuilding)
|
||||
)
|
||||
|
||||
// PlansBuilding is the local name of the bucket of the plans building, as a reader names it in `reads`
|
||||
// (`mesh-controller.plans-building`).
|
||||
const PlansBuilding = "plans-building"
|
||||
|
||||
// PlansBuildingKeptFor is the age of a plan-building key: the controller puts it again at PlansBuildingRefresh, so a
|
||||
// controller that stopped leaves checks held at most this long (ADR 0483 clause 2).
|
||||
const (
|
||||
PlansBuildingKeptFor = 2 * time.Minute
|
||||
PlansBuildingRefresh = 30 * time.Second
|
||||
)
|
||||
|
||||
// DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
|
||||
@@ -204,14 +224,14 @@ func IsControllerBucket(bucket string) bool {
|
||||
bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
|
||||
bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
|
||||
bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
|
||||
bucket == delivery.HaveBucket
|
||||
bucket == PlansBuildingBucket || bucket == delivery.HaveBucket
|
||||
}
|
||||
|
||||
// ControllerBuckets are the controller's own buckets, in the order they are asserted.
|
||||
func ControllerBuckets() []string {
|
||||
return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
|
||||
AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
|
||||
DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
|
||||
DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
@@ -354,6 +374,22 @@ func (j *JetStream) EnsureControllerBuckets() error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
|
||||
}
|
||||
// **The plans building: keys that age** (ADR 0483 clause 2, issue 1687). One message a key, each refresh a new one, so the
|
||||
// age of the bucket is how long a key outlives a controller that stopped refreshing it.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: PlansBuildingBucket,
|
||||
Description: "the delivery plans in their build phase (novox/hq ADR 0483): `<plan id>` holds the plan and when its " +
|
||||
"build phase began; put when the plan's first build is asked and about every thirty seconds, deleted when the " +
|
||||
"phase ends, gone within two minutes of a controller that stopped; read by the build seat's holders, which " +
|
||||
"take no new check while a key is there; written by the controller alone",
|
||||
History: 1,
|
||||
TTL: PlansBuildingKeptFor,
|
||||
MaxValueSize: 4 << 10,
|
||||
MaxBytes: 8 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", PlansBuildingBucket, err)
|
||||
}
|
||||
// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: DeliveryMergesBucket,
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The object buckets a declaration's large files travel in (novox/hq ADR 0486, issues 1696 and 1697): **one bucket for each
|
||||
// node**, `declaration-files-<node>`.
|
||||
//
|
||||
// **The controller alone writes them, and a node-engine reads an object by its name from its own bucket** (ADR 0486 §2): its
|
||||
// account may neither list the bucket nor write it, no account may touch another node's, and no other account has any access.
|
||||
// Why a bucket each: a subject pattern cannot tell a name from a wildcard. A grant of `...M.*` matches a publish to the literal
|
||||
// subject `...M.*` too (probed on nats-server 2.11.17 by the review of this issue: a node created a consumer filtered on
|
||||
// `$O.<bucket>.C.*` and read every object of the bucket), so in one shared bucket a node could read every other node's files.
|
||||
// With a bucket each, the same trick reads only the node's own files, which are the files of its own declaration.
|
||||
//
|
||||
// A bucket of the bus's object store is a stream, OBJ_<bucket>, whose messages are the objects' metadata
|
||||
// (`$O.<bucket>.M.<base64 of the name>`) and chunks (`$O.<bucket>.C.<nuid>`).
|
||||
|
||||
// DeclarationFilesBucket is the prefix of the buckets' names, which mesh-sdk fixes for the node-engine and the controller alike.
|
||||
const DeclarationFilesBucket = declaration.FileBucketPrefix
|
||||
|
||||
// DeclarationFilesBucketOf is the bucket a node's files are in: mesh-sdk's FileBucketFor, the one place the name is composed. A
|
||||
// node name it refuses (one that could carry a subject wildcard or separator into a grant) is an error, never a bucket.
|
||||
func DeclarationFilesBucketOf(node string) (string, error) { return declaration.FileBucketFor(node) }
|
||||
|
||||
// DeclarationFilesStreamOf is the stream under a node's bucket: mesh-sdk's FileStreamFor.
|
||||
func DeclarationFilesStreamOf(node string) (string, error) { return declaration.FileStreamFor(node) }
|
||||
|
||||
// DeclarationFilesMaxBytes is a bucket's size: the objects are immutable and regenerable, and one node's hold well under a hundred
|
||||
// megabytes.
|
||||
const DeclarationFilesMaxBytes = 1 << 30
|
||||
|
||||
// DeclarationFilesKeptFor is how long an object stays after no node's latest declaration names it (ADR 0486 §3).
|
||||
const DeclarationFilesKeptFor = 24 * time.Hour
|
||||
|
||||
// DeclarationFilesWrites is the one pattern the controller may publish to write a node's bucket: every object's metadata and
|
||||
// chunks. The server's API for the stream is the controller's `$JS.API.>`.
|
||||
func DeclarationFilesWrites(node string) (string, error) {
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "$O." + bucket + ".>", nil
|
||||
}
|
||||
|
||||
// DeclarationFilesReads is what a node-engine's account may publish to read an object of its own bucket by its name, and nothing
|
||||
// more. The set is the one node-engine's own call (mesh-host internal/link FileObject) makes, for B the node's bucket and S its
|
||||
// stream (issue 1697, notes 4 and 5):
|
||||
//
|
||||
// - the object's metadata, by a direct get of the last message of the one subject that names it. **Only the suffixed
|
||||
// form**: the bare `$JS.API.DIRECT.GET.S` takes a sequence number and would read any message of the stream.
|
||||
// - the ordered consumer on that object's chunks: `$JS.API.CONSUMER.CREATE.S.<name>.$O.B.C.<nuid>`. **Not `S.>`**:
|
||||
// a consumer filtered on `$O.B.M.>` would read every object's name.
|
||||
// - the flow-control replies of that consumer, without which a large object stalls at the flow-control window.
|
||||
// - the consumer's deletion, best effort: the consumer's inactivity threshold removes it if this is refused.
|
||||
//
|
||||
// **Never granted:** STREAM.INFO (a `subjects_filter` of `>` lists every name), STREAM.MSG.GET (any message by its sequence), a
|
||||
// publish to `$O.B.>` (a write), and any subject of another node's stream. A literal `*` in a subject matches the grant's `*`: it
|
||||
// reaches the objects of this node's own bucket and no other (above).
|
||||
func DeclarationFilesReads(node string) ([]string, error) {
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stream, err := DeclarationFilesStreamOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{
|
||||
"$JS.API.DIRECT.GET." + stream + ".$O." + bucket + ".M.*",
|
||||
"$JS.API.CONSUMER.CREATE." + stream + ".*.$O." + bucket + ".C.*",
|
||||
"$JS.API.CONSUMER.DELETE." + stream + ".*",
|
||||
"$JS.FC." + stream + ".>",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// EnsureDeclarationFiles creates the buckets of these nodes if absent and brings their options to match. A bucket exists whenever the
|
||||
// bus does, so a node that is starting finds it (ADR 0486 §4). An update, never a delete.
|
||||
func (j *JetStream) EnsureDeclarationFiles(nodes []string) error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
return EnsureDeclarationFiles(ctx, js, nodes...)
|
||||
}
|
||||
|
||||
// EnsureDeclarationFiles is the same on a JetStream the caller holds.
|
||||
func EnsureDeclarationFiles(ctx context.Context, js jetstream.JetStream, nodes ...string) error {
|
||||
for _, node := range nodes {
|
||||
cfg, err := DeclarationFilesConfig(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := js.CreateOrUpdateObjectStore(ctx, cfg); err != nil {
|
||||
return fmt.Errorf("asserting the object bucket %s: %w", cfg.Bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeclarationFilesConfig is a node's bucket as the controller asserts it.
|
||||
func DeclarationFilesConfig(node string) (jetstream.ObjectStoreConfig, error) {
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return jetstream.ObjectStoreConfig{}, err
|
||||
}
|
||||
return jetstream.ObjectStoreConfig{
|
||||
Bucket: bucket,
|
||||
Description: "the contents of the files over 64 KiB of the declaration of " + node + ", each under the hex of its sha256 " +
|
||||
"digest (novox/hq ADR 0486): written by the controller alone, read by the node-engine of that node by name, removed " +
|
||||
"a day after the node's latest declaration no longer names it",
|
||||
MaxBytes: DeclarationFilesMaxBytes,
|
||||
Storage: jetstream.FileStorage,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
//go:build integration
|
||||
|
||||
package broker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// nodeEnginesFileObject is the node-engine's own call (mesh-host internal/link files.go, OverNATS.FileObject), as the
|
||||
// engine makes it: the object's metadata by a direct get of one subject, then its chunks through an ordered consumer
|
||||
// bound to the stream. It is copied here so the grants are judged by the call that has to work, and its subjects are the
|
||||
// ones the grant names (issue 1697, notes 4 and 5); the node-engine's own integration test holds the other copy.
|
||||
func nodeEnginesFileObject(ctx context.Context, nc *nats.Conn, node, name string) ([]byte, error) {
|
||||
js, err := nc.JetStream(nats.MaxWait(10 * time.Second))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stream, err := DeclarationFilesStreamOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
|
||||
nats.DirectGet(), nats.Context(ctx))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var info struct {
|
||||
NUID string `json:"nuid"`
|
||||
Size uint64 `json:"size"`
|
||||
}
|
||||
if err := json.Unmarshal(meta.Data, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sub, err := js.SubscribeSync("$O."+bucket+".C."+info.NUID, nats.OrderedConsumer(), nats.BindStream(stream))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
body := make([]byte, 0, info.Size)
|
||||
for uint64(len(body)) < info.Size {
|
||||
msg, err := sub.NextMsgWithContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body = append(body, msg.Data...)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// **The buckets' grants on a real server, judged by the node-engine's own call** (novox/hq ADR 0486 §2, issue 1697, 1697/AC2): the
|
||||
// controller's account writes an object into each node's bucket, a node-engine's account reads one of its own bucket by its name,
|
||||
// a large object included, and is refused a list of the bucket (a stream info asking for every subject), a message get by
|
||||
// sequence, a direct get by sequence, a consumer on the names, and a write; it is refused every one of those and the read by name on
|
||||
// another node's bucket, **the literal-`*` forms included** (a published subject `...M.*` or `...C.*` matches the grant's `*`: on
|
||||
// its own bucket that reaches its own files, on another's it is refused); another module's account is refused all of them.
|
||||
func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testing.T) { // 1697/AC2
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
users, err := Users(Records{Nodes: []string{"alpha", "beta"}, Assigned: map[string][]Declared{
|
||||
"alpha": {{Module: RuntimeModule}, {Module: "other-module"}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range users {
|
||||
users[i].PasswordHash = string(hash)
|
||||
}
|
||||
accounts, err := ComposeAccounts(users)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
connect := func(user string) (*nats.Conn, chan error) {
|
||||
t.Helper()
|
||||
refused := make(chan error, 16)
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc, refused
|
||||
}
|
||||
|
||||
// The controller makes the buckets (as it does at its start) and writes into each: for alpha a small object and one of
|
||||
// several chunks, which needs the consumer's flow control; for beta one that holds a marker alpha must never read.
|
||||
controller, _ := connect(ControllerName)
|
||||
cjs, err := jetstream.New(controller)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
|
||||
t.Fatalf("the controller cannot make the buckets: %v", err)
|
||||
}
|
||||
if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
|
||||
t.Fatalf("asserting them again: %v", err)
|
||||
}
|
||||
put := func(node string, content []byte) string {
|
||||
t.Helper()
|
||||
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(content)
|
||||
name := hex.EncodeToString(sum[:])
|
||||
if _, err := store.PutBytes(ctx, name, content); err != nil {
|
||||
t.Fatalf("the controller cannot write the object %s of %s: %v", name[:8], node, err)
|
||||
}
|
||||
return name
|
||||
}
|
||||
small := bytes.Repeat([]byte("small file\n"), 100)
|
||||
large := make([]byte, 900<<10)
|
||||
for i := range large {
|
||||
large[i] = byte(i*7 + i>>8)
|
||||
}
|
||||
secret := bytes.Repeat([]byte("BETA-SECRET "), 50)
|
||||
smallName, largeName, betaName := put("alpha", small), put("alpha", large), put("beta", secret)
|
||||
|
||||
alpha, alphaRefused := connect("node.alpha")
|
||||
for name, want := range map[string][]byte{smallName: small, largeName: large} {
|
||||
got, err := nodeEnginesFileObject(ctx, alpha, "alpha", name)
|
||||
if err != nil {
|
||||
t.Fatalf("a node-engine cannot read the object %s of its own bucket by its name: %v", name[:8], err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("the node-engine read %d bytes of the object %s, want %d", len(got), name[:8], len(want))
|
||||
}
|
||||
}
|
||||
beta, _ := connect("node.beta")
|
||||
if got, err := nodeEnginesFileObject(ctx, beta, "beta", betaName); err != nil || !bytes.Equal(got, secret) {
|
||||
t.Fatalf("beta's node-engine reads its own object: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-alphaRefused:
|
||||
t.Fatalf("a node-engine's own read was refused something: %v", err)
|
||||
default:
|
||||
}
|
||||
|
||||
// What each account is refused, judged by the server.
|
||||
alphaStream, alphaBucket := mustStream(t, "alpha"), mustBucket(t, "alpha")
|
||||
betaStream, betaBucket := mustStream(t, "beta"), mustBucket(t, "beta")
|
||||
listing := []byte(`{"subjects_filter":">"}`)
|
||||
bySeq := []byte(`{"seq":1}`)
|
||||
type refusal struct {
|
||||
what string
|
||||
do func(nc *nats.Conn)
|
||||
}
|
||||
onBucket := func(stream, bucket, which string) []refusal {
|
||||
return []refusal{
|
||||
{which + ": a list of the bucket (stream info asking for every subject)", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.STREAM.INFO."+stream, listing, 300*time.Millisecond)
|
||||
}},
|
||||
{which + ": a message get by sequence", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.STREAM.MSG.GET."+stream, bySeq, 300*time.Millisecond)
|
||||
}},
|
||||
{which + ": a direct get by sequence", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+stream, bySeq, 300*time.Millisecond)
|
||||
}},
|
||||
{which + ": a consumer on every object's name", func(nc *nats.Conn) {
|
||||
js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
|
||||
if sub, err := js.SubscribeSync("$O."+bucket+".M.>", nats.OrderedConsumer(), nats.BindStream(stream)); err == nil {
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
}},
|
||||
{which + ": a write of an object's metadata", func(nc *nats.Conn) {
|
||||
_ = nc.Publish("$O."+bucket+".M.abc", []byte(`{}`))
|
||||
_ = nc.Flush()
|
||||
}},
|
||||
{which + ": a write of a chunk", func(nc *nats.Conn) {
|
||||
_ = nc.Publish("$O."+bucket+".C.abc", []byte("x"))
|
||||
_ = nc.Flush()
|
||||
}},
|
||||
}
|
||||
}
|
||||
refusedWith := func(who string, nc *nats.Conn, ch chan error, r refusal) {
|
||||
t.Helper()
|
||||
for len(ch) > 0 {
|
||||
<-ch
|
||||
}
|
||||
r.do(nc)
|
||||
_ = nc.Flush()
|
||||
deadline := time.After(3 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case err := <-ch:
|
||||
if errors.Is(err, nats.ErrPermissionViolation) {
|
||||
return
|
||||
}
|
||||
case <-deadline:
|
||||
t.Errorf("the server did not refuse %s %s", who, r.what)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
// Alpha's account on its own bucket (list, sequence reads, names, writes) and on beta's (all of that, and the read by name,
|
||||
// and the literal-`*` probes of the review).
|
||||
reachBeta := []refusal{
|
||||
{"beta's bucket: a read of one object by its name", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M."+
|
||||
base64.URLEncoding.EncodeToString([]byte(betaName)), nil, 300*time.Millisecond)
|
||||
}},
|
||||
{"beta's bucket: a direct get with a literal `*` for the name", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M.*", nil, 300*time.Millisecond)
|
||||
}},
|
||||
{"beta's bucket: a consumer with a literal `*` for the chunks", func(nc *nats.Conn) {
|
||||
js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
|
||||
if sub, err := js.SubscribeSync("$O."+betaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(betaStream)); err == nil {
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
}},
|
||||
}
|
||||
all := append(append(onBucket(alphaStream, alphaBucket, "its own bucket"), onBucket(betaStream, betaBucket, "beta's bucket")...), reachBeta...)
|
||||
for _, r := range all {
|
||||
nodeNC, nodeCh := connect("node.alpha")
|
||||
refusedWith("a node-engine", nodeNC, nodeCh, r)
|
||||
}
|
||||
// Another module is refused all of that on both buckets, and the reads by name on both.
|
||||
for _, r := range append(append(onBucket(alphaStream, alphaBucket, "alpha's bucket"), reachBeta...),
|
||||
refusal{"alpha's bucket: a read of one object by its name", func(nc *nats.Conn) {
|
||||
_, _ = nc.Request("$JS.API.DIRECT.GET."+alphaStream+".$O."+alphaBucket+".M."+
|
||||
base64.URLEncoding.EncodeToString([]byte(smallName)), nil, 300*time.Millisecond)
|
||||
}}) {
|
||||
otherNC, otherCh := connect("alpha.other-module")
|
||||
refusedWith("another module", otherNC, otherCh, r)
|
||||
}
|
||||
|
||||
// The literal `*` on alpha's own bucket reaches alpha's own files and never beta's: the chunks it is given hold no marker of beta's.
|
||||
js, err := alpha.JetStream(nats.MaxWait(2 * time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sub, err := js.SubscribeSync("$O."+alphaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(alphaStream)); err == nil {
|
||||
for i := 0; i < 20; i++ {
|
||||
msg, err := sub.NextMsg(300 * time.Millisecond)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if bytes.Contains(msg.Data, []byte("BETA-SECRET")) {
|
||||
t.Fatal("a literal `*` on alpha's bucket gave it a chunk of beta's file")
|
||||
}
|
||||
}
|
||||
sub.Unsubscribe()
|
||||
}
|
||||
|
||||
// Nothing was written by the refused: each bucket holds what the controller wrote.
|
||||
for node, want := range map[string]int{"alpha": 2, "beta": 1} {
|
||||
store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
infos, err := store.List(ctx)
|
||||
if err != nil || len(infos) != want {
|
||||
t.Errorf("the bucket of %s holds %d objects (%v), want the %d the controller wrote", node, len(infos), err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mustBucket(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
b, err := DeclarationFilesBucketOf(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func mustStream(t *testing.T, node string) string {
|
||||
t.Helper()
|
||||
b, err := DeclarationFilesStreamOf(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1697/AC1: the composed accounts give a node-engine exactly the four reads of one object by its name, on its own node's bucket and
|
||||
// no other's; the controller the write of every node's bucket; and no other principal anything of any bucket.
|
||||
func TestEachNodesBucketIsWrittenByTheControllerAndReadByNameByThatNodesEngine(t *testing.T) { // 1697/AC1
|
||||
node, err := PermissionsFor(Principal{Kind: KindNode, Node: "alpha", PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mine []string
|
||||
for _, s := range node.Publish {
|
||||
if strings.Contains(s, "declaration-files") {
|
||||
mine = append(mine, s)
|
||||
}
|
||||
}
|
||||
want := []string{
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.*.$O.declaration-files-alpha.C.*",
|
||||
"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-alpha.*",
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.*",
|
||||
"$JS.FC.OBJ_declaration-files-alpha.>",
|
||||
}
|
||||
if !slices.Equal(mine, want) {
|
||||
t.Fatalf("a node-engine may publish %q of the buckets, want exactly %q", mine, want)
|
||||
}
|
||||
// What would make a list, a write, or a read of another node's files: none of it, in any pattern a node holds. (The real
|
||||
// server's judgement of the wildcard forms is the live test's.)
|
||||
for _, refused := range []string{
|
||||
"$JS.API.STREAM.INFO.OBJ_declaration-files-alpha",
|
||||
"$JS.API.STREAM.MSG.GET.OBJ_declaration-files-alpha",
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha",
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.abc.def",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj.more",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha",
|
||||
"$O.declaration-files-alpha.M.abc",
|
||||
"$O.declaration-files-alpha.C.abc",
|
||||
// another node's bucket, however it is named
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-beta.$O.declaration-files-beta.M.YWJj",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-beta.h4sh.$O.declaration-files-beta.C.NUID",
|
||||
"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-beta.h4sh",
|
||||
"$JS.FC.OBJ_declaration-files-beta.x",
|
||||
// and the old shared bucket
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.YWJj",
|
||||
} {
|
||||
if MayPublish(node, refused) {
|
||||
t.Errorf("a node-engine may publish %s, which lists or writes a bucket or reads another node's", refused)
|
||||
}
|
||||
}
|
||||
for _, kept := range []string{
|
||||
"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.YWJj",
|
||||
"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.h4sh.$O.declaration-files-alpha.C.NUID",
|
||||
} {
|
||||
if !MayPublish(node, kept) {
|
||||
t.Errorf("a node-engine may not publish %s, which reads one object of its own bucket by its name", kept)
|
||||
}
|
||||
}
|
||||
|
||||
controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x", FilesFor: []string{"alpha", "beta"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []string{"$O.declaration-files-alpha.M.abc", "$O.declaration-files-beta.C.nuid"} {
|
||||
if !MayPublish(controller, s) {
|
||||
t.Errorf("the controller may not write %s", s)
|
||||
}
|
||||
}
|
||||
if MayPublish(controller, "$O.declaration-files-gamma.M.abc") {
|
||||
t.Error("the controller may write the bucket of a node it was not given")
|
||||
}
|
||||
|
||||
for _, p := range []Principal{
|
||||
{Kind: KindModule, Module: "mesh-vault", PasswordHash: "x"},
|
||||
{Kind: KindNodeTools, Node: "alpha", PasswordHash: "x"},
|
||||
{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
|
||||
} {
|
||||
perms, err := PermissionsFor(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) {
|
||||
if strings.Contains(s, "declaration-files") {
|
||||
t.Errorf("%s holds %s, and no account but the controller's and the node-engines' has any access", p.Username(), s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The users the controller composes carry each node's bucket to the controller and to that node alone.
|
||||
func TestTheComposedUsersGiveEachNodeItsOwnBucketAndTheControllerAll(t *testing.T) {
|
||||
users, err := Users(Records{Nodes: []string{"alpha", "beta"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var controller Principal
|
||||
for _, u := range users {
|
||||
if u.Kind == KindController {
|
||||
controller = u
|
||||
}
|
||||
}
|
||||
if !slices.Equal(controller.FilesFor, []string{"alpha", "beta"}) {
|
||||
t.Fatalf("the controller writes the buckets of %v", controller.FilesFor)
|
||||
}
|
||||
}
|
||||
|
||||
// A node name that could carry a subject wildcard or separator into a grant is refused with an error, never composed into a bucket
|
||||
// (ADR 0488): the grants of such a name would reach another node's bucket.
|
||||
func TestANodeNameThatCarriesAWildcardGetsNoBucketAndNoGrant(t *testing.T) {
|
||||
for _, node := range []string{"", "*", ">", "a.b", "a b", "a*"} {
|
||||
if b, err := DeclarationFilesBucketOf(node); err == nil {
|
||||
t.Errorf("node %q got the bucket %q", node, b)
|
||||
}
|
||||
if _, err := DeclarationFilesReads(node); err == nil {
|
||||
t.Errorf("node %q got read grants", node)
|
||||
}
|
||||
if _, err := DeclarationFilesWrites(node); err == nil {
|
||||
t.Errorf("node %q got write grants", node)
|
||||
}
|
||||
if _, err := DeclarationFilesConfig(node); err == nil {
|
||||
t.Errorf("node %q got a bucket config", node)
|
||||
}
|
||||
}
|
||||
if b, err := DeclarationFilesBucketOf("g14"); err != nil || b != "declaration-files-g14" {
|
||||
t.Errorf("g14: %q %v", b, err)
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,10 @@ type Principal struct {
|
||||
// Username and inbox are <node>.<module>.<process id>.
|
||||
Process string
|
||||
|
||||
// FilesFor are the nodes whose declaration-file buckets this principal writes, for the controller alone (novox/hq ADR 0486 §2):
|
||||
// every node of the mesh, one bucket each.
|
||||
FilesFor []string
|
||||
|
||||
Emits []string
|
||||
Consumes []string
|
||||
Serves []string
|
||||
@@ -408,6 +412,11 @@ var DegradableGrants = []DegradableGrant{
|
||||
Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
|
||||
"holders do (ADR 0406 §3, issue 974)",
|
||||
},
|
||||
{
|
||||
Subject: "$KV." + PlansBuildingBucket + ".>",
|
||||
Reason: "refused: the controller then says no plan builds, and no check is held while a delivery builds — a check run " +
|
||||
"wrongly costs a delivery a place, a check held wrongly costs a pull request its turn (ADR 0483 clause 2, issue 1687)",
|
||||
},
|
||||
{
|
||||
Subject: "mesh.seat." + ControllerSeat + ".event.sent",
|
||||
Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
|
||||
@@ -736,6 +745,15 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
}
|
||||
pub = append(pub, "$KV."+bucket+".>")
|
||||
}
|
||||
// **And the object bucket a declaration's large files travel in** (novox/hq ADR 0486 §2, issue 1697): the
|
||||
// controller alone writes it, as a publish to the bucket's own subjects, which `$JS.API.>` does not cover.
|
||||
for _, node := range p.FilesFor {
|
||||
writes, err := DeclarationFilesWrites(node)
|
||||
if err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, writes)
|
||||
}
|
||||
// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
|
||||
// consumer that gave up on a message, or one that was deleted. The server already publishes
|
||||
// both in the mesh's own account; the controller says each as a condition in the mesh's words.
|
||||
@@ -835,6 +853,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, haveKeys...)
|
||||
// And an object of the declaration bucket by its name, and nothing of the bucket's list or writing
|
||||
// (novox/hq ADR 0486 §2, issue 1697).
|
||||
fileReads, err := DeclarationFilesReads(p.Node)
|
||||
if err != nil {
|
||||
return Permissions{}, err
|
||||
}
|
||||
pub = append(pub, fileReads...)
|
||||
// And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the
|
||||
// values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call.
|
||||
if p.MovesIntoTheVault {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
//go:build integration
|
||||
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// **A holder of the build seat reads the plans building and the server refuses its write** (novox/hq ADR 0483 clause 2, issue
|
||||
// 1687): on a real server whose users are composed as the controller composes them, the controller makes the bucket and puts a
|
||||
// plan's key; the build agent's credential lists the keys and gets one, and its put of a key is refused with a permissions
|
||||
// violation. It also reads the first waiting ask of a check queue by a direct get, which takes nothing from the queue.
|
||||
func TestNatsTheBuildSeatsHoldersReadThePlansBuildingAndTheServerRefusesTheirWrite(t *testing.T) { // 1687
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
|
||||
users, err := Users(Records{Nodes: []string{"alpha"}, Assigned: map[string][]Declared{
|
||||
"alpha": {{Module: RuntimeModule}, {Module: "build-agent", Holds: []Seat{seat}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := range users {
|
||||
users[i].PasswordHash = string(hash)
|
||||
}
|
||||
accounts, err := ComposeAccounts(users)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(s.Shutdown)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
connect := func(user string) (*nats.Conn, chan error) {
|
||||
t.Helper()
|
||||
refused := make(chan error, 8)
|
||||
nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot connect: %v", user, err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc, refused
|
||||
}
|
||||
controller, _ := connect(ControllerName)
|
||||
cj := OnConn(controller)
|
||||
if err := AssertMeshStreams(cj); err != nil {
|
||||
t.Fatalf("the controller cannot make the streams: %v", err)
|
||||
}
|
||||
if err := cj.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatalf("the controller cannot make its buckets: %v", err)
|
||||
}
|
||||
cjs, _ := jetstream.New(controller)
|
||||
ckv, err := cjs.KeyValue(ctx, PlansBuildingBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ckv.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatalf("the controller cannot put a plan's key: %v", err)
|
||||
}
|
||||
|
||||
holder, refused := connect("alpha.build-agent")
|
||||
hjs, err := jetstream.New(holder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := hjs.KeyValue(ctx, PlansBuildingBucket)
|
||||
if err != nil {
|
||||
t.Fatalf("a holder of the build seat cannot bind %s: %v", PlansBuildingBucket, err)
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if err != nil || len(keys) != 1 || keys[0] != "plan-1" {
|
||||
t.Fatalf("the holder lists %v (%v), want plan-1", keys, err)
|
||||
}
|
||||
if e, err := kv.Get(ctx, "plan-1"); err != nil || e == nil {
|
||||
t.Fatalf("the holder cannot get the key: %v", err)
|
||||
}
|
||||
// Its write is refused by the server.
|
||||
if err := holder.Publish("$KV."+PlansBuildingBucket+".plan-2", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = holder.Flush()
|
||||
select {
|
||||
case err := <-refused:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("the server said %v, want a permissions violation", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("the server did not refuse the holder's put of a plan's key")
|
||||
}
|
||||
if _, err := ckv.Get(ctx, "plan-2"); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
t.Errorf("the refused put left a key: %v", err)
|
||||
}
|
||||
|
||||
// And the direct get of a check queue's first waiting ask, which is granted.
|
||||
reply, err := holder.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+ChecksOtherStream, []byte(`{"seq":1,"next_by_subj":"`+
|
||||
CheckSubjectPrefix(TheBuildSeat, false)+`>"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("the holder's direct get of %s was not answered: %v", ChecksOtherStream, err)
|
||||
}
|
||||
if reply.Header.Get("Status") != "404" {
|
||||
t.Errorf("an empty queue answered %q, want 404", reply.Header.Get("Status"))
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -29,7 +29,7 @@ accounts {
|
||||
]
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_plans-building.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
@@ -38,7 +38,7 @@ accounts {
|
||||
subscribe: { allow: ["_INBOX.enrol.one.>"] }
|
||||
} }
|
||||
{ user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
|
||||
publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-one.*.$O.declaration-files-one.C.*", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-one.*", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-one.$O.declaration-files-one.M.*", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.FC.OBJ_declaration-files-one.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
|
||||
subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
|
||||
@@ -135,7 +135,7 @@ type Records struct {
|
||||
// correct.
|
||||
func Users(r Records) ([]Principal, error) {
|
||||
out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r),
|
||||
AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig)}}
|
||||
AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig), FilesFor: sortedCopy(r.Nodes)}}
|
||||
|
||||
for _, node := range sortedCopy(r.Nodes) {
|
||||
witness := false
|
||||
|
||||
@@ -177,6 +177,10 @@ var WritersTable = []WriterRow{
|
||||
KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
|
||||
Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
|
||||
Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
|
||||
{State: "a delivery plan in its build phase", Writer: "controller (lease holder), at the first build's ask and at each refresh",
|
||||
KeptIn: "key-value " + PlansBuildingBucket + ", aged out when not refreshed",
|
||||
Others: "the build seat's holders read it and take no new check while a key is there (ADR 0483)",
|
||||
Subjects: kvOf(PlansBuildingBucket), Writes: isController},
|
||||
{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
|
||||
KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
|
||||
// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
|
||||
@@ -219,6 +223,13 @@ var WritersTable = []WriterRow{
|
||||
{State: "a machine's have", Writer: "the node-engine of the machine, under its own keys `<node>.>`",
|
||||
KeptIn: "key-value " + delivery.HaveBucket + ", twenty revisions of each key",
|
||||
Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys},
|
||||
// The contents of a declaration's files over 64 KiB (novox/hq ADR 0486 §2): the controller alone writes the bucket, each
|
||||
// object under the digest of its content; a node-engine reads one by its name, with no list and no write.
|
||||
{State: "a declaration's large files", Writer: "controller, each object under the sha256 digest of its content",
|
||||
KeptIn: "the object bucket " + DeclarationFilesBucket + "-<node> of each node, an object removed a day after the " +
|
||||
"node's latest declaration no longer names it",
|
||||
Others: "a node-engine reads an object of its own bucket by its name and may neither list the bucket nor write it",
|
||||
Subjects: []string{"$O.*.>"}, Writes: isController},
|
||||
{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
|
||||
KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
|
||||
Subjects: kvOf(DeclaredConfigBucket), Writes: isController},
|
||||
|
||||
@@ -29,6 +29,8 @@ var designRows = []string{
|
||||
"builds and their outcomes",
|
||||
"a pull request's check asked",
|
||||
"a pull request's check running",
|
||||
// The plans in their build phase, a bucket the build seat's holders read (ADR 0483 clause 2, issue 1687).
|
||||
"a delivery plan in its build phase",
|
||||
"a merge announced",
|
||||
"a pull request's head announced",
|
||||
"a pull request's merge check",
|
||||
@@ -43,6 +45,8 @@ var designRows = []string{
|
||||
"the rollouts of the chain",
|
||||
// What each machine has of each module, written by its node-engine under its own keys (ADR 0429 §1, issue 1136).
|
||||
"a machine's have",
|
||||
// The contents of a declaration's large files, in the bus's object bucket (ADR 0486 §2, issue 1697).
|
||||
"a declaration's large files",
|
||||
"the declared-config mirror",
|
||||
"the predefined-keys mirror",
|
||||
// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).
|
||||
|
||||
@@ -1773,9 +1773,24 @@ func whatFailed(s string) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The controller's module check refuses with a count line and one line per problem, then prints its
|
||||
// warning counts: the refusal is what failed, not the count printed last (novox/hq issue 1310).
|
||||
for i, line := range lines {
|
||||
if line = strings.TrimSpace(line); problemsLine.MatchString(line) {
|
||||
if i+1 < len(lines) {
|
||||
if first := strings.TrimSpace(lines[i+1]); first != "" {
|
||||
return line + ": " + first
|
||||
}
|
||||
}
|
||||
return line
|
||||
}
|
||||
}
|
||||
return lastLine(s)
|
||||
}
|
||||
|
||||
// problemsLine is the module check's refusal: "mesh-controller: 3 problem(s) in 139 manifest(s)".
|
||||
var problemsLine = regexp.MustCompile(`^\S+: [1-9]\d* problem\(s\) in \d+ manifest\(s\)$`)
|
||||
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
|
||||
@@ -199,3 +199,29 @@ func TestOnlyATrysSelectionReachesTheScript(t *testing.T) {
|
||||
t.Fatalf("a try's selection: %v", env)
|
||||
}
|
||||
}
|
||||
|
||||
// **Issue 1310**: the controller's module check prints its warning counts after its refusal; a failed check
|
||||
// is said by the refusal, never by the count printed last.
|
||||
func TestAFailedModuleCheckIsSaidByItsRefusalNotItsWarningCount(t *testing.T) {
|
||||
log := "mesh-controller: 3 problem(s) in 139 manifest(s)\n" +
|
||||
"mesh-files claims node-files and says it serves upload, download, which that seat's protocol does not promise\n" +
|
||||
"minio claims the seat \"object-store\", which no module declares and the mesh does not define\n" +
|
||||
"only-office claims the seat \"document-converter\", which no module declares and the mesh does not define\n" +
|
||||
"adwaita: ok, 4 tool(s)\nlong-running resources without health: 59\n" +
|
||||
"files asking for a setting without saying whether it is trusted: 36\n"
|
||||
// 1310/AC1
|
||||
want := "mesh-controller: 3 problem(s) in 139 manifest(s): mesh-files claims node-files and says it serves upload, " +
|
||||
"download, which that seat's protocol does not promise"
|
||||
if got := whatFailed(log); got != want {
|
||||
t.Errorf("1310/AC1: the captured log is said as %q, not %q", got, want)
|
||||
}
|
||||
// 1310/AC2
|
||||
if got, want := whatFailed("ok \tx/a\t1s\n--- FAIL: TestX (0.01s)\n x_test.go:12: got 1\nFAIL\nlong-running resources without health: 59\n"),
|
||||
"--- FAIL: TestX (0.01s)"; got != want {
|
||||
t.Errorf("1310/AC2: a failing test is said as %q, not %q", got, want)
|
||||
}
|
||||
// 1310/AC3
|
||||
if got, want := whatFailed("starting\nsomething odd happened\n"), "something odd happened"; got != want {
|
||||
t.Errorf("1310/AC3: a log with no failure line is said as %q, not %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
+206
-57
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -162,11 +163,21 @@ type natsMachine struct {
|
||||
// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
|
||||
subs []*takeQueue
|
||||
opts MachineOptions
|
||||
// gate is what the loop asks of the bus to hold checks while a delivery builds; nil is the bus's (a test sets a fake).
|
||||
gate checkGate
|
||||
// said is the faults of the hold already said, letKey whether this holder created the let-through key for the check it
|
||||
// took last (ADR 0483 clause 3).
|
||||
said map[string]bool
|
||||
letKey bool
|
||||
// mu guards the subscriptions of subs, bound and let go of by the taking loop and closed by Close.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// takeQueue is one queue a machine pulls from.
|
||||
type takeQueue struct {
|
||||
sub *nats.Subscription
|
||||
// pull is what the queue is fetched through: the subscription once bound (a test sets a fake).
|
||||
pull puller
|
||||
// kind is what its asks are: a build, a ready check, or another check.
|
||||
kind string
|
||||
// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
|
||||
@@ -174,6 +185,11 @@ type takeQueue struct {
|
||||
bound bool
|
||||
}
|
||||
|
||||
// puller is the fetch of a queue.
|
||||
type puller interface {
|
||||
Fetch(batch int, opts ...nats.PullOpt) ([]*nats.Msg, error)
|
||||
}
|
||||
|
||||
// The kinds of queue a machine takes from, in the order it takes from them.
|
||||
const (
|
||||
queueBuilds = "builds"
|
||||
@@ -195,6 +211,9 @@ type MachineOptions struct {
|
||||
// the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or
|
||||
// the other, never neither. Nil claims every one.
|
||||
Claim func() bool
|
||||
// LetThroughAfter is how long a check waits while a delivery builds before it is let through; zero is ADR 0483's thirty
|
||||
// minutes. Set by a test.
|
||||
LetThroughAfter time.Duration
|
||||
}
|
||||
|
||||
// MachineOverNATS takes build work from the current build role.
|
||||
@@ -222,6 +241,8 @@ const (
|
||||
)
|
||||
|
||||
func (m *natsMachine) Close() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for _, q := range m.subs {
|
||||
if q.sub != nil {
|
||||
_ = q.sub.Unsubscribe()
|
||||
@@ -244,6 +265,9 @@ func (m *natsMachine) Close() {
|
||||
// **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
|
||||
// other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
|
||||
// and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
|
||||
//
|
||||
// **Unless a delivery builds** (novox/hq ADR 0483): while the controller's bucket of the plans building holds a key, no check queue
|
||||
// is fetched from, but for the one check that waited LetThroughAfter, which goes first (see next).
|
||||
func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
|
||||
worker, found := broker.HolderConsumerFor(m.on, "build-agent",
|
||||
broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
|
||||
@@ -288,6 +312,19 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The seat's own queue is bound again when a running ask made the machine let go of it (quiesce): a machine that cannot
|
||||
// is not taking, which is said and tried again.
|
||||
if !m.subs[0].bound {
|
||||
if err := m.bind(m.subs[0]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this machine cannot bind %s again after a build, and tries again: %v\n", m.seat, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(pausedPoll):
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if time.Now().After(bindAgain) {
|
||||
for _, q := range m.subs[1:] {
|
||||
if !q.bound {
|
||||
@@ -312,7 +349,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// say why (2026-10-03, the first build agents). Asked again.
|
||||
continue
|
||||
default:
|
||||
if m.subs[0].sub.IsValid() {
|
||||
if m.subs[0].sub != nil && m.subs[0].sub.IsValid() {
|
||||
// A transient fault in asking — a reconnect, a slow server — is asked past rather
|
||||
// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
|
||||
time.Sleep(time.Second)
|
||||
@@ -320,62 +357,138 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
}
|
||||
return fmt.Errorf("the bus stopped delivering build work: %w", err)
|
||||
}
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
continue
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
continue
|
||||
}
|
||||
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||
if q.kind == queueOther {
|
||||
if why := m.judgedAlready(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
continue
|
||||
}
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
continue
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
m.handOver(ctx, msg, q, do)
|
||||
// And the queues it let go of are bound again at once, not after the half minute.
|
||||
bindAgain = time.Time{}
|
||||
}
|
||||
}
|
||||
|
||||
// handOver settles one fetched ask: dropped, handed back or built. **A check let through keeps its key for as long as it
|
||||
// stays here** (ADR 0483 clause 3): refreshed while it runs and deleted when it ends, however it ends.
|
||||
func (m *natsMachine) handOver(ctx context.Context, msg *nats.Msg, q *takeQueue, do func(context.Context, Build)) {
|
||||
if m.letKey {
|
||||
defer m.keepLetThrough(ctx, heartbeatWhileWorking)()
|
||||
}
|
||||
// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
|
||||
// the one that arrived in that moment. Handed back at once for another holder — counted as
|
||||
// a delivery, which a pause landing exactly then costs and nothing else does.
|
||||
if m.opts.Paused != nil && m.opts.Paused() {
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
var request BuildRequest
|
||||
if err := json.Unmarshal(msg.Data, &request); err != nil {
|
||||
// Unreadable: terminated rather than retried, because the next attempt reads the same
|
||||
// bytes. Nobody waiting hears an answer, which is right — there was no request.
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
|
||||
// id into the seat's cancelled set before deleting the ask, so one taken in between is
|
||||
// ended here, terminated rather than redelivered, and its outcome said as failed — never
|
||||
// built. A set that cannot be read is said and the ask built: a cancel is a person's
|
||||
// exception, and a holder that refused every build while its grant was missing would
|
||||
// stop the mesh building for it.
|
||||
build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
|
||||
if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
|
||||
} else if cancelled {
|
||||
endCancelled(ctx, build)
|
||||
return
|
||||
}
|
||||
// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
|
||||
// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
|
||||
if q.kind == queueOther {
|
||||
if why := m.judgedAlready(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
return
|
||||
}
|
||||
}
|
||||
// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
|
||||
// landing then is.
|
||||
if m.opts.Claim != nil && !m.opts.Claim() {
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
// **A duplicate of a check another builder is running is dropped** (issue 1649): an ask redelivered to this machine
|
||||
// because its first holder did not say, in time, that it had it. The running key names that builder and this same
|
||||
// ask; running it a second time would spend a second place on one check.
|
||||
if why := m.runningElsewhere(ctx, request); why != "" {
|
||||
fmt.Fprintf(os.Stderr, "%s is handed out again while another builder runs it and is dropped: %s\n", request.ID, why)
|
||||
_ = msg.Ack()
|
||||
return
|
||||
}
|
||||
// A build outlives the acknowledgement window many times over; said while it runs,
|
||||
// as the controller says it for its own long handlers, so the server neither hands
|
||||
// the ask to a second machine nor counts the wait against its deliveries.
|
||||
working := make(chan struct{})
|
||||
go stillWorking(msg, working, heartbeatWhileWorking)
|
||||
// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
|
||||
// deleted when it ends, whatever the outcome.
|
||||
stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
|
||||
// **And holds nothing else while it runs** (issue 1747): see quiesce.
|
||||
m.quiesce(q)
|
||||
do(ctx, build)
|
||||
stopRunning()
|
||||
close(working)
|
||||
// **The queue it came from too, once the ask is settled**: a fetch that returned a late message leaves its pull request
|
||||
// open, and the server can deliver one more into the subscription (issue 1747). Take binds every queue again first.
|
||||
m.quiesce(nil)
|
||||
}
|
||||
|
||||
// quiesce lets go of every queue but the one the running ask came from, for as long as it runs. **A builder never holds a
|
||||
// message it does not start** (issue 1747). A message the server sends after a fetch gave up, within the few milliseconds its
|
||||
// pull request still lived, lands in that queue's subscription and stays there until the next fetch of it: held, never
|
||||
// acknowledged as in progress, for as long as the builder runs its current ask, five or ten minutes for a check, while the
|
||||
// holders that could build it are all busy and a delivery waits behind it. The legacy pull subscription cannot be read
|
||||
// without sending a new pull request, which could take one more, so it cannot be searched for such a message. It is dropped
|
||||
// instead: the subscription is closed, a message in it or on its way is lost to this builder and the server hands it out again
|
||||
// when its worker's wait for an acknowledgement ends (a minute), to any holder that is free, and the queue is bound again
|
||||
// before the next fetch (Take). The queue the running ask came from keeps its subscription, which its acknowledgements use.
|
||||
func (m *natsMachine) quiesce(keep *takeQueue) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for _, q := range m.subs {
|
||||
if q == keep || !q.bound {
|
||||
continue
|
||||
}
|
||||
if q.sub != nil {
|
||||
_ = q.sub.Unsubscribe()
|
||||
}
|
||||
q.sub, q.pull, q.bound = nil, nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// runningElsewhere says why an ask need not run here, or nothing: the bucket of the checks running says another builder
|
||||
// runs this very ask (its id, its head and its trunk commit). A key that cannot be read runs the check.
|
||||
func (m *natsMachine) runningElsewhere(ctx context.Context, r BuildRequest) string {
|
||||
if r.Check == nil {
|
||||
return ""
|
||||
}
|
||||
running, ok, err := ReadRunning(ctx, m.js.Conn(), r)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether %s is running elsewhere, so it runs: %v\n", r.ID, err)
|
||||
return ""
|
||||
}
|
||||
return duplicateOf(running, ok, r, m.on)
|
||||
}
|
||||
|
||||
// runningFreshFor is how recent a running key's last heartbeat must be for its builder to be taken as alive: two missed
|
||||
// heartbeats of ten seconds are not yet a dead builder, and a redelivery after a crash comes a minute after the last one.
|
||||
const runningFreshFor = 25 * time.Second
|
||||
|
||||
// duplicateOf says why an ask is a second hand-out of one another builder is running, or nothing.
|
||||
func duplicateOf(running CheckRunning, found bool, r BuildRequest, me string) string {
|
||||
// Seen is the server's time and runningFreshFor is judged against this builder's clock: a skew over about 35 s would matter.
|
||||
// A key not renewed for runningFreshFor is a builder that has gone: its ask is redelivered after the ack wait, and
|
||||
// that is the recovery, not a duplicate (the key itself lives two minutes).
|
||||
if found && running.Build == r.ID && running.Builder != me && running.Same(r) && time.Since(running.Seen) < runningFreshFor {
|
||||
return fmt.Sprintf("%s is running it as %s since %s", running.Builder, running.Build, running.Started.Local().Format("15:04:05"))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// bind binds a queue's worker, which exists and is not created here.
|
||||
//
|
||||
// **The consumer's own filter, not the one subject this machine cares about.** The client checks what is asked for
|
||||
@@ -383,25 +496,39 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
|
||||
// subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned
|
||||
// twice now, on two different consumers, which is why it is written down here.
|
||||
func (m *natsMachine) bind(q *takeQueue) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name,
|
||||
nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q.sub, q.bound = sub, true
|
||||
q.sub, q.pull, q.bound = sub, sub, true
|
||||
return nil
|
||||
}
|
||||
|
||||
// next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
|
||||
// on for a second. Nothing, with no error, when there was nothing.
|
||||
func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
// **While a delivery builds, only delivery builds are taken, but for one check that waited** (novox/hq ADR 0483): the
|
||||
// bucket of the plans building is read before either check queue is fetched from, and a key there holds both. The check
|
||||
// that waited LetThroughAfter goes first, ahead of the builds that wait; the rest of the holders take builds only.
|
||||
if m.seat == broker.TheBuildSeat && len(m.subs) > 1 {
|
||||
if plans := m.building(ctx); len(plans) > 0 {
|
||||
if msg, q := m.letOneThrough(ctx); msg != nil {
|
||||
fmt.Fprintf(os.Stderr, "a check that waited %s is let through while %s builds\n", m.letThroughAfter(), whyHeld(plans))
|
||||
return msg, q, nil
|
||||
}
|
||||
return m.nextBuild(ctx)
|
||||
}
|
||||
}
|
||||
var last error
|
||||
for _, q := range m.subs {
|
||||
if !q.bound {
|
||||
continue
|
||||
}
|
||||
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||
fetched, err := q.sub.Fetch(1, nats.Context(trying))
|
||||
fetched, err := q.pull.Fetch(1, nats.Context(trying))
|
||||
end()
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], q, nil
|
||||
@@ -417,9 +544,18 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
if last != nil {
|
||||
return nil, nil, last
|
||||
}
|
||||
return m.nextBuild(ctx)
|
||||
}
|
||||
|
||||
// nextBuild waits for a second at most for a delivery build, the seat's own queue.
|
||||
func (m *natsMachine) nextBuild(ctx context.Context) (*nats.Msg, *takeQueue, error) {
|
||||
if !m.subs[0].bound {
|
||||
// Let go of while an ask ran and not bound again yet (Take binds it before it asks): nothing to take from.
|
||||
return nil, nil, nil
|
||||
}
|
||||
waiting, end := context.WithTimeout(ctx, takeIdleWait)
|
||||
defer end()
|
||||
fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
|
||||
fetched, err := m.subs[0].pull.Fetch(1, nats.Context(waiting))
|
||||
if len(fetched) > 0 {
|
||||
return fetched[0], m.subs[0], nil
|
||||
}
|
||||
@@ -582,3 +718,16 @@ func endCancelled(ctx context.Context, b *natsBuild) {
|
||||
}
|
||||
_ = b.msg.Term()
|
||||
}
|
||||
|
||||
// boundKinds are the kinds of queue this machine holds a subscription to now.
|
||||
func (m *natsMachine) boundKinds() []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []string
|
||||
for _, q := range m.subs {
|
||||
if q.bound {
|
||||
out = append(out, q.kind)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -24,3 +24,6 @@ func ConnectNats(js *broker.JetStream, enroller Enroller, listener Listener) *Se
|
||||
// Bus is the controller's outbound, whichever transport it connected over. Callers that send a
|
||||
// declaration or ask a tool use this rather than the channel, which one transport does not have.
|
||||
func (s *Server) Bus() Bus { return s.bus }
|
||||
|
||||
// SendingOn is a server that only sends, over the bus given: for a test of a sender with a bus it holds in its hand.
|
||||
func SendingOn(bus Bus) *Server { return &Server{bus: bus} }
|
||||
|
||||
@@ -253,11 +253,25 @@ func watchPullIn(ctx context.Context, buckets []string, open func(ctx context.Co
|
||||
|
||||
// checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it
|
||||
// waits on the first when all three are empty.
|
||||
//
|
||||
// **A fetch's pull request must end before the fetch does** (issue 1649). nats.go gives the server-side request an expiry a tenth
|
||||
// shorter than the fetch's wait (fetchExpiryOf), and the server then answers with a status message at the expiry; a wait so
|
||||
// short that the answer can land after the fetch gave up left a status message in the pull's buffer, and anything a request
|
||||
// could still deliver would be held by a machine that is busy. Each wait leaves at least minFetchMargin between the two.
|
||||
const (
|
||||
checkTakeWait = 30 * time.Millisecond
|
||||
checkTakeWait = 100 * time.Millisecond
|
||||
takeIdleWait = time.Second
|
||||
)
|
||||
|
||||
// minFetchMargin is the least time between a fetch's server-side expiry and the end of its wait.
|
||||
const minFetchMargin = 10 * time.Millisecond
|
||||
|
||||
// fetchExpiryOf is the expiry nats.go gives the pull request of a fetch that waits this long: the wait less a tenth of it, or
|
||||
// less five seconds.
|
||||
func fetchExpiryOf(wait time.Duration) time.Duration {
|
||||
return wait - min(wait/10, 5*time.Second)
|
||||
}
|
||||
|
||||
// CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406
|
||||
// §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf).
|
||||
type CheckRunning struct {
|
||||
@@ -268,15 +282,21 @@ type CheckRunning struct {
|
||||
// Started is when the check started, kept on every heartbeat: the entry's own time is the last heartbeat's, and a
|
||||
// reader that asks how long a check has run reads this (novox/hq ADR 0433 §6).
|
||||
Started time.Time `json:"started"`
|
||||
// Seen is when the key was last put, which is the last heartbeat: read from the entry, never written (issue 1649).
|
||||
Seen time.Time `json:"-"`
|
||||
}
|
||||
|
||||
// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
|
||||
// composed check; none for a try, which no other request replaces.
|
||||
// composed check; `try.<id>` for a try, which no other request replaces.
|
||||
func RunningKeyOf(r BuildRequest) (string, bool) {
|
||||
c := r.Check
|
||||
switch {
|
||||
case c == nil, c.Try:
|
||||
case c == nil:
|
||||
return "", false
|
||||
case c.Try:
|
||||
// A try says it runs under its own id, which no other request shares: so a reader of the bucket can tell a try that runs
|
||||
// from a check left waiting for a redelivery (ADR 0483 clause 3).
|
||||
return "try." + tokenOf(r.ID), true
|
||||
case c.Group != "":
|
||||
return "group." + tokenOf(c.Group), true
|
||||
case c.Number <= 0:
|
||||
@@ -319,6 +339,7 @@ func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRun
|
||||
if err := json.Unmarshal(e.Value(), &running); err != nil {
|
||||
return CheckRunning{}, false, nil // a key of a builder from before this means no match
|
||||
}
|
||||
running.Seen = e.Created()
|
||||
return running, true, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -510,3 +510,113 @@ func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
|
||||
// An ask handed out a second time to another builder while the first still runs it is acknowledged and not run: the
|
||||
// running key names the other builder and this same ask (issue 1649).
|
||||
func TestAnAskHandedOutAgainWhileAnotherBuilderRunsItIsNotRunTwice(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
aKeyBucket(t, js)
|
||||
aRunningBucket(t, js, 2*time.Minute)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
running := make(chan string, 4)
|
||||
release := make(chan struct{})
|
||||
a := MachineOverNATS(js, "builder-a")
|
||||
defer a.Close()
|
||||
go func() {
|
||||
_ = a.Take(ctx, func(ctx context.Context, work Build) {
|
||||
running <- work.Request().ID
|
||||
<-release
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
check := aPullCheck("build-twice", 7, "head-7-000000000", "trunk-1-000000", false)
|
||||
if err := asks.Ask(ctx, check); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-running:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("builder a took nothing")
|
||||
}
|
||||
// The bus's redelivery, as it reaches the other machine: the same ask, the same id.
|
||||
if err := asks.Ask(ctx, check); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := MachineOverNATS(js, "builder-b")
|
||||
defer b.Close()
|
||||
took := make(chan string, 4)
|
||||
go func() {
|
||||
_ = b.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case id := <-took:
|
||||
t.Fatalf("builder b ran %s, which builder a is running", id)
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
|
||||
t.Errorf("%d request(s) still wait; builder b was to acknowledge the duplicate", n)
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
// A busy machine fetches nothing, so no ask waiting in the queue is taken by it, whatever status message sits in its pulls
|
||||
// (issue 1649). A pull request of our own that expires at once leaves such a message in the other checks' pull.
|
||||
func TestABusyMachineTakesNoWaitingAskWhateverItsPullsHold(t *testing.T) {
|
||||
js := aBusWithTheBuildRole(t)
|
||||
aKeyBucket(t, js)
|
||||
aRunningBucket(t, js, 2*time.Minute)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
busy := make(chan string, 4)
|
||||
release := make(chan struct{})
|
||||
a := MachineOverNATS(js, "builder-a").(*natsMachine)
|
||||
defer a.Close()
|
||||
go func() {
|
||||
_ = a.Take(ctx, func(ctx context.Context, work Build) {
|
||||
busy <- work.Request().ID
|
||||
<-release
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
if err := asks.Ask(ctx, BuildRequest{ID: "busy", Repository: "/d"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case <-busy:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("builder a took nothing")
|
||||
}
|
||||
// **A busy machine holds no pull at all** (issues 1649 and 1747): the queues the running ask did not come from are let go of,
|
||||
// so no message, a status message or a late delivery, can sit in one while it runs.
|
||||
if kinds := a.boundKinds(); len(kinds) != 1 || kinds[0] != queueBuilds {
|
||||
t.Fatalf("builder a holds %v while it builds; want only the queue its build came from", kinds)
|
||||
}
|
||||
other := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
for i, id := range []string{"wait-1", "wait-2"} {
|
||||
if err := asks.Ask(ctx, aPullCheck(id, 20+i, "head-x-000000000", "trunk-1-000000", false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
info, err := js.Context().ConsumerInfo(other.worker.Stream, other.worker.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumAckPending != 0 || info.NumRedelivered != 0 || info.Delivered.Consumer != 0 {
|
||||
t.Fatalf("a busy machine took from the queue: %d delivered, %d pending, %d redelivered", info.Delivered.Consumer,
|
||||
info.NumAckPending, info.NumRedelivered)
|
||||
}
|
||||
if n := len(queueOf(t, js).Of(AskWaiting)); n != 2 {
|
||||
t.Fatalf("%d asks wait; want the 2 nobody took", n)
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
//go:build integration
|
||||
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/nats-io/nats.go"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687, 1688), against a real bus.
|
||||
|
||||
// aBusWithTheBuckets is the bus with the build role and the controller's buckets, the bucket of the checks running with an age
|
||||
// of two seconds so a key a holder stopped refreshing is seen to expire.
|
||||
func aBusWithTheBuckets(t *testing.T) (*broker.JetStream, jetstream.KeyValue) {
|
||||
t.Helper()
|
||||
js := aBusWithTheBuildRole(t)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := t.Context()
|
||||
if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: 2 * time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plans, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.PlansBuildingBucket, History: 1, TTL: broker.PlansBuildingKeptFor})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js, plans
|
||||
}
|
||||
|
||||
func aCheckAskedAgo(ago time.Duration, number int, ready bool) BuildRequest {
|
||||
r := aPullCheck(fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano()), number, fmt.Sprintf("head-%d-0000000", number), "trunk-0-000000", ready)
|
||||
return r
|
||||
}
|
||||
|
||||
// 1652/AC6: a plan's builds asked in two tiers with a check waiting, the check is handed out only after the last tier's builds,
|
||||
// and not in the gap between the tiers.
|
||||
func TestNatsACheckIsHandedOutOnlyAfterTheLastTierOfAPlan(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asks.Ask(ctx, aCheckAskedAgo(time.Minute, 7, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asks.Ask(ctx, BuildRequest{ID: "tier-1-a", Repository: "/a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
took := make(chan string, 16)
|
||||
machine := MachineOverNATS(js, "anchor")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
took <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
expect := func(want string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case got := <-took:
|
||||
if got != want {
|
||||
t.Fatalf("taken %s, want %s", got, want)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatalf("%s was not taken", want)
|
||||
}
|
||||
}
|
||||
expect("tier-1-a")
|
||||
// The gap between two tiers: the delivery's queue is empty, the plan's key is there, and the check is not taken.
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s was taken in the gap between two tiers", got)
|
||||
case <-time.After(3 * time.Second):
|
||||
}
|
||||
if err := asks.Ask(ctx, BuildRequest{ID: "tier-2-a", Repository: "/b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expect("tier-2-a")
|
||||
select {
|
||||
case got := <-took:
|
||||
t.Fatalf("%s was taken before the plan's last build was built and its key deleted", got)
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
// The phase ends: the key is deleted, and the check is handed out.
|
||||
if err := plans.Delete(ctx, "plan-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case got := <-took:
|
||||
if got == "tier-1-a" || got == "tier-2-a" {
|
||||
t.Fatalf("a build was taken twice: %s", got)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("the check was not handed out once the plan's key was deleted")
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: two holders racing for checks that waited past the bound: exactly one is let through at a time, by a create that fails
|
||||
// while the key is there, and the key is deleted when its check ends, after which the next oldest goes.
|
||||
func TestNatsTwoHoldersRacingForOldChecksLetExactlyOneThrough(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
// The buckets' age is two seconds here, so the keys are refreshed well within it.
|
||||
was := heartbeatWhileWorking
|
||||
heartbeatWhileWorking = 400 * time.Millisecond
|
||||
t.Cleanup(func() { heartbeatWhileWorking = was })
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, ago := range []time.Duration{50 * time.Minute, 40 * time.Minute, 35 * time.Minute} {
|
||||
if err := asks.Ask(ctx, aCheckAskedAgo(ago, i+1, false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
running, most := 0, 0
|
||||
started := make(chan string, 8)
|
||||
release := make(chan struct{})
|
||||
do := func(ctx context.Context, work Build) {
|
||||
mu.Lock()
|
||||
running++
|
||||
most = max(most, running)
|
||||
mu.Unlock()
|
||||
started <- work.Request().Check.Repo + fmt.Sprint(work.Request().Check.Number)
|
||||
<-release
|
||||
mu.Lock()
|
||||
running--
|
||||
mu.Unlock()
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "x", Failed: "no"})
|
||||
_ = work.Done()
|
||||
}
|
||||
for _, on := range []string{"ace", "shanks"} {
|
||||
machine := MachineOverNATS(js, on)
|
||||
defer machine.Close()
|
||||
go func() { _ = machine.Take(ctx, do) }()
|
||||
}
|
||||
var order []string
|
||||
for range 3 {
|
||||
select {
|
||||
case id := <-started:
|
||||
order = append(order, id)
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatalf("started so far: %v", order)
|
||||
}
|
||||
// While it runs, no second check starts and the key is there.
|
||||
select {
|
||||
case id := <-started:
|
||||
t.Fatalf("%s started beside a check let through", id)
|
||||
case <-time.After(2500 * time.Millisecond):
|
||||
}
|
||||
kv := runningKV(t, js)
|
||||
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
|
||||
t.Fatalf("the let-through key is not there while the check runs: %v", err)
|
||||
}
|
||||
release <- struct{}{}
|
||||
// It is deleted when the check ends: the next check let through creates the key again at once, and a create fails
|
||||
// while the key is there, so its start (the next round) is the proof; after the last one the key stays gone.
|
||||
if len(order) == 3 {
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
if _, err := kv.Get(ctx, LetThroughKey); err != nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the let-through key was not deleted when the last check ended")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
if want := []string{"a1", "a2", "a3"}; !reflect.DeepEqual(order, want) {
|
||||
t.Fatalf("checks went %v, want the oldest first %v", order, want)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if most != 1 {
|
||||
t.Fatalf("%d checks ran at once while a delivery built", most)
|
||||
}
|
||||
}
|
||||
|
||||
func runningKV(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
|
||||
t.Helper()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(t.Context(), broker.CheckRunningBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// 1688: the create of the let-through key succeeds for exactly one of holders asking at once, and a key its holder stopped
|
||||
// refreshing expires, after which the next create succeeds.
|
||||
func TestNatsOneCreateOfTheLetThroughKeySucceedsAndAStoppedHoldersKeyExpires(t *testing.T) {
|
||||
js, _ := aBusWithTheBuckets(t)
|
||||
gate := busGate{conn: js.Conn()}
|
||||
ctx := t.Context()
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
won := 0
|
||||
for i := range 8 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
ok, err := gate.TakeLetThrough(ctx, fmt.Sprint("holder-", i))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
mu.Lock()
|
||||
won++
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if won != 1 {
|
||||
t.Fatalf("%d of 8 holders created the let-through key, want exactly one", won)
|
||||
}
|
||||
// The holder stopped: nothing refreshes the key, and the bucket's age (two seconds here) removes it.
|
||||
deadline := time.Now().Add(15 * time.Second)
|
||||
for {
|
||||
ok, err := gate.TakeLetThrough(ctx, "next")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the key of a stopped holder did not expire")
|
||||
}
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
// And a refreshed key does not expire.
|
||||
for range 6 {
|
||||
if err := gate.KeepLetThrough(ctx, "next"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
}
|
||||
if ok, _ := gate.TakeLetThrough(ctx, "other"); ok {
|
||||
t.Fatal("a refreshed key expired")
|
||||
}
|
||||
var v LetThrough
|
||||
e, err := runningKV(t, js).Get(ctx, LetThroughKey)
|
||||
if err != nil || json.Unmarshal(e.Value(), &v) != nil || v.Builder != "next" {
|
||||
t.Fatalf("the key says %+v (%v)", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1687: the oldest waiting ask is read without being taken: no delivery is counted against it.
|
||||
func TestNatsTheOldestWaitingAskIsReadWithoutTakingIt(t *testing.T) {
|
||||
js, _ := aBusWithTheBuckets(t)
|
||||
ctx := t.Context()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
old := aCheckAskedAgo(45*time.Minute, 1, false)
|
||||
if err := asks.Ask(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gate := busGate{conn: js.Conn()}
|
||||
q := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
|
||||
at, found, err := gate.OldestWaiting(ctx, q)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the first waiting ask was not read: %v %v", found, err)
|
||||
}
|
||||
if want, _ := BuildAskedAt(old.ID); !at.Equal(want) {
|
||||
t.Fatalf("read %v, asked %v", at, want)
|
||||
}
|
||||
for range 3 {
|
||||
if _, _, err := gate.OldestWaiting(ctx, q); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
info, err := js.Context().ConsumerInfo(q.worker.Stream, q.worker.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Delivered.Consumer != 0 || info.NumAckPending != 0 || info.NumPending != 1 {
|
||||
t.Fatalf("reading the age took the ask: delivered %d, pending acks %d, waiting %d", info.Delivered.Consumer, info.NumAckPending, info.NumPending)
|
||||
}
|
||||
// And an empty queue reads as none.
|
||||
if _, found, err := gate.OldestWaiting(ctx, &takeQueue{kind: queueReady, worker: broker.CheckWorkers()[0]}); err != nil || found {
|
||||
t.Fatalf("an empty queue read as %v (%v)", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1747: a message that reaches a builder's subscription outside a fetch is not held while the builder runs another ask. A
|
||||
// stray check lands in the other queue's subscription of builder A, which then runs a long check; the server hands the stray
|
||||
// out again when its wait for an acknowledgement ends (two seconds here), builder B runs it, and A, finished, does not run it a
|
||||
// second time from the subscription it held.
|
||||
func TestNatsAMessageDeliveredOutsideAFetchIsNotHeldWhileTheBuilderRunsAnotherAsk(t *testing.T) { // 1747
|
||||
js := aBusWithTheBuildRole(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
for _, w := range broker.CheckWorkers() {
|
||||
w.AckWaitSeconds = 2
|
||||
if err := js.EnsureConsumer(w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The long check says it is alive well within the ack wait of two seconds.
|
||||
was := heartbeatWhileWorking
|
||||
heartbeatWhileWorking = 400 * time.Millisecond
|
||||
t.Cleanup(func() { heartbeatWhileWorking = was })
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
long := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()), 1, "head-1-0000000", "trunk-0-000000", true)
|
||||
stray := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()+1), 2, "head-2-0000000", "trunk-0-000000", false)
|
||||
for _, r := range []BuildRequest{long, stray} {
|
||||
if err := asks.Ask(ctx, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
runs := map[string][]string{} // ask id -> the machines that ran it
|
||||
release := make(chan struct{})
|
||||
started := make(chan string, 8)
|
||||
do := func(on string) func(context.Context, Build) {
|
||||
return func(ctx context.Context, work Build) {
|
||||
id := work.Request().ID
|
||||
mu.Lock()
|
||||
runs[id] = append(runs[id], on)
|
||||
mu.Unlock()
|
||||
started <- on + ":" + id
|
||||
if id == long.ID {
|
||||
<-release
|
||||
}
|
||||
_ = work.Announce(ctx, BuildResult{ID: id, On: on, Failed: "no"})
|
||||
_ = work.Done()
|
||||
}
|
||||
}
|
||||
|
||||
var a *natsMachine
|
||||
strayLanded := make(chan error, 1)
|
||||
a = MachineOverNATSWith(js, "ace", TheBuildMachine, MachineOptions{Claim: func() bool {
|
||||
// The moment the long check is claimed: the server hands the other queue's waiting check to A's subscription there, as
|
||||
// one that arrives after a fetch gave up does. Done once.
|
||||
select {
|
||||
case <-strayLanded:
|
||||
return true
|
||||
default:
|
||||
}
|
||||
other := a.subs[2]
|
||||
inbox := strings.TrimSuffix(other.sub.Subject, "*") + "stray"
|
||||
body, _ := json.Marshal(map[string]any{"batch": 1, "expires": int64(30 * time.Second)})
|
||||
if err := js.Conn().PublishRequest("$JS.API.CONSUMER.MSG.NEXT."+other.worker.Stream+"."+other.worker.Name, inbox, body); err != nil {
|
||||
strayLanded <- err
|
||||
return true
|
||||
}
|
||||
_ = js.Conn().Flush()
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
|
||||
if n, _, _ := other.sub.Pending(); n > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
strayLanded <- nil
|
||||
return true
|
||||
}}).(*natsMachine)
|
||||
defer a.Close()
|
||||
go func() { _ = a.Take(ctx, do("ace")) }()
|
||||
|
||||
select {
|
||||
case got := <-started:
|
||||
if got != "ace:"+long.ID {
|
||||
t.Fatalf("started %s, want the long check on ace first", got)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("the long check did not start")
|
||||
}
|
||||
if err := <-strayLanded; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// B, free, runs the stray once the server hands it out again.
|
||||
b := MachineOverNATS(js, "shanks")
|
||||
defer b.Close()
|
||||
go func() { _ = b.Take(ctx, do("shanks")) }()
|
||||
select {
|
||||
case got := <-started:
|
||||
if got != "shanks:"+stray.ID {
|
||||
t.Fatalf("started %s, want the stray on shanks", got)
|
||||
}
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("the stray was held by the builder that ran the long check and nobody ran it")
|
||||
}
|
||||
// A finishes, and does not run the stray from the subscription it held.
|
||||
release <- struct{}{}
|
||||
select {
|
||||
case got := <-started:
|
||||
t.Fatalf("%s started after the long check: the stray ran twice", got)
|
||||
case <-time.After(6 * time.Second):
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(runs[stray.ID]) != 1 {
|
||||
t.Fatalf("the stray ran on %v, want once", runs[stray.ID])
|
||||
}
|
||||
}
|
||||
|
||||
// Review of 1688: a young check pending redelivery (handed back by a Nak) stands ahead of a check older than 30 minutes in the
|
||||
// same queue while a plan builds. The let-through fetch would return the young one first, so nothing is fetched: the young
|
||||
// check's delivery count stays 1, it is not handed back every second until it is dead-lettered, and it is not run.
|
||||
func TestNatsAYoungRedeliveryAheadOfAnOldCheckIsNeverFetchedByTheLetThrough(t *testing.T) {
|
||||
js, plans := aBusWithTheBuckets(t)
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
defer stop()
|
||||
asks := BuildsOn(js, TheBuildMachine)
|
||||
young := aCheckAskedAgo(time.Minute, 1, false)
|
||||
if err := asks.Ask(ctx, young); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Someone took it and handed it back: it is pending redelivery.
|
||||
worker := broker.CheckWorkers()[1]
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
|
||||
if err != nil || len(got) != 1 {
|
||||
t.Fatalf("the young check was not fetched to be handed back: %v", err)
|
||||
}
|
||||
if err := got[0].Nak(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = sub.Unsubscribe()
|
||||
old := aCheckAskedAgo(45*time.Minute, 2, false)
|
||||
if err := asks.Ask(ctx, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := make(chan string, 4)
|
||||
machine := MachineOverNATS(js, "ace")
|
||||
defer machine.Close()
|
||||
go func() {
|
||||
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
|
||||
started <- work.Request().ID
|
||||
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "ace", Failed: "no"})
|
||||
_ = work.Done()
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case id := <-started:
|
||||
t.Fatalf("%s was run while the young redelivery stood ahead of it", id)
|
||||
case <-time.After(6 * time.Second):
|
||||
}
|
||||
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Delivered.Consumer != 1 || info.NumRedelivered != 0 {
|
||||
t.Fatalf("the young check was handed out %d time(s), %d redelivered: the let-through fetched it", info.Delivered.Consumer, info.NumRedelivered)
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
|
||||
return fmt.Errorf("cannot sign a declaration for %s: %w", node, err)
|
||||
}
|
||||
|
||||
body, err := json.Marshal(Signed{Declaration: declaration, Signature: signature})
|
||||
body, err := Envelope(declaration, signature)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -66,8 +66,16 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
|
||||
return bus.PublishDeclaration(publish, node, body)
|
||||
}
|
||||
|
||||
// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it.
|
||||
const NearTheLimit = 0.9
|
||||
// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON
|
||||
// document. Its length is what the bus's largest message bounds.
|
||||
func Envelope(declaration, signature []byte) ([]byte, error) {
|
||||
return json.Marshal(Signed{Declaration: declaration, Signature: signature})
|
||||
}
|
||||
|
||||
// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that
|
||||
// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486
|
||||
// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline).
|
||||
const NearTheLimit = 0.75
|
||||
|
||||
// DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message,
|
||||
// so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689).
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// An ask a machine holds but does not start gets no heartbeat, and the bus hands it to another machine after the ack
|
||||
// wait, so two machines run one check (issue 1649).
|
||||
|
||||
// A fetch's server-side pull request ends clearly before the fetch does, so the server's answer and any message it could still
|
||||
// deliver come inside the fetch and never into a buffer nobody reads while the machine builds.
|
||||
func TestAFetchsPullRequestEndsClearlyBeforeTheFetchDoes(t *testing.T) {
|
||||
for name, wait := range map[string]time.Duration{"a check queue": checkTakeWait, "the idle wait": takeIdleWait} {
|
||||
if margin := wait - fetchExpiryOf(wait); margin < minFetchMargin {
|
||||
t.Errorf("%s: the pull request ends %v before a fetch of %v does; want at least %v", name, margin, wait, minFetchMargin)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskAnotherBuilderIsRunningIsADuplicateAndYoursIsNot(t *testing.T) {
|
||||
r := aPullCheck("build-1", 423, "head-4771ee53aa", "trunk-0123456789", false)
|
||||
run := CheckRunning{Head: r.Ref, Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-1", Started: time.Now().Add(-time.Minute), Seen: time.Now().Add(-4 * time.Second)}
|
||||
dead := run
|
||||
dead.Seen = time.Now().Add(-60 * time.Second)
|
||||
for name, c := range map[string]struct {
|
||||
running CheckRunning
|
||||
found bool
|
||||
me string
|
||||
dup bool
|
||||
}{
|
||||
"another builder runs this very ask": {run, true, "ace", true},
|
||||
"its builder stopped renewing the key a minute ago": {dead, true, "ace", false},
|
||||
"this builder runs it itself": {run, true, "shanks", false},
|
||||
"nobody runs it": {CheckRunning{}, false, "ace", false},
|
||||
"another ask of the same head": {CheckRunning{Head: r.Ref, Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-0"}, true, "ace", false},
|
||||
"the same ask at another head": {CheckRunning{Head: "head-other-0000", Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-1"}, true, "ace", false},
|
||||
} {
|
||||
if got := duplicateOf(c.running, c.found, r, c.me) != ""; got != c.dup {
|
||||
t.Errorf("%s: duplicate = %v; want %v", name, got, c.dup)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687 and 1688).
|
||||
//
|
||||
// **The layout other parts read** (ADR 0484 clause 12 has the checker read the same key; a layout shared between modules
|
||||
// belongs in mesh-sdk, and this one is not there yet):
|
||||
//
|
||||
// - bucket `mesh-controller_plans-building` (broker.PlansBuildingBucket): one key per delivery plan in its build phase, named
|
||||
// by the plan's id (`plan-<digits>`), value a PlanBuilding as JSON. Written by the controller (lease holder) alone: put when
|
||||
// the plan's first build is asked, put again about every thirty seconds, deleted when the phase ends; the bucket's age
|
||||
// (two minutes) removes the key of a controller that stopped. A reader lists the keys and needs no value to decide.
|
||||
// - key `let-through` in bucket `mesh-controller_checks-running` (broker.CheckRunningBucket), value a LetThrough as JSON:
|
||||
// created by the holder that takes a check which waited LetThroughAfter, refreshed at each heartbeat of that check, deleted
|
||||
// when it ends. At most one such check runs at a time on the build seat, because the create fails while the key is there.
|
||||
|
||||
// LetThroughAfter is how long a check waits, while a delivery builds, before it is let through (ADR 0483 clause 3).
|
||||
const LetThroughAfter = 30 * time.Minute
|
||||
|
||||
// LetThroughKey is the key of the one check let through, in the bucket of the checks running. It has no dot, so it is no
|
||||
// pull request's key (`<owner>.<repo>.<number>`) and no group's (`group.<name>`).
|
||||
const LetThroughKey = "let-through"
|
||||
|
||||
// PlanBuilding is the value of a plan's key in the bucket of the plans building.
|
||||
type PlanBuilding struct {
|
||||
Plan string `json:"plan"`
|
||||
Since time.Time `json:"since"`
|
||||
Seen time.Time `json:"seen"`
|
||||
}
|
||||
|
||||
// LetThrough is the value of the let-through key: the builder that runs the check and when it took it.
|
||||
type LetThrough struct {
|
||||
Builder string `json:"builder"`
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// checkGate is what a taking loop asks of the bus to hold checks while a delivery builds; a variable of the machine so a test
|
||||
// hands out against fakes.
|
||||
type checkGate interface {
|
||||
// Building is the ids of the plans in their build phase: the keys of the bucket. An error is a bucket that cannot be read.
|
||||
Building(ctx context.Context) ([]string, error)
|
||||
// OldestWaiting is when the first waiting ask of a check queue was asked, read without taking it.
|
||||
OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error)
|
||||
// TakeLetThrough creates the let-through key for a builder: false when it is there (another builder runs a check let through).
|
||||
TakeLetThrough(ctx context.Context, builder string) (bool, error)
|
||||
// KeepLetThrough puts the key again, and DropLetThrough deletes it.
|
||||
KeepLetThrough(ctx context.Context, builder string) error
|
||||
DropLetThrough(ctx context.Context) error
|
||||
}
|
||||
|
||||
// busGate is the gate over a connection.
|
||||
type busGate struct{ conn *nats.Conn }
|
||||
|
||||
func (g busGate) Building(ctx context.Context) ([]string, error) {
|
||||
return BuildingPlans(ctx, g.conn)
|
||||
}
|
||||
|
||||
// BuildingPlans is the ids of the delivery plans in their build phase, from the controller's bucket. A reader that is not the
|
||||
// build seat's (the checker, ADR 0484 clause 12) calls this. A bucket that cannot be read is an error; none is none building.
|
||||
func BuildingPlans(ctx context.Context, conn *nats.Conn) ([]string, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the plans building: %w", err)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// OldestWaiting is when the check a fetch of this queue would return was asked, read without taking anything.
|
||||
//
|
||||
// **It sees redeliveries** (review of issue 1688). A work queue keeps only what is not acknowledged: what the worker handed out
|
||||
// (sequence up to its delivered mark) is running or waiting to be handed out again, and a fetch returns those before anything
|
||||
// new. An ask there that is not said running (a key of the checks running names its build) is such a redelivery. If one is
|
||||
// younger than the bound, the queue's next fetch would return it, and a young check is never let through: the queue is not a
|
||||
// candidate, and nothing is fetched, so nothing is handed back and no delivery is spent. Otherwise the candidate is the oldest
|
||||
// of the redeliveries and the first ask after the delivered mark.
|
||||
func (g busGate) OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error) {
|
||||
info, err := jsOf(g.conn).ConsumerInfo(q.worker.Stream, q.worker.Name, nats.Context(ctx))
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("the %s worker: %w", q.kind, err)
|
||||
}
|
||||
var oldest time.Time
|
||||
note := func(at time.Time) {
|
||||
if oldest.IsZero() || at.Before(oldest) {
|
||||
oldest = at
|
||||
}
|
||||
}
|
||||
for seq := uint64(1); seq <= info.Delivered.Stream; {
|
||||
data, stored, got, err := g.directGet(ctx, q, seq)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
if got == 0 || got > info.Delivered.Stream {
|
||||
break
|
||||
}
|
||||
seq = got + 1
|
||||
var r BuildRequest
|
||||
if json.Unmarshal(data, &r) != nil {
|
||||
continue
|
||||
}
|
||||
running, ok, err := ReadRunning(ctx, g.conn, r)
|
||||
if err != nil {
|
||||
return time.Time{}, false, fmt.Errorf("the checks running: %w", err)
|
||||
}
|
||||
if ok && running.Build == r.ID {
|
||||
continue // running now
|
||||
}
|
||||
at := askedAtOf(data, stored)
|
||||
if time.Since(at) < LetThroughAfter {
|
||||
return time.Time{}, false, nil // a young redelivery is next in line
|
||||
}
|
||||
note(at)
|
||||
}
|
||||
data, stored, got, err := g.directGet(ctx, q, info.Delivered.Stream+1)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
if got != 0 {
|
||||
note(askedAtOf(data, stored))
|
||||
}
|
||||
return oldest, !oldest.IsZero(), nil
|
||||
}
|
||||
|
||||
// directGet reads the first ask at or after a sequence of a check queue's stream, without taking it: its body, the time the
|
||||
// stream stored it, and its sequence (0 when there is none).
|
||||
func (g busGate) directGet(ctx context.Context, q *takeQueue, seq uint64) ([]byte, string, uint64, error) {
|
||||
body, _ := json.Marshal(map[string]any{"seq": seq, "next_by_subj": q.worker.Filters[0]})
|
||||
reply, err := g.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+q.worker.Stream, body)
|
||||
if err != nil {
|
||||
return nil, "", 0, fmt.Errorf("the asks of the %s queue: %w", q.kind, err)
|
||||
}
|
||||
if st := reply.Header.Get("Status"); st == "404" || st == "408" {
|
||||
return nil, "", 0, nil
|
||||
}
|
||||
got, _ := strconv.ParseUint(reply.Header.Get("Nats-Sequence"), 10, 64)
|
||||
return reply.Data, reply.Header.Get("Nats-Time-Stamp"), got, nil
|
||||
}
|
||||
|
||||
// askedAtOf is when a request was asked: its id says (NewBuildID), the stream's own time otherwise, and a request that says
|
||||
// neither is taken as asked now, so it is never let through on a guess.
|
||||
func askedAtOf(data []byte, stored string) time.Time {
|
||||
var r BuildRequest
|
||||
if json.Unmarshal(data, &r) == nil {
|
||||
if at, ok := BuildAskedAt(r.ID); ok {
|
||||
return at
|
||||
}
|
||||
}
|
||||
if at, err := time.Parse(time.RFC3339Nano, stored); err == nil {
|
||||
return at
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func jsOf(conn *nats.Conn) nats.JetStreamContext {
|
||||
js, _ := conn.JetStream()
|
||||
return js
|
||||
}
|
||||
|
||||
func (g busGate) TakeLetThrough(ctx context.Context, builder string) (bool, error) {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
|
||||
if _, err := kv.Create(ctx, LetThroughKey, body); err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (g busGate) KeepLetThrough(ctx context.Context, builder string) error {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
|
||||
_, err = kv.Put(ctx, LetThroughKey, body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (g busGate) DropLetThrough(ctx context.Context) error {
|
||||
kv, err := runningBucket(ctx, g.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := kv.Delete(ctx, LetThroughKey); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckHold is what a reader of the queue (the controller's own verb) says of why checks wait: the plans building and the check
|
||||
// let through now, if one.
|
||||
type CheckHold struct {
|
||||
Plans []string
|
||||
LetThrough *LetThrough
|
||||
}
|
||||
|
||||
// ReadCheckHold reads the plans building and the let-through key. A bucket that cannot be read is an error the caller says.
|
||||
func ReadCheckHold(ctx context.Context, conn *nats.Conn) (CheckHold, error) {
|
||||
var h CheckHold
|
||||
plans, err := BuildingPlans(ctx, conn)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
h.Plans = plans
|
||||
kv, err := runningBucket(ctx, conn)
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
e, err := kv.Get(ctx, LetThroughKey)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return h, nil
|
||||
}
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
var l LetThrough
|
||||
if json.Unmarshal(e.Value(), &l) == nil {
|
||||
h.LetThrough = &l
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// gateOf is the machine's gate: the one a test set, or the bus's.
|
||||
func (m *natsMachine) gateOf() checkGate {
|
||||
if m.gate != nil {
|
||||
return m.gate
|
||||
}
|
||||
return busGate{conn: m.js.Conn()}
|
||||
}
|
||||
|
||||
// sayOnce says a fault of the hold the first time it happens and again once it has recovered and happened anew, so a bucket
|
||||
// that cannot be read is in the log once and not every second.
|
||||
func (m *natsMachine) sayOnce(kind string, err error) {
|
||||
if m.said == nil {
|
||||
m.said = map[string]bool{}
|
||||
}
|
||||
if err == nil {
|
||||
delete(m.said, kind)
|
||||
return
|
||||
}
|
||||
if !m.said[kind] {
|
||||
m.said[kind] = true
|
||||
fmt.Fprintf(os.Stderr, "%s; read as empty, so no check is held for it: %v\n", kind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// building says whether a delivery builds: any key in the bucket of the plans building. **A bucket that cannot be read is said
|
||||
// and read as empty** (ADR 0483 clause 2): a check wrongly held costs a pull request its turn, a fault that held every check
|
||||
// would stop the mesh checking.
|
||||
func (m *natsMachine) building(ctx context.Context) []string {
|
||||
plans, err := m.gateOf().Building(ctx)
|
||||
m.sayOnce("the bucket of the plans building cannot be read", err)
|
||||
if err != nil {
|
||||
// empty-on-error: ADR 0483 clause 2 — a bucket that cannot be read is said (sayOnce) and read as empty, because a check wrongly held costs a pull request its turn and a fault that held every check would stop the mesh checking.
|
||||
return nil
|
||||
}
|
||||
return plans
|
||||
}
|
||||
|
||||
// letOneThrough takes the oldest waiting check of the two check queues once it waited LetThroughAfter, by the one let-through
|
||||
// key: nil when none waited that long, when another holder runs the check let through, or when the check was taken by
|
||||
// another holder in the moment. The caller owns the key while it returns a message and releases it with dropLetThrough.
|
||||
func (m *natsMachine) letOneThrough(ctx context.Context) (*nats.Msg, *takeQueue) {
|
||||
gate := m.gateOf()
|
||||
var oldest *takeQueue
|
||||
var oldestAt time.Time
|
||||
for _, q := range m.subs[1:] {
|
||||
if !q.bound {
|
||||
continue
|
||||
}
|
||||
at, found, err := gate.OldestWaiting(ctx, q)
|
||||
m.sayOnce("the first waiting check of the "+q.kind+" queue cannot be read", err)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if oldest == nil || at.Before(oldestAt) {
|
||||
oldest, oldestAt = q, at
|
||||
}
|
||||
}
|
||||
if oldest == nil || time.Since(oldestAt) < m.letThroughAfter() {
|
||||
return nil, nil
|
||||
}
|
||||
created, err := gate.TakeLetThrough(ctx, m.on)
|
||||
m.sayOnce("the let-through key cannot be taken", err)
|
||||
if err != nil || !created {
|
||||
return nil, nil
|
||||
}
|
||||
trying, end := context.WithTimeout(ctx, checkTakeWait)
|
||||
defer end()
|
||||
fetched, err := oldest.pull.Fetch(1, nats.Context(trying))
|
||||
if len(fetched) == 0 {
|
||||
if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.Canceled) {
|
||||
m.sayOnce("the "+oldest.kind+" check queue cannot be taken from", err)
|
||||
}
|
||||
m.dropLetThrough(ctx)
|
||||
return nil, nil
|
||||
}
|
||||
// Another holder took the one that was oldest in the moment between the read and the fetch: what came is a younger check,
|
||||
// handed back and the key released, never run during a delivery.
|
||||
if askedAtOf(fetched[0].Data, "").Add(m.letThroughAfter()).After(time.Now()) {
|
||||
_ = fetched[0].NakWithDelay(time.Minute) // a delay keeps it off the queue of redeliveries a fetch reads first
|
||||
m.dropLetThrough(ctx)
|
||||
return nil, nil
|
||||
}
|
||||
m.letKey = true
|
||||
return fetched[0], oldest
|
||||
}
|
||||
|
||||
func (m *natsMachine) letThroughAfter() time.Duration {
|
||||
if m.opts.LetThroughAfter > 0 {
|
||||
return m.opts.LetThroughAfter
|
||||
}
|
||||
return LetThroughAfter
|
||||
}
|
||||
|
||||
// dropLetThrough deletes the let-through key this holder created.
|
||||
func (m *natsMachine) dropLetThrough(ctx context.Context) {
|
||||
m.letKey = false
|
||||
if err := m.gateOf().DropLetThrough(context.WithoutCancel(ctx)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not take the let-through key away; it ages out: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// keepLetThrough refreshes the key at every heartbeat while the check it was created for runs, and returns the end of that: it
|
||||
// stops the refreshing and deletes the key.
|
||||
func (m *natsMachine) keepLetThrough(ctx context.Context, every time.Duration) (end func()) {
|
||||
done := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
defer close(finished)
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-tick.C:
|
||||
putting, stop := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
if err := m.gateOf().KeepLetThrough(putting, m.on); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not keep the let-through key: %v\n", err)
|
||||
}
|
||||
stop()
|
||||
}
|
||||
}
|
||||
}()
|
||||
return func() {
|
||||
close(done)
|
||||
<-finished
|
||||
m.dropLetThrough(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// whyHeld words the plans a check is held for, for the log.
|
||||
func whyHeld(plans []string) string { return strings.Join(plans, ", ") }
|
||||
@@ -0,0 +1,307 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1687 and 1688): the order the taking loop hands out in, against
|
||||
// fake queues and a fake bus.
|
||||
|
||||
// fakeQueue is a queue the loop fetches from: the asks waiting, handed out in order, and how often it was fetched from.
|
||||
type fakeQueue struct {
|
||||
mu sync.Mutex
|
||||
waiting []*nats.Msg
|
||||
fetched int
|
||||
}
|
||||
|
||||
func (f *fakeQueue) Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.fetched++
|
||||
if len(f.waiting) == 0 {
|
||||
return nil, nats.ErrTimeout
|
||||
}
|
||||
m := f.waiting[0]
|
||||
f.waiting = f.waiting[1:]
|
||||
return []*nats.Msg{m}, nil
|
||||
}
|
||||
|
||||
func (f *fakeQueue) fetches() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.fetched
|
||||
}
|
||||
|
||||
// askedAgo is a request asked this long ago, as the id says, for a check of this pull request.
|
||||
func askedAgo(ago time.Duration, number int, ready bool) *nats.Msg {
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano())
|
||||
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "r", Check: &CheckRequest{Owner: "o", Repo: "r", Number: number, Ready: ready}})
|
||||
return &nats.Msg{Data: body}
|
||||
}
|
||||
|
||||
func buildAsk() *nats.Msg {
|
||||
body, _ := json.Marshal(BuildRequest{ID: fmt.Sprintf("build-%d", time.Now().UnixNano()), Repository: "r"})
|
||||
return &nats.Msg{Data: body}
|
||||
}
|
||||
|
||||
// fakeGate is the bus as the loop sees it for the hold.
|
||||
type fakeGate struct {
|
||||
mu sync.Mutex
|
||||
plans []string
|
||||
plansErr error
|
||||
letKey bool
|
||||
taken int
|
||||
kept int
|
||||
dropped int
|
||||
oldestFrom map[*takeQueue]*fakeQueue
|
||||
}
|
||||
|
||||
func (g *fakeGate) Building(context.Context) ([]string, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.plans, g.plansErr
|
||||
}
|
||||
|
||||
// OldestWaiting reads the first waiting ask of the fake queue without taking it.
|
||||
func (g *fakeGate) OldestWaiting(_ context.Context, q *takeQueue) (time.Time, bool, error) {
|
||||
f := g.oldestFrom[q]
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if len(f.waiting) == 0 {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
return askedAtOf(f.waiting[0].Data, ""), true, nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) TakeLetThrough(context.Context, string) (bool, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.letKey {
|
||||
return false, nil
|
||||
}
|
||||
g.letKey = true
|
||||
g.taken++
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) KeepLetThrough(context.Context, string) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.kept++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) DropLetThrough(context.Context) error {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.letKey = false
|
||||
g.dropped++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *fakeGate) setPlans(plans ...string) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.plans = plans
|
||||
}
|
||||
|
||||
// aMachine is a loop on the build seat with a delivery queue and the two check queues, all fake.
|
||||
func aMachine() (m *natsMachine, builds, ready, other *fakeQueue, gate *fakeGate) {
|
||||
builds, ready, other = &fakeQueue{}, &fakeQueue{}, &fakeQueue{}
|
||||
subs := []*takeQueue{
|
||||
{kind: queueBuilds, pull: builds, bound: true},
|
||||
{kind: queueReady, pull: ready, bound: true},
|
||||
{kind: queueOther, pull: other, bound: true},
|
||||
}
|
||||
gate = &fakeGate{oldestFrom: map[*takeQueue]*fakeQueue{subs[1]: ready, subs[2]: other}}
|
||||
m = &natsMachine{on: "ace", seat: broker.TheBuildSeat, subs: subs, gate: gate}
|
||||
return m, builds, ready, other, gate
|
||||
}
|
||||
|
||||
func nextAsk(t *testing.T, m *natsMachine) (*nats.Msg, *takeQueue) {
|
||||
t.Helper()
|
||||
msg, q, err := m.next(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return msg, q
|
||||
}
|
||||
|
||||
// 1652/AC6, 1687: a waiting check is not taken while a plan key is there, also when the delivery's queue is empty for the moment
|
||||
// (between two tiers).
|
||||
func TestNoCheckIsTakenWhileAPlanBuilds(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(5*time.Minute, 2, false)}
|
||||
// The delivery's queue is empty: between two tiers.
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatalf("a check was taken while plan-1 builds: %s", msg.Data)
|
||||
}
|
||||
if ready.fetches() != 0 || other.fetches() != 0 {
|
||||
t.Fatalf("the check queues were fetched from while plan-1 builds: ready %d, other %d", ready.fetches(), other.fetches())
|
||||
}
|
||||
if builds.fetches() == 0 {
|
||||
t.Fatal("the delivery's queue was not looked at")
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery build is still taken first, and the check waits behind it.
|
||||
func TestADeliveryBuildIsTakenWhileAPlanBuilds(t *testing.T) {
|
||||
m, builds, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueBuilds {
|
||||
t.Fatalf("the delivery build was not taken first: %v %v", msg, q)
|
||||
}
|
||||
if ready.fetches() != 0 {
|
||||
t.Fatal("a check queue was fetched from")
|
||||
}
|
||||
}
|
||||
|
||||
// It is taken once the key is gone (deleted at the phase's end, or expired).
|
||||
func TestACheckIsTakenOnceTheKeyIsGone(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatal("taken while the key is there")
|
||||
}
|
||||
gate.setPlans()
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueReady {
|
||||
t.Fatalf("the check was not taken once the key was gone: %v", q)
|
||||
}
|
||||
if m.letKey {
|
||||
t.Fatal("a check taken without a delivery building took the let-through key")
|
||||
}
|
||||
}
|
||||
|
||||
// 1687: a bucket that cannot be read is said and read as empty, so the checks are taken.
|
||||
func TestAnUnreadableBucketHoldsNoCheck(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.plansErr = errors.New("permissions violation")
|
||||
gate.plans = []string{"plan-1"}
|
||||
ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueReady {
|
||||
t.Fatal("a check was held because the bucket could not be read")
|
||||
}
|
||||
if !m.said["the bucket of the plans building cannot be read"] {
|
||||
t.Fatalf("the fault was not said: %v", m.said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: a check asked 30 minutes ago is let through while a key is there; the oldest across both queues by ask time, ahead of a
|
||||
// waiting delivery build; a younger one is never taken before it.
|
||||
func TestTheOldestCheckPastTheBoundGoesFirstAcrossBothQueues(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
ready.waiting = []*nats.Msg{askedAgo(31*time.Minute, 1, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(50*time.Minute, 2, false)}
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil || q.kind != queueOther || !strings.Contains(string(msg.Data), `"number":2`) {
|
||||
t.Fatalf("the oldest check (50 minutes, other queue) was not let through first: %v", q)
|
||||
}
|
||||
if !m.letKey || gate.taken != 1 {
|
||||
t.Fatalf("the let-through key was not created for it: held %v, created %d", m.letKey, gate.taken)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: several checks past the bound go one at a time, oldest first: while the key is there every other holder takes builds.
|
||||
func TestChecksPastTheBoundGoOneAtATimeOldestFirst(t *testing.T) {
|
||||
m, builds, ready, other, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(33*time.Minute, 1, true), askedAgo(31*time.Minute, 3, true)}
|
||||
other.waiting = []*nats.Msg{askedAgo(40*time.Minute, 2, false)}
|
||||
var order []string
|
||||
for range 3 {
|
||||
msg, q := nextAsk(t, m)
|
||||
if msg == nil {
|
||||
t.Fatalf("no check let through after %v", order)
|
||||
}
|
||||
var r BuildRequest
|
||||
_ = json.Unmarshal(msg.Data, &r)
|
||||
order = append(order, fmt.Sprint(r.Check.Number, q.kind))
|
||||
// While it runs another holder (the same fake gate, key held) takes builds only.
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
other2 := &natsMachine{on: "shanks", seat: broker.TheBuildSeat, subs: m.subs, gate: gate}
|
||||
if next, nq := nextAsk(t, other2); next == nil || nq.kind != queueBuilds {
|
||||
t.Fatalf("a second holder did not take a delivery build while the let-through key is held: %v", nq)
|
||||
}
|
||||
m.dropLetThrough(context.Background())
|
||||
}
|
||||
if want := []string{"2other", "1ready", "3ready"}; strings.Join(order, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("checks went %v, want %v (oldest first, one at a time)", order, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: a check younger than the bound is never let through, and no key is created for it.
|
||||
func TestACheckYoungerThanTheBoundIsHeld(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(29*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatal("a check of 29 minutes was let through")
|
||||
}
|
||||
if gate.taken != 0 {
|
||||
t.Fatal("a let-through key was created for a young check")
|
||||
}
|
||||
if ready.fetches() != 0 {
|
||||
t.Fatal("the age was read by a fetch; it must be read without taking")
|
||||
}
|
||||
}
|
||||
|
||||
// 1688: the holder that lost the race for the key, or the check, takes builds only; and a younger check that arrives in the
|
||||
// moment between the read and the fetch is handed back with the key released.
|
||||
func TestAYoungerCheckFetchedInTheRaceIsHandedBackAndTheKeyReleased(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
// The read sees the old check; before the fetch another holder takes it and a young one is first.
|
||||
gate.oldestFrom[m.subs[1]] = &fakeQueue{waiting: []*nats.Msg{askedAgo(40*time.Minute, 1, true)}}
|
||||
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
|
||||
if msg, _ := nextAsk(t, m); msg != nil {
|
||||
t.Fatalf("a young check was run during a delivery: %s", msg.Data)
|
||||
}
|
||||
if gate.letKey || m.letKey {
|
||||
t.Fatal("the let-through key was left held")
|
||||
}
|
||||
}
|
||||
|
||||
// The key of a check let through is refreshed while it runs and deleted when it ends.
|
||||
func TestTheLetThroughKeyLivesAsLongAsTheCheckRuns(t *testing.T) {
|
||||
m, _, ready, _, gate := aMachine()
|
||||
gate.setPlans("plan-1")
|
||||
ready.waiting = []*nats.Msg{askedAgo(40*time.Minute, 1, true)}
|
||||
if msg, _ := nextAsk(t, m); msg == nil {
|
||||
t.Fatal("not let through")
|
||||
}
|
||||
end := m.keepLetThrough(context.Background(), 20*time.Millisecond)
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
gate.mu.Lock()
|
||||
held, kept := gate.letKey, gate.kept
|
||||
gate.mu.Unlock()
|
||||
if !held || kept == 0 {
|
||||
t.Errorf("while the check ran the key was held %v and refreshed %d times", held, kept)
|
||||
}
|
||||
end()
|
||||
gate.mu.Lock()
|
||||
defer gate.mu.Unlock()
|
||||
if gate.letKey || m.letKey {
|
||||
t.Error("the key was not deleted when the check ended")
|
||||
}
|
||||
}
|
||||
@@ -88,6 +88,40 @@ type Queue struct {
|
||||
AckPending int `json:"ack-pending"`
|
||||
MaxDeliver int `json:"max-deliver"`
|
||||
Asks []QueuedAsk `json:"asks"`
|
||||
// Hold is why the checks wait: the delivery plans building and the check let through now (ADR 0483, issues 1687 and 1688);
|
||||
// HoldFault says the bucket could not be read, in which case Hold is empty.
|
||||
Hold CheckHold `json:"hold"`
|
||||
HoldFault string `json:"hold-fault,omitempty"`
|
||||
}
|
||||
|
||||
// LetThroughOf says which ask is let through under ADR 0483 clause 3: the check that waited LetThroughAfter or longer and is next
|
||||
// (the oldest waiting check across the two check queues, while a delivery builds), and the check in flight that was let through
|
||||
// (the let-through key names the machine that runs it). Empty when none.
|
||||
func (q Queue) LetThroughOf(now time.Time) (next, running string) {
|
||||
if len(q.Hold.Plans) == 0 && q.Hold.LetThrough == nil {
|
||||
return "", ""
|
||||
}
|
||||
var oldest *QueuedAsk
|
||||
for i := range q.Asks {
|
||||
a := &q.Asks[i]
|
||||
if a.State != AskWaiting || a.Request.Check == nil || a.AskedAt.IsZero() {
|
||||
continue
|
||||
}
|
||||
if oldest == nil || a.AskedAt.Before(oldest.AskedAt) {
|
||||
oldest = a
|
||||
}
|
||||
}
|
||||
if oldest != nil && len(q.Hold.Plans) > 0 && now.Sub(oldest.AskedAt) >= LetThroughAfter {
|
||||
next = oldest.ID
|
||||
}
|
||||
if q.Hold.LetThrough != nil {
|
||||
for _, a := range q.Asks {
|
||||
if a.State == AskInFlight && a.Request.Check != nil && a.On == q.Hold.LetThrough.Builder {
|
||||
running = a.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
return next, running
|
||||
}
|
||||
|
||||
// Of is the asks in one state, in queue order.
|
||||
@@ -319,6 +353,16 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
|
||||
for _, r := range reads {
|
||||
q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
|
||||
}
|
||||
// And why the checks wait, for the build seat that has them (ADR 0483 clause 5). A bucket that cannot be read is said by the
|
||||
// queue, which then names no plan.
|
||||
if seat == broker.TheBuildSeat {
|
||||
hold, err := ReadCheckHold(ctx, js.Conn())
|
||||
if err != nil {
|
||||
q.HoldFault = err.Error()
|
||||
} else {
|
||||
q.Hold = hold
|
||||
}
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// Issue 1747: a builder never holds a message it does not start. A message the server sends after a fetch gave up lands in that
|
||||
// queue's subscription; while an ask runs, the queues it does not come from are let go of, so such a message is not held
|
||||
// unheartbeated for the whole run, and the queues are bound again before the next fetch.
|
||||
|
||||
func TestARunningAskLetsGoOfTheQueuesItDidNotComeFrom(t *testing.T) {
|
||||
m, builds, _, other, _ := aMachine()
|
||||
// A delivery build reached the builds queue's subscription after its fetch gave up: it sits in the buffer.
|
||||
builds.waiting = []*nats.Msg{buildAsk()}
|
||||
took := m.subs[1] // the running ask came from the ready queue
|
||||
|
||||
m.quiesce(took)
|
||||
|
||||
if !took.bound || took.pull == nil {
|
||||
t.Error("the queue the running ask came from was let go of; its acknowledgements need it")
|
||||
}
|
||||
for _, q := range []*takeQueue{m.subs[0], m.subs[2]} {
|
||||
if q.bound || q.pull != nil || q.sub != nil {
|
||||
t.Errorf("the %s queue is still held while an ask runs", q.kind)
|
||||
}
|
||||
}
|
||||
// Nothing is fetched from a queue that was let go of: a message in it cannot be taken unheartbeated.
|
||||
if msg, _, _ := m.next(t.Context()); msg != nil {
|
||||
t.Fatalf("a message was taken from a queue that was let go of: %s", msg.Data)
|
||||
}
|
||||
if builds.fetches() != 0 || other.fetches() != 0 {
|
||||
t.Errorf("fetched from a queue that was let go of: builds %d, other %d", builds.fetches(), other.fetches())
|
||||
}
|
||||
}
|
||||
|
||||
// Once the ask is settled the queue it came from is let go of too: a second message the server delivered into its subscription
|
||||
// on a pull request still open is not run after the ask ends (review of 1747).
|
||||
func TestTheQueueTheAskCameFromIsLetGoOfWhenTheAskEnds(t *testing.T) {
|
||||
m, _, ready, _, _ := aMachine()
|
||||
ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
|
||||
m.quiesce(nil)
|
||||
for _, q := range m.subs {
|
||||
if q.bound || q.pull != nil {
|
||||
t.Errorf("the %s queue is still held after the ask ended", q.kind)
|
||||
}
|
||||
}
|
||||
m.subs[0].bound, m.subs[0].pull = true, &fakeQueue{}
|
||||
if msg, _, _ := m.next(t.Context()); msg != nil || ready.fetches() != 0 {
|
||||
t.Fatal("a message held in the kept queue's subscription was taken after the ask ended")
|
||||
}
|
||||
}
|
||||
Vendored
+1
-1
@@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself.
|
||||
|
||||
mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile
|
||||
mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock
|
||||
mesh-sdk d4deecdb380b69358aae2fdaedcfcc06e1571d42 conformance/events/module-event.json
|
||||
mesh-sdk 725590e06e75c67b96ce58d015b3056c2d261249 conformance/events/module-event.json
|
||||
|
||||
To move them, from this repository's root, with the two repositories checked out beside it:
|
||||
|
||||
|
||||
@@ -326,7 +326,7 @@ type File struct {
|
||||
|
||||
// Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as
|
||||
// the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes
|
||||
// (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucket, checks what it
|
||||
// (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucketFor(node), checks what it
|
||||
// received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes,
|
||||
// VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine
|
||||
// refuses a field it does not know, whole.
|
||||
|
||||
@@ -14,13 +14,38 @@ const (
|
||||
// FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer
|
||||
// stays in the declaration as it is.
|
||||
FileReferenceThreshold = 64 * 1024
|
||||
// FileBucket is the object bucket of the bus that holds the contents, each under the hex of its sha256 digest
|
||||
// (ObjectName). The controller alone writes it; a node-engine reads an object by name.
|
||||
FileBucket = "declaration-files"
|
||||
// FileBucketPrefix starts the name of a node's object bucket, FileBucketFor(node), which holds the contents of the
|
||||
// files of that node's declaration, each under the hex of its sha256 digest (ObjectName). One bucket per node,
|
||||
// so a grant names it and a literal `*` is never needed (ADR 0486 §2, changed by ADR 0488 §1 and §2). The controller alone writes it; the
|
||||
// node's engine reads an object by name.
|
||||
FileBucketPrefix = "declaration-files-"
|
||||
// FileStreamPrefix starts the name of the stream behind such a bucket, which the bus's grants name.
|
||||
FileStreamPrefix = "OBJ_"
|
||||
// FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size.
|
||||
FileReferencesCapability = "file-references"
|
||||
)
|
||||
|
||||
var nodeToken = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
|
||||
|
||||
// FileBucketFor is the name of the object bucket of a node's files, "declaration-files-<node>" (novox/hq ADR 0488
|
||||
// §4, which replaces the single bucket constant of issue 1694). The node name must be a name the bus accepts in a
|
||||
// bucket and in a subject token: letters, digits, '_' and '-', so no '.', '*', '>' or space.
|
||||
func FileBucketFor(node string) (string, error) {
|
||||
if !nodeToken.MatchString(node) {
|
||||
return "", fmt.Errorf("node %q; a node's bucket is named by letters, digits, '_' and '-' only", node)
|
||||
}
|
||||
return FileBucketPrefix + node, nil
|
||||
}
|
||||
|
||||
// FileStreamFor is the name of the stream behind a node's bucket, "OBJ_" and the bucket's name.
|
||||
func FileStreamFor(node string) (string, error) {
|
||||
b, err := FileBucketFor(node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return FileStreamPrefix + b, nil
|
||||
}
|
||||
|
||||
var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>".
|
||||
|
||||
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021144048-d4deecdb380b
|
||||
# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021153908-725590e06e75
|
||||
## explicit; go 1.26.0
|
||||
git.novox.be/novox/mesh-sdk/go/asks
|
||||
git.novox.be/novox/mesh-sdk/go/catalogue
|
||||
|
||||
Reference in New Issue
Block a user