Compare commits
108
Commits
2bcd85d2df
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5c281ad99b | ||
|
|
29dfb28f38 | ||
|
|
88b16e9be2 | ||
|
|
b564c315d0 | ||
|
|
5bd03b3984 | ||
|
|
e02fcad739 | ||
|
|
4a92ec6102 | ||
|
|
909cc25169 | ||
|
|
272d5ebd61 | ||
|
|
a6386ff23a | ||
|
|
b51105908a | ||
|
|
8b4d536a11 | ||
|
|
dc90b3f5bd | ||
|
|
0e9c4c8299 | ||
|
|
14d10051e5 | ||
|
|
c58b311ea7 | ||
|
|
7a169eae76 | ||
|
|
a07621d761 | ||
|
|
d791052f77 | ||
|
|
22935b9202 | ||
|
|
efe745f2be | ||
|
|
ff912d0972 | ||
|
|
4c9be8efc8 | ||
|
|
62c05c681c | ||
|
|
bfb728730f | ||
|
|
159db37618 | ||
|
|
b7c3868980 | ||
|
|
ddae6729c1 | ||
|
|
a4426190b0 | ||
|
|
2154ea8e3c | ||
|
|
812c528479 | ||
|
|
8b32e04e2f | ||
|
|
79f44386d2 | ||
|
|
6fe9efa712 | ||
|
|
d96542cd25 | ||
|
|
80dc6980b7 | ||
|
|
80df1b6361 | ||
|
|
816cf96459 | ||
|
|
3f05df5d1a | ||
|
|
e04846e9fa | ||
|
|
020c6a99b0 | ||
|
|
5a9766d4a4 | ||
|
|
f46a08772f | ||
|
|
1c2adfdca9 | ||
|
|
0fbb5ef359 | ||
|
|
8f84731896 | ||
|
|
fbaa463a38 | ||
|
|
2dafbe60e9 | ||
|
|
8d58afea6c | ||
|
|
b611f7375a | ||
|
|
253bbcea2d | ||
|
|
2c6382346f | ||
|
|
b8cafccaf6 | ||
|
|
82d212a2a1 | ||
|
|
df8ebf6a77 | ||
|
|
dc414fbfef | ||
|
|
fd891688ac | ||
|
|
9d8a5a7ede | ||
|
|
4e9e5c27d4 | ||
|
|
7c8a2e03dd | ||
|
|
89f52a6504 | ||
|
|
fb2f5a2962 | ||
|
|
2ecfbb3a29 | ||
|
|
9ad8715871 | ||
|
|
e1a55f099b | ||
|
|
ccc7ba334c | ||
|
|
65d5ed3367 | ||
|
|
08a261ac1a | ||
|
|
23226384e5 | ||
|
|
350f8ebdfe | ||
|
|
09d2cc5056 | ||
|
|
47e64f0284 | ||
|
|
6927b02d4d | ||
|
|
fda4859a95 | ||
|
|
a81717e53c | ||
|
|
d01839c29c | ||
|
|
f67d15536b | ||
|
|
7cd4d18bd5 | ||
|
|
fbf14b68db | ||
|
|
79ac62ffbc | ||
|
|
40034be479 | ||
|
|
045f25bd13 | ||
|
|
ff3cc47027 | ||
|
|
31a2bcdb3c | ||
|
|
2df47d7b1c | ||
|
|
666d1b682e | ||
|
|
e40b1e70db | ||
|
|
6b116586fc | ||
|
|
bf20d9d8cf | ||
|
|
81e335e42d | ||
|
|
2407bfb9ff | ||
|
|
850f4b4d26 | ||
|
|
e7673e3bd3 | ||
|
|
3e39ccf958 | ||
|
|
94bb74966f | ||
|
|
bfb271a91b | ||
|
|
9ff47f6aaa | ||
|
|
ca5f74eadb | ||
|
|
c3db00ceac | ||
|
|
8e02c4726e | ||
|
|
cc5c9ae888 | ||
|
|
c7e31640cf | ||
|
|
c75d726205 | ||
|
|
a4cd5ba375 | ||
|
|
62802cd966 | ||
|
|
e9a885cda9 | ||
|
|
2814262ac0 | ||
|
|
0a536d7b6d |
@@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
"github.com/novox/mesh-controller/internal/lease"
|
"github.com/novox/mesh-controller/internal/lease"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
@@ -72,6 +73,14 @@ type actor struct {
|
|||||||
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
|
||||||
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
|
||||||
renewEvery, poll time.Duration
|
renewEvery, poll time.Duration
|
||||||
|
// releasing is held for the whole of a release, so a second caller — the stop signal's and serve's own
|
||||||
|
// deferred one — waits until the lease is given back and its epoch recorded, never past it to the store's
|
||||||
|
// close (novox/hq issue 906).
|
||||||
|
releasing sync.Mutex
|
||||||
|
// keeper and inv are where a takeover is said (sayTakeover): set once the serving controller's condition
|
||||||
|
// store opens; nil before, and in a command.
|
||||||
|
keeper *conditions.Keeper
|
||||||
|
inv *inventory.Inventory
|
||||||
}
|
}
|
||||||
|
|
||||||
// theLease is this process's standing to the lease.
|
// theLease is this process's standing to the lease.
|
||||||
@@ -174,8 +183,11 @@ func (a *actor) forACommand(ctx context.Context) (uint64, error) {
|
|||||||
return holder.Epoch, nil
|
return holder.Epoch, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// release gives back what this process holds, at its end.
|
// release gives back what this process holds, at its end — or at the stop signal, before the rest of the
|
||||||
|
// process is taken down (serveUnderTheLease). A second call waits for the first to finish.
|
||||||
func (a *actor) release() {
|
func (a *actor) release() {
|
||||||
|
a.releasing.Lock()
|
||||||
|
defer a.releasing.Unlock()
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
closing := a.close
|
closing := a.close
|
||||||
a.close = nil
|
a.close = nil
|
||||||
@@ -266,20 +278,40 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
|
|||||||
}
|
}
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
a.close = func() {
|
a.close = func() {
|
||||||
if held, err := l.Epoch(); err == nil {
|
// Given back first, then recorded: an epoch is recorded released only when its key is gone, and
|
||||||
|
// whatever the gate says by then — a shutdown longer than half the key's age closed it, and the epoch
|
||||||
|
// was left open for the next holder to find "expired" (novox/hq issue 906, 2026-10-26).
|
||||||
|
if given := l.Release(context.Background()); given != 0 {
|
||||||
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
ending, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
if err := inv.EndEpoch(ending, held, inventory.EpochReleased); err != nil {
|
if err := inv.EndEpoch(ending, given, inventory.EpochReleased); err != nil {
|
||||||
say("how epoch %d ended could not be recorded: %v", held, err)
|
say("how epoch %d ended could not be recorded: %v", given, err)
|
||||||
}
|
}
|
||||||
cancel()
|
cancel()
|
||||||
}
|
}
|
||||||
l.Release(context.Background())
|
|
||||||
js.Close()
|
js.Close()
|
||||||
}
|
}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
// **The stop signal gives the lease back first** (novox/hq issue 906): the moment ctx ends, before the rest
|
||||||
|
// of the process is taken down, the gate closes and the key goes, so the next controller takes the lease
|
||||||
|
// at once and the epoch is recorded released. Waiting for the process's own end left the key unrenewed for
|
||||||
|
// as long as the shutdown took — ten seconds on 2026-10-26, past the gate's half-age.
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
if a.holds(l) {
|
||||||
|
say("stopping: the controller lease is given back first")
|
||||||
|
}
|
||||||
|
a.release()
|
||||||
|
}()
|
||||||
return gone, nil
|
return gone, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// holds is whether this process holds l now.
|
||||||
|
func (a *actor) holds(l *lease.Lease) bool {
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
return a.held == l
|
||||||
|
}
|
||||||
|
|
||||||
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
// took is the lease taken: recorded, earlier epochs nobody gave back ended as expired, kept.
|
||||||
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Inventory, epoch uint64, gone chan struct{}) {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -288,15 +320,19 @@ func (a *actor) took(ctx context.Context, l *lease.Lease, inv *inventory.Invento
|
|||||||
h := holderOf(instance)
|
h := holderOf(instance)
|
||||||
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
recording, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
expired, err := inv.TookEpoch(recording, inventory.Epoch{Epoch: epoch, Instance: h.Instance, Host: h.Host,
|
||||||
Build: h.Build, Taken: time.Now()})
|
Build: h.Build, Taken: time.Now()}, l.GivenBack())
|
||||||
cancel()
|
cancel()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
fmt.Printf("epoch %d could not be recorded as taken, so a stale refusal from it will not name it: %v\n", epoch, err)
|
||||||
}
|
}
|
||||||
|
if given := l.GivenBack(); given != 0 {
|
||||||
|
fmt.Printf("the lease was given back by the controller of epoch %d: taken at once, at epoch %d\n", given, epoch)
|
||||||
|
}
|
||||||
for _, e := range expired {
|
for _, e := range expired {
|
||||||
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
fmt.Printf("the controller of epoch %d (%s) stopped renewing the lease without giving it back: it is "+
|
||||||
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
"taken over at epoch %d\n", e.Epoch, e.Instance, epoch)
|
||||||
}
|
}
|
||||||
|
a.sayTakeover(ctx, epoch)
|
||||||
go l.Keep(ctx)
|
go l.Keep(ctx)
|
||||||
go func() {
|
go func() {
|
||||||
<-l.Lost()
|
<-l.Lost()
|
||||||
|
|||||||
@@ -292,6 +292,22 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
|
|||||||
answer += "\n " + strings.TrimSpace(line)
|
answer += "\n " + strings.TrimSpace(line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Its bus accounts on the node go with it once it no longer resolves there (a module another requires keeps running
|
||||||
|
// and keeps them), so an assignment again issues them afresh (issue 1945). The node's own provisioner account stays.
|
||||||
|
if after.err == nil {
|
||||||
|
for _, module := range modules {
|
||||||
|
if slices.Contains(after.modules, module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
forgot, err := open.inventory.ForgetBusAccountsOf(ctx, node, module, false)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
answer += fmt.Sprintf("\n its bus accounts on %s are not forgotten: %v", node, err)
|
||||||
|
case len(forgot) > 0:
|
||||||
|
answer += fmt.Sprintf("\n its bus accounts on %s forgotten: %s", node, strings.Join(forgot, ", "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
// What it leaves behind that is irreplaceable is kept and retired, never removed (novox/hq ADR 0233).
|
||||||
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
for _, line := range keptOnUnassign(ctx, open.inventory, node, modules) {
|
||||||
answer += "\n " + line
|
answer += "\n " + line
|
||||||
@@ -399,6 +415,8 @@ func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
|||||||
// purpose, never by re-assigning); and when the bus cannot be reached from here, the line names the
|
// purpose, never by re-assigning); and when the bus cannot be reached from here, the line names the
|
||||||
// verb and the push that would refuse the module until it is run — never a silent placeholder (novox/hq issue 203).
|
// verb and the push that would refuse the module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||||
|
// Mint only what no row names: the assignments' watch hears the same write and may issue at once (ADR 0543).
|
||||||
|
ctx = onlyIfAbsent(ctx)
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
shelf, err := inv.Catalogue(ctx)
|
shelf, err := inv.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -419,9 +437,9 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
|
|||||||
// A module running the provisioner client alone is minted the client's account (issue 917).
|
// A module running the provisioner client alone is minted the client's account (issue 917).
|
||||||
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
|
||||||
}
|
}
|
||||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
|
if _, sealed, err := inv.BusAccountIssued(ctx, user); err != nil {
|
||||||
return ""
|
return ""
|
||||||
} else if minted {
|
} else if sealed { // a row whose credential is not sealed to the node is not issued (issue 1945)
|
||||||
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
|
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
|
||||||
return processAccountsOnAssign(ctx, inv, m, node)
|
return processAccountsOnAssign(ctx, inv, m, node)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,12 +78,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
|||||||
|
|
||||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
|
// Issue 1126: the placement export over this fresh mesh, before anything below changes it.
|
||||||
|
placementExportIsEachNodesPlacementFromTheStore(t, open)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
raw, err := os.ReadFile("../../module.json")
|
raw, err := os.ReadFile("../../module.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
m, err := catalogue.ParseManifest(raw)
|
m, err := catalogue.Decode(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -111,6 +113,11 @@ func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T)
|
|||||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// Seeded with the credential the controller already uses, so it is sealed: an account counts as issued only once
|
||||||
|
// sealed (ADR 0543 §2, PR #467), and genesis's seed is the credential in use.
|
||||||
|
if err := open.inventory.MarkBusAccountSealed(ctx, "anchor.mesh-controller"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
@@ -27,7 +28,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// askerFrom is the serving controller's asker; nil in any other process.
|
// askerFrom is the serving controller's asker; nil in any other process.
|
||||||
var askerFrom *asker
|
var askerFrom atomic.Pointer[asker]
|
||||||
|
|
||||||
// askWithin is how long a verb a warrant chose is given to answer.
|
// askWithin is how long a verb a warrant chose is given to answer.
|
||||||
const askWithin = time.Minute
|
const askWithin = time.Minute
|
||||||
@@ -573,8 +574,8 @@ func startAsking(ctx context.Context, open *stores, server *link.Server, conn *n
|
|||||||
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
fmt.Printf("the operator's answers cannot be heard, so nothing is asked: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
askerFrom = a
|
askerFrom.Store(a)
|
||||||
go a.keep(ctx)
|
goServing(ctx, func() { a.keep(ctx) })
|
||||||
}
|
}
|
||||||
|
|
||||||
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
// servingAsker is the serving controller's asker, wired to the mesh: what it reads, publishes, performs and
|
||||||
|
|||||||
@@ -0,0 +1,489 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/chain"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An assignment that stands issues its node's bus accounts and the module's grant (novox/hq ADR 0543, issue 1945; a
|
||||||
|
// step of research 072's grants: the memberships and credentials heard from placement, not from a send or a build).
|
||||||
|
//
|
||||||
|
// ADR 0539 clause 7 issues a module's credential and grant when its build is done. A module already built and assigned
|
||||||
|
// to one more node never has a build done again, so nothing issued that node's account: on 2026-10-25 at 17:19
|
||||||
|
// mesh-build-node and mesh-build-go were assigned to ace, composed for it, and the bus's user list never named
|
||||||
|
// ace.mesh-build-node nor ace.mesh-build-go; one runtime found no credential, the other was refused by the bus, both
|
||||||
|
// restarting every 5 s. The assign act composes the assigned node alone (composedAfterAct): never the node holding the
|
||||||
|
// bus, never the grant.
|
||||||
|
//
|
||||||
|
// So the write of an assignments key is the event. When a key `<node>.<module>` of the assignments bucket comes to stand
|
||||||
|
// (a new key, or one that was ended or pending), the lease holder issues the module's accounts on that node that are not
|
||||||
|
// minted (idempotent: a minted account is never issued again, a rotation is on purpose), composes the node holding the
|
||||||
|
// bus so its user list carries them, and writes the grants. A rewrite of an assignment that already stood (the
|
||||||
|
// provisioner's plan kept in its value) does nothing. A starting controller looks at every standing assignment once,
|
||||||
|
// from one reading of the catalogue: it issues what is missing, then composes the bus's node and writes the grants
|
||||||
|
// once. The assign act issues too; both mint only an account no row names, in one statement, so exactly one of them
|
||||||
|
// mints and seals it (mintBusUser, the review of issue 1945).
|
||||||
|
|
||||||
|
// assignmentAccountSteps are what an assignment that comes to stand does, each a step a test may stand in for.
|
||||||
|
type assignmentAccountSteps struct {
|
||||||
|
// holds says this controller holds the lease: only its holder issues.
|
||||||
|
holds func(ctx context.Context) bool
|
||||||
|
// forgetStale forgets every module's accounts on a node where the module no longer resolves, and answers a line for
|
||||||
|
// each (issue 1945: rows of an unassignment before this release, marked sealed by migration 0108).
|
||||||
|
forgetStale func(ctx context.Context) ([]string, error)
|
||||||
|
// catalogue reads the catalogue once, for every assignment a look issues for.
|
||||||
|
catalogue func(ctx context.Context) (map[string]catalogue.Manifest, error)
|
||||||
|
// issue issues the module's accounts on the node that are not minted, from the catalogue read; issued says it
|
||||||
|
// minted one, line what it did.
|
||||||
|
issue func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (issued bool, line string, err error)
|
||||||
|
// busHolder is the node holding the bus, "" for none.
|
||||||
|
busHolder func(ctx context.Context) (string, error)
|
||||||
|
// compose writes the nodes' compositions; grants writes the grants issued.
|
||||||
|
compose func(ctx context.Context, nodes []string) error
|
||||||
|
grants func(ctx context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// settle issues what the standing assignment of module on node lacks. fresh says the assignment just came to stand: the
|
||||||
|
// bus's node is composed and the grants written even when every account was minted before (a module assigned again
|
||||||
|
// keeps its account, and the bus's user list left it out while it was unassigned). It answers what it did, "" for
|
||||||
|
// nothing.
|
||||||
|
func (s assignmentAccountSteps) settle(ctx context.Context, node, module string) (string, error) {
|
||||||
|
if module == controllerModule {
|
||||||
|
// The controller's own module is issued at the terminal alone: its `broker` secret is the credential it is
|
||||||
|
// using (rollout.go), and it is assigned and unassigned there only (refusedMovingTheController).
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if !s.holds(ctx) {
|
||||||
|
return "", errNotHolding
|
||||||
|
}
|
||||||
|
shelf, err := s.catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
_, line, err := s.issue(ctx, shelf, node, module)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("its bus accounts on %s are not issued: %w", node, err)
|
||||||
|
}
|
||||||
|
said, err := s.cover(ctx)
|
||||||
|
prefix := module + " on " + node + ": "
|
||||||
|
if line != "" {
|
||||||
|
prefix += line + "; "
|
||||||
|
}
|
||||||
|
return prefix + said, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// look is the start's look: the accounts every standing assignment lacks, issued one after another from one reading of
|
||||||
|
// the catalogue, then the bus's node composed and the grants written once, when it looked at any assignment, whatever
|
||||||
|
// it issued (an assignment that came to stand while no controller heard it may have its accounts and still lack its
|
||||||
|
// place in the bus's user list or its grant). It answers what it did.
|
||||||
|
func (s assignmentAccountSteps) look(ctx context.Context, standing []chain.Assignment) (string, error) {
|
||||||
|
var forgot []string
|
||||||
|
if len(standing) > 0 && s.forgetStale != nil {
|
||||||
|
// Only from a reading of the assignments that holds some: an empty or failed one would forget every account.
|
||||||
|
if !s.holds(ctx) {
|
||||||
|
return "", errNotHolding
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if forgot, err = s.forgetStale(ctx); err != nil {
|
||||||
|
return "", fmt.Errorf("the accounts of modules no longer on their nodes are not forgotten: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var looked []chain.Assignment
|
||||||
|
for _, a := range standing {
|
||||||
|
if a.Module != controllerModule {
|
||||||
|
looked = append(looked, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(looked) == 0 {
|
||||||
|
return strings.Join(forgot, "\n"), nil
|
||||||
|
}
|
||||||
|
if !s.holds(ctx) {
|
||||||
|
return "", errNotHolding
|
||||||
|
}
|
||||||
|
shelf, err := s.catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the catalogue cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
for _, a := range looked {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return "", ctx.Err()
|
||||||
|
}
|
||||||
|
if _, line, err := s.issue(ctx, shelf, a.Node, a.Module); err != nil {
|
||||||
|
return "", fmt.Errorf("%s's bus accounts on %s are not issued: %w", a.Module, a.Node, err)
|
||||||
|
} else if line != "" {
|
||||||
|
lines = append(lines, a.Module+" on "+a.Node+": "+line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
said, err := s.cover(ctx)
|
||||||
|
lines = append(append(forgot, lines...), fmt.Sprintf("%d standing assignment(s) looked at; %s", len(looked), said))
|
||||||
|
return strings.Join(lines, "\n"), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// errNotHolding is a controller that does not hold the lease: it issues nothing, and tries again until it holds it.
|
||||||
|
var errNotHolding = errors.New("this controller does not hold the lease")
|
||||||
|
|
||||||
|
// cover composes the node holding the bus and writes the grants, and says so.
|
||||||
|
func (s assignmentAccountSteps) cover(ctx context.Context) (string, error) {
|
||||||
|
holder, err := s.busHolder(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the node holding the bus cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if holder != "" {
|
||||||
|
if err := s.compose(ctx, []string{holder}); err != nil {
|
||||||
|
return "", fmt.Errorf("the bus's user list on %s is not composed: %w", holder, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := s.grants(ctx); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if holder != "" {
|
||||||
|
return "the bus's user list composed on " + holder + ", the grants written", nil
|
||||||
|
}
|
||||||
|
return "the grants written", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignmentStands says a value of the assignments bucket holds an assignment that stands: made (a module only pulled in
|
||||||
|
// by another names no time assigned), not ended and in no state (chain.Assignment.Stands).
|
||||||
|
func assignmentStands(a chain.Assignment) bool {
|
||||||
|
return !a.Assigned.IsZero() && a.Stands() && a.Node != "" && a.Module != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignmentTurns keeps which keys of the assignments bucket stood when last heard, to tell an assignment that comes to
|
||||||
|
// stand from a rewrite of one that stood.
|
||||||
|
type assignmentTurns struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
stood map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// heard records a value of key (nil for a delete or a purge) and answers whether it holds an assignment that just came
|
||||||
|
// to stand.
|
||||||
|
func (t *assignmentTurns) heard(key string, value []byte) (chain.Assignment, bool) {
|
||||||
|
var a chain.Assignment
|
||||||
|
stands := false
|
||||||
|
if value != nil && json.Unmarshal(value, &a) == nil {
|
||||||
|
stands = assignmentStands(a)
|
||||||
|
}
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if t.stood == nil {
|
||||||
|
t.stood = map[string]bool{}
|
||||||
|
}
|
||||||
|
was := t.stood[key]
|
||||||
|
t.stood[key] = stands
|
||||||
|
return a, stands && !was
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownAccountOf is the username of the account a module speaks on the bus with on node, its own and no process's: the
|
||||||
|
// module's when it reads a `broker` own secret, else its provisioner client's when it runs one; false when it has none
|
||||||
|
// (a module whose accounts are its processes' alone, or one that does not speak on the bus).
|
||||||
|
func ownAccountOf(m catalogue.Manifest, node string) (string, bool) {
|
||||||
|
if _, reads := m.OwnSecrets["broker"]; reads {
|
||||||
|
return broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username(), true
|
||||||
|
}
|
||||||
|
if runsTheProvisionerClient(m) {
|
||||||
|
return broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username(), true
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
// issueAssignmentAccounts issues module's accounts on node that are not minted: its own (issueOnTheNewBus) and its
|
||||||
|
// processes' (issueMissingProcessAccounts). A module the catalogue does not hold, or one with nowhere to deliver an
|
||||||
|
// account (mayIssue), has nothing to issue; an account minted before is kept.
|
||||||
|
func issueAssignmentAccounts(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest,
|
||||||
|
node, module string) (bool, string, error) {
|
||||||
|
// Mint only what no row names: the assign act hears its own write and may issue at once (the review of issue 1945).
|
||||||
|
ctx = onlyIfAbsent(ctx)
|
||||||
|
m, known := shelf[module]
|
||||||
|
if !known || mayIssue(m) != nil {
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
if _, own := ownAccountOf(m, node); own {
|
||||||
|
minted, err := inv.BusMinted(ctx) // sealed accounts alone: a row with no sealed credential is not issued
|
||||||
|
if err != nil {
|
||||||
|
return false, "", fmt.Errorf("the issued accounts cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if missingAccounts(m, node, minted) {
|
||||||
|
busAddress, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
// Its processes' and provisioner client's accounts too: issueOnTheNewBus issues each that is not issued.
|
||||||
|
ctx, got := withIssued(ctx)
|
||||||
|
if err := issueOnTheNewBus(ctx, inv, m, node, busAddress); err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
if issued := got.list(); len(issued) > 0 {
|
||||||
|
return true, "issued and sealed to " + node + ": " + strings.Join(issued, ", "), nil
|
||||||
|
}
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
return false, "its bus accounts on " + node + " stood issued and sealed", nil
|
||||||
|
}
|
||||||
|
line, err := issueMissingProcessAccounts(ctx, inv, m, node)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
return line != "", line, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// missingAccounts says some account of m on node is not issued: its own, a process's, or its provisioner client's.
|
||||||
|
func missingAccounts(m catalogue.Manifest, node string, minted []string) bool {
|
||||||
|
if user, own := ownAccountOf(m, node); own && !slices.Contains(minted, user) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if runsTheProvisionerClient(m) &&
|
||||||
|
!slices.Contains(minted, broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return len(processAccountsMissing(m, node, minted)) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// liveAssignmentAccountSteps are the steps on the controller's own stores and the bus: the composer's passes, as the
|
||||||
|
// done keys' take-ins use them, so a node asked for by both is composed once a pass.
|
||||||
|
func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps {
|
||||||
|
inv := open.inventory
|
||||||
|
return assignmentAccountSteps{
|
||||||
|
holds: func(ctx context.Context) bool {
|
||||||
|
// The lease's gate, as every act's: a serving controller the bus does not grant the lease yet acts unleased.
|
||||||
|
_, err := theLease.epoch(ctx)
|
||||||
|
return err == nil
|
||||||
|
},
|
||||||
|
forgetStale: func(ctx context.Context) ([]string, error) { return forgetStaleAccounts(ctx, open) },
|
||||||
|
catalogue: func(ctx context.Context) (map[string]catalogue.Manifest, error) { return inv.Catalogue(ctx) },
|
||||||
|
issue: func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (bool, string, error) {
|
||||||
|
return issueAssignmentAccounts(ctx, inv, shelf, node, module)
|
||||||
|
},
|
||||||
|
busHolder: func(ctx context.Context) (string, error) {
|
||||||
|
holders, err := seatHolders(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return holders[catalogue.BrokerSeat].Node, nil
|
||||||
|
},
|
||||||
|
compose: func(ctx context.Context, nodes []string) error {
|
||||||
|
return composerFor(open).compose(ctx, nodes, false)
|
||||||
|
},
|
||||||
|
grants: func(ctx context.Context) error {
|
||||||
|
return composerFor(open).compose(ctx, nil, true)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignmentAccountsRetry is how long a settle that failed waits before it tries again.
|
||||||
|
var assignmentAccountsRetry = 30 * time.Second
|
||||||
|
|
||||||
|
// untilSettled runs settle until it works or ctx ends: a failure, or a controller not holding the lease, is said and
|
||||||
|
// tried again every assignmentAccountsRetry, never dropped, since no later write of the key may come.
|
||||||
|
func untilSettled(ctx context.Context, what string, settle func(context.Context) (string, error)) {
|
||||||
|
for {
|
||||||
|
said, err := settle(ctx)
|
||||||
|
if err == nil {
|
||||||
|
if said != "" {
|
||||||
|
for _, line := range strings.Split(said, "\n") {
|
||||||
|
fmt.Printf("assignments: %s\n", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Printf("assignments: %s: %v; tried again in %s\n", what, err, assignmentAccountsRetry)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(assignmentAccountsRetry):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchAssignmentAccounts settles every assignment of the assignments bucket that comes to stand, for as long as ctx: the
|
||||||
|
// bucket's current values first (the start's look, settled as not fresh), then each write. A watch that ends, or a
|
||||||
|
// bucket not there yet, is watched again a minute later.
|
||||||
|
func watchAssignmentAccounts(ctx context.Context, open *stores) {
|
||||||
|
steps := liveAssignmentAccountSteps(open)
|
||||||
|
goServing(ctx, func() {
|
||||||
|
runAssignmentWatches(ctx, open.inventory.WatchAssignments, func(a chain.Assignment) {
|
||||||
|
inBackground(ctx, func() {
|
||||||
|
untilSettled(ctx, a.Module+" on "+a.Node, func(ctx context.Context) (string, error) {
|
||||||
|
return steps.settle(ctx, a.Node, a.Module)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}, func(look []chain.Assignment) {
|
||||||
|
// The start's look, once in the process: one reading of the catalogue, one assignment after another,
|
||||||
|
// then one composition of the bus's node and the grants (issues 1945 and 1277).
|
||||||
|
inBackground(ctx, func() {
|
||||||
|
untilSettled(ctx, "the start's look", func(ctx context.Context) (string, error) {
|
||||||
|
return steps.look(ctx, look)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}, func() bool {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return false
|
||||||
|
case <-time.After(time.Minute):
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// runAssignmentWatches watches the assignments again each time a watch ends, until ctx ends or pause says stop: each
|
||||||
|
// assignment that comes to stand is handed to fresh, and the standing ones of the first watch that reads them all (its
|
||||||
|
// Caught mark) to look, once. A watch that ends, or fails to open, before the start's look ran is no start's look: the
|
||||||
|
// next one is, with what it hears from nothing (a fresh assignmentTurns), so no key it handed is dropped and no burst of
|
||||||
|
// fresh settles replaces the look (the review of issue 1945).
|
||||||
|
func runAssignmentWatches(ctx context.Context, watch func(context.Context) (<-chan chain.Entry, error),
|
||||||
|
fresh func(chain.Assignment), look func([]chain.Assignment), pause func() bool) {
|
||||||
|
turns := &assignmentTurns{}
|
||||||
|
first := true
|
||||||
|
for {
|
||||||
|
var atStart []chain.Assignment
|
||||||
|
if err := watchAssignmentsOnce(ctx, watch, turns, first, func(a chain.Assignment, isFresh bool) {
|
||||||
|
if !isFresh {
|
||||||
|
atStart = append(atStart, a)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fresh(a)
|
||||||
|
}, func() {
|
||||||
|
first = false
|
||||||
|
look(atStart)
|
||||||
|
atStart = nil
|
||||||
|
}); err != nil && ctx.Err() == nil {
|
||||||
|
fmt.Printf("assignments: the assignments are not heard, so a new assignment's bus accounts wait: %v\n", err)
|
||||||
|
}
|
||||||
|
if first {
|
||||||
|
turns = &assignmentTurns{} // the start's look has not run: the next watch hears every standing key anew
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil || !pause() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchAssignmentsOnce watches the assignments until the watch or ctx ends, handing each assignment that comes to stand
|
||||||
|
// to settle. On the process's first watch, the current values are the start's look (not fresh) and read is called once
|
||||||
|
// they are all handed (the chain's Caught mark); every other value, and every value of a watch opened again after one
|
||||||
|
// ended, is fresh: an assignment that came to stand while no watch heard it is settled whole, never as the start's look
|
||||||
|
// (the review of issue 1945).
|
||||||
|
func watchAssignmentsOnce(ctx context.Context, watch func(context.Context) (<-chan chain.Entry, error),
|
||||||
|
turns *assignmentTurns, first bool, settle func(a chain.Assignment, fresh bool), read func()) error {
|
||||||
|
ctx, stop := context.WithCancel(ctx)
|
||||||
|
defer stop()
|
||||||
|
entries, err := watch(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
current := true
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case e, ok := <-entries:
|
||||||
|
if !ok {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("the watch ended")
|
||||||
|
}
|
||||||
|
if e.Caught {
|
||||||
|
if current {
|
||||||
|
current = false // the current values are read: what follows is written from now on
|
||||||
|
if first {
|
||||||
|
read()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var value []byte
|
||||||
|
if !e.Deleted {
|
||||||
|
value = e.Value
|
||||||
|
}
|
||||||
|
if a, turned := turns.heard(e.Key, value); turned {
|
||||||
|
settle(a, !current || !first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleAccounts is, by node, each module some account of which stands on a node where the module no longer resolves:
|
||||||
|
// the accounts of a module's own, its processes' and its provisioner client's. A node whose resolution is unknown is
|
||||||
|
// left alone.
|
||||||
|
func staleAccounts(users map[string]inventory.BusUser, resolved map[string]map[string]bool) map[string][]string {
|
||||||
|
seen := map[string]map[string]bool{}
|
||||||
|
for _, u := range users {
|
||||||
|
switch u.Kind {
|
||||||
|
case inventory.BusModule, inventory.BusNodeTools, inventory.BusProcess, inventory.BusProvisioner:
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
on, known := resolved[u.Node]
|
||||||
|
if u.Node == "" || u.Module == "" || !known || on[u.Module] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if seen[u.Node] == nil {
|
||||||
|
seen[u.Node] = map[string]bool{}
|
||||||
|
}
|
||||||
|
seen[u.Node][u.Module] = true
|
||||||
|
}
|
||||||
|
out := map[string][]string{}
|
||||||
|
for node, modules := range seen {
|
||||||
|
out[node] = slices.Sorted(maps.Keys(modules))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetStaleAccounts forgets the accounts of every module on a node where it no longer resolves (assigned or
|
||||||
|
// required), from each node's own resolution; a node that does not resolve is left alone.
|
||||||
|
func forgetStaleAccounts(ctx context.Context, open *stores) ([]string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
users, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ctx = withInputCache(ctx) // one reading of the mesh for every node resolved
|
||||||
|
resolved := map[string]map[string]bool{}
|
||||||
|
for _, u := range users {
|
||||||
|
if _, done := resolved[u.Node]; done || u.Node == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
plan, _, err := resolveNode(ctx, open, u.Node)
|
||||||
|
if err != nil {
|
||||||
|
continue // unknown: nothing of this node is forgotten
|
||||||
|
}
|
||||||
|
on := map[string]bool{}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
on[m.Module] = true
|
||||||
|
}
|
||||||
|
resolved[u.Node] = on
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
stale := staleAccounts(users, resolved)
|
||||||
|
for _, node := range slices.Sorted(maps.Keys(stale)) {
|
||||||
|
for _, module := range stale[node] {
|
||||||
|
forgot, err := inv.ForgetBusAccountsOf(ctx, node, module, true)
|
||||||
|
if err != nil {
|
||||||
|
return lines, err
|
||||||
|
}
|
||||||
|
if len(forgot) > 0 {
|
||||||
|
lines = append(lines, fmt.Sprintf("%s no longer runs on %s: its bus accounts forgotten: %s", module, node,
|
||||||
|
strings.Join(forgot, ", ")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return lines, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,498 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/chain"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An assignment that comes to stand issues its node's bus accounts and the module's grant (ADR 0543, issue 1945): on
|
||||||
|
// 2026-10-25 at 17:19 mesh-build-node and mesh-build-go, built long before, were assigned to ace, and nothing issued
|
||||||
|
// ace's accounts nor put them in the bus's user list.
|
||||||
|
|
||||||
|
func anAssignmentValue(t *testing.T, a chain.Assignment) []byte {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := json.Marshal(a)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only a write that makes the key stand turns it: a rewrite of a standing assignment (the provisioner's plan kept in its
|
||||||
|
// value), an ended one, a pending one or a module only pulled in by another does not. The process's first watch hands
|
||||||
|
// the assignments standing when it starts to the start's look (not fresh), says once that they are all read, and hands
|
||||||
|
// each assignment written afterwards as fresh; a watch opened again after one ended hands every standing assignment it
|
||||||
|
// has not seen stand as fresh and never calls the start's look (the review of issue 1945).
|
||||||
|
func TestAnAssignmentThatComesToStand(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 25, 16, 19, 42, 0, time.UTC)
|
||||||
|
stands := chain.Assignment{Node: "ace", Module: "mesh-build-node", Assigned: at}
|
||||||
|
ended := stands
|
||||||
|
ended.Ended = at.Add(time.Hour)
|
||||||
|
pending := stands
|
||||||
|
pending.State = chain.StatePending
|
||||||
|
pulled := chain.Assignment{Node: "ace", Module: "mesh-build-node", RequiredBy: "mesh-build-go"}
|
||||||
|
replanned := stands
|
||||||
|
replanned.Provisioning = json.RawMessage(`{"plan":1}`)
|
||||||
|
|
||||||
|
type write struct {
|
||||||
|
value *chain.Assignment // nil for a delete
|
||||||
|
turning bool
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
writes []write
|
||||||
|
}{
|
||||||
|
{"a new assignment turns", []write{{&stands, true}}},
|
||||||
|
{"a rewrite of one that stands does not", []write{{&stands, true}, {&replanned, false}, {&stands, false}}},
|
||||||
|
{"an ended one does not", []write{{&ended, false}}},
|
||||||
|
{"assigned again after it ended turns again", []write{{&stands, true}, {&ended, false}, {&stands, true}}},
|
||||||
|
{"a pending one does not, and turns once made", []write{{&pending, false}, {&stands, true}}},
|
||||||
|
{"a module only pulled in does not", []write{{&pulled, false}}},
|
||||||
|
{"one deleted and written again turns again", []write{{&stands, true}, {nil, false}, {&stands, true}}},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
turns := &assignmentTurns{}
|
||||||
|
for i, w := range c.writes {
|
||||||
|
var value []byte
|
||||||
|
if w.value != nil {
|
||||||
|
value = anAssignmentValue(t, *w.value)
|
||||||
|
}
|
||||||
|
a, turned := turns.heard("ace.mesh-build-node", value)
|
||||||
|
if turned != w.turning {
|
||||||
|
t.Fatalf("write %d: turned %v, want %v", i, turned, w.turning)
|
||||||
|
}
|
||||||
|
if turned && (a.Node != "ace" || a.Module != "mesh-build-node") {
|
||||||
|
t.Fatalf("write %d: the assignment turned is %s on %s", i, a.Module, a.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type handed struct {
|
||||||
|
key string
|
||||||
|
fresh bool
|
||||||
|
}
|
||||||
|
for _, first := range []bool{true, false} {
|
||||||
|
t.Run(fmt.Sprintf("the watch, first %v", first), func(t *testing.T) {
|
||||||
|
bucket := chain.NewMemory()
|
||||||
|
put := func(a chain.Assignment) {
|
||||||
|
t.Helper()
|
||||||
|
value := anAssignmentValue(t, a)
|
||||||
|
if e, held, _ := bucket.Get(t.Context(), a.Key()); held {
|
||||||
|
if _, err := bucket.Update(t.Context(), a.Key(), value, e.Revision); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := bucket.Create(t.Context(), a.Key(), value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
put(chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at})
|
||||||
|
got := make(chan handed, 8)
|
||||||
|
read := make(chan struct{}, 8)
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- watchAssignmentsOnce(ctx, bucket.Watch, &assignmentTurns{}, first,
|
||||||
|
func(a chain.Assignment, fresh bool) { got <- handed{a.Key(), fresh} },
|
||||||
|
func() { read <- struct{}{} })
|
||||||
|
}()
|
||||||
|
next := func() handed {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case h := <-got:
|
||||||
|
return h
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("nothing handed")
|
||||||
|
return handed{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Standing when the watch opened: the start's look on the first watch, fresh on any other.
|
||||||
|
if h := next(); h != (handed{"g14.mesh-build-go", !first}) {
|
||||||
|
t.Fatalf("the standing assignment was handed %+v", h)
|
||||||
|
}
|
||||||
|
put(chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at, Provisioning: json.RawMessage(`{"plan":2}`)})
|
||||||
|
put(chain.Assignment{Node: "ace", Module: "mesh-build-node", Assigned: at})
|
||||||
|
if h := next(); h != (handed{"ace.mesh-build-node", true}) {
|
||||||
|
t.Fatalf("handed %+v, want ace.mesh-build-node as fresh (and nothing for the rewrite before it)", h)
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
if err := <-done; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if want := map[bool]int{true: 1, false: 0}[first]; len(read) != want {
|
||||||
|
t.Fatalf("the start's look was called %d time(s), want %d", len(read), want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// A first watch that fails to open, or ends before its Caught mark, is no start's look: the next watch is, and hands
|
||||||
|
// every standing assignment to it once, none as fresh (the review of issue 1945: a burst of fresh settles, or keys
|
||||||
|
// handed and dropped).
|
||||||
|
for _, firstWatch := range []string{"fails to open", "ends before its Caught mark"} {
|
||||||
|
t.Run("a first watch that "+firstWatch, func(t *testing.T) {
|
||||||
|
bucket := chain.NewMemory()
|
||||||
|
standing := chain.Assignment{Node: "g14", Module: "mesh-build-go", Assigned: at}
|
||||||
|
if _, err := bucket.Create(t.Context(), standing.Key(), anAssignmentValue(t, standing)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
watches := 0
|
||||||
|
watch := func(ctx context.Context) (<-chan chain.Entry, error) {
|
||||||
|
watches++
|
||||||
|
if watches > 1 {
|
||||||
|
return bucket.Watch(ctx)
|
||||||
|
}
|
||||||
|
if firstWatch == "fails to open" {
|
||||||
|
return nil, errors.New("the chain is not open yet")
|
||||||
|
}
|
||||||
|
cut := make(chan chain.Entry, 1)
|
||||||
|
cut <- chain.Entry{Key: standing.Key(), Value: anAssignmentValue(t, standing)}
|
||||||
|
close(cut)
|
||||||
|
return cut, nil
|
||||||
|
}
|
||||||
|
var looks [][]chain.Assignment
|
||||||
|
var fresh []chain.Assignment
|
||||||
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
|
defer stop()
|
||||||
|
runAssignmentWatches(ctx, watch, func(a chain.Assignment) { fresh = append(fresh, a) },
|
||||||
|
func(look []chain.Assignment) { looks = append(looks, look); stop() },
|
||||||
|
func() bool { return ctx.Err() == nil })
|
||||||
|
if len(looks) != 1 || len(looks[0]) != 1 || looks[0][0].Key() != standing.Key() || len(fresh) != 0 {
|
||||||
|
t.Fatalf("after %d watch(es): looks %v, fresh %v; want one look at %s and nothing fresh",
|
||||||
|
watches, looks, fresh, standing.Key())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// settle issues what is missing and then puts it on the bus: the node holding the bus composed (its user list) and the
|
||||||
|
// grants written, even when every account was minted before (a module assigned again keeps its account). The start's
|
||||||
|
// look reads the catalogue once, issues for each standing assignment, and composes the bus's node and writes the grants
|
||||||
|
// once when it looked at any. Neither acts for a controller that does not hold the lease, nor for the controller's own
|
||||||
|
// module.
|
||||||
|
func TestSettlingAndTheStartsLook(t *testing.T) {
|
||||||
|
ace := func(m string) chain.Assignment { return chain.Assignment{Node: "ace", Module: m} }
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
look []chain.Assignment // nil: settle mesh-build-node on ace
|
||||||
|
module string
|
||||||
|
notHolding bool
|
||||||
|
issued bool
|
||||||
|
issueErr, composeE error
|
||||||
|
holder string
|
||||||
|
wantIssues int
|
||||||
|
wantComposed []string
|
||||||
|
wantGrants bool
|
||||||
|
wantErr string
|
||||||
|
stale []string // what forgetting the accounts of modules no longer on their nodes says
|
||||||
|
}{
|
||||||
|
{name: "an account not minted", module: "mesh-build-node", issued: true, holder: "novox", wantIssues: 1,
|
||||||
|
wantComposed: []string{"novox"}, wantGrants: true},
|
||||||
|
{name: "an account minted before: the bus and the grants still", module: "mesh-build-go", holder: "novox",
|
||||||
|
wantIssues: 1, wantComposed: []string{"novox"}, wantGrants: true},
|
||||||
|
{name: "no lease: nothing, and tried again", module: "mesh-build-node", notHolding: true, issued: true,
|
||||||
|
holder: "novox", wantErr: errNotHolding.Error()},
|
||||||
|
{name: "the controller's own module is the terminal's", module: controllerModule, issued: true, holder: "novox"},
|
||||||
|
{name: "no bus holder: the grants still", module: "mesh-build-node", issued: true, wantIssues: 1, wantGrants: true},
|
||||||
|
{name: "an issue that fails is the error, nothing composed", module: "mesh-build-node",
|
||||||
|
issueErr: errors.New("the bus cannot be reached"), holder: "novox", wantIssues: 1, wantErr: "the bus cannot be reached"},
|
||||||
|
{name: "a composition that fails is the error, no grant", module: "mesh-build-node", issued: true, holder: "novox",
|
||||||
|
composeE: errors.New("store refused"), wantIssues: 1, wantComposed: []string{"novox"}, wantErr: "store refused"},
|
||||||
|
{name: "the start's look: each issued, the bus and the grants once",
|
||||||
|
look: []chain.Assignment{ace("mesh-build-node"), ace(controllerModule), ace("mesh-build-go")}, issued: true,
|
||||||
|
holder: "novox", wantIssues: 2, wantComposed: []string{"novox"}, wantGrants: true},
|
||||||
|
{name: "the start's look with every account minted still covers once",
|
||||||
|
look: []chain.Assignment{ace("mesh-build-node"), ace("mesh-build-go")}, holder: "novox", wantIssues: 2,
|
||||||
|
wantComposed: []string{"novox"}, wantGrants: true},
|
||||||
|
{name: "the start's look at nothing does nothing", look: []chain.Assignment{ace(controllerModule)}, holder: "novox"},
|
||||||
|
{name: "the start's look forgets the accounts of modules no longer on their nodes first",
|
||||||
|
look: []chain.Assignment{ace("mesh-build-node")}, holder: "novox", wantIssues: 1, wantComposed: []string{"novox"},
|
||||||
|
wantGrants: true, stale: []string{"mesh-build-go no longer runs on ace: its bus accounts forgotten: ace.mesh-build-go"}},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
var composed []string
|
||||||
|
issues, catalogues, grants := 0, 0, 0
|
||||||
|
forgets := 0
|
||||||
|
steps := assignmentAccountSteps{
|
||||||
|
holds: func(context.Context) bool { return !c.notHolding },
|
||||||
|
forgetStale: func(context.Context) ([]string, error) {
|
||||||
|
forgets++
|
||||||
|
return c.stale, nil
|
||||||
|
},
|
||||||
|
catalogue: func(context.Context) (map[string]catalogue.Manifest, error) {
|
||||||
|
catalogues++
|
||||||
|
return map[string]catalogue.Manifest{}, nil
|
||||||
|
},
|
||||||
|
issue: func(context.Context, map[string]catalogue.Manifest, string, string) (bool, string, error) {
|
||||||
|
issues++
|
||||||
|
if c.issued {
|
||||||
|
return true, "its bus credential issued", c.issueErr
|
||||||
|
}
|
||||||
|
return false, "", c.issueErr
|
||||||
|
},
|
||||||
|
busHolder: func(context.Context) (string, error) { return c.holder, nil },
|
||||||
|
compose: func(_ context.Context, nodes []string) error {
|
||||||
|
composed = append(composed, nodes...)
|
||||||
|
return c.composeE
|
||||||
|
},
|
||||||
|
grants: func(context.Context) error { grants++; return nil },
|
||||||
|
}
|
||||||
|
var said string
|
||||||
|
var err error
|
||||||
|
if c.look != nil {
|
||||||
|
said, err = steps.look(t.Context(), c.look)
|
||||||
|
} else {
|
||||||
|
said, err = steps.settle(t.Context(), "ace", c.module)
|
||||||
|
}
|
||||||
|
if c.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.wantErr) {
|
||||||
|
t.Fatalf("error %v, want one saying %q", err, c.wantErr)
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if issues != c.wantIssues || catalogues > 1 {
|
||||||
|
t.Fatalf("%d issue(s) from %d reading(s) of the catalogue, want %d from at most one", issues, catalogues, c.wantIssues)
|
||||||
|
}
|
||||||
|
if strings.Join(composed, ",") != strings.Join(c.wantComposed, ",") {
|
||||||
|
t.Fatalf("composed %v, want %v", composed, c.wantComposed)
|
||||||
|
}
|
||||||
|
if want := map[bool]int{true: 1}[c.wantGrants]; grants != want {
|
||||||
|
t.Fatalf("grants written %d time(s), want %d", grants, want)
|
||||||
|
}
|
||||||
|
if want := map[bool]int{true: 1}[len(c.look) > 0 && !c.notHolding]; forgets != want {
|
||||||
|
t.Fatalf("forgot stale accounts %d time(s), want %d (only from a reading that holds assignments)", forgets, want)
|
||||||
|
}
|
||||||
|
for _, line := range c.stale {
|
||||||
|
if !strings.Contains(said, line) {
|
||||||
|
t.Fatalf("said %q, want it to say %q", said, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.wantGrants && c.wantErr == "" && !strings.Contains(said, "the grants written") {
|
||||||
|
t.Fatalf("said %q, want it to name the grants", said)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The accounts a module speaks with on a node: the module's own when it reads `broker`, its provisioner client's when
|
||||||
|
// it runs one alone, none when its accounts are its processes' or it does not speak on the bus. A process's account is
|
||||||
|
// held in the provisioner client's form, named for the process and carrying no claims, and may be issued only when
|
||||||
|
// the process declares its own secret broker-<process id> (ADR 0436 §2; issue 1181).
|
||||||
|
func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) {
|
||||||
|
client := aProvisioner()
|
||||||
|
delete(client.OwnSecrets, "broker")
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
m catalogue.Manifest
|
||||||
|
user string
|
||||||
|
own bool
|
||||||
|
}{
|
||||||
|
{"a module reading broker", catalogue.Manifest{Module: "mesh-build-node",
|
||||||
|
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh-build-node/broker"}}},
|
||||||
|
broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-build-node"}.Username(), true},
|
||||||
|
{"a module reading broker and running the client", aProvisioner(),
|
||||||
|
broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-provisioner"}.Username(), true},
|
||||||
|
{"the provisioner client alone", client,
|
||||||
|
broker.Principal{Kind: broker.KindProvisioner, Node: "ace", Module: "mesh-provisioner"}.Username(), true},
|
||||||
|
{"no broker, no client", catalogue.Manifest{Module: "hello-web"}, "", false},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
user, own := ownAccountOf(c.m, "ace")
|
||||||
|
if user != c.user || own != c.own {
|
||||||
|
t.Fatalf("got %q %v, want %q %v", user, own, c.user, c.own)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("a process's account", func(t *testing.T) {
|
||||||
|
user := broker.Principal{Kind: broker.KindProcess, Node: "novox", Module: "mesh-mail", Process: "send"}.Username()
|
||||||
|
if user != "novox.mesh-mail.send" {
|
||||||
|
t.Fatalf("the process's username: %q", user)
|
||||||
|
}
|
||||||
|
held, err := heldProcessAccount(broker.Broker{Fingerprint: "abc"}, "novox.example:4222", "novox", "mesh-mail",
|
||||||
|
user, "a-password")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var got map[string]any
|
||||||
|
if err := json.Unmarshal(held, &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := map[string]any{"url": "nats://novox.example:4222", "fingerprint": "abc", "node": "novox",
|
||||||
|
"module": "mesh-mail", "user": "novox.mesh-mail.send", "password": "a-password"}
|
||||||
|
for k, v := range want {
|
||||||
|
if got[k] != v {
|
||||||
|
t.Errorf("%s: %v, want %v", k, got[k], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Errorf("the held account carries %v; want exactly %v (never claims — a process holds no seat)", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// An issue mints and seals an account only when it is not issued: no row, or a row with no sealed credential (issue
|
||||||
|
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go had a row from 17:19 and no credential), under the lock on
|
||||||
|
// the account; a seal that fails leaves it unsealed for the next issue. A rotation always mints.
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
only bool
|
||||||
|
minted, sealed bool
|
||||||
|
sealErr error
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"no row: minted unsealed, sealed, marked", true, false, false, nil, "lock mint-unsealed seal mark release"},
|
||||||
|
{"a row sealed: nothing", true, true, true, nil, "lock release"},
|
||||||
|
{"a row with no sealed credential: minted and sealed again", true, true, false, nil, "lock mint-unsealed seal mark release"},
|
||||||
|
{"a seal that fails: not marked", true, false, false, errors.New("the vault did not take it"), "lock mint-unsealed seal release"},
|
||||||
|
{"a rotation mints whatever stands", false, true, true, nil, "mint seal"},
|
||||||
|
} {
|
||||||
|
t.Run("issuing an account: "+c.name, func(t *testing.T) {
|
||||||
|
acts := &anAccountStore{minted: c.minted, sealed: c.sealed}
|
||||||
|
ctx, got := withIssued(t.Context())
|
||||||
|
if c.only {
|
||||||
|
ctx = onlyIfAbsent(ctx)
|
||||||
|
}
|
||||||
|
err := issueAccount(ctx, acts, inventory.BusUser{Username: "ace.mesh-build-go", Kind: inventory.BusModule},
|
||||||
|
func(password string) error {
|
||||||
|
acts.did("seal")
|
||||||
|
if password == "" {
|
||||||
|
t.Error("sealed no password")
|
||||||
|
}
|
||||||
|
return c.sealErr
|
||||||
|
}, func() error { acts.did("after"); return nil })
|
||||||
|
if (err != nil) != (c.sealErr != nil) {
|
||||||
|
t.Fatalf("error %v, want %v", err, c.sealErr)
|
||||||
|
}
|
||||||
|
if strings.Join(acts.acts, " ") != c.want {
|
||||||
|
t.Fatalf("acts %q, want %q", strings.Join(acts.acts, " "), c.want)
|
||||||
|
}
|
||||||
|
if issued := len(got.list()) == 1; issued != (c.sealErr == nil && strings.Contains(c.want, "seal")) {
|
||||||
|
t.Fatalf("said issued %v", got.list())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 774: where a vault is installed the controller mints nothing; `module issue` and an assignment's issue are
|
||||||
|
// forwarded to the vault's issue-account, and what the account needs beside its value still runs. A refusal fails
|
||||||
|
// `module issue` and only says itself on an assignment's issue: the vault mints on the want when it hears it.
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
only bool
|
||||||
|
answer string
|
||||||
|
refuse error
|
||||||
|
want string
|
||||||
|
fails bool
|
||||||
|
}{
|
||||||
|
{"module issue: forwarded, minted by the vault", false, `{"minted":true,"account":{"secret":"own/ace/mesh-build-go/broker","generation":1}}`, nil, "after", false},
|
||||||
|
{"an assignment's issue: forwarded, held already", true, `{"minted":false,"why":"held already"}`, nil, "after", false},
|
||||||
|
{"an assignment's issue: refused, said", true, "", errors.New("nobody wants it yet"), "after", false},
|
||||||
|
{"module issue: refused, failed", false, "", errors.New("nobody wants it"), "", true},
|
||||||
|
} {
|
||||||
|
t.Run("forwarded to the vault: "+c.name, func(t *testing.T) {
|
||||||
|
was := vaultIssuer
|
||||||
|
t.Cleanup(func() { vaultIssuer = was })
|
||||||
|
var asked []string
|
||||||
|
vaultIssuer = func(context.Context, accountIssuer) (vaultAsker, bool, error) {
|
||||||
|
return func(verb string, args map[string]any) (json.RawMessage, error) {
|
||||||
|
asked = append(asked, verb+" "+fmt.Sprint(args["user"]))
|
||||||
|
return json.RawMessage(c.answer), c.refuse
|
||||||
|
}, true, nil
|
||||||
|
}
|
||||||
|
acts := &anAccountStore{}
|
||||||
|
ctx, got := withIssued(t.Context())
|
||||||
|
if c.only {
|
||||||
|
ctx = onlyIfAbsent(ctx)
|
||||||
|
}
|
||||||
|
err := issueAccount(ctx, acts, inventory.BusUser{Username: "ace.mesh-build-go", Kind: inventory.BusModule},
|
||||||
|
func(string) error { acts.did("seal"); return nil }, func() error { acts.did("after"); return nil })
|
||||||
|
if (err != nil) != c.fails || strings.Join(acts.acts, " ") != c.want {
|
||||||
|
t.Fatalf("error %v, acts %q; want failed %v, acts %q (nothing minted, locked or sealed here)", err,
|
||||||
|
strings.Join(acts.acts, " "), c.fails, c.want)
|
||||||
|
}
|
||||||
|
if strings.Join(asked, ",") != "issue-account ace.mesh-build-go" {
|
||||||
|
t.Errorf("the vault was asked %v", asked)
|
||||||
|
}
|
||||||
|
if minted := len(got.list()) == 1; minted != strings.Contains(c.answer, `"minted":true`) {
|
||||||
|
t.Errorf("said issued %v", got.list())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The accounts of a module that no longer resolves on its node are stale, its provisioner client's among them; a
|
||||||
|
// module still resolved (assigned or required), an account of another kind, and a node whose resolution is unknown
|
||||||
|
// are left alone (issue 1945: ace's mesh-build-go rows from 17:19, marked sealed by migration 0108).
|
||||||
|
t.Run("stale accounts", func(t *testing.T) {
|
||||||
|
u := func(kind, node, module string) inventory.BusUser {
|
||||||
|
return inventory.BusUser{Username: node + "." + module + "." + kind, Kind: kind, Node: node, Module: module}
|
||||||
|
}
|
||||||
|
users := map[string]inventory.BusUser{}
|
||||||
|
for _, b := range []inventory.BusUser{u(inventory.BusModule, "ace", "mesh-build-go"),
|
||||||
|
u(inventory.BusProcess, "ace", "mesh-build-go"), u(inventory.BusProvisioner, "ace", "mesh-provisioner"),
|
||||||
|
u(inventory.BusModule, "ace", "mesh-build-node"), u(inventory.BusNode, "ace", ""),
|
||||||
|
u(inventory.BusModule, "g14", "mesh-build-go")} {
|
||||||
|
users[b.Username] = b
|
||||||
|
}
|
||||||
|
got := staleAccounts(users, map[string]map[string]bool{"ace": {"mesh-build-node": true}})
|
||||||
|
if want := map[string][]string{"ace": {"mesh-build-go", "mesh-provisioner"}}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("stale %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a process may be issued only with its own broker secret", func(t *testing.T) {
|
||||||
|
// mesh-mail's real shape: no `broker` of its own, its process declaring bus (review round 1 of issue 1181).
|
||||||
|
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
||||||
|
t.Fatalf("a module with no broker of its own, whose process declares bus, was refused: %v", err)
|
||||||
|
}
|
||||||
|
m := mailSenderWithProcess()
|
||||||
|
delete(m.OwnSecrets, "broker-send")
|
||||||
|
err := mayIssue(m)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a process with no broker-<id> own secret was issued an account")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"mesh-mail", "send", "broker-send", "own-secrets"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// anAccountStore is an accountIssuer that writes down what was asked of it.
|
||||||
|
type anAccountStore struct {
|
||||||
|
minted, sealed bool
|
||||||
|
acts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *anAccountStore) did(act string) { a.acts = append(a.acts, act) }
|
||||||
|
func (a *anAccountStore) LockBusAccount(context.Context, string) (func(), error) {
|
||||||
|
a.did("lock")
|
||||||
|
return func() { a.did("release") }, nil
|
||||||
|
}
|
||||||
|
func (a *anAccountStore) BusAccountIssued(context.Context, string) (bool, bool, error) {
|
||||||
|
return a.minted, a.sealed, nil
|
||||||
|
}
|
||||||
|
func (a *anAccountStore) MintBusPassword(context.Context, inventory.BusUser) (string, error) {
|
||||||
|
a.did("mint")
|
||||||
|
return "a-password", nil
|
||||||
|
}
|
||||||
|
func (a *anAccountStore) MintBusPasswordUnsealed(context.Context, inventory.BusUser) (string, error) {
|
||||||
|
a.did("mint-unsealed")
|
||||||
|
return "a-password", nil
|
||||||
|
}
|
||||||
|
func (a *anAccountStore) MarkBusAccountSealed(context.Context, string) error {
|
||||||
|
a.did("mark")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -5,10 +5,11 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
@@ -127,17 +128,13 @@ type takenBuild struct {
|
|||||||
carried func() error
|
carried func() error
|
||||||
}
|
}
|
||||||
|
|
||||||
// registerJudged judges a recorded build and registers its module, or says why not.
|
// registerJudged registers its module, or says why not.
|
||||||
func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild) (catalogue.Manifest, error) {
|
func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild) (catalogue.Manifest, error) {
|
||||||
manifest, kept := j.manifest, j.kept
|
manifest, kept := j.manifest, j.kept
|
||||||
notRegistered := func(err error) (catalogue.Manifest, error) {
|
notRegistered := func(err error) (catalogue.Manifest, error) {
|
||||||
return manifest, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", j.by, manifest.Module,
|
return manifest, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", j.by, manifest.Module,
|
||||||
short(j.commit), err)
|
short(j.commit), err)
|
||||||
}
|
}
|
||||||
if err := namesNoInstallation(manifest); err != nil {
|
|
||||||
return manifest, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
|
||||||
j.by, j.what, short(j.commit), err)
|
|
||||||
}
|
|
||||||
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
||||||
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
||||||
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
||||||
@@ -150,18 +147,12 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild)
|
|||||||
return notRegistered(err)
|
return notRegistered(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
// The manifest was judged once, by the indexer, and its verdict taken before this (buildOfKey; ADR 0578 §3).
|
||||||
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
|
||||||
if j.carried != nil {
|
if j.carried != nil {
|
||||||
if err := j.carried(); err != nil {
|
if err := j.carried(); err != nil {
|
||||||
return manifest, err
|
return manifest, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if manifest.NamesPredefined() {
|
|
||||||
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
|
|
||||||
return notRegistered(fmt.Errorf("judged against the list of predefined keys the mesh publishes, %w", err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||||
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
// what the build the mesh stands on was made from registers that build's artifacts at the new
|
||||||
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
|
||||||
@@ -172,7 +163,7 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild)
|
|||||||
return manifest, err
|
return manifest, err
|
||||||
}
|
}
|
||||||
if stands != "" && stands != kept.ID && len(raw) > 0 {
|
if stands != "" && stands != kept.ID && len(raw) > 0 {
|
||||||
if was, err := catalogue.ParseManifest(raw); err == nil && was.Module == manifest.Module {
|
if was, err := catalogue.Decode(raw); err == nil && was.Module == manifest.Module {
|
||||||
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
|
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
|
||||||
manifest.Module, short(j.commit), stands)
|
manifest.Module, short(j.commit), stands)
|
||||||
manifest = was
|
manifest = was
|
||||||
@@ -273,4 +264,7 @@ type answers struct {
|
|||||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||||
pending []inventory.PendingAssignment
|
pending []inventory.PendingAssignment
|
||||||
pendingUnread string
|
pendingUnread string
|
||||||
|
// takeIn is the take-in's backlog of done keys on the bus (novox/hq ADR 0544 §6), nil where it cannot be read
|
||||||
|
// (a process with no bus, or a consumer not made yet).
|
||||||
|
takeIn *takeInBacklog
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,8 +49,10 @@ func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
|||||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
// Not registered from a source the catalogue does not build it from. That it names an installation is no longer the
|
||||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
// controller's to judge (ADR 0578 §3): the indexer's judge refuses it (mesh-delivery-indexer judge's installation rows).
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "recorded and not registered") {
|
||||||
|
t.Fatalf("a definition from a source the catalogue does not build it from was taken in: %v", err)
|
||||||
}
|
}
|
||||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||||
t.Fatal("the refused module was registered anyway")
|
t.Fatal("the refused module was registered anyway")
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ import (
|
|||||||
type builderOnTheBus struct {
|
type builderOnTheBus struct {
|
||||||
js *broker.JetStream
|
js *broker.JetStream
|
||||||
builds, modules, manifests jetstream.KeyValue
|
builds, modules, manifests jetstream.KeyValue
|
||||||
|
// verdicts is the indexer's state of verdicts on manifests (ADR 0578 §1).
|
||||||
|
verdicts jetstream.KeyValue
|
||||||
}
|
}
|
||||||
|
|
||||||
func aBuilderOnTheBus(t *testing.T) builderOnTheBus {
|
func aBuilderOnTheBus(t *testing.T) builderOnTheBus {
|
||||||
@@ -40,6 +42,7 @@ func aBuilderOnTheBus(t *testing.T) builderOnTheBus {
|
|||||||
{&b.builds, sdkdelivery.BuilderBuildsBucket, sdkdelivery.BuilderBuildsHistory},
|
{&b.builds, sdkdelivery.BuilderBuildsBucket, sdkdelivery.BuilderBuildsHistory},
|
||||||
{&b.modules, sdkdelivery.ModulesBucket, sdkdelivery.ModulesHistory},
|
{&b.modules, sdkdelivery.ModulesBucket, sdkdelivery.ModulesHistory},
|
||||||
{&b.manifests, sdkdelivery.ManifestsBucket, 1},
|
{&b.manifests, sdkdelivery.ManifestsBucket, 1},
|
||||||
|
{&b.verdicts, sdkdelivery.VerdictsBucket, sdkdelivery.VerdictsHistory},
|
||||||
} {
|
} {
|
||||||
kv, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: c.bucket, History: c.history})
|
kv, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: c.bucket, History: c.history})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
|||||||
// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
|
// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
|
||||||
if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
|
if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
|
||||||
fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
|
fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
|
||||||
go layBuildsOutSoon(ctx, inv)
|
goServing(ctx, func() { layBuildsOutSoon(ctx, inv) })
|
||||||
}
|
}
|
||||||
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
||||||
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
||||||
@@ -80,7 +80,11 @@ func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
|
|||||||
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
||||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
if ctx.Err() != nil {
|
||||||
|
return // stopped: the next controller lays them out
|
||||||
|
}
|
||||||
|
// A cancelled layout is tried again and raises nothing: a cancellation is no fault of the chain (issue 1945).
|
||||||
|
if keeper := conditionsFrom; keeper != nil && !failed && !errors.Is(err, context.Canceled) {
|
||||||
o.Summary += err.Error()
|
o.Summary += err.Error()
|
||||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||||
failed = true
|
failed = true
|
||||||
|
|||||||
@@ -1,337 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
|
||||||
//
|
|
||||||
// **The same functions registration runs, and nothing the command line adds** (ADR 0035): the strict
|
|
||||||
// parse with every per-manifest problem, then the rules no single manifest can be judged against,
|
|
||||||
// over exactly the manifests given. Somebody describing their own application in their own
|
|
||||||
// repository runs this before pushing and finds out there, rather than when a running mesh refuses
|
|
||||||
// the registration or, later, when a machine applies something that resolved and should not have.
|
|
||||||
//
|
|
||||||
// **What it cannot know without a store, it says.** The mesh's own seat set is the store's (ADR
|
|
||||||
// 0122); this binary carries a compiled copy that the store overrides when loaded, so a claim on a
|
|
||||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
|
||||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
|
||||||
// refused what it could not see would teach people to ignore it.
|
|
||||||
//
|
|
||||||
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
|
||||||
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
|
||||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
|
||||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
|
||||||
// provider's machine when a real machine's name first meets the module's.
|
|
||||||
// SomeManifestsVar, set by the merge gate, says the manifests given are only some of their repository's.
|
|
||||||
const SomeManifestsVar = "MESH_MODULE_CHECK_SOME"
|
|
||||||
|
|
||||||
func moduleCheck(paths []string, out io.Writer) error {
|
|
||||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
|
||||||
if len(paths) == 0 {
|
|
||||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
|
||||||
"manifest of a repository together so the rules between them are checked too")
|
|
||||||
}
|
|
||||||
shelf := catalogue.Shelf{}
|
|
||||||
faulted := map[string]bool{}
|
|
||||||
failed := 0
|
|
||||||
// The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read
|
|
||||||
// once per repository: a manifest naming a key is judged against the list its own tree holds.
|
|
||||||
lists := map[string]catalogue.PredefinedList{}
|
|
||||||
listOf := func(path string) (catalogue.PredefinedList, error) {
|
|
||||||
root := catalogue.RepositoryRoot(filepath.Dir(path))
|
|
||||||
if root == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if list, read := lists[root]; read {
|
|
||||||
return list, nil
|
|
||||||
}
|
|
||||||
list, err := catalogue.ReadPredefined(root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
lists[root] = list
|
|
||||||
return list, nil
|
|
||||||
}
|
|
||||||
for _, path := range paths {
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
list, err := listOf(path)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
m, err := catalogue.ParseManifestWith(raw, list)
|
|
||||||
if err == nil {
|
|
||||||
// Each template's text read from the tree beside the manifest, and judged as inline content is
|
|
||||||
// (novox/hq ADR 0349 §2): a missing template is refused by name.
|
|
||||||
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if first, twice := shelf[m.Module]; twice {
|
|
||||||
_ = first
|
|
||||||
fmt.Fprintf(out, "%s: %s was already given; two manifests name one module\n", path, m.Module)
|
|
||||||
failed++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
|
||||||
// catalogue-wide test, and at registration, which refuses in the same words.
|
|
||||||
if wrong := catalogue.TrustProblems(m); len(wrong) > 0 {
|
|
||||||
for _, p := range wrong {
|
|
||||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
|
||||||
}
|
|
||||||
failed += len(wrong)
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
// A module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326 §3,
|
|
||||||
// issue 574): judged where a definition is written, as at registration, never where a stored
|
|
||||||
// build is read again.
|
|
||||||
if wrong := catalogue.WrittenSettingsProblems(m); len(wrong) > 0 {
|
|
||||||
for _, p := range wrong {
|
|
||||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
|
||||||
}
|
|
||||||
failed += len(wrong)
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
// A module lives whole in one repository (novox/hq ADR 0420 §1, issue 1019): when the check is told the
|
|
||||||
// repository it runs for (MESH_CHECK_REPOSITORY, set by the build's check), an artifact building from
|
|
||||||
// another is refused, as the merge gate refuses it. A run by hand names no repository and judges none.
|
|
||||||
if repo := os.Getenv("MESH_CHECK_REPOSITORY"); repo != "" && m.Build != nil {
|
|
||||||
for _, a := range m.Build.Artifacts {
|
|
||||||
if a.Context != nil && a.Context.Repository != "" && !sameRepository(a.Context.Repository,
|
|
||||||
link.SourceMoved{Owner: ownerOf(repo), Repo: repoOf(repo)}) {
|
|
||||||
fmt.Fprintf(out, "%s: %s builds artifact %s from %s: a module lives whole in one repository "+
|
|
||||||
"(novox/hq ADR 0420 §1)\n", path, m.Module, a.Name, a.Context.Repository)
|
|
||||||
failed++
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
|
||||||
for _, p := range named {
|
|
||||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
|
||||||
}
|
|
||||||
failed += len(named)
|
|
||||||
faulted[m.Module] = true
|
|
||||||
}
|
|
||||||
shelf[m.Module] = m
|
|
||||||
}
|
|
||||||
|
|
||||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
|
||||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
|
||||||
// has already been said and would be said again here in a worse form.
|
|
||||||
//
|
|
||||||
// **Over some of a repository's manifests, a seat none of them declares is a note** (novox/hq issue 364), as
|
|
||||||
// this command's own word says above: the merge gate passes only the manifests a change touches, and says so
|
|
||||||
// with SomeManifestsVar, so a module that uses or claims a seat another module declares (the operator
|
|
||||||
// channel's, a channel bench) was refused there for a manifest it was not given. Given every manifest — the
|
|
||||||
// catalogue's own check, and registration — it stays a refusal.
|
|
||||||
some := os.Getenv(SomeManifestsVar) != ""
|
|
||||||
problems := catalogue.CatalogueProblems(shelf)
|
|
||||||
sort.Strings(problems)
|
|
||||||
for _, p := range problems {
|
|
||||||
if some && catalogue.IsUndeclaredSeat(p) {
|
|
||||||
fmt.Fprintf(out, "note: %s among the manifests given; registration judges it against the whole catalogue, "+
|
|
||||||
"and passing the declaring module's manifest too judges it here\n", p)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fmt.Fprintln(out, p)
|
|
||||||
failed++
|
|
||||||
}
|
|
||||||
|
|
||||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
|
||||||
// only the provider's manifest states.
|
|
||||||
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
|
||||||
sort.Strings(identities)
|
|
||||||
for _, p := range identities {
|
|
||||||
fmt.Fprintln(out, p)
|
|
||||||
}
|
|
||||||
failed += len(identities)
|
|
||||||
|
|
||||||
// And the data each module keeps (novox/hq ADR 0233): a provider that grants says what it keeps for
|
|
||||||
// its consumers, a directory a container writes is declared, and no backup line is written by hand.
|
|
||||||
data := catalogue.DataProblems(shelf)
|
|
||||||
sort.Strings(data)
|
|
||||||
for _, p := range data {
|
|
||||||
fmt.Fprintln(out, p)
|
|
||||||
}
|
|
||||||
failed += len(data)
|
|
||||||
|
|
||||||
var names []string
|
|
||||||
for name := range shelf {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
|
|
||||||
// **Every long-running resource says how it is ready** (novox/hq ADR 0240 rule 8): warned until the
|
|
||||||
// date, refused from it. The count is the catalogue's: its merge check keeps the number and lets a
|
|
||||||
// change lower it, never raise it.
|
|
||||||
undeclared := 0
|
|
||||||
required := !checkNow().Before(catalogue.HealthRequiredFrom)
|
|
||||||
for _, name := range names {
|
|
||||||
missing := catalogue.Undeclared(shelf[name])
|
|
||||||
undeclared += len(missing)
|
|
||||||
if len(missing) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if required {
|
|
||||||
for _, id := range missing {
|
|
||||||
fmt.Fprintf(out, "%s: %s stays up and does not say how it is ready: a long-running resource declares "+
|
|
||||||
"health since %s (novox/hq ADR 0240 rule 8)\n", name, id, catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
|
||||||
}
|
|
||||||
failed += len(missing)
|
|
||||||
faulted[name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A file that asks for a setting and does not say whether it is trusted counts as trusted** (novox/hq issue
|
|
||||||
// 339): listed and counted, never refused, so an author can opt out a file nothing trusts.
|
|
||||||
unsaid := 0
|
|
||||||
for _, name := range names {
|
|
||||||
unsaid += len(catalogue.UnsaidTrust(shelf[name]))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
m := shelf[name]
|
|
||||||
if faulted[name] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, "%s: ok", name)
|
|
||||||
if n := len(m.Tools); n > 0 {
|
|
||||||
fmt.Fprintf(out, ", %d tool(s)", n)
|
|
||||||
}
|
|
||||||
if len(m.Invokes) > 0 {
|
|
||||||
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
|
||||||
}
|
|
||||||
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
|
||||||
if len(m.State) > 0 {
|
|
||||||
kept := make([]string, 0, len(m.State))
|
|
||||||
for _, s := range m.State {
|
|
||||||
kept = append(kept, s.Name)
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
// And the logs it keeps, with their caps (novox/hq ADR 0297).
|
|
||||||
if len(m.Logs) > 0 {
|
|
||||||
kept := make([]string, 0, len(m.Logs))
|
|
||||||
for _, l := range m.Logs {
|
|
||||||
kept = append(kept, fmt.Sprintf("%s (%d MiB)", l.Name, l.Cap()))
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps log %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
if len(m.Reads) > 0 {
|
|
||||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
|
||||||
}
|
|
||||||
// The data it keeps, by class, so a reviewer sees what the mesh will protect and how.
|
|
||||||
if items := m.DataItems(); len(items) > 0 {
|
|
||||||
kept := make([]string, 0, len(items))
|
|
||||||
for _, it := range items {
|
|
||||||
kept = append(kept, it.ID+" ("+it.Class+")")
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, ", keeps %s", strings.Join(kept, ", "))
|
|
||||||
}
|
|
||||||
// How what it runs is ready (ADR 0240): each declared check, and what is judged by liveness alone.
|
|
||||||
var checks []string
|
|
||||||
for _, r := range m.Resources {
|
|
||||||
if h, has, _ := catalogue.ReadHealth(r); has {
|
|
||||||
checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// And its own runtime's, where it runs as its own account (novox/hq ADR 0366).
|
|
||||||
if own, has := catalogue.OwnRuntimeHealth(m); has {
|
|
||||||
if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" {
|
|
||||||
checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(checks) > 0 {
|
|
||||||
fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; "))
|
|
||||||
}
|
|
||||||
if missing := catalogue.UnsaidTrust(m); len(missing) > 0 {
|
|
||||||
fmt.Fprintf(out, "; WARNING: %s ask(s) for a setting and do(es) not say whether it is trusted, so it counts as "+
|
|
||||||
"trusted: set at the terminal alone; say %q false where nothing trusts it (novox/hq issue 339)",
|
|
||||||
strings.Join(missing, ", "), catalogue.TrustedField)
|
|
||||||
}
|
|
||||||
if missing := catalogue.Undeclared(m); len(missing) > 0 {
|
|
||||||
fmt.Fprintf(out, "; WARNING: %s stay(s) up and say(s) not how it is ready — judged by liveness alone, "+
|
|
||||||
"refused from %s (ADR 0240 rule 8)", strings.Join(missing, ", "), catalogue.HealthRequiredFrom.Format("2006-01-02"))
|
|
||||||
}
|
|
||||||
fmt.Fprintln(out)
|
|
||||||
}
|
|
||||||
// The count the catalogue keeps (ADR 0240 rule 8), in a line its merge check reads.
|
|
||||||
fmt.Fprintf(out, "%s %d\n", UndeclaredHealthLine, undeclared)
|
|
||||||
fmt.Fprintf(out, "%s %d\n", UnsaidTrustLine, unsaid)
|
|
||||||
if failed > 0 {
|
|
||||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
|
||||||
}
|
|
||||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
|
||||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
|
||||||
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
|
||||||
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// UndeclaredHealthLine starts the line `module check` says the count of long-running resources without
|
|
||||||
// `health` in, over the manifests given: the catalogue's merge check compares it with the number it keeps.
|
|
||||||
const UndeclaredHealthLine = "long-running resources without health:"
|
|
||||||
|
|
||||||
// UnsaidTrustLine starts the line `module check` says the count of files that ask for a setting and do not say
|
|
||||||
// whether it is trusted, and so count as trusted (novox/hq issue 339).
|
|
||||||
const UnsaidTrustLine = "files asking for a setting without saying whether it is trusted:"
|
|
||||||
|
|
||||||
// checkNow is the clock `module check` judges the date by; a test sets it.
|
|
||||||
var checkNow = time.Now
|
|
||||||
|
|
||||||
func joinInvokes(invokes []string) string {
|
|
||||||
if len(invokes) == 1 && invokes[0] == "*" {
|
|
||||||
return "every tool"
|
|
||||||
}
|
|
||||||
s := ""
|
|
||||||
for i, t := range invokes {
|
|
||||||
if i > 0 {
|
|
||||||
s += ", "
|
|
||||||
}
|
|
||||||
s += t
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// manifestsUnder lists every module.json below a directory, for `module check <dir>`.
|
|
||||||
func manifestsUnder(dir string) ([]string, error) {
|
|
||||||
var found []string
|
|
||||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git" || d.Name() == "dist") {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
if !d.IsDir() && d.Name() == "module.json" {
|
|
||||||
found = append(found, path)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
sort.Strings(found)
|
|
||||||
return found, err
|
|
||||||
}
|
|
||||||
@@ -1,161 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"github.com/novox/mesh-controller/internal/beside"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
|
||||||
// with a known fault is named, and one without passes, with no store opened.
|
|
||||||
func TestModuleCheckNamesAFaultAndNeedsNoMesh(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
good := filepath.Join(dir, "good.json")
|
|
||||||
bad := filepath.Join(dir, "bad.json")
|
|
||||||
os.WriteFile(good, []byte(`{"module":"shop","version":"1","tools":["price"],"invokes":["mesh-catalog.catalog_modules"]}`), 0o600)
|
|
||||||
os.WriteFile(bad, []byte(`{"module":"till","version":"1","invokes":["shop"]}`), 0o600)
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{good}, &out); err != nil {
|
|
||||||
t.Fatalf("a sound manifest was refused: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "shop: ok, 1 tool(s), invokes mesh-catalog.catalog_modules") {
|
|
||||||
t.Fatalf("the report does not say what it checked:\n%s", out.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
out.Reset()
|
|
||||||
err := moduleCheck([]string{good, bad}, &out)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("a manifest invoking a module and no tool passed")
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), `till invokes "shop", which does not name a tool`) {
|
|
||||||
t.Fatalf("the fault is not named in the manifest's words:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The rules between manifests run over what was given together: a seat two modules declare is
|
|
||||||
// refused, which no single-manifest check can see.
|
|
||||||
func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
a := filepath.Join(dir, "a.json")
|
|
||||||
b := filepath.Join(dir, "b.json")
|
|
||||||
os.WriteFile(a, []byte(`{"module":"a","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
|
||||||
os.WriteFile(b, []byte(`{"module":"b","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{a, b}, &out); err == nil {
|
|
||||||
t.Fatalf("two declarations of one seat passed:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "a seat name means one protocol") {
|
|
||||||
t.Fatalf("the cross-manifest rule was not the one named:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
|
||||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
|
||||||
root := beside.Catalogue(t)
|
|
||||||
paths, err := manifestsUnder(root)
|
|
||||||
if err != nil || len(paths) == 0 {
|
|
||||||
t.Fatalf("no manifests under %s: %v", root, err)
|
|
||||||
}
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck(paths, &out); err != nil {
|
|
||||||
t.Fatalf("the catalogue does not pass its own check: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
|
|
||||||
// novox/hq ADR 0155: the check moves to registration once the catalogue passes it. Both
|
|
||||||
// ways in — `module add` and a build's result — go through this, and a name declared on
|
|
||||||
// purpose passes with its reason.
|
|
||||||
named := catalogue.Manifest{Module: "idp", Resources: []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "image": "x@sha256:aa",
|
|
||||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}},
|
|
||||||
}}
|
|
||||||
err := namesNoInstallation(named)
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "login.mesh-one.be") ||
|
|
||||||
!strings.Contains(err.Error(), catalogue.NamesOnPurpose) {
|
|
||||||
t.Fatalf("a definition naming an installation is refused with the name and the way out; got %v", err)
|
|
||||||
}
|
|
||||||
meant := catalogue.Manifest{Module: "site", Resources: []map[string]any{
|
|
||||||
{"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc",
|
|
||||||
catalogue.NamesOnPurpose: map[string]any{
|
|
||||||
"registry.mesh-one.be": "built outside the mesh until its repository is a build source here"}},
|
|
||||||
}}
|
|
||||||
if err := namesNoInstallation(meant); err != nil {
|
|
||||||
t.Fatalf("a name declared on purpose passes; got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The controller's own manifest names every verb of its seat** (novox/hq ADR 0132): its tools lagged
|
|
||||||
// the seat's verbs for weeks, and `module check` — the gate's first step for a change touching it —
|
|
||||||
// refused it. Held here, so a verb added to the table without the manifest fails this repository's
|
|
||||||
// own suite rather than its next pull request's gate.
|
|
||||||
func TestTheControllersManifestServesEveryVerbOfItsSeat(t *testing.T) {
|
|
||||||
var out strings.Builder
|
|
||||||
if err := moduleCheck([]string{"../../module.json"}, &out); err != nil {
|
|
||||||
t.Fatalf("the controller's own module.json fails module check: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A module that uses a seat another module declares (novox/hq issue 364): refused over the whole catalogue when the
|
|
||||||
// declarer is missing, a note when the gate says it gives only the manifests a change touches.
|
|
||||||
func TestASeatAnotherModuleDeclaresIsANoteOverSomeManifests(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
user := filepath.Join(dir, "user.json")
|
|
||||||
os.WriteFile(user, []byte(`{"module":"asker","version":"1","uses":["operator-channel"]}`), 0o600)
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{user}, &out); err == nil {
|
|
||||||
t.Fatalf("a use of a seat nothing given declares passed the whole-catalogue check:\n%s", out.String())
|
|
||||||
}
|
|
||||||
t.Setenv(SomeManifestsVar, "1")
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{user}, &out); err != nil {
|
|
||||||
t.Fatalf("over some manifests the use was refused:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "note: asker uses the seat \"operator-channel\"") {
|
|
||||||
t.Fatalf("the note was not said:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): `module check`, told the
|
|
||||||
// repository it runs for, refuses an artifact whose build context names another; run by hand, it does not know.
|
|
||||||
func TestModuleCheckRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
write := func(name, context string) string {
|
|
||||||
path := filepath.Join(dir, name+".json")
|
|
||||||
raw := `{"module":"` + name + `","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"` + context + `}]}}`
|
|
||||||
if err := os.WriteFile(path, []byte(raw), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
other := write("other", `,"context":{"seat":"git","repository":"novox/mesh-controller","ref":"main"}`)
|
|
||||||
own := write("own", `,"context":{"seat":"git","repository":"novox/mesh-catalog","ref":"main"}`)
|
|
||||||
none := write("none", ``)
|
|
||||||
|
|
||||||
t.Setenv("MESH_CHECK_REPOSITORY", "novox/mesh-catalog")
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{other}, &out); err == nil {
|
|
||||||
t.Fatalf("a context naming another repository passed:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "other builds artifact server from novox/mesh-controller: a module lives whole in one repository (novox/hq ADR 0420 §1)") {
|
|
||||||
t.Errorf("the refusal is not in the words of the gate:\n%s", out.String())
|
|
||||||
}
|
|
||||||
for _, path := range []string{own, none} {
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
|
||||||
t.Errorf("%s was refused: %v\n%s", path, err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Setenv("MESH_CHECK_REPOSITORY", "")
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{other}, &out); err != nil {
|
|
||||||
t.Errorf("a local run, with no repository named, refused it: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -217,6 +217,11 @@ func TestTheAdaptersInputCarriesTheOwnSecretsTheBoxesTheLicencesAndTheBus(t *tes
|
|||||||
Module: "objects"}, "a-password"); err != nil {
|
Module: "objects"}, "a-password"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// Minted and sealed to its node: an account counts as issued, and is named among the minted, only once sealed
|
||||||
|
// (ADR 0543 §2, PR #467).
|
||||||
|
if err := inv.MarkBusAccountSealed(ctx, "anchor.objects"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if err := inv.PutBusMembership(ctx, "laptop", "sealed-membership"); err != nil {
|
if err := inv.PutBusMembership(ctx, "laptop", "sealed-membership"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
// configForm is a module in the config form, shaped as a real one (inventory's config_move_test).
|
// configForm is a module in the config form, shaped as a real one (inventory's config_move_test).
|
||||||
func configForm(t *testing.T, name string) catalogue.Manifest {
|
func configForm(t *testing.T, name string) catalogue.Manifest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
m, err := catalogue.ParseManifest([]byte(`{"module": "` + name + `", "version": "1",
|
m, err := catalogue.Decode([]byte(`{"module": "` + name + `", "version": "1",
|
||||||
"config": {"level": {"type": "string", "description": "how much it says", "default": "info"}},
|
"config": {"level": {"type": "string", "description": "how much it says", "default": "info"}},
|
||||||
"resources": [{"id": "conf", "type": "file", "path": "/etc/` + name + `.json", "mode": "0640",
|
"resources": [{"id": "conf", "type": "file", "path": "/etc/` + name + `.json", "mode": "0640",
|
||||||
"content": "{\"level\": \"info\"}", "merge": "json", "takes-config": ["level"]}]}`))
|
"content": "{\"level\": \"info\"}", "merge": "json", "takes-config": ["level"]}]}`))
|
||||||
|
|||||||
@@ -86,8 +86,8 @@ func perform(ctx context.Context, open *stores, in *meshInput, node string, d co
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The node's own secrets: a module's superuser password, an administrator, an account. The composition refused what the
|
// The node's own secrets: a module's superuser password, an administrator, an account. A bus credential nobody issued is
|
||||||
// send refuses (a bus credential nobody issued, a given value under an old key).
|
// held out of the composition before this (holdBackUnissuedBusAccounts), a given value under an old key by the composition.
|
||||||
for _, one := range d.StoodIn {
|
for _, one := range d.StoodIn {
|
||||||
module, name, ok := strings.Cut(one, "/")
|
module, name, ok := strings.Cut(one, "/")
|
||||||
if !ok || strings.Contains(one, " ") {
|
if !ok || strings.Contains(one, " ") {
|
||||||
|
|||||||
@@ -184,8 +184,10 @@ func composeQuestion(ctx context.Context, open *stores, node string, mode compos
|
|||||||
if err := refusedBus(in, node); err != nil {
|
if err := refusedBus(in, node); err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
}
|
}
|
||||||
|
held := holdBackUnissuedBusAccounts(in, node, &d)
|
||||||
sayFound(ctx, d)
|
sayFound(ctx, d)
|
||||||
out := sendableOf(d)
|
out := sendableOf(d)
|
||||||
|
out.heldForCredential = held
|
||||||
out.fileReferences = readsFileReferences(in.Input, node)
|
out.fileReferences = readsFileReferences(in.Input, node)
|
||||||
if !withOrder {
|
if !withOrder {
|
||||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||||
@@ -212,6 +214,82 @@ func refusedBus(in *meshInput, node string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// holdBackUnissuedBusAccounts holds out of d every module that stands in for a bus credential nobody issued: the module's
|
||||||
|
// `broker` own secret (its account `<node>.<module>`) or the provisioner client's (`provisioner.<node>`). A bus credential is
|
||||||
|
// minted, never invented: a send that made one would seal a placeholder its process cannot read (novox/hq issue 203).
|
||||||
|
// mesh-sdk's composition stands one in, in the Foreseeing mode, since a node's answer to a proposal is asked before the vault
|
||||||
|
// mints it (mesh-sdk #141, issue 774).
|
||||||
|
//
|
||||||
|
// **That one module is held, never the node.** Its resources, its stand-ins and its build leave this composition, it is named
|
||||||
|
// among the left out with issue 203's words (said where the composition's other lines go), and the rest of the node composes:
|
||||||
|
// a refusal of the whole node stopped every take-in on the machine that runs the bus for one account (2026-11-01, novox's
|
||||||
|
// mesh-hand-acts, which the vault had in fact minted; the controller cannot read the vault's accounts yet, so such a module
|
||||||
|
// stays held until its account is in the controller's store). The modules held are answered, sorted.
|
||||||
|
func holdBackUnissuedBusAccounts(in *meshInput, node string, d *compose.Declared) []string {
|
||||||
|
why := map[string]string{}
|
||||||
|
for _, one := range d.StoodIn {
|
||||||
|
module, name, ok := strings.Cut(one, "/")
|
||||||
|
if !ok || strings.Contains(one, " ") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var user string
|
||||||
|
switch name {
|
||||||
|
case "broker":
|
||||||
|
user = node + "." + module
|
||||||
|
case compose.ProvisionerSecret:
|
||||||
|
user = "provisioner." + node
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, said := why[module]; said || slices.Contains(in.Input.BusMinted, user) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
why[module] = fmt.Sprintf("it has no bus credential: nothing was issued for %s, and a push would seal a placeholder "+
|
||||||
|
"its process cannot read (novox/hq issue 203). `module issue %s --node %s`, then push again", user, module, node)
|
||||||
|
}
|
||||||
|
if len(why) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
held := make([]string, 0, len(why))
|
||||||
|
for module := range why {
|
||||||
|
held = append(held, module)
|
||||||
|
}
|
||||||
|
sort.Strings(held)
|
||||||
|
ofHeld := func(s string) bool {
|
||||||
|
module, _, _ := strings.Cut(s, "/")
|
||||||
|
_, out := why[module]
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
kept := d.Resources[:0:0]
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
module := d.ModuleOf[fmt.Sprint(r["id"])]
|
||||||
|
if module == "" {
|
||||||
|
module, _ = r["module"].(string)
|
||||||
|
}
|
||||||
|
if _, out := why[module]; !out {
|
||||||
|
kept = append(kept, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.Resources = kept
|
||||||
|
d.StoodIn = slices.DeleteFunc(d.StoodIn, ofHeld)
|
||||||
|
d.Foreseen = slices.DeleteFunc(d.Foreseen, ofHeld)
|
||||||
|
d.Modules = slices.DeleteFunc(d.Modules, func(m string) bool { _, out := why[m]; return out })
|
||||||
|
if d.LeftOutWhy == nil {
|
||||||
|
d.LeftOutWhy = map[string]string{}
|
||||||
|
}
|
||||||
|
for _, module := range held {
|
||||||
|
delete(d.BuildIDs, module)
|
||||||
|
d.LeftOutWhy[module] = why[module]
|
||||||
|
if !slices.Contains(d.LeftOut, module) {
|
||||||
|
d.LeftOut = append(d.LeftOut, module)
|
||||||
|
}
|
||||||
|
d.Said = append(d.Said, fmt.Sprintf("%s on %s is held out of %s's composition, and the rest of %s composes: %s",
|
||||||
|
module, node, node, node, why[module]))
|
||||||
|
}
|
||||||
|
sort.Strings(d.LeftOut)
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
|
||||||
// markUnresolvable is an error of a node whose own set does not compose, as planFor marked it.
|
// markUnresolvable is an error of a node whose own set does not compose, as planFor marked it.
|
||||||
func markUnresolvable(err error) error {
|
func markUnresolvable(err error) error {
|
||||||
if compose.IsUnresolvable(err) {
|
if compose.IsUnresolvable(err) {
|
||||||
@@ -256,6 +334,9 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
|
|||||||
if err := refusedBus(in, node); err != nil {
|
if err := refusedBus(in, node); err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
}
|
}
|
||||||
|
// Before perform makes what the composition stands in for: a bus credential is never one of them. The module
|
||||||
|
// whose account nobody issued is held out of this composition, and the rest of the node composes.
|
||||||
|
held := holdBackUnissuedBusAccounts(in, node, &d)
|
||||||
if len(byController) > 0 {
|
if len(byController) > 0 {
|
||||||
if d, problems, err = sealedByTheController(in, node, d, byController); err != nil {
|
if d, problems, err = sealedByTheController(in, node, d, byController); err != nil {
|
||||||
return sendable{}, nil, err
|
return sendable{}, nil, err
|
||||||
@@ -272,6 +353,7 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
|
|||||||
if len(left) == 0 && len(in.wouldMake) == 0 {
|
if len(left) == 0 && len(in.wouldMake) == 0 {
|
||||||
sayFound(ctx, d)
|
sayFound(ctx, d)
|
||||||
out := sendableOf(d)
|
out := sendableOf(d)
|
||||||
|
out.heldForCredential = held
|
||||||
out.fileReferences = readsFileReferences(in.Input, node)
|
out.fileReferences = readsFileReferences(in.Input, node)
|
||||||
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
|
// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
|
||||||
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
out.Sequence, out.Epoch, out.Generation = 0, 0, 0
|
||||||
@@ -326,9 +408,13 @@ func restOfTheMesh(ctx context.Context, open *stores, shelf map[string]catalogue
|
|||||||
return world, nil
|
return world, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// restOfTheMeshOf is restOfTheMesh for a caller that holds the inventory alone: the other stores the Input reads are opened for
|
// restOfTheMeshOf is restOfTheMesh for a caller that holds the inventory alone: on the serving controller's stores where it
|
||||||
// the call and let go of again.
|
// lends them (sharedStores), never a pool per call; elsewhere (a verb run as its own process) the other stores the Input
|
||||||
|
// reads are opened for the call and let go of again.
|
||||||
func restOfTheMeshOf(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
|
func restOfTheMeshOf(ctx context.Context, inv *inventory.Inventory, shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
|
||||||
|
if shared, ok := sharedStoresOf(inv); ok {
|
||||||
|
return restOfTheMeshOn(ctx, shared, shelf, exclude)
|
||||||
|
}
|
||||||
open := &stores{inventory: inv}
|
open := &stores{inventory: inv}
|
||||||
defer func() {
|
defer func() {
|
||||||
if open.licences != nil {
|
if open.licences != nil {
|
||||||
@@ -338,9 +424,12 @@ func restOfTheMeshOf(ctx context.Context, inv *inventory.Inventory, shelf map[st
|
|||||||
open.identity.Close()
|
open.identity.Close()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
return restOfTheMesh(ctx, open, shelf, exclude)
|
return restOfTheMeshOn(ctx, open, shelf, exclude)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// restOfTheMeshOn is restOfTheMesh; a variable so a test reads which stores it is handed.
|
||||||
|
var restOfTheMeshOn = restOfTheMesh
|
||||||
|
|
||||||
// sayFound says the lines a composition found while composing, once each: a machine left out of the rest of the mesh, a seat
|
// sayFound says the lines a composition found while composing, once each: a machine left out of the rest of the mesh, a seat
|
||||||
// that two claimants could take, the bus's users left out for want of a credential.
|
// that two claimants could take, the bus's users left out for want of a credential.
|
||||||
func sayFound(ctx context.Context, d compose.Declared) {
|
func sayFound(ctx context.Context, d compose.Declared) {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A declaration as the SDK composes it is the one the controller sends, byte for byte, whatever it carries: the body is the
|
// A declaration as the SDK composes it is the one the controller sends, byte for byte, whatever it carries: the body is the
|
||||||
@@ -85,3 +86,73 @@ func TestADeclarationThatStandsInForAValueIsStillForeseen(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 2026-11-01 (issue 203's refusal after the 1585 group): a module standing in for a bus account nobody issued is held out of
|
||||||
|
// the composition, alone; the rest of the node composes, with no stand-in left. An issued account holds nothing. And a held
|
||||||
|
// module with no fingerprint of the new form keeps the node's own composition as last written.
|
||||||
|
func TestAnUnissuedBusAccountHoldsItsModuleAloneAndNeverTheNode(t *testing.T) {
|
||||||
|
declared := func() compose.Declared {
|
||||||
|
return compose.Declared{
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "hands.broker", "type": "file", "sealed": catalogue.ForeseenSealed},
|
||||||
|
{"id": "hands.unit", "type": "service"},
|
||||||
|
{"id": "client.broker", "type": "file", "sealed": catalogue.ForeseenSealed},
|
||||||
|
{"id": "web.config", "type": "file", "content": "ok"},
|
||||||
|
},
|
||||||
|
ModuleOf: map[string]string{"hands.broker": "hands", "hands.unit": "hands", "client.broker": "client",
|
||||||
|
"web.config": "web"},
|
||||||
|
BuildIDs: map[string]string{"hands": "b-1", "client": "b-2", "web": "b-3"},
|
||||||
|
Modules: []string{"client", "hands", "web"},
|
||||||
|
StoodIn: []string{"hands/broker", "client/" + compose.ProvisionerSecret},
|
||||||
|
Foreseen: []string{"hands/broker", "client/" + compose.ProvisionerSecret},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
minted []string
|
||||||
|
held []string
|
||||||
|
}{
|
||||||
|
"issued by nobody": {held: []string{"client", "hands"}},
|
||||||
|
"the module's account issued": {minted: []string{"anchor.hands"}, held: []string{"client"}},
|
||||||
|
"both issued": {minted: []string{"anchor.hands", "provisioner.anchor"}},
|
||||||
|
} {
|
||||||
|
in := &meshInput{Input: compose.Input{BusMinted: c.minted}}
|
||||||
|
d := declared()
|
||||||
|
held := holdBackUnissuedBusAccounts(in, "anchor", &d)
|
||||||
|
if !reflect.DeepEqual(held, c.held) {
|
||||||
|
t.Errorf("%s: held %v, want %v", name, held, c.held)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(c.held) < 2 {
|
||||||
|
continue // an issued account's stand-in is the send's to make (perform), not this one's
|
||||||
|
}
|
||||||
|
if left := stillForeseen(d); len(left) != 0 {
|
||||||
|
t.Errorf("%s: the rest of the node still stands in for %v", name, left)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(d.Modules, []string{"web"}) || len(d.Resources) != 1 || d.BuildIDs["hands"] != "" {
|
||||||
|
t.Errorf("%s: the rest of the node is not what composes: %v %v %v", name, d.Modules, d.Resources, d.BuildIDs)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(d.LeftOut, []string{"client", "hands"}) ||
|
||||||
|
!strings.Contains(d.LeftOutWhy["hands"], "module issue hands --node anchor") ||
|
||||||
|
!strings.Contains(d.LeftOutWhy["client"], "provisioner.anchor") || len(d.Said) != 2 {
|
||||||
|
t.Errorf("%s: the held modules are not said in issue 203's words: %v %v %v", name, d.LeftOut, d.LeftOutWhy, d.Said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
own, err := sdkdelivery.ComposedNodeKey("anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
fps map[string]string
|
||||||
|
keepOwn bool
|
||||||
|
}{
|
||||||
|
"a held module of the new form": {fps: map[string]string{"hands": sdkdelivery.SourcePrefix2 + "x"}, keepOwn: true},
|
||||||
|
"a held module of the former form": {fps: map[string]string{}},
|
||||||
|
} {
|
||||||
|
comps := map[string]sdkdelivery.Composed{own: {Node: "anchor"}}
|
||||||
|
said := withoutNodeOwnForHeld("anchor", comps, c.fps, []string{"hands"})
|
||||||
|
if _, kept := comps[own]; kept != c.keepOwn || (said == "") != c.keepOwn {
|
||||||
|
t.Errorf("%s: the node's own composition kept %v (want %v), said %q", name, kept, c.keepOwn, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||||
"github.com/novox/mesh-controller/internal/identity"
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
@@ -76,13 +77,18 @@ func aComposingMesh(t *testing.T) *composingMesh {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// trunkKeys are a done key and the indexer's trunk value naming it.
|
// trunkKeys are a done key and the indexer's trunk value naming it. The indexer holds that value at the trunk's commit
|
||||||
|
// alone, as the real one does: answering it for any commit made a module registered at an older commit look already
|
||||||
|
// registered at the trunk's fingerprint, so the take-in only moved its commit and recorded no build (issue 1945).
|
||||||
type trunkKeys struct {
|
type trunkKeys struct {
|
||||||
keyBuckets
|
keyBuckets
|
||||||
trunk sdkdelivery.Module
|
trunk sdkdelivery.Module
|
||||||
}
|
}
|
||||||
|
|
||||||
func (k trunkKeys) ModuleAt(context.Context, string, string) (sdkdelivery.Module, error) {
|
func (k trunkKeys) ModuleAt(_ context.Context, _ string, commit string) (sdkdelivery.Module, error) {
|
||||||
|
if commit != "" && !sameCommitID(commit, k.trunk.Commit) {
|
||||||
|
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
||||||
|
}
|
||||||
return k.trunk, nil
|
return k.trunk, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,8 +108,10 @@ func TestTheDeclarationsStreamGoesOnceEveryNodeRunsTheNewEngine(t *testing.T) {
|
|||||||
// The new node-engine registered at its fingerprint, from its done key.
|
// The new node-engine registered at its fingerprint, from its done key.
|
||||||
n6 := fpOf('6')
|
n6 := fpOf('6')
|
||||||
keys, trunk := aKeyFor("mesh-host", "6", n6, "novox/mesh-host", "", "f00d0123456789ab")
|
keys, trunk := aKeyFor("mesh-host", "6", n6, "novox/mesh-host", "", "f00d0123456789ab")
|
||||||
if _, err := liveDoneKeySteps(m.open, trunkKeys{keyBuckets: keys, trunk: trunk}).take(ctx, "mesh-host"); err != nil {
|
settled, _ := liveTakeInSteps(m.open, trunkKeys{keyBuckets: keys, trunk: trunk}, m.js).batch(ctx,
|
||||||
t.Fatal(err)
|
[]delivered{{module: "mesh-host", again: true}}, map[string]string{})
|
||||||
|
if len(settled) != 1 || settled[0].err != nil || settled[0].verdict != acknowledged {
|
||||||
|
t.Fatalf("the take-in of mesh-host: %+v", settled)
|
||||||
}
|
}
|
||||||
if _, err := m.js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: sdkdelivery.HaveBucket, History: sdkdelivery.HaveHistory}); err != nil {
|
if _, err := m.js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: sdkdelivery.HaveBucket, History: sdkdelivery.HaveHistory}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -140,4 +148,35 @@ func TestTheDeclarationsStreamGoesOnceEveryNodeRunsTheNewEngine(t *testing.T) {
|
|||||||
if !declarationsRetiredNow(ctx, m.open.inventory) {
|
if !declarationsRetiredNow(ctx, m.open.inventory) {
|
||||||
t.Errorf("a raise after the retirement would assert the stream again")
|
t.Errorf("a raise after the retirement would assert the stream again")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 1945: a node-engine other than the registered one landing on laptop (a newer build on its way) leaves the
|
||||||
|
// declarations retired, so what the health check, the probes and healer H3 expect of the bus names neither the
|
||||||
|
// stream nor a node's consumer on it, and asserting the bus's objects again does not make the stream anew.
|
||||||
|
have("laptop", fpOf('7'))
|
||||||
|
if !declarationsRetiredNow(ctx, m.open.inventory) {
|
||||||
|
t.Fatalf("with laptop on another node-engine after the stream went, the declarations are no longer retired")
|
||||||
|
}
|
||||||
|
streams, consumers, err := expectedBusObjects(ctx, m.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range streams {
|
||||||
|
if s.Name == broker.DeclarationsStream {
|
||||||
|
t.Errorf("the bus's health check and probe D7 still expect the stream %s", s.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range consumers {
|
||||||
|
if c.Stream == broker.DeclarationsStream {
|
||||||
|
t.Errorf("the bus's health check and probe D6 still expect %s on %s", c.Name, c.Stream)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bus, err := broker.Dial(m.conn.ConnectedUrl())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(bus.Close)
|
||||||
|
_ = assertOnSend(ctx, m.open.inventory, bus, " ")
|
||||||
|
if _, err := m.js.Stream(ctx, broker.DeclarationsStream); !errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
t.Errorf("healer H3's assertion of the bus's objects made the stream anew: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -12,205 +14,285 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||||
"github.com/novox/mesh-controller/internal/conditions"
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A new fingerprint of a registered module, from its done key to its compositions (novox/hq ADR 0539 clauses 5 and 7,
|
// A new fingerprint of a registered module, from its done key to its compositions (novox/hq ADR 0539 clauses 5 and 7,
|
||||||
// ADR 0501 §2). With no delivery plan left to take a module's done key in, a write of the builder's or the indexer's
|
// ADR 0501 §2, ADR 0544). With no delivery plan left to take a module's done key in, a write of the builder's or the
|
||||||
// keys does it: the controller takes in the done key of the fingerprint the indexer's trunk value names (registers the
|
// indexer's keys does it: the controller takes in the done key of the fingerprint the indexer's trunk value names
|
||||||
// module at it, as a plan's take-in did), composes every node the module is assigned to (which makes what a module's
|
// (registers the module at it, as a plan's take-in did), composes every node the module is assigned to (which makes what
|
||||||
// new manifest needs, a credential among it, as a send made it), composes the node holding the bus (its user list now
|
// a module's new manifest needs, a credential among it), composes the node holding the bus (its user list now carries
|
||||||
// carries the module's permissions at the new fingerprint: a new revision of the bus module's composition alone,
|
// the module's permissions at the new fingerprint), and writes `grants.<module>.<fp>`. The assigner's sixth and seventh
|
||||||
// written into its want at once by the assigner), and writes `grants.<module>.<fp>`. The assigner's sixth and seventh
|
// rules then let a want name the fingerprint.
|
||||||
// rules then let a want name the fingerprint. **No timer**: woken by the keys, and a starting controller looks at every
|
//
|
||||||
// module once (doneKeysAtStart).
|
// **A durable pull consumer, not a watch** (ADR 0544; research 072, milestones 2 and 4). The writes are read from two
|
||||||
|
// durable consumers (take_in_consumer.go): the bus keeps what is not taken in yet, so a restart takes in only what the
|
||||||
|
// controller before it left unacknowledged, and the start's look is the consumers' own catch-up. A batch of keys asks the
|
||||||
|
// composer twice whatever its size, and a key with nothing new composes nothing.
|
||||||
|
|
||||||
// doneKeySteps are what taking a module's done key in does, each a step a test may stand in for.
|
// takeInSteps are what taking a batch of done keys in does, each a step a test may stand in for.
|
||||||
type doneKeySteps struct {
|
type takeInSteps struct {
|
||||||
// held says the catalogue holds the module (a new one is the new-module take-in's, new_modules.go).
|
// catalogue is every module the catalogue holds, with the nodes it is assigned to: read once a batch. A module it
|
||||||
held func(ctx context.Context, module string) (bool, error)
|
// does not hold is the new-module take-in's (new_modules.go).
|
||||||
|
catalogue func(ctx context.Context) (map[string][]string, error)
|
||||||
// trunk is the indexer's trunk value of the module.
|
// trunk is the indexer's trunk value of the module.
|
||||||
trunk func(ctx context.Context, module string) (sdkdelivery.Module, error)
|
trunk func(ctx context.Context, module string) (sdkdelivery.Module, error)
|
||||||
// registeredAt says the module is registered at the commit.
|
// registered says whether the module is registered at the trunk's commit, and whether at its fingerprint: a commit
|
||||||
registeredAt func(ctx context.Context, module, commit string) (bool, error)
|
// that moves while the fingerprint stays (a catalogue merge that leaves the module's sources alone) is no new build.
|
||||||
// takeIn registers the module from the builder's done key of the trunk's fingerprint (takeInKey); errKeyWaits when
|
registered func(ctx context.Context, module string, trunk sdkdelivery.Module) (atCommit, atFP bool, err error)
|
||||||
// the key is not done yet.
|
// recordCommit records the module's registration at the trunk's commit, its fingerprint unchanged: nothing composed.
|
||||||
|
recordCommit func(ctx context.Context, module, commit string) error
|
||||||
|
// takeIn registers the module from the builder's done key of the trunk's fingerprint (takeInKey).
|
||||||
takeIn func(ctx context.Context, module string, trunk sdkdelivery.Module) error
|
takeIn func(ctx context.Context, module string, trunk sdkdelivery.Module) error
|
||||||
// nodesOf are the nodes the module is assigned to; busHolder the node holding the bus, "" for none.
|
// taken says the module's compositions at fp exist for every node and its grant is written: nothing to compose.
|
||||||
nodesOf func(ctx context.Context, module string) ([]string, error)
|
taken func(ctx context.Context, module, fp string, nodes []string) (bool, error)
|
||||||
|
// busHolder is the node holding the bus, "" for none.
|
||||||
busHolder func(ctx context.Context) (string, error)
|
busHolder func(ctx context.Context) (string, error)
|
||||||
// compose writes the nodes' compositions; grants writes the grants issued.
|
// compose composes the nodes, and writes the grants after when grants: each node's error, and the grants'.
|
||||||
compose func(ctx context.Context, nodes []string) error
|
compose func(ctx context.Context, nodes []string, grants bool) (map[string]error, error, error)
|
||||||
grants func(ctx context.Context) error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// take takes module's done key in and composes what it changed. It answers what it did in a line, "" when there was
|
// delivered is one key a consumer delivered: its module, and whether it was delivered before (a take-in that may have
|
||||||
// nothing to do.
|
// stopped half-way, so it is taken in whole).
|
||||||
func (s doneKeySteps) take(ctx context.Context, module string) (string, error) {
|
type delivered struct {
|
||||||
held, err := s.held(ctx, module)
|
module string
|
||||||
if err != nil || !held {
|
again bool
|
||||||
return "", err
|
// deliveries is how often the bus delivered it, this time included: what a refusal's delay grows with.
|
||||||
}
|
deliveries uint64
|
||||||
trunk, err := s.trunk(ctx, module)
|
|
||||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(trunk.FP2, sdkdelivery.SourcePrefix2) {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
at, err := s.registeredAt(ctx, module, trunk.Commit)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
said := ""
|
|
||||||
if !at {
|
|
||||||
if err := s.takeIn(ctx, module, trunk); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
said = fmt.Sprintf("%s registered from the builder's done key %s (%s)", module, short(trunk.FP2), short(trunk.Commit))
|
|
||||||
}
|
|
||||||
nodes, err := s.nodesOf(ctx, module)
|
|
||||||
if err != nil {
|
|
||||||
return said, err
|
|
||||||
}
|
|
||||||
if len(nodes) == 0 {
|
|
||||||
return said, nil
|
|
||||||
}
|
|
||||||
// The module's nodes first: composing them makes what the new manifest needs, its credentials among it.
|
|
||||||
if err := s.compose(ctx, nodes); err != nil {
|
|
||||||
return said, err
|
|
||||||
}
|
|
||||||
// Then the bus's user list, which carries the module's permissions at its new fingerprint.
|
|
||||||
holder, err := s.busHolder(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return said, err
|
|
||||||
}
|
|
||||||
if holder != "" && !inList(nodes, holder) {
|
|
||||||
if err := s.compose(ctx, []string{holder}); err != nil {
|
|
||||||
return said, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Then the grant, which with the composition lets a want name the fingerprint (rules 6 and 7).
|
|
||||||
if err := s.grants(ctx); err != nil {
|
|
||||||
return said, err
|
|
||||||
}
|
|
||||||
if said == "" {
|
|
||||||
said = fmt.Sprintf("%s at %s", module, short(trunk.FP2))
|
|
||||||
}
|
|
||||||
return said + "; composed for " + strings.Join(nodes, ", ") + ", its grant written", nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func inList(list []string, s string) bool {
|
// verdict is what becomes of a module's delivered keys.
|
||||||
for _, v := range list {
|
type verdict int
|
||||||
if v == s {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// doneKeyTakeOnce is one take-in; a variable so a test may stand in for it. doneKeysSay says a line of the take-ins.
|
const (
|
||||||
var (
|
// acknowledged: taken in, nothing new, a wait a later write ends, or a build key that failed.
|
||||||
doneKeyTakeOnce = takeDoneKeyOnce
|
acknowledged verdict = iota
|
||||||
doneKeysSay = func(s string) { fmt.Println(s) }
|
// refused: delivered again by the bus after doneKeyRetry (a wait on a read, or a failure).
|
||||||
|
refused
|
||||||
)
|
)
|
||||||
|
|
||||||
// doneKeysBeforeLetGo is called just before a follower lets go of its module; a variable so a test can wake the module
|
// settledKey is what a batch did with one module.
|
||||||
// at that moment. Nil does nothing.
|
type settledKey struct {
|
||||||
var doneKeysBeforeLetGo func(module string)
|
module string
|
||||||
|
verdict verdict
|
||||||
// doneKeysLookAgain marks a module woken while a take-in of it ran: that take-in goes round once more.
|
// taken: registered, composed and granted now (not nothing new); its condition is cleared.
|
||||||
var doneKeysLookAgain sync.Map
|
taken bool
|
||||||
|
// nothingNew: registered and composed before, so nothing was done.
|
||||||
// doneKeysMu keeps one take-in of a module's done key at a time in this process, per module.
|
nothingNew bool
|
||||||
var doneKeysMu sync.Map
|
// waits is why it waits, "" when it does not.
|
||||||
|
waits string
|
||||||
// takeDoneKey is a woken module's take-in on the live stores and buckets, said. A take-in that fails (its compositions
|
// err is a failure: raised as the module's condition.
|
||||||
// or its grant not written) is tried again every doneKeyRetry while it fails, loudly: printed each time, and an urgent
|
err error
|
||||||
// condition raised by the first until it has worked, as the chain's own moves are (layBuildsOutSoon). A key not done
|
// said is the line said of it, "" for none.
|
||||||
// yet is no failure: it is said once for each reason it waits for and looked at again every doneKeyRetry.
|
said string
|
||||||
func takeDoneKey(ctx context.Context, open *stores, module string) {
|
|
||||||
mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
|
|
||||||
for {
|
|
||||||
if !mu.(*sync.Mutex).TryLock() {
|
|
||||||
// A take-in of the module already follows it: it is told to look again once it is done, so a newer key
|
|
||||||
// written meanwhile is never lost (the review of issue 1945).
|
|
||||||
doneKeysLookAgain.Store(module, true)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
followDoneKey(ctx, open, module)
|
|
||||||
if doneKeysBeforeLetGo != nil {
|
|
||||||
doneKeysBeforeLetGo(module)
|
|
||||||
}
|
|
||||||
mu.(*sync.Mutex).Unlock()
|
|
||||||
// A wake between this follower's last look and its letting go failed its TryLock and only marked the module:
|
|
||||||
// looked at again now that it is let go, and taken once more (the review of issue 1945).
|
|
||||||
if _, again := doneKeysLookAgain.LoadAndDelete(module); !again || ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// followDoneKey takes a module's done key in, under the module's lock, until it is taken in or ctx ends: a failure or a
|
// batch takes the delivered keys in, each module once: it answers one settledKey per module in the order first
|
||||||
// wait is tried again every doneKeyRetry.
|
// delivered, and the number of composer passes it asked. memo is, per module, the trunk (fingerprint and commit) this
|
||||||
func followDoneKey(ctx context.Context, open *stores, module string) {
|
// process took in last; a module at the same trunk delivered for the first time has nothing new.
|
||||||
failed := false
|
func (s takeInSteps) batch(ctx context.Context, keys []delivered, memo map[string]string) ([]settledKey, int) {
|
||||||
saidWait := ""
|
var order []string
|
||||||
for {
|
again := map[string]bool{}
|
||||||
doneKeysLookAgain.Delete(module) // what this look reads covers every wake before it
|
for _, k := range keys {
|
||||||
err := doneKeyTakeOnce(ctx, open, module)
|
if _, seen := again[k.module]; !seen {
|
||||||
o := doneKeyObservation(module)
|
order = append(order, k.module)
|
||||||
if err == nil {
|
|
||||||
if keeper := conditionsFrom; keeper != nil && failed {
|
|
||||||
_, _ = keeper.Clear(ctx, o.Key(), "composed and granted")
|
|
||||||
failed = false
|
|
||||||
}
|
|
||||||
if _, again := doneKeysLookAgain.LoadAndDelete(module); again {
|
|
||||||
continue // a wake came while this one ran: once more, for the key written meanwhile
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
if errors.Is(err, errKeyWaits) {
|
again[k.module] = again[k.module] || k.again
|
||||||
// A take-in that waits is said once for each reason, and tried again every doneKeyRetry: the key write that
|
}
|
||||||
// would wake it may never come (issue 1945, live on 2026-10-25: mesh-host's take-in waited unsaid).
|
out := make([]settledKey, 0, len(order))
|
||||||
if why := err.Error(); why != saidWait {
|
all := func(v verdict, err error) ([]settledKey, int) {
|
||||||
doneKeysSay(fmt.Sprintf("done keys: %s waits: %s; looked at again every %s", module,
|
for _, m := range order {
|
||||||
strings.TrimPrefix(why, errKeyWaits.Error()+": "), doneKeyRetry))
|
out = append(out, settledKey{module: m, verdict: v, err: err})
|
||||||
saidWait = why
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
doneKeysSay(fmt.Sprintf("done keys: %s: %v; tried again in %s", module, err, doneKeyRetry))
|
|
||||||
}
|
}
|
||||||
if errors.Is(err, errKeyWaits) {
|
return out, 0
|
||||||
select {
|
}
|
||||||
case <-ctx.Done():
|
held, err := s.catalogue(ctx)
|
||||||
return
|
if err != nil {
|
||||||
case <-time.After(doneKeyRetry):
|
return all(refused, fmt.Errorf("the catalogue cannot be read: %w", err))
|
||||||
}
|
}
|
||||||
|
type toCompose struct {
|
||||||
|
at int
|
||||||
|
nodes []string
|
||||||
|
mark string
|
||||||
|
said string
|
||||||
|
}
|
||||||
|
var need []toCompose
|
||||||
|
for _, m := range order {
|
||||||
|
r := settledKey{module: m}
|
||||||
|
nodes, isHeld := held[m]
|
||||||
|
if !isHeld {
|
||||||
|
out = append(out, r)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
trunk, err := s.trunk(ctx, m)
|
||||||
o.Summary += err.Error()
|
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
out = append(out, r)
|
||||||
failed = true
|
continue
|
||||||
|
}
|
||||||
|
if err == nil && !strings.HasPrefix(trunk.FP2, sdkdelivery.SourcePrefix2) {
|
||||||
|
// The indexer's next write names one: said once, as every wait for a write (ADR 0544 §4).
|
||||||
|
why := trunk.None2
|
||||||
|
if why == "" {
|
||||||
|
why = "the indexer wrote no fingerprint of the new form"
|
||||||
|
}
|
||||||
|
r.waits = fmt.Sprintf("%v: %s at %.8s has no fingerprint to take a build by: %s", errWaitsForAWrite, m,
|
||||||
|
trunk.Commit, why)
|
||||||
|
out = append(out, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
r.verdict, r.err = refused, err
|
||||||
|
out = append(out, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
mark := trunk.FP2 + "@" + trunk.Commit
|
||||||
|
if !again[m] && memo[m] == mark {
|
||||||
|
r.nothingNew = true
|
||||||
|
out = append(out, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at, atFP, err := s.registered(ctx, m, trunk)
|
||||||
|
if err == nil && !at && atFP {
|
||||||
|
// Only the commit moved (issue 1945: every catalogue merge took about 200 modules in again, 25 minutes): the
|
||||||
|
// new commit is recorded, and the module is judged as one registered, composed only if something is missing.
|
||||||
|
if err = s.recordCommit(ctx, m, trunk.Commit); err == nil {
|
||||||
|
at = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
select {
|
if err != nil {
|
||||||
case <-ctx.Done():
|
r.verdict, r.err = refused, err
|
||||||
return
|
out = append(out, r)
|
||||||
case <-time.After(doneKeyRetry):
|
continue
|
||||||
|
}
|
||||||
|
said := ""
|
||||||
|
if !at {
|
||||||
|
if err := s.takeIn(ctx, m, trunk); err != nil {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, errWaitsForAWrite):
|
||||||
|
r.waits = err.Error()
|
||||||
|
case errors.Is(err, errKeyWaits):
|
||||||
|
r.verdict, r.waits = refused, err.Error()
|
||||||
|
case errors.Is(err, errKeyFailed):
|
||||||
|
r.err = err // acknowledged: only a new build ends it, and its key is delivered then
|
||||||
|
default:
|
||||||
|
r.verdict, r.err = refused, err
|
||||||
|
}
|
||||||
|
out = append(out, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
said = fmt.Sprintf("%s registered from the builder's done key %s (%s)", m, short(trunk.FP2), short(trunk.Commit))
|
||||||
|
} else if !again[m] {
|
||||||
|
// Registered at its trunk already: composed and granted by a take-in before this one, or at this
|
||||||
|
// controller's start, unless that take-in stopped half-way, which leaves a composition or the grant missing.
|
||||||
|
if done, err := s.taken(ctx, m, trunk.FP2, nodes); err == nil && done {
|
||||||
|
memo[m] = mark
|
||||||
|
r.nothingNew = true
|
||||||
|
out = append(out, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
memo[m] = mark
|
||||||
|
r.taken, r.said = true, said
|
||||||
|
out = append(out, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if said == "" {
|
||||||
|
said = fmt.Sprintf("%s at %s", m, short(trunk.FP2))
|
||||||
|
}
|
||||||
|
need = append(need, toCompose{at: len(out), nodes: nodes, mark: mark, said: said})
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
if len(need) == 0 {
|
||||||
|
return out, 0
|
||||||
|
}
|
||||||
|
refuseNeed := func(err error) ([]settledKey, int) {
|
||||||
|
for _, n := range need {
|
||||||
|
out[n.at].verdict, out[n.at].err = refused, err
|
||||||
|
}
|
||||||
|
return out, 0
|
||||||
|
}
|
||||||
|
holder, err := s.busHolder(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return refuseNeed(fmt.Errorf("the node holding the bus cannot be read: %w", err))
|
||||||
|
}
|
||||||
|
// The modules' nodes first, in one pass: composing them makes what their new manifests need, credentials among it.
|
||||||
|
var union []string
|
||||||
|
for _, n := range need {
|
||||||
|
for _, node := range n.nodes {
|
||||||
|
if node != holder && !slices.Contains(union, node) {
|
||||||
|
union = append(union, node)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sort.Strings(union)
|
||||||
|
passes := 0
|
||||||
|
failed := map[string]error{}
|
||||||
|
if len(union) > 0 {
|
||||||
|
f, _, err := s.compose(ctx, union, false)
|
||||||
|
passes++
|
||||||
|
if err != nil {
|
||||||
|
return refuseNeed(err)
|
||||||
|
}
|
||||||
|
maps.Copy(failed, f)
|
||||||
|
}
|
||||||
|
// Then the bus's user list, which carries the modules' permissions at their new fingerprints, and the grants, which
|
||||||
|
// with the compositions let a want name each fingerprint (rules 6 and 7).
|
||||||
|
var bus []string
|
||||||
|
if holder != "" {
|
||||||
|
bus = []string{holder}
|
||||||
|
}
|
||||||
|
f, grantsErr, err := s.compose(ctx, bus, true)
|
||||||
|
passes++
|
||||||
|
if err != nil {
|
||||||
|
return refuseNeed(err)
|
||||||
|
}
|
||||||
|
maps.Copy(failed, f)
|
||||||
|
for _, n := range need {
|
||||||
|
var errs []error
|
||||||
|
for _, node := range append(slices.Clone(n.nodes), bus...) {
|
||||||
|
if e := failed[node]; e != nil && !slices.ContainsFunc(errs, func(x error) bool { return x == e }) {
|
||||||
|
errs = append(errs, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if grantsErr != nil {
|
||||||
|
errs = append(errs, grantsErr)
|
||||||
|
}
|
||||||
|
r := &out[n.at]
|
||||||
|
if len(errs) > 0 {
|
||||||
|
r.verdict, r.err = refused, errors.Join(errs...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
memo[r.module] = n.mark
|
||||||
|
r.taken = true
|
||||||
|
r.said = n.said + "; composed for " + strings.Join(n.nodes, ", ") + ", its grant written"
|
||||||
|
}
|
||||||
|
return out, passes
|
||||||
}
|
}
|
||||||
|
|
||||||
// doneKeyRetry is how long a failed take-in of a done key waits before it tries again.
|
// doneKeysSay says a line of the take-ins; a variable so a test may hear it.
|
||||||
|
var doneKeysSay = func(s string) { fmt.Println(s) }
|
||||||
|
|
||||||
|
// doneKeyRetry is how long the bus waits before it delivers again a key whose take-in was refused the first time; each
|
||||||
|
// refusal after doubles it, up to doneKeyRetryMost, so a key refused for long (a manifest word this controller does not
|
||||||
|
// read yet, a node whose composition keeps failing) costs a take-in every few minutes, not every half minute.
|
||||||
var doneKeyRetry = 30 * time.Second
|
var doneKeyRetry = 30 * time.Second
|
||||||
|
|
||||||
|
// doneKeyRetryMost is the longest a refused key waits before it is delivered again.
|
||||||
|
const doneKeyRetryMost = 10 * time.Minute
|
||||||
|
|
||||||
|
// refusalDelay is how long the bus waits before it delivers again a key refused on its deliveries-th delivery.
|
||||||
|
func refusalDelay(deliveries uint64) time.Duration {
|
||||||
|
d := doneKeyRetry
|
||||||
|
for i := uint64(1); i < deliveries && d < doneKeyRetryMost; i++ {
|
||||||
|
d *= 2
|
||||||
|
}
|
||||||
|
return min(d, doneKeyRetryMost)
|
||||||
|
}
|
||||||
|
|
||||||
// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
|
// kindNotComposed is the condition of a module's new fingerprint whose compositions or grant are not written: until
|
||||||
// they are, no want can name it.
|
// they are, no want can name it.
|
||||||
const kindNotComposed = "fingerprint-not-composed"
|
const kindNotComposed = "fingerprint-not-composed"
|
||||||
@@ -221,40 +303,37 @@ func doneKeyObservation(module string) conditions.Observation {
|
|||||||
Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
|
Summary: module + "'s new fingerprint is not composed or granted, so no want can name it: "}
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeDoneKeyOnce is one take-in of a module's done key.
|
// liveTakeInSteps are the steps on the controller's own stores and the bus.
|
||||||
func takeDoneKeyOnce(ctx context.Context, open *stores, module string) error {
|
func liveTakeInSteps(open *stores, keys registerKeys, js jetstream.JetStream) takeInSteps {
|
||||||
keys, closeKeys, err := openBuilderKeys()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the builder's keys cannot be read: %w", err)
|
|
||||||
}
|
|
||||||
defer closeKeys()
|
|
||||||
said, err := liveDoneKeySteps(open, keys).take(ctx, module)
|
|
||||||
if err == nil && said != "" {
|
|
||||||
fmt.Printf("done keys: %s\n", said)
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// liveDoneKeySteps are the steps on the controller's own stores and the bus.
|
|
||||||
func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
|
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
return doneKeySteps{
|
return takeInSteps{
|
||||||
held: func(ctx context.Context, module string) (bool, error) {
|
catalogue: func(ctx context.Context) (map[string][]string, error) {
|
||||||
_, err := inv.SourceOf(ctx, module)
|
entries, err := inv.Catalogued(ctx)
|
||||||
if errors.Is(err, inventory.ErrNoSuchModule) {
|
if err != nil {
|
||||||
return false, nil
|
return nil, err
|
||||||
}
|
}
|
||||||
return err == nil, err
|
out := make(map[string][]string, len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
out[e.Manifest.Module] = e.On
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
},
|
},
|
||||||
trunk: func(ctx context.Context, module string) (sdkdelivery.Module, error) {
|
trunk: func(ctx context.Context, module string) (sdkdelivery.Module, error) {
|
||||||
return keys.ModuleAt(ctx, module, "")
|
return keys.ModuleAt(ctx, module, "")
|
||||||
},
|
},
|
||||||
registeredAt: func(ctx context.Context, module, commit string) (bool, error) {
|
registered: func(ctx context.Context, module string, trunk sdkdelivery.Module) (bool, bool, error) {
|
||||||
src, err := inv.SourceOf(ctx, module)
|
src, err := inv.SourceOf(ctx, module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, false, err
|
||||||
}
|
}
|
||||||
return sameCommitID(src.BuiltFrom, commit), nil
|
if sameCommitID(src.BuiltFrom, trunk.Commit) {
|
||||||
|
return true, true, nil
|
||||||
|
}
|
||||||
|
fp, err := registeredFingerprint(ctx, inv, keys, module, src.BuiltFrom)
|
||||||
|
return false, err == nil && fp != "" && fp == trunk.FP2, nil
|
||||||
|
},
|
||||||
|
recordCommit: func(ctx context.Context, module, commit string) error {
|
||||||
|
return inv.RecordBuiltFrom(ctx, module, commit)
|
||||||
},
|
},
|
||||||
takeIn: func(ctx context.Context, module string, trunk sdkdelivery.Module) error {
|
takeIn: func(ctx context.Context, module string, trunk sdkdelivery.Module) error {
|
||||||
_, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
_, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||||
@@ -263,17 +342,8 @@ func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
|
|||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
},
|
},
|
||||||
nodesOf: func(ctx context.Context, module string) ([]string, error) {
|
taken: func(ctx context.Context, module, fp string, nodes []string) (bool, error) {
|
||||||
entries, err := inv.Catalogued(ctx)
|
return composedAndGranted(ctx, js, module, fp, nodes)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.Manifest.Module == module {
|
|
||||||
return e.On, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
},
|
},
|
||||||
busHolder: func(ctx context.Context) (string, error) {
|
busHolder: func(ctx context.Context) (string, error) {
|
||||||
holders, err := seatHolders(ctx, inv)
|
holders, err := seatHolders(ctx, inv)
|
||||||
@@ -284,107 +354,64 @@ func liveDoneKeySteps(open *stores, keys registerKeys) doneKeySteps {
|
|||||||
},
|
},
|
||||||
// The composer's, never the take-in's own: one pass at a time composes each node asked for once, and writes the
|
// The composer's, never the take-in's own: one pass at a time composes each node asked for once, and writes the
|
||||||
// grants once (issue 1945, composer.go).
|
// grants once (issue 1945, composer.go).
|
||||||
compose: func(ctx context.Context, nodes []string) error {
|
compose: func(ctx context.Context, nodes []string, grants bool) (map[string]error, error, error) {
|
||||||
return composerFor(open).compose(ctx, nodes, false)
|
return composerFor(open).composeEach(ctx, nodes, grants)
|
||||||
},
|
|
||||||
grants: func(ctx context.Context) error {
|
|
||||||
return composerFor(open).compose(ctx, nil, true)
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// watchDoneKeys wakes the take-in of a module's done key for each module the builder or the indexer writes, until ctx
|
// registeredFingerprint is the fingerprint a module is registered at, as currentFingerprints reads it: its commit's
|
||||||
// ends: never on the watch's own goroutine.
|
// newest build's, or the indexer's value at the commit when the commit has no build of its own; "" when neither says.
|
||||||
func watchDoneKeys(ctx context.Context, open *stores) {
|
func registeredFingerprint(ctx context.Context, inv *inventory.Inventory, keys registerKeys, module, commit string) (string, error) {
|
||||||
keys, closeKeys, err := openBuilderKeys()
|
if commit == "" {
|
||||||
if err != nil {
|
return "", nil
|
||||||
fmt.Printf("done keys: the builder's keys are not watched: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
if err := builderkeys.Watch(ctx, keys.JS, func(module string) {
|
if id, err := inv.BuildOf(ctx, module, commit); err == nil && id != "" {
|
||||||
inBackground(func() { takeDoneKey(ctx, open, module) })
|
if b, found, err := inv.BuildByID(ctx, id); err == nil && found && b.SourceFingerprint != "" {
|
||||||
}); err != nil {
|
return b.SourceFingerprint, nil
|
||||||
closeKeys()
|
|
||||||
fmt.Printf("done keys: the builder's keys are not watched: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
<-ctx.Done()
|
|
||||||
closeKeys()
|
|
||||||
}()
|
|
||||||
// And every module once now, after the watch is open so no write falls between the two.
|
|
||||||
inBackground(func() {
|
|
||||||
doneKeysAtStart(ctx, catalogedModules(open), func(module string) { takeDoneKeyAtStart(ctx, open, module) })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// doneKeysAtStart takes in the done key of every catalogued module once, one after another: the start's look the watch of
|
|
||||||
// new writes cannot give (it hears updates only). A done key written while no controller served, or whose take-in the
|
|
||||||
// controller this one replaced was stopped in, is taken here; without it the module stays in the node's own composition
|
|
||||||
// with no credential until its keys are written again (issue 1945, live on 2026-10-25 at 02:34). A take-in with nothing
|
|
||||||
// to do does nothing; a catalogue that cannot be read is said.
|
|
||||||
func doneKeysAtStart(ctx context.Context, modules func(context.Context) ([]string, error), take func(module string)) {
|
|
||||||
names, err := modules(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("done keys: the modules are not looked at once at the start: %v\n", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sort.Strings(names)
|
|
||||||
// Side by side, a few at a time: their compositions meet in the composer's passes, each node composed once per pass
|
|
||||||
// (issue 1945), so the start's look is not one module's composition after another.
|
|
||||||
sem := make(chan struct{}, startTakeIns)
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, m := range names {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
sem <- struct{}{}
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer func() { <-sem; wg.Done() }()
|
|
||||||
take(m)
|
|
||||||
}()
|
|
||||||
}
|
}
|
||||||
wg.Wait()
|
at, err := keys.ModuleAt(ctx, module, commit)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return at.FP2, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// startTakeIns is how many take-ins the start's look runs at once.
|
// composedAndGranted says the bus holds module's grant at fp and its composition at fp for every node: a few reads of
|
||||||
const startTakeIns = 16
|
// keys, never a composition.
|
||||||
|
func composedAndGranted(ctx context.Context, js jetstream.JetStream, module, fp string, nodes []string) (bool, error) {
|
||||||
// takeDoneKeyAtStart is one take-in of the start's look, under the module's lock; one that fails is handed to takeDoneKey,
|
grants, err := js.KeyValue(ctx, broker.GrantsBucket)
|
||||||
// which tries again every doneKeyRetry, loudly, so a failing module never holds up the look at the others.
|
if err != nil {
|
||||||
func takeDoneKeyAtStart(ctx context.Context, open *stores, module string) {
|
return false, err
|
||||||
mu, _ := doneKeysMu.LoadOrStore(module, &sync.Mutex{})
|
|
||||||
if !mu.(*sync.Mutex).TryLock() {
|
|
||||||
// A take-in already follows the module: told to look again, it takes what the start would have (issue 1945).
|
|
||||||
doneKeysLookAgain.Store(module, true)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
doneKeysLookAgain.Delete(module) // what this look reads covers every wake before it
|
key, err := sdkdelivery.ControllerGrantKey(module, fp)
|
||||||
err := doneKeyTakeOnce(ctx, open, module)
|
if err != nil {
|
||||||
mu.(*sync.Mutex).Unlock()
|
return false, err
|
||||||
// Read after letting go, as every holder of a module's lock does: a wake during this look only marked the module
|
|
||||||
// (the review of issue 1945).
|
|
||||||
_, again := doneKeysLookAgain.LoadAndDelete(module)
|
|
||||||
if err != nil || again {
|
|
||||||
// A failure, a wait or a wake meanwhile is followed by takeDoneKey, which says it and looks again.
|
|
||||||
inBackground(func() { takeDoneKey(ctx, open, module) })
|
|
||||||
}
|
}
|
||||||
}
|
if _, err := grants.Get(ctx, key); err != nil {
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
// catalogedModules are the names of every module the catalogue holds.
|
return false, nil
|
||||||
func catalogedModules(open *stores) func(context.Context) ([]string, error) {
|
}
|
||||||
return func(ctx context.Context) ([]string, error) {
|
return false, err
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
}
|
||||||
|
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
for _, node := range nodes {
|
||||||
|
key, err := sdkdelivery.ComposedKey(node, module, fp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return false, err
|
||||||
}
|
}
|
||||||
out := make([]string, 0, len(entries))
|
if _, err := digests.Get(ctx, key); err != nil {
|
||||||
for _, e := range entries {
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
out = append(out, e.Manifest.Module)
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, err
|
||||||
}
|
}
|
||||||
return out, nil
|
|
||||||
}
|
}
|
||||||
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// doneKeysComposers are the composers of the done keys' take-ins in this process, one for each set of stores (a test
|
// doneKeysComposers are the composers of the done keys' take-ins in this process, one for each set of stores (a test
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"reflect"
|
|
||||||
"sort"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A starting controller takes in the done key of every catalogued module once, beside the watch of new writes: a done key
|
|
||||||
// written while no controller served, or whose take-in the controller it replaced was stopped in, is otherwise never
|
|
||||||
// taken, and its module stays in the node's own composition with no credential (issue 1945, live on 2026-10-25 at 02:34:
|
|
||||||
// the controller replaced mid take-in left mesh-provisioner-client, gitea, postgres and many more unregistered at their
|
|
||||||
// new fingerprints, and the new one, watching updates only, never looked).
|
|
||||||
func TestAStartingControllerTakesEveryModulesDoneKeyOnce(t *testing.T) {
|
|
||||||
var took []string
|
|
||||||
var mu sync.Mutex
|
|
||||||
doneKeysAtStart(context.Background(), func(context.Context) ([]string, error) {
|
|
||||||
return []string{"mesh-provisioner-client", "gitea", "postgres"}, nil
|
|
||||||
}, func(module string) { mu.Lock(); took = append(took, module); mu.Unlock() })
|
|
||||||
sort.Strings(took)
|
|
||||||
if want := []string{"gitea", "mesh-provisioner-client", "postgres"}; !reflect.DeepEqual(took, want) {
|
|
||||||
t.Fatalf("took %v, want %v", took, want)
|
|
||||||
}
|
|
||||||
// A catalogue that cannot be read takes nothing and is said, never fatal.
|
|
||||||
took = nil
|
|
||||||
doneKeysAtStart(context.Background(), func(context.Context) ([]string, error) { return nil, errors.New("down") },
|
|
||||||
func(module string) { took = append(took, module) })
|
|
||||||
if len(took) != 0 {
|
|
||||||
t.Fatalf("took %v from a catalogue that could not be read", took)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -4,84 +4,305 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||||
)
|
)
|
||||||
|
|
||||||
// stepsFor are doneKeySteps over a module registered at registered, whose trunk value names fp at commit, assigned to
|
// aTakeIn is a mesh for the take-in's steps: per module its registered commit, its trunk (commit, fingerprint), its
|
||||||
// nodes, the bus held on holder; every act is written to acts in order.
|
// nodes, whether its compositions and grant exist, and what its take-in answers; every act is written to acts.
|
||||||
func stepsFor(acts *[]string, registered, commit, fp string, nodes []string, holder string) doneKeySteps {
|
type aTakeIn struct {
|
||||||
return doneKeySteps{
|
registered map[string]string
|
||||||
held: func(_ context.Context, module string) (bool, error) { return module == "mesh-mail", nil },
|
sameFP map[string]bool // registered at another commit, at the trunk's fingerprint
|
||||||
trunk: func(context.Context, string) (sdkdelivery.Module, error) {
|
trunks map[string]string // module -> commit; no entry: no trunk value
|
||||||
if fp == "" {
|
nodes map[string][]string
|
||||||
|
taken map[string]bool
|
||||||
|
takeInErr map[string]error
|
||||||
|
composeErr map[string]error
|
||||||
|
grantsErr error
|
||||||
|
holder string
|
||||||
|
acts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *aTakeIn) steps() takeInSteps {
|
||||||
|
fp := func(module string) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(module[:1], 64) }
|
||||||
|
return takeInSteps{
|
||||||
|
catalogue: func(context.Context) (map[string][]string, error) {
|
||||||
|
m.acts = append(m.acts, "catalogue")
|
||||||
|
out := map[string][]string{}
|
||||||
|
for k, v := range m.nodes {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
},
|
||||||
|
trunk: func(_ context.Context, module string) (sdkdelivery.Module, error) {
|
||||||
|
commit, ok := m.trunks[module]
|
||||||
|
if !ok {
|
||||||
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
||||||
}
|
}
|
||||||
return sdkdelivery.Module{Commit: commit, FP2: fp}, nil
|
return sdkdelivery.Module{Commit: commit, FP2: fp(module)}, nil
|
||||||
|
},
|
||||||
|
registered: func(_ context.Context, module string, trunk sdkdelivery.Module) (bool, bool, error) {
|
||||||
|
if m.registered[module] == trunk.Commit {
|
||||||
|
return true, true, nil
|
||||||
|
}
|
||||||
|
return false, m.sameFP[module], nil
|
||||||
|
},
|
||||||
|
recordCommit: func(_ context.Context, module, commit string) error {
|
||||||
|
m.acts = append(m.acts, "record "+module+" at "+commit)
|
||||||
|
return nil
|
||||||
},
|
},
|
||||||
registeredAt: func(_ context.Context, _ string, c string) (bool, error) { return c == registered, nil },
|
|
||||||
takeIn: func(_ context.Context, module string, trunk sdkdelivery.Module) error {
|
takeIn: func(_ context.Context, module string, trunk sdkdelivery.Module) error {
|
||||||
*acts = append(*acts, "take in "+module+" at "+trunk.Commit)
|
m.acts = append(m.acts, "take in "+module+" at "+trunk.Commit)
|
||||||
return nil
|
return m.takeInErr[module]
|
||||||
},
|
},
|
||||||
nodesOf: func(context.Context, string) ([]string, error) { return nodes, nil },
|
taken: func(_ context.Context, module, _ string, _ []string) (bool, error) { return m.taken[module], nil },
|
||||||
busHolder: func(context.Context) (string, error) { return holder, nil },
|
busHolder: func(context.Context) (string, error) { return m.holder, nil },
|
||||||
compose: func(_ context.Context, ns []string) error {
|
compose: func(_ context.Context, nodes []string, grants bool) (map[string]error, error, error) {
|
||||||
*acts = append(*acts, "compose "+strings.Join(ns, ","))
|
act := "compose " + strings.Join(nodes, ",")
|
||||||
return nil
|
if grants {
|
||||||
|
act += " and grants"
|
||||||
|
}
|
||||||
|
m.acts = append(m.acts, act)
|
||||||
|
failed := map[string]error{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if e := m.composeErr[n]; e != nil {
|
||||||
|
failed[n] = e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if grants {
|
||||||
|
return failed, m.grantsErr, nil
|
||||||
|
}
|
||||||
|
return failed, nil, nil
|
||||||
},
|
},
|
||||||
grants: func(context.Context) error { *acts = append(*acts, "grants"); return nil },
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC3 (unit half): a done key of a new fingerprint of an assigned module is taken in, then the module's nodes are
|
// A batch of done keys takes each module in once and asks the composer at most twice, whatever its size: once for the
|
||||||
// composed, then the node holding the bus (its user list), then the grants: the grant and the composition exist
|
// nodes of every module with something new, then once for the node holding the bus with the grants (ADR 0544 §5). A
|
||||||
// before the assigner's rules let a want name the fingerprint. A module registered at the trunk's commit already is
|
// module registered at its trunk whose compositions and grant exist, or that this process took in at the same trunk,
|
||||||
// composed and granted without a second take-in; one assigned nowhere, unregistered, or with no trunk value is left
|
// composes nothing (§3): the old take-in composed it three times over, every key and every start (issue 1945, 17:36 on
|
||||||
// alone.
|
// 2026-10-25). A key delivered again is taken in whole. A wait for a write is acknowledged, a wait on a read and a
|
||||||
func TestADoneKeyIsTakenInComposedAndGranted(t *testing.T) {
|
// failure refused, a failed build key acknowledged with its condition (§4).
|
||||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
|
func TestATakeInBatch(t *testing.T) {
|
||||||
var acts []string
|
waitsForAWrite := fmt.Errorf("%w: the key is building", errWaitsForAWrite)
|
||||||
said, err := stepsFor(&acts, "old", "new", fp, []string{"ace", "g14"}, "novox").take(context.Background(), "mesh-mail")
|
waitsOnARead := fmt.Errorf("%w: the store is away", errKeyWaits)
|
||||||
if err != nil {
|
keyFailed := fmt.Errorf("%w: the build failed", errKeyFailed)
|
||||||
t.Fatal(err)
|
type want struct {
|
||||||
|
verdict verdict
|
||||||
|
taken, nothingNew bool
|
||||||
|
waits, err bool
|
||||||
}
|
}
|
||||||
if got := strings.Join(acts, "; "); got != "take in mesh-mail at new; compose ace,g14; compose novox; grants" {
|
cases := []struct {
|
||||||
t.Errorf("the acts were %q", got)
|
name string
|
||||||
|
mesh aTakeIn
|
||||||
|
keys []delivered
|
||||||
|
memo map[string]string
|
||||||
|
acts []string
|
||||||
|
passes int
|
||||||
|
want map[string]want
|
||||||
|
}{
|
||||||
|
{name: "a new fingerprint: taken in, its nodes, then the bus's node and the grants",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{"mail": "old"}, trunks: map[string]string{"mail": "new"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace", "g14", "novox"}}, holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail"}},
|
||||||
|
acts: []string{"catalogue", "take in mail at new", "compose ace,g14", "compose novox and grants"}, passes: 2,
|
||||||
|
want: map[string]want{"mail": {taken: true}}},
|
||||||
|
{name: "registered, composed and granted: nothing composed (fails on the old take-in)",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{"mail": "new", "nats": "n"}, trunks: map[string]string{"mail": "new", "nats": "n"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace"}, "nats": {"novox"}}, taken: map[string]bool{"mail": true, "nats": true},
|
||||||
|
holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail"}, {module: "nats"}, {module: "mail"}},
|
||||||
|
acts: []string{"catalogue"},
|
||||||
|
want: map[string]want{"mail": {nothingNew: true}, "nats": {nothingNew: true}}},
|
||||||
|
{name: "a catalogue merge that moved the commit and not the fingerprint: recorded, nothing composed",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{"mail": "a85d9037"}, sameFP: map[string]bool{"mail": true},
|
||||||
|
trunks: map[string]string{"mail": "2df05030"}, nodes: map[string][]string{"mail": {"ace"}},
|
||||||
|
taken: map[string]bool{"mail": true}, holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail"}},
|
||||||
|
acts: []string{"catalogue", "record mail at 2df05030"}, want: map[string]want{"mail": {nothingNew: true}}},
|
||||||
|
{name: "taken in by this process at the same trunk: nothing new, not even a look at its keys",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail"}},
|
||||||
|
memo: map[string]string{"mail": sdkdelivery.SourcePrefix2 + strings.Repeat("m", 64) + "@new"},
|
||||||
|
acts: []string{"catalogue"}, want: map[string]want{"mail": {nothingNew: true}}},
|
||||||
|
{name: "delivered again: taken in whole, even when it looks done",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace"}}, taken: map[string]bool{"mail": true}, holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail", again: true}},
|
||||||
|
acts: []string{"catalogue", "compose ace", "compose novox and grants"}, passes: 2,
|
||||||
|
want: map[string]want{"mail": {taken: true}}},
|
||||||
|
{name: "a take-in that stopped half-way: composed",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{"mail": "new"}, trunks: map[string]string{"mail": "new"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail"}},
|
||||||
|
acts: []string{"catalogue", "compose ace", "compose novox and grants"}, passes: 2,
|
||||||
|
want: map[string]want{"mail": {taken: true}}},
|
||||||
|
{name: "three modules, one pass for their nodes and one for the bus",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "web": "2", "bus": "3"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace", "novox"}, "web": {"g14", "ace"}, "bus": {"novox"}}, holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail"}, {module: "web"}, {module: "bus"}},
|
||||||
|
acts: []string{"catalogue", "take in mail at 1", "take in web at 2", "take in bus at 3", "compose ace,g14",
|
||||||
|
"compose novox and grants"}, passes: 2,
|
||||||
|
want: map[string]want{"mail": {taken: true}, "web": {taken: true}, "bus": {taken: true}}},
|
||||||
|
{name: "assigned nowhere, held by no catalogue, no trunk value: nothing composed",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "odd": "2"},
|
||||||
|
nodes: map[string][]string{"mail": nil, "none": {"ace"}}, holder: "novox"},
|
||||||
|
keys: []delivered{{module: "mail"}, {module: "odd"}, {module: "none"}},
|
||||||
|
acts: []string{"catalogue", "take in mail at 1"},
|
||||||
|
want: map[string]want{"mail": {taken: true}, "odd": {}, "none": {}}},
|
||||||
|
{name: "a grant not written refuses the key, never swallowed",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace"}}, holder: "novox", grantsErr: errors.New("the bucket mesh-controller_grants: not found")},
|
||||||
|
keys: []delivered{{module: "mail"}},
|
||||||
|
acts: []string{"catalogue", "take in mail at 1", "compose ace", "compose novox and grants"}, passes: 2,
|
||||||
|
want: map[string]want{"mail": {verdict: refused, err: true}}},
|
||||||
|
{name: "a node that fails refuses only its modules",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"mail": "1", "web": "2"},
|
||||||
|
nodes: map[string][]string{"mail": {"ace"}, "web": {"g14"}}, holder: "novox",
|
||||||
|
composeErr: map[string]error{"ace": errors.New("store refused")}},
|
||||||
|
keys: []delivered{{module: "mail"}, {module: "web"}},
|
||||||
|
acts: []string{"catalogue", "take in mail at 1", "take in web at 2", "compose ace,g14", "compose novox and grants"},
|
||||||
|
passes: 2, want: map[string]want{"mail": {verdict: refused, err: true}, "web": {taken: true}}},
|
||||||
|
{name: "waits and failures: a write's wait acknowledged, a read's refused, a failed key acknowledged",
|
||||||
|
mesh: aTakeIn{registered: map[string]string{}, trunks: map[string]string{"a": "1", "b": "2", "c": "3"},
|
||||||
|
nodes: map[string][]string{"a": {"ace"}, "b": {"ace"}, "c": {"ace"}}, holder: "novox",
|
||||||
|
takeInErr: map[string]error{"a": waitsForAWrite, "b": waitsOnARead, "c": keyFailed}},
|
||||||
|
keys: []delivered{{module: "a"}, {module: "b"}, {module: "c"}},
|
||||||
|
acts: []string{"catalogue", "take in a at 1", "take in b at 2", "take in c at 3"},
|
||||||
|
want: map[string]want{"a": {waits: true}, "b": {verdict: refused, waits: true}, "c": {err: true}}},
|
||||||
}
|
}
|
||||||
if !strings.Contains(said, "registered from the builder's done key") {
|
for _, c := range cases {
|
||||||
t.Errorf("said %q", said)
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
memo := c.memo
|
||||||
|
if memo == nil {
|
||||||
|
memo = map[string]string{}
|
||||||
|
}
|
||||||
|
settled, passes := c.mesh.steps().batch(t.Context(), c.keys, memo)
|
||||||
|
if !slices.Equal(c.mesh.acts, c.acts) {
|
||||||
|
t.Fatalf("the acts were\n %q\nwant\n %q", c.mesh.acts, c.acts)
|
||||||
|
}
|
||||||
|
if passes != c.passes {
|
||||||
|
t.Fatalf("%d composer pass(es), want %d", passes, c.passes)
|
||||||
|
}
|
||||||
|
if len(settled) != len(c.want) {
|
||||||
|
t.Fatalf("settled %d module(s), want %d: %+v", len(settled), len(c.want), settled)
|
||||||
|
}
|
||||||
|
for _, r := range settled {
|
||||||
|
w := c.want[r.module]
|
||||||
|
got := want{verdict: r.verdict, taken: r.taken, nothingNew: r.nothingNew, waits: r.waits != "", err: r.err != nil}
|
||||||
|
if got != w {
|
||||||
|
t.Errorf("%s: %+v, want %+v (%v)", r.module, got, w, r.err)
|
||||||
|
}
|
||||||
|
if r.taken && memo[r.module] == "" {
|
||||||
|
t.Errorf("%s was taken in and not remembered", r.module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
acts = nil
|
// And each delivered key is acknowledged or refused by its module's verdict, the refused with the retry's delay.
|
||||||
if _, err := stepsFor(&acts, "new", "new", fp, []string{"novox"}, "novox").take(context.Background(), "mesh-mail"); err != nil {
|
t.Run("acknowledged and refused", func(t *testing.T) {
|
||||||
t.Fatal(err)
|
mesh := aTakeIn{registered: map[string]string{}, trunks: map[string]string{"a": "1", "b": "2"},
|
||||||
}
|
nodes: map[string][]string{"a": {"ace"}, "b": {"ace"}}, takeInErr: map[string]error{"b": waitsOnARead}}
|
||||||
if got := strings.Join(acts, "; "); got != "compose novox; grants" {
|
var acked, naked []string
|
||||||
t.Errorf("a module registered at its trunk already: %q", got)
|
msg := func(module string) takeInMsg {
|
||||||
}
|
return takeInMsg{delivered: delivered{module: module, deliveries: 3}, ack: func() { acked = append(acked, module) },
|
||||||
|
nak: func(d time.Duration) {
|
||||||
|
if d != refusalDelay(3) {
|
||||||
|
t.Errorf("refused with a delay of %s", d)
|
||||||
|
}
|
||||||
|
naked = append(naked, module)
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
state := &takeInState{memo: map[string]string{}, raised: map[string]bool{}, saidWait: map[string]string{}}
|
||||||
|
said := state.take(t.Context(), mesh.steps(), []takeInMsg{msg("a"), msg("b"), msg("a")})
|
||||||
|
if !slices.Equal(acked, []string{"a", "a"}) || !slices.Equal(naked, []string{"b"}) {
|
||||||
|
t.Fatalf("acknowledged %v and refused %v", acked, naked)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "1 taken in") || !strings.Contains(said, "1 waiting") {
|
||||||
|
t.Fatalf("said %q", said)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
for name, s := range map[string]doneKeySteps{
|
// A key refused for long holds no slot the others need: each consumer leaves far more keys unacknowledged than a
|
||||||
"assigned nowhere": stepsFor(&acts, "old", "new", fp, nil, "novox"),
|
// batch takes, so 8 keys refused on each stop no ninth (the review of ADR 0544: with 8 a consumer, they stopped it
|
||||||
"no trunk value": stepsFor(&acts, "old", "new", "", []string{"ace"}, "novox"),
|
// for good), and a refusal waits longer each time, up to ten minutes.
|
||||||
|
t.Run("refused keys stop no other", func(t *testing.T) {
|
||||||
|
for _, bucket := range []string{sdkdelivery.ModulesBucket, sdkdelivery.BuilderBuildsBucket} {
|
||||||
|
if c := takeInConfig(bucket); c.MaxAckPending < 8+startTakeIns || c.AckPolicy != jetstream.AckExplicitPolicy {
|
||||||
|
t.Errorf("%s: %d unacknowledged at most (%v): 8 refused keys and a batch do not fit", bucket,
|
||||||
|
c.MaxAckPending, c.AckPolicy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
was := time.Duration(0)
|
||||||
|
for n := uint64(1); n <= 8; n++ {
|
||||||
|
d := refusalDelay(n)
|
||||||
|
if d < was || d > doneKeyRetryMost || (n == 1 && d != doneKeyRetry) {
|
||||||
|
t.Fatalf("the %d. refusal waits %s (the one before %s)", n, d, was)
|
||||||
|
}
|
||||||
|
was = d
|
||||||
|
}
|
||||||
|
if was != doneKeyRetryMost {
|
||||||
|
t.Fatalf("the 8. refusal waits %s, want %s", was, doneKeyRetryMost)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The take-in stalls when it holds work (keys not delivered yet, or a batch handed and not settled) and settles none
|
||||||
|
// for five minutes; refused keys waiting for their delay are no work, so a quiet mesh is no stall. The first look
|
||||||
|
// always says what it found, so a stall raised before a restart is cleared once this controller moves (ADR 0544 §6
|
||||||
|
// and its second review).
|
||||||
|
func TestATakeInStallsWhenItHoldsWorkAndSettlesNone(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 25, 17, 36, 0, 0, time.UTC)
|
||||||
|
type reading struct {
|
||||||
|
after time.Duration
|
||||||
|
pending, taking uint64
|
||||||
|
handed, settled uint64
|
||||||
|
stalled, changed bool
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
readings []reading
|
||||||
|
}{
|
||||||
|
{"keys not delivered and nothing settled", []reading{
|
||||||
|
{0, 190, 8, 50, 50, false, true}, // the first look always says what it found
|
||||||
|
{4 * time.Minute, 190, 8, 50, 50, false, false},
|
||||||
|
{5 * time.Minute, 190, 8, 50, 50, true, true},
|
||||||
|
{6 * time.Minute, 190, 8, 50, 50, true, false},
|
||||||
|
{7 * time.Minute, 180, 8, 60, 60, false, true},
|
||||||
|
{20 * time.Minute, 0, 0, 60, 60, false, false},
|
||||||
|
}},
|
||||||
|
{"refused keys waiting alone are no stall", []reading{
|
||||||
|
{0, 0, 3, 40, 40, false, true},
|
||||||
|
{30 * time.Minute, 0, 3, 40, 40, false, false},
|
||||||
|
}},
|
||||||
|
{"every key settled, one of them refused by the bus, nothing pending: no stall", []reading{
|
||||||
|
{0, 0, 1, 3, 3, false, true},
|
||||||
|
{6 * time.Minute, 0, 1, 3, 3, false, false},
|
||||||
|
}},
|
||||||
|
{"a batch handed and never settled is a stall", []reading{
|
||||||
|
{0, 0, 16, 56, 40, false, true},
|
||||||
|
{5 * time.Minute, 0, 16, 56, 40, true, true},
|
||||||
|
}},
|
||||||
} {
|
} {
|
||||||
acts = nil
|
t.Run(c.name, func(t *testing.T) {
|
||||||
if _, err := s.take(context.Background(), "mesh-mail"); err != nil {
|
w := &stallWatch{}
|
||||||
t.Fatal(err)
|
for i, r := range c.readings {
|
||||||
}
|
stalled, changed := w.look(takeInBacklog{Pending: r.pending, Taking: r.taking}, r.handed, r.settled,
|
||||||
if name == "assigned nowhere" && strings.Join(acts, "; ") != "take in mesh-mail at new" {
|
at.Add(r.after))
|
||||||
t.Errorf("%s: %v", name, acts)
|
if stalled != r.stalled || changed != r.changed {
|
||||||
}
|
t.Fatalf("reading %d: stalled %v changed %v, want %v %v", i, stalled, changed, r.stalled, r.changed)
|
||||||
if name == "no trunk value" && len(acts) != 0 {
|
}
|
||||||
t.Errorf("%s: %v", name, acts)
|
}
|
||||||
}
|
})
|
||||||
}
|
|
||||||
acts = nil
|
|
||||||
if _, err := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox").take(context.Background(), "unknown"); err != nil || len(acts) != 0 {
|
|
||||||
t.Errorf("a module the catalogue does not hold: %v %v", acts, err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,15 +336,3 @@ func TestAChangedUserListIsANewRevisionOfTheBusModulesCompositionAlone(t *testin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945 (the review's D1): a grant that is not written is the take-in's error, never swallowed: with no send left,
|
|
||||||
// nothing else would write it, and no want could name the fingerprint.
|
|
||||||
func TestAGrantNotWrittenIsTheTakeInsError(t *testing.T) {
|
|
||||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("c", 64)
|
|
||||||
var acts []string
|
|
||||||
s := stepsFor(&acts, "old", "new", fp, []string{"ace"}, "novox")
|
|
||||||
s.grants = func(context.Context) error { return errors.New("the bucket mesh-controller_grants: not found") }
|
|
||||||
if _, err := s.take(context.Background(), "mesh-mail"); err == nil || !strings.Contains(err.Error(), "grants") {
|
|
||||||
t.Errorf("a grant not written: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,184 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A take-in that waits is said once for each reason it waits for, and tried again every doneKeyRetry until it is taken
|
|
||||||
// in: never left for a key write that may not come (issue 1945, live on 2026-10-25: mesh-host's take-in at 26bd waited,
|
|
||||||
// said nothing, and nothing woke it until another write).
|
|
||||||
func TestAWaitingTakeInIsSaidOnceAndTriedAgain(t *testing.T) {
|
|
||||||
was, wasRetry, wasSay := doneKeyTakeOnce, doneKeyRetry, doneKeysSay
|
|
||||||
t.Cleanup(func() { doneKeyTakeOnce, doneKeyRetry, doneKeysSay = was, wasRetry, wasSay })
|
|
||||||
doneKeyRetry = 5 * time.Millisecond
|
|
||||||
var mu sync.Mutex
|
|
||||||
var said []string
|
|
||||||
doneKeysSay = func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }
|
|
||||||
calls := 0
|
|
||||||
doneKeyTakeOnce = func(context.Context, *stores, string) error {
|
|
||||||
calls++
|
|
||||||
switch {
|
|
||||||
case calls <= 2:
|
|
||||||
return fmt.Errorf("%w: the artifact store could not hold mesh-host's artifact", errKeyWaits)
|
|
||||||
case calls == 3:
|
|
||||||
return fmt.Errorf("%w: mesh-host's key cannot be read", errKeyWaits)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() { takeDoneKey(context.Background(), nil, "mesh-host"); close(done) }()
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("a waiting take-in was not tried again until it was taken in")
|
|
||||||
}
|
|
||||||
if calls != 4 {
|
|
||||||
t.Fatalf("tried %d times, want 4", calls)
|
|
||||||
}
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
waits := 0
|
|
||||||
for _, s := range said {
|
|
||||||
if strings.Contains(s, "mesh-host waits") {
|
|
||||||
waits++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if waits != 2 {
|
|
||||||
t.Fatalf("said %q: want each of the two reasons once", said)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A wake while a take-in of the module already follows it adds no second one: the follower looks again itself.
|
|
||||||
func TestAWakeWhileATakeInFollowsAddsNone(t *testing.T) {
|
|
||||||
was, wasRetry, wasSay := doneKeyTakeOnce, doneKeyRetry, doneKeysSay
|
|
||||||
t.Cleanup(func() { doneKeyTakeOnce, doneKeyRetry, doneKeysSay = was, wasRetry, wasSay })
|
|
||||||
doneKeyRetry = 20 * time.Millisecond
|
|
||||||
doneKeysSay = func(string) {}
|
|
||||||
var mu sync.Mutex
|
|
||||||
calls := 0
|
|
||||||
release := make(chan struct{})
|
|
||||||
doneKeyTakeOnce = func(context.Context, *stores, string) error {
|
|
||||||
mu.Lock()
|
|
||||||
calls++
|
|
||||||
mu.Unlock()
|
|
||||||
select {
|
|
||||||
case <-release:
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%w: the key says building", errKeyWaits)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
follower := make(chan struct{})
|
|
||||||
go func() { takeDoneKey(context.Background(), nil, "board"); close(follower) }()
|
|
||||||
time.Sleep(5 * time.Millisecond)
|
|
||||||
returned := make(chan struct{})
|
|
||||||
go func() { takeDoneKey(context.Background(), nil, "board"); close(returned) }()
|
|
||||||
select {
|
|
||||||
case <-returned:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("a second wake waited for the follower")
|
|
||||||
}
|
|
||||||
close(release)
|
|
||||||
<-follower
|
|
||||||
}
|
|
||||||
|
|
||||||
// A wake while a take-in of the module follows it is not lost: the follower goes round once more after it succeeds, so
|
|
||||||
// a newer build's key written meanwhile is taken in (the review of issue 1945).
|
|
||||||
func TestAWakeDuringATakeInIsTakenAfterIt(t *testing.T) {
|
|
||||||
was, wasRetry, wasSay := doneKeyTakeOnce, doneKeyRetry, doneKeysSay
|
|
||||||
t.Cleanup(func() { doneKeyTakeOnce, doneKeyRetry, doneKeysSay = was, wasRetry, wasSay })
|
|
||||||
doneKeyRetry = 20 * time.Millisecond
|
|
||||||
doneKeysSay = func(string) {}
|
|
||||||
var mu sync.Mutex
|
|
||||||
calls := 0
|
|
||||||
inFirst := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
doneKeyTakeOnce = func(context.Context, *stores, string) error {
|
|
||||||
mu.Lock()
|
|
||||||
calls++
|
|
||||||
n := calls
|
|
||||||
mu.Unlock()
|
|
||||||
if n == 1 {
|
|
||||||
close(inFirst)
|
|
||||||
<-release
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
follower := make(chan struct{})
|
|
||||||
go func() { takeDoneKey(context.Background(), nil, "mesh-host"); close(follower) }()
|
|
||||||
<-inFirst
|
|
||||||
takeDoneKey(context.Background(), nil, "mesh-host") // the newer key's wake, while the first runs
|
|
||||||
close(release)
|
|
||||||
<-follower
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if calls != 2 {
|
|
||||||
t.Fatalf("taken %d times: the wake during the take-in was lost", calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A wake that comes after the follower's last look but before it lets go of the module is not lost: the follower looks
|
|
||||||
// at the mark again once it has let go, and takes the module once more (the review of issue 1945).
|
|
||||||
func TestAWakeJustBeforeTheFollowerLetsGoIsTaken(t *testing.T) {
|
|
||||||
was, wasRetry, wasSay, wasHook := doneKeyTakeOnce, doneKeyRetry, doneKeysSay, doneKeysBeforeLetGo
|
|
||||||
t.Cleanup(func() {
|
|
||||||
doneKeyTakeOnce, doneKeyRetry, doneKeysSay, doneKeysBeforeLetGo = was, wasRetry, wasSay, wasHook
|
|
||||||
})
|
|
||||||
doneKeyRetry = 10 * time.Millisecond
|
|
||||||
doneKeysSay = func(string) {}
|
|
||||||
var mu sync.Mutex
|
|
||||||
calls := 0
|
|
||||||
doneKeyTakeOnce = func(context.Context, *stores, string) error {
|
|
||||||
mu.Lock()
|
|
||||||
calls++
|
|
||||||
mu.Unlock()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
woke := false
|
|
||||||
doneKeysBeforeLetGo = func(module string) {
|
|
||||||
if !woke {
|
|
||||||
woke = true
|
|
||||||
takeDoneKey(context.Background(), nil, module) // fails its TryLock and marks the module
|
|
||||||
}
|
|
||||||
}
|
|
||||||
takeDoneKey(context.Background(), nil, "mesh-host")
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if calls != 2 {
|
|
||||||
t.Fatalf("taken %d times: the wake just before the follower let go was lost", calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A wake during the start's take-in of a module is not lost: the start pass reads the mark once it lets go of the module,
|
|
||||||
// and hands the module to takeDoneKey, which takes it once more (the review of issue 1945: the third dropped wake of its
|
|
||||||
// kind).
|
|
||||||
func TestAWakeDuringTheStartsTakeInIsTaken(t *testing.T) {
|
|
||||||
was, wasRetry, wasSay := doneKeyTakeOnce, doneKeyRetry, doneKeysSay
|
|
||||||
t.Cleanup(func() { doneKeyTakeOnce, doneKeyRetry, doneKeysSay = was, wasRetry, wasSay })
|
|
||||||
doneKeyRetry = 10 * time.Millisecond
|
|
||||||
doneKeysSay = func(string) {}
|
|
||||||
var mu sync.Mutex
|
|
||||||
calls := 0
|
|
||||||
doneKeyTakeOnce = func(context.Context, *stores, string) error {
|
|
||||||
mu.Lock()
|
|
||||||
calls++
|
|
||||||
n := calls
|
|
||||||
mu.Unlock()
|
|
||||||
if n == 1 {
|
|
||||||
takeDoneKey(context.Background(), nil, "mesh-host") // the newer key's wake, during the start's take-in
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
takeDoneKeyAtStart(context.Background(), nil, "mesh-host")
|
|
||||||
background.Wait()
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if calls != 2 {
|
|
||||||
t.Fatalf("taken %d times: the wake during the start's take-in was lost", calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -124,6 +124,9 @@ func composeModulesByNode(ctx context.Context, open *stores, nodes []string) (ma
|
|||||||
errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
|
errsOf[node] = append(errsOf[node], fmt.Errorf("%s: %w", node, err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if out := withoutNodeOwnForHeld(node, comps, fps, declared.heldForCredential); out != "" {
|
||||||
|
fmt.Printf("compositions: %s\n", out)
|
||||||
|
}
|
||||||
if len(moved) > 0 {
|
if len(moved) > 0 {
|
||||||
fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
|
fmt.Printf("compositions: %s's %s compose with the node's own: no fingerprint of the new form names them\n",
|
||||||
node, strings.Join(moved, ", "))
|
node, strings.Join(moved, ", "))
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ func watchHavesForJudgement(ctx context.Context, open *stores) {
|
|||||||
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
fmt.Printf("compositions: the haves are not watched: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
go func() {
|
goServing(ctx, func() {
|
||||||
defer bus.Close()
|
defer bus.Close()
|
||||||
defer w.Stop()
|
defer w.Stop()
|
||||||
for {
|
for {
|
||||||
@@ -152,7 +152,7 @@ func watchHavesForJudgement(ctx context.Context, open *stores) {
|
|||||||
if err != nil || isNode {
|
if err != nil || isNode {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
inBackground(func() {
|
inBackground(ctx, func() {
|
||||||
if err := rejudgeOn(ctx, open, []string{node}, module); err != nil {
|
if err := rejudgeOn(ctx, open, []string{node}, module); err != nil {
|
||||||
fmt.Printf("compositions: %s on %s is not judged again: %v\n", module, node, err)
|
fmt.Printf("compositions: %s on %s is not judged again: %v\n", module, node, err)
|
||||||
}
|
}
|
||||||
@@ -165,5 +165,5 @@ func watchHavesForJudgement(ctx context.Context, open *stores) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -608,8 +608,11 @@ func keepWhatServeSets(t *testing.T) {
|
|||||||
t.Setenv(servedVar, "")
|
t.Setenv(servedVar, "")
|
||||||
unheld, status, given, check, follow, keeper, hand := logUnheldChanges, statusFrom, givenEvents, checkEvents,
|
unheld, status, given, check, follow, keeper, hand := logUnheldChanges, statusFrom, givenEvents, checkEvents,
|
||||||
link.Calls.Follow, conditionsFrom, handActConn
|
link.Calls.Follow, conditionsFrom, handActConn
|
||||||
|
doctor, asker := doctorFrom, askerFrom.Load()
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
logUnheldChanges, statusFrom, givenEvents, checkEvents, link.Calls.Follow, conditionsFrom, handActConn =
|
logUnheldChanges, statusFrom, givenEvents, checkEvents, link.Calls.Follow, conditionsFrom, handActConn =
|
||||||
unheld, status, given, check, follow, keeper, hand
|
unheld, status, given, check, follow, keeper, hand
|
||||||
|
doctorFrom = doctor
|
||||||
|
askerFrom.Store(asker)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -135,9 +135,18 @@ func declarationsDone(nodes []string, runs map[string]string, current string) bo
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// declarationsRetiredOn reads whether the declarations are done on the mesh now: every node's have of the node-engine
|
// declarationsRetiredOn reads whether the declarations are done on the mesh now: their stream gone from the bus, or
|
||||||
// against the fingerprint it is registered at.
|
// every node's have of the node-engine against the fingerprint it is registered at.
|
||||||
|
//
|
||||||
|
// **A stream that is gone stays gone** (issue 1945). Release 6 sends no declaration (ADR 0539 clauses 5 and 11), so a
|
||||||
|
// stream made again would hold no node's last declaration and help no node; and the done test alone is not lasting:
|
||||||
|
// a node-engine newer than the one registered landing, or a node asleep on an older have, makes it false again after
|
||||||
|
// the start deleted the stream, and then the raise, the bus's health check (H-bus), the probes (D6, D7) and healer H3
|
||||||
|
// expected the stream and every node's consumer again, raised them missing and made them anew (2026-10-25 14:52).
|
||||||
func declarationsRetiredOn(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream) (bool, error) {
|
func declarationsRetiredOn(ctx context.Context, inv *inventory.Inventory, js jetstream.JetStream) (bool, error) {
|
||||||
|
if _, err := js.Stream(ctx, broker.DeclarationsStream); errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
fps, err := currentFingerprints(ctx, inv)
|
fps, err := currentFingerprints(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
@@ -162,7 +171,8 @@ func declarationsRetiredOn(ctx context.Context, inv *inventory.Inventory, js jet
|
|||||||
}
|
}
|
||||||
|
|
||||||
// retireDeclarations deletes the declarations' stream, with every node's consumer on it, once every node runs the new
|
// retireDeclarations deletes the declarations' stream, with every node's consumer on it, once every node runs the new
|
||||||
// node-engine, and says it once; a stream already gone is said by nobody. It answers whether the declarations are done.
|
// node-engine, and says it once; a stream already gone is said by nobody, and is done. It answers whether the
|
||||||
|
// declarations are done.
|
||||||
func retireDeclarations(ctx context.Context, open *stores) (bool, error) {
|
func retireDeclarations(ctx context.Context, open *stores) (bool, error) {
|
||||||
if _, err := theLease.epoch(ctx); err != nil {
|
if _, err := theLease.epoch(ctx); err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
@@ -306,26 +316,33 @@ func composeReplacedController(ctx context.Context, open *stores) error {
|
|||||||
// replacedComposition is the controller's composition on r's node at r's fingerprint: in composed with r's manifest
|
// replacedComposition is the controller's composition on r's node at r's fingerprint: in composed with r's manifest
|
||||||
// in place of the controller's registered one.
|
// in place of the controller's registered one.
|
||||||
func replacedComposition(in compose.Input, r replacedController) (sdkdelivery.Composed, error) {
|
func replacedComposition(in compose.Input, r replacedController) (sdkdelivery.Composed, error) {
|
||||||
|
return keptComposition(in, r.node, controllerModule, r.fp, r.manifest)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptComposition is module's composition on node at fp: in composed with manifest, that fingerprint's own, in place of
|
||||||
|
// the module's registered one, in the reading mode (nothing made). in is not changed.
|
||||||
|
func keptComposition(in compose.Input, node, module, fp string, manifest catalogue.Manifest) (sdkdelivery.Composed, error) {
|
||||||
shelf := make(map[string]catalogue.Manifest, len(in.Catalogue))
|
shelf := make(map[string]catalogue.Manifest, len(in.Catalogue))
|
||||||
for k, v := range in.Catalogue {
|
for k, v := range in.Catalogue {
|
||||||
shelf[k] = v
|
shelf[k] = v
|
||||||
}
|
}
|
||||||
shelf[controllerModule] = r.manifest
|
shelf[module] = manifest
|
||||||
in.Catalogue = shelf
|
in.Catalogue = shelf
|
||||||
all, _, err := compose.Compositions(in, r.node, map[string]string{controllerModule: r.fp}, compose.Reading)
|
all, _, err := compose.Compositions(in, node, map[string]string{module: fp}, compose.Reading)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sdkdelivery.Composed{}, err
|
return sdkdelivery.Composed{}, err
|
||||||
}
|
}
|
||||||
key, _ := sdkdelivery.ComposedKey(r.node, controllerModule, r.fp)
|
key, _ := sdkdelivery.ComposedKey(node, module, fp)
|
||||||
c, ok := all[key]
|
c, ok := all[key]
|
||||||
if !ok {
|
if !ok {
|
||||||
return sdkdelivery.Composed{}, fmt.Errorf("%s composes no %s with the replaced build's manifest", r.node, controllerModule)
|
return sdkdelivery.Composed{}, fmt.Errorf("%s composes no %s with the manifest of %s", node, module, short(fp))
|
||||||
}
|
}
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// declarationsRetiredNow says the declarations are done on the mesh now, for a raise of the bus's objects; a bus or a
|
// declarationsRetiredNow says the declarations are done on the mesh now, for a raise of the bus's objects and for what
|
||||||
// store it cannot read is not done, so the stream is asserted as before.
|
// the health check, the probes and the healers expect of the bus; a bus or a store it cannot read is not done, so a
|
||||||
|
// stream still there is asserted as before.
|
||||||
func declarationsRetiredNow(ctx context.Context, inv *inventory.Inventory) bool {
|
func declarationsRetiredNow(ctx context.Context, inv *inventory.Inventory) bool {
|
||||||
bus, err := aBus()
|
bus, err := aBus()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -45,51 +47,97 @@ func TestTheFirstStartDeletesThePlansBucketsAndSaysEachOnce(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC6 (unit half): the controller this one replaced is composed on its node from its own build's manifest at its
|
// An earlier fingerprint is composed from its own build's manifest with today's inputs: the controller this one
|
||||||
// own fingerprint: the composition holds what that manifest declares, not what the controller registered now declares.
|
// replaced (1945/AC6), and every kept composition of a module whose config changed (ADR 0545: on 2026-10-25 a want back
|
||||||
func TestTheReplacedControllerIsComposedFromItsOwnManifest(t *testing.T) {
|
// to an earlier mesh-build-go on g14 applied the capacity of 0 its kept composition was made with). The composition
|
||||||
now := catalogue.Manifest{Module: controllerModule, Version: "6", Resources: []map[string]any{
|
// holds what that manifest declares and today's config, the kept keys are those of the touched modules at a
|
||||||
{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "six\n"}}}
|
// fingerprint other than the registered one, the build of a fingerprint is its newest that worked, and a write of
|
||||||
before := catalogue.Manifest{Module: controllerModule, Version: "5", Resources: []map[string]any{
|
// mesh-config's resolved values names the node and module it touches.
|
||||||
{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings", "content": "five\n"}}}
|
func TestAnEarlierFingerprintIsComposedFromItsOwnManifest(t *testing.T) {
|
||||||
in := compose.Input{Catalogue: map[string]catalogue.Manifest{controllerModule: now},
|
fp := func(c string) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||||
Nodes: []compose.Node{{Name: "anchor", Assigned: []string{controllerModule}, Stated: true}}}
|
t.Run("from its own manifest, with today's inputs", func(t *testing.T) {
|
||||||
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("5", 64)
|
manifest := func(version string) catalogue.Manifest {
|
||||||
c, err := replacedComposition(in, replacedController{node: "anchor", fp: fp, manifest: before})
|
return catalogue.Manifest{Module: controllerModule, Version: version,
|
||||||
if err != nil {
|
Settings: map[string]catalogue.SettingDeclaration{
|
||||||
t.Fatal(err)
|
"capacity": {Kind: catalogue.KindPreference, Default: float64(0), Why: "how many builds at once"}},
|
||||||
}
|
Resources: []map[string]any{{"id": "settings", "type": "file", "path": "/etc/mesh-controller/settings",
|
||||||
if c.Module != controllerModule || c.FP != fp || c.Node != "anchor" || len(c.Resources) != 1 {
|
"content": version + " ${setting:capacity}\n"}}}
|
||||||
t.Fatalf("composed %s at %s on %s with %d resources", c.Module, c.FP, c.Node, len(c.Resources))
|
}
|
||||||
}
|
now, before := manifest("six"), manifest("five")
|
||||||
var f struct{ Content string }
|
in := compose.Input{Catalogue: map[string]catalogue.Manifest{controllerModule: now},
|
||||||
_ = json.Unmarshal(c.Resources[0], &f)
|
Nodes: []compose.Node{{Name: "anchor", Assigned: []string{controllerModule}, Stated: true,
|
||||||
if f.Content != "five\n" {
|
Settings: map[string][]catalogue.Layer{controllerModule: {{From: "mesh", Values: map[string]any{"capacity": float64(4)}}}}}}}
|
||||||
t.Errorf("the replaced controller's composition holds %q, its own manifest says five", f.Content)
|
for name, compose := range map[string]func() (sdkdelivery.Composed, error){
|
||||||
}
|
"the replaced controller": func() (sdkdelivery.Composed, error) {
|
||||||
if in.Catalogue[controllerModule].Version != "6" {
|
return replacedComposition(in, replacedController{node: "anchor", fp: fp("5"), manifest: before})
|
||||||
t.Errorf("composing the replaced controller changed the Input's catalogue")
|
},
|
||||||
}
|
"a kept composition": func() (sdkdelivery.Composed, error) {
|
||||||
}
|
return keptComposition(in, "anchor", controllerModule, fp("5"), before)
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
c, err := compose()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
if c.Module != controllerModule || c.FP != fp("5") || c.Node != "anchor" || len(c.Resources) != 1 {
|
||||||
|
t.Fatalf("%s: composed %s at %s on %s with %d resources", name, c.Module, c.FP, c.Node, len(c.Resources))
|
||||||
|
}
|
||||||
|
var f struct{ Content string }
|
||||||
|
_ = json.Unmarshal(c.Resources[0], &f)
|
||||||
|
if f.Content != "five 4\n" {
|
||||||
|
t.Errorf("%s holds %q, want its own manifest's five with today's capacity 4", name, f.Content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if in.Catalogue[controllerModule].Version != "six" {
|
||||||
|
t.Errorf("composing an earlier fingerprint changed the Input's catalogue")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
// 1945/AC6 (unit half): the controller replaced is the newest earlier build of the controller that worked and names a
|
t.Run("which build, which keys, which touch", func(t *testing.T) {
|
||||||
// fingerprint of the new form, not the one registered now.
|
m := []byte(`{"module":"mesh-controller","version":"5"}`)
|
||||||
func TestTheReplacedControllerIsItsNewestEarlierBuild(t *testing.T) {
|
builds := []inventory.Build{
|
||||||
now, before, older := sdkdelivery.SourcePrefix2+strings.Repeat("6", 64), sdkdelivery.SourcePrefix2+strings.Repeat("5", 64),
|
{ID: "c6", SourceFingerprint: fp("6"), Manifest: m},
|
||||||
sdkdelivery.SourcePrefix2+strings.Repeat("4", 64)
|
{ID: "c5-failed", SourceFingerprint: fp("5"), Failed: "no", Manifest: m},
|
||||||
m := []byte(`{"module":"mesh-controller","version":"5"}`)
|
{ID: "c5", SourceFingerprint: fp("5"), Manifest: m},
|
||||||
builds := []inventory.Build{
|
{ID: "c4", SourceFingerprint: fp("4"), Manifest: m},
|
||||||
{ID: "c6", SourceFingerprint: now, Manifest: m},
|
}
|
||||||
{ID: "c5-failed", SourceFingerprint: before, Failed: "no", Manifest: m},
|
if b, ok := replacedBuild(builds, fp("6")); !ok || b.ID != "c5" {
|
||||||
{ID: "c5", SourceFingerprint: before, Manifest: m},
|
t.Errorf("the replaced build is %+v (%v), want c5", b, ok)
|
||||||
{ID: "c4", SourceFingerprint: older, Manifest: m},
|
}
|
||||||
}
|
if _, ok := replacedBuild(builds[:1], fp("6")); ok {
|
||||||
if b, ok := replacedBuild(builds, now); !ok || b.ID != "c5" {
|
t.Errorf("a controller with no earlier build replaced one")
|
||||||
t.Errorf("the replaced build is %+v (%v), want c5", b, ok)
|
}
|
||||||
}
|
if b, ok := buildAt(builds, fp("5")); !ok || b.ID != "c5" {
|
||||||
if _, ok := replacedBuild(builds[:1], now); ok {
|
t.Errorf("the build at 5 is %+v (%v), want c5, the one that worked", b, ok)
|
||||||
t.Errorf("a controller with no earlier build replaced one")
|
}
|
||||||
}
|
if _, ok := buildAt(builds, fp("3")); ok {
|
||||||
|
t.Errorf("a fingerprint with no build has one")
|
||||||
|
}
|
||||||
|
key := func(node, module, f string) string { k, _ := sdkdelivery.ComposedKey(node, module, f); return k }
|
||||||
|
keys := []string{key("g14", "mesh-build-go", fp("2")), key("g14", "mesh-build-go", fp("1")),
|
||||||
|
key("g14", "mesh-build-go", fp("3")), key("g14", "hello-web", fp("1")), key("ace", "mesh-build-go", fp("1"))}
|
||||||
|
registered := map[string]string{"mesh-build-go": fp("3"), "hello-web": fp("9")}
|
||||||
|
got := keptToFollow(keys, "g14", map[string]bool{"mesh-build-go": true}, registered)
|
||||||
|
if want := []keptKey{{"mesh-build-go", fp("1")}, {"mesh-build-go", fp("2")}}; !slices.Equal(got, want) {
|
||||||
|
t.Errorf("kept to follow %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
if got := keptToFollow(keys, "g14", nil, registered); len(got) != 3 {
|
||||||
|
t.Errorf("every module's kept compositions: %v", got)
|
||||||
|
}
|
||||||
|
for k, want := range map[string][2]string{"g14.mesh-build-go.capacity": {"g14", "mesh-build-go"},
|
||||||
|
"g14.mesh-build-go.limits.cpu": {"g14", "mesh-build-go"}, "g14.mesh-build-go": {}, "": {}} {
|
||||||
|
node, module, ok := configTouched(k)
|
||||||
|
if [2]string{node, module} != want || ok != (want[0] != "") {
|
||||||
|
t.Errorf("%q touches %s %s (%v), want %v", k, node, module, ok, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A value set before the consumer was made reaches the kept compositions: it reads each key once at creation.
|
||||||
|
if c := configFollowConfig(); c.DeliverPolicy != jetstream.DeliverLastPerSubjectPolicy ||
|
||||||
|
c.AckPolicy != jetstream.AckExplicitPolicy {
|
||||||
|
t.Errorf("the config consumer delivers %v, acknowledged %v; want each key's last value, acknowledged",
|
||||||
|
c.DeliverPolicy, c.AckPolicy)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC14 (unit half): the declarations' stream stays while one node's have names another node-engine than the one
|
// 1945/AC14 (unit half): the declarations' stream stays while one node's have names another node-engine than the one
|
||||||
@@ -110,3 +158,28 @@ func TestTheDeclarationsGoOnlyOnceEveryNodeRunsTheNewEngine(t *testing.T) {
|
|||||||
t.Errorf("done with no fingerprint registered, or no node")
|
t.Errorf("done with no fingerprint registered, or no node")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// streamsGone is a bus that holds none of the streams asked for; every other part of it is unread.
|
||||||
|
type streamsGone struct{ jetstream.JetStream }
|
||||||
|
|
||||||
|
func (streamsGone) Stream(context.Context, string) (jetstream.Stream, error) {
|
||||||
|
return nil, jetstream.ErrStreamNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 1945 (the alarm of a stream the start deleted): once the declarations' stream is gone from the bus, the
|
||||||
|
// declarations are retired whatever the nodes run now, so no raise, health check, probe or healer expects the stream
|
||||||
|
// or a node's consumer on it again. A node-engine newer than the registered one, or a node asleep on an older have,
|
||||||
|
// made the done test false after the stream went, and the bus's objects named it missing again (2026-10-25 14:52).
|
||||||
|
func TestTheDeclarationsStayRetiredOnceTheirStreamIsGone(t *testing.T) {
|
||||||
|
retired, err := func() (retired bool, err error) {
|
||||||
|
defer func() {
|
||||||
|
if p := recover(); p != nil {
|
||||||
|
err = fmt.Errorf("it read what the nodes run although the stream is gone: %v", p)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return declarationsRetiredOn(t.Context(), nil, streamsGone{})
|
||||||
|
}()
|
||||||
|
if err != nil || !retired {
|
||||||
|
t.Fatalf("with no declarations' stream on the bus: retired %v (%v), want retired", retired, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -486,3 +486,29 @@ func controllerSigner(ctx context.Context, open *stores) (func(sdkdelivery.Compo
|
|||||||
return sdkdelivery.SignComposedBy(c, public, func(m []byte) ([]byte, error) { return ident.Sign(ctx, m) })
|
return sdkdelivery.SignComposedBy(c, public, func(m []byte) ([]byte, error) { return ident.Sign(ctx, m) })
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withoutNodeOwnForHeld takes the node's own composition out of comps when a module held for a bus credential nobody
|
||||||
|
// issued (holdBackUnissuedBusAccounts) has no fingerprint of the new form: its resources would have been in the node's own,
|
||||||
|
// and the node's own written without them would have the node-engine remove a module that may be running. The node's
|
||||||
|
// own stays as last written; every module's composition is written. Answers what it said, "" when nothing was taken.
|
||||||
|
func withoutNodeOwnForHeld(node string, comps map[string]sdkdelivery.Composed, fps map[string]string, held []string) string {
|
||||||
|
var old []string
|
||||||
|
for _, module := range held {
|
||||||
|
if !strings.HasPrefix(fps[module], sdkdelivery.SourcePrefix2) {
|
||||||
|
old = append(old, module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(old) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
own, err := sdkdelivery.ComposedNodeKey(node)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if _, ok := comps[own]; !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
delete(comps, own)
|
||||||
|
return fmt.Sprintf("%s's own composition is not written this pass: %s, held for a bus account nobody issued, would "+
|
||||||
|
"leave it, and the node-engine would remove what may be running", node, strings.Join(old, ", "))
|
||||||
|
}
|
||||||
|
|||||||
@@ -29,7 +29,13 @@ type composer struct {
|
|||||||
type waiter struct {
|
type waiter struct {
|
||||||
nodes []string
|
nodes []string
|
||||||
grants bool
|
grants bool
|
||||||
done chan error
|
done chan passResult
|
||||||
|
}
|
||||||
|
|
||||||
|
// passResult is a waiter's answer: the error of each of its nodes that failed, and of the grants when it asked for them.
|
||||||
|
type passResult struct {
|
||||||
|
failed map[string]error
|
||||||
|
grants error
|
||||||
}
|
}
|
||||||
|
|
||||||
func newComposer(compose func(context.Context, []string) map[string]error, grants func(context.Context) error) *composer {
|
func newComposer(compose func(context.Context, []string) map[string]error, grants func(context.Context) error) *composer {
|
||||||
@@ -39,7 +45,23 @@ func newComposer(compose func(context.Context, []string) map[string]error, grant
|
|||||||
// compose asks for the nodes to be composed, and the grants written after when grants is true, and waits for the pass
|
// compose asks for the nodes to be composed, and the grants written after when grants is true, and waits for the pass
|
||||||
// that does it, with the errors of its own nodes alone.
|
// that does it, with the errors of its own nodes alone.
|
||||||
func (c *composer) compose(ctx context.Context, nodes []string, grants bool) error {
|
func (c *composer) compose(ctx context.Context, nodes []string, grants bool) error {
|
||||||
done := make(chan error, 1)
|
failed, grantsErr, err := c.composeEach(ctx, nodes, grants)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var errs []error
|
||||||
|
for _, n := range nodes {
|
||||||
|
if e := failed[n]; e != nil {
|
||||||
|
errs = append(errs, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.Join(append(errs, grantsErr)...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeEach is compose answering each node's error apart, and the grants' (ADR 0544 §5: a batch of take-ins asks one
|
||||||
|
// pass and settles each module by its own nodes); err is ctx's when it ends first.
|
||||||
|
func (c *composer) composeEach(ctx context.Context, nodes []string, grants bool) (map[string]error, error, error) {
|
||||||
|
done := make(chan passResult, 1)
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
c.pending[n] = true
|
c.pending[n] = true
|
||||||
@@ -52,10 +74,10 @@ func (c *composer) compose(ctx context.Context, nodes []string, grants bool) err
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case err := <-done:
|
case r := <-done:
|
||||||
return err
|
return r.failed, r.grants, nil
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return nil, nil, ctx.Err()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -90,16 +112,16 @@ func (c *composer) run(ctx context.Context) {
|
|||||||
grantsErr = c.writeGrants(ctx)
|
grantsErr = c.writeGrants(ctx)
|
||||||
}
|
}
|
||||||
for _, w := range waiters {
|
for _, w := range waiters {
|
||||||
var errs []error
|
r := passResult{failed: map[string]error{}}
|
||||||
for _, n := range w.nodes {
|
for _, n := range w.nodes {
|
||||||
if err := failed[n]; err != nil {
|
if err := failed[n]; err != nil {
|
||||||
errs = append(errs, err)
|
r.failed[n] = err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if w.grants && grantsErr != nil {
|
if w.grants {
|
||||||
errs = append(errs, grantsErr)
|
r.grants = grantsErr
|
||||||
}
|
}
|
||||||
w.done <- errors.Join(errs...)
|
w.done <- r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func keeperOn(ctx context.Context, conn *nats.Conn, looks conditions.LookStore)
|
|||||||
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
Say: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) },
|
||||||
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
// What status leads with changed: composed again soon (a nudge outside the serving controller
|
||||||
// does nothing).
|
// does nothing).
|
||||||
Changed: func() { statusFrom.nudge(); askerFrom.nudge() },
|
Changed: func() { statusFrom.nudge(); askerFrom.Load().nudge() },
|
||||||
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
// Written under the lease, carrying its epoch (novox/hq to-be 45 §6).
|
||||||
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
Epoch: func() (uint64, error) { return theLease.epoch(context.WithoutCancel(ctx)) }}), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
m, err := catalogue.ParseManifest(text)
|
m, err := catalogue.Decode(text)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("%s: %v", name, err)
|
t.Fatalf("%s: %v", name, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ const n8nWithAGroup = `{"module":"n8n","version":"1",
|
|||||||
|
|
||||||
// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named.
|
// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named.
|
||||||
func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) {
|
func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) {
|
||||||
m, err := catalogue.ParseManifest([]byte(n8nWithAGroup))
|
m, err := catalogue.Decode([]byte(n8nWithAGroup))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -39,17 +40,23 @@ func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
|
|||||||
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
||||||
}
|
}
|
||||||
|
|
||||||
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
|
// And the push holds talker out, naming the same verb, rather than sealing a placeholder; the rest of the
|
||||||
|
// machine composes (2026-11-01: one unissued account refused the whole node that runs the bus).
|
||||||
plan, settings, err := resolveNode(ctx, open, "laptop")
|
plan, settings, err := resolveNode(ctx, open, "laptop")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, err = declarationFor(ctx, open, "laptop", plan, settings)
|
held, err := declarationFor(ctx, open, "laptop", plan, settings)
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
|
t.Fatalf("one unissued account refused the whole machine: %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
|
why := held.leftOutWhy["talker"]
|
||||||
t.Fatalf("the refusal does not say what to run: %v", err)
|
if !slices.Contains(held.LeftOut, "talker") || !strings.Contains(why, "module issue talker --node laptop") ||
|
||||||
|
!strings.Contains(why, "issue 203") {
|
||||||
|
t.Fatalf("talker is not held out in issue 203's words: %v %q", held.LeftOut, why)
|
||||||
|
}
|
||||||
|
if body, err := held.Body(); err != nil || strings.Contains(string(body), catalogue.ForeseenSealed) {
|
||||||
|
t.Fatalf("a push would seal a placeholder where talker's bus credential belongs: %v\n%s", err, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
|||||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil {
|
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
house, err := catalogue.Decode([]byte(houseManifest))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ func shelfFor(t *testing.T, manifests ...string) map[string]catalogue.Manifest {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
out := map[string]catalogue.Manifest{}
|
out := map[string]catalogue.Manifest{}
|
||||||
for _, raw := range manifests {
|
for _, raw := range manifests {
|
||||||
m, err := catalogue.ParseManifest([]byte(raw))
|
m, err := catalogue.Decode([]byte(raw))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -225,6 +225,9 @@ func filesNotPutObservation(node, summary string) conditions.Observation {
|
|||||||
// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
|
// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
|
||||||
// clears the count and the condition.
|
// clears the count and the condition.
|
||||||
func notePut(ctx context.Context, node string, put error) {
|
func notePut(ctx context.Context, node string, put error) {
|
||||||
|
if errors.Is(put, context.Canceled) || ctx.Err() != nil {
|
||||||
|
return // a send cut by a cancellation or a stop says nothing of the bucket (issue 1945)
|
||||||
|
}
|
||||||
filesFailMu.Lock()
|
filesFailMu.Lock()
|
||||||
if put == nil {
|
if put == nil {
|
||||||
delete(filesFailed, node)
|
delete(filesFailed, node)
|
||||||
|
|||||||
@@ -415,6 +415,7 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi
|
|||||||
was := conditionsFrom
|
was := conditionsFrom
|
||||||
conditionsFrom = k
|
conditionsFrom = k
|
||||||
t.Cleanup(func() { conditionsFrom = was })
|
t.Cleanup(func() { conditionsFrom = was })
|
||||||
|
forgetPutFailures(t, "cond-node")
|
||||||
key := filesNotPutObservation("cond-node", "").Key()
|
key := filesNotPutObservation("cond-node", "").Key()
|
||||||
open := func() bool {
|
open := func() bool {
|
||||||
_, found, err := conditions.ReadOne(t.Context(), store, key)
|
_, found, err := conditions.ReadOne(t.Context(), store, key)
|
||||||
@@ -442,6 +443,36 @@ func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A send cut by a cancellation counts as no failure to put its files, however often (issue 1945: a cancellation is no fault
|
||||||
|
// of the bucket).
|
||||||
|
func TestAPutCutByACancellationRaisesNothing(t *testing.T) {
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}})
|
||||||
|
t.Cleanup(func() { k.Close(context.Background()) })
|
||||||
|
was := conditionsFrom
|
||||||
|
conditionsFrom = k
|
||||||
|
t.Cleanup(func() { conditionsFrom = was })
|
||||||
|
forgetPutFailures(t, "cut-node")
|
||||||
|
cut := fmt.Errorf("the files could not be put: %w", context.Canceled)
|
||||||
|
for range filesNotPutAfter + 1 {
|
||||||
|
notePut(t.Context(), "cut-node", cut)
|
||||||
|
}
|
||||||
|
if _, found, err := conditions.ReadOne(t.Context(), store, filesNotPutObservation("cut-node", "").Key()); err != nil || found {
|
||||||
|
t.Fatalf("sends cut by a cancellation raised the condition (found %v, %v)", found, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetPutFailures starts the node's count of failed puts at nothing and leaves it so, as a run of the test again finds it.
|
||||||
|
func forgetPutFailures(t *testing.T, node string) {
|
||||||
|
forget := func() {
|
||||||
|
filesFailMu.Lock()
|
||||||
|
delete(filesFailed, node)
|
||||||
|
filesFailMu.Unlock()
|
||||||
|
}
|
||||||
|
forget()
|
||||||
|
t.Cleanup(forget)
|
||||||
|
}
|
||||||
|
|
||||||
// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
|
// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
|
||||||
type aBusThatSays struct {
|
type aBusThatSays struct {
|
||||||
link.Bus
|
link.Bus
|
||||||
|
|||||||
@@ -87,3 +87,29 @@ func TestTheDeclaredPackagesMirrorNamesWhatEachNodesModulesDeclare(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1945/AC-mirror-5: the digest the declared-packages mirror skips an unchanged pass by is made from a real reading of the
|
||||||
|
// stores, is the same over two readings of the same records, and moves when an assignment does.
|
||||||
|
func TestTheInputDigestOfTheStoresIsStable(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "viewer", Version: "1", Resources: []map[string]any{
|
||||||
|
{"id": "pkg-curl", "type": "package", "package": "curl"}}})
|
||||||
|
read := func() string {
|
||||||
|
in, err := readComposeInput(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return inputDigest(in)
|
||||||
|
}
|
||||||
|
first, second := read(), read()
|
||||||
|
if first == "" || first != second {
|
||||||
|
t.Fatalf("two readings of the same records gave the digests %q and %q", first, second)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "viewer"); err != nil {
|
||||||
|
t.Fatalf("assign: %v", err)
|
||||||
|
}
|
||||||
|
if read() == first {
|
||||||
|
t.Errorf("an assignment left the digest of the stores' Input at %s", first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -169,7 +169,7 @@ func watchHavesForGiven(ctx context.Context, open *stores) {
|
|||||||
log.Printf("given values: the haves are not watched: %v", err)
|
log.Printf("given values: the haves are not watched: %v", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
go func() {
|
goServing(ctx, func() {
|
||||||
defer bus.Close()
|
defer bus.Close()
|
||||||
defer w.Stop()
|
defer w.Stop()
|
||||||
for {
|
for {
|
||||||
@@ -195,8 +195,8 @@ func watchHavesForGiven(ctx context.Context, open *stores) {
|
|||||||
if at.IsZero() {
|
if at.IsZero() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
inBackground(func() { replaceGiven(ctx, open, node, map[string]time.Time{module: at}) })
|
inBackground(ctx, func() { replaceGiven(ctx, open, node, map[string]time.Time{module: at}) })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,10 +23,13 @@ import (
|
|||||||
// writes `<module>.<fingerprint hex>` then, naming the accounts it composed for the module, never removed. A module one
|
// writes `<module>.<fingerprint hex>` then, naming the accounts it composed for the module, never removed. A module one
|
||||||
// of whose accounts has no password yet (the list leaves it out) has no grant until a later send carries it.
|
// of whose accounts has no password yet (the list leaves it out) has no grant until a later send carries it.
|
||||||
|
|
||||||
// grantsFrom is the grant of every module whose accounts all have a password in hashes, at the fingerprint current names
|
// grantsFrom is the grant of every module at the fingerprint current names for it, naming its accounts that have a
|
||||||
// for it; a module whose current build names no fingerprint of the new form has none. principals are the bus's users
|
// sealed password in hashes; a module whose current build names no fingerprint of the new form has none. A module one
|
||||||
// as broker.Users composes them.
|
// of whose accounts has none is granted its sealed accounts alone, and when it has none, its grant is dropped (drop):
|
||||||
func grantsFrom(principals []broker.Principal, hashes map[string]string, current map[string]string) map[string]sdkdelivery.ControllerGrant {
|
// an older grant naming a node whose account is gone would let the assigner want it (issue 1945, rule 6 per node), and
|
||||||
|
// an empty grant names every node. principals are the bus's users as broker.Users composes them.
|
||||||
|
func grantsFrom(principals []broker.Principal, hashes map[string]string, current map[string]string) (
|
||||||
|
map[string]sdkdelivery.ControllerGrant, []string) {
|
||||||
accounts, missing := map[string][]string{}, map[string]bool{}
|
accounts, missing := map[string][]string{}, map[string]bool{}
|
||||||
for _, p := range principals {
|
for _, p := range principals {
|
||||||
if p.Module == "" || (p.Kind != broker.KindModule && p.Kind != broker.KindProcess) {
|
if p.Module == "" || (p.Kind != broker.KindModule && p.Kind != broker.KindProcess) {
|
||||||
@@ -39,8 +42,9 @@ func grantsFrom(principals []broker.Principal, hashes map[string]string, current
|
|||||||
accounts[p.Module] = append(accounts[p.Module], p.Username())
|
accounts[p.Module] = append(accounts[p.Module], p.Username())
|
||||||
}
|
}
|
||||||
out := map[string]sdkdelivery.ControllerGrant{}
|
out := map[string]sdkdelivery.ControllerGrant{}
|
||||||
|
var drop []string
|
||||||
for module, fp := range current {
|
for module, fp := range current {
|
||||||
if missing[module] || !strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
|
if !strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
key, err := sdkdelivery.ControllerGrantKey(module, fp)
|
key, err := sdkdelivery.ControllerGrantKey(module, fp)
|
||||||
@@ -48,10 +52,15 @@ func grantsFrom(principals []broker.Principal, hashes map[string]string, current
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
names := accounts[module]
|
names := accounts[module]
|
||||||
|
if missing[module] && len(names) == 0 {
|
||||||
|
drop = append(drop, key)
|
||||||
|
continue
|
||||||
|
}
|
||||||
sort.Strings(names)
|
sort.Strings(names)
|
||||||
out[key] = sdkdelivery.ControllerGrant{Accounts: names}
|
out[key] = sdkdelivery.ControllerGrant{Accounts: names}
|
||||||
}
|
}
|
||||||
return out
|
sort.Strings(drop)
|
||||||
|
return out, drop
|
||||||
}
|
}
|
||||||
|
|
||||||
// currentFingerprints is, by module, the fingerprint of the build each registered module is registered at: its
|
// currentFingerprints is, by module, the fingerprint of the build each registered module is registered at: its
|
||||||
@@ -248,13 +257,27 @@ func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.Je
|
|||||||
}
|
}
|
||||||
hashes := make(map[string]string, len(kept))
|
hashes := make(map[string]string, len(kept))
|
||||||
for name, u := range kept {
|
for name, u := range kept {
|
||||||
hashes[name] = u.PasswordHash
|
// A grant names an account whose credential is sealed to its node, never one only minted: the assigner wants a
|
||||||
|
// node only once the grant names its account (issue 1945, rule 6 per node).
|
||||||
|
if u.Sealed {
|
||||||
|
hashes[name] = u.PasswordHash
|
||||||
|
}
|
||||||
}
|
}
|
||||||
current, err := currentFingerprints(ctx, inv)
|
current, err := currentFingerprints(ctx, inv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the grants issued are not written: %w", err)
|
return fmt.Errorf("the grants issued are not written: %w", err)
|
||||||
}
|
}
|
||||||
wrote, err := keepGrants(ctx, kv, grantsFrom(principals, hashes, current))
|
grants, drop := grantsFrom(principals, hashes, current)
|
||||||
|
wrote, err := keepGrants(ctx, kv, grants)
|
||||||
|
for _, key := range drop {
|
||||||
|
if _, gerr := kv.Get(ctx, key); gerr == nil {
|
||||||
|
if derr := kv.Delete(ctx, key); derr != nil {
|
||||||
|
err = errors.Join(err, fmt.Errorf("%s: %w", key, derr))
|
||||||
|
} else {
|
||||||
|
fmt.Printf("%sthe grant %s dropped: none of its accounts is sealed to its node\n", indent, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if wrote > 0 {
|
if wrote > 0 {
|
||||||
fmt.Printf("%s%d grant(s) issued written for the assigner\n", indent, wrote)
|
fmt.Printf("%s%d grant(s) issued written for the assigner\n", indent, wrote)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,12 +23,25 @@ func TestTheGrantsAListIssues(t *testing.T) {
|
|||||||
y := broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-y"}
|
y := broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-y"}
|
||||||
node := broker.Principal{Kind: broker.KindNode, Node: "novox"}
|
node := broker.Principal{Kind: broker.KindNode, Node: "novox"}
|
||||||
hashes := map[string]string{x.Username(): "h1", xp.Username(): "h2", node.Username(): "h3"}
|
hashes := map[string]string{x.Username(): "h1", xp.Username(): "h2", node.Username(): "h3"}
|
||||||
got := grantsFrom([]broker.Principal{x, xp, y, node}, hashes,
|
// mesh-z is sealed on novox and not on ace (issue 1945: an older grant must not name ace): its grant names novox alone.
|
||||||
map[string]string{"mesh-x": grantFP('a'), "mesh-y": grantFP('b'), "web": grantFP('c'), "old": "src1:abc"})
|
zn := broker.Principal{Kind: broker.KindModule, Node: "novox", Module: "mesh-z"}
|
||||||
|
za := broker.Principal{Kind: broker.KindModule, Node: "ace", Module: "mesh-z"}
|
||||||
|
hashes[zn.Username()] = "h4"
|
||||||
|
got, drop := grantsFrom([]broker.Principal{x, xp, y, node, zn, za}, hashes,
|
||||||
|
map[string]string{"mesh-x": grantFP('a'), "mesh-y": grantFP('b'), "web": grantFP('c'), "old": "src1:abc",
|
||||||
|
"mesh-z": grantFP('d')})
|
||||||
xKey, _ := sdkdelivery.ControllerGrantKey("mesh-x", grantFP('a'))
|
xKey, _ := sdkdelivery.ControllerGrantKey("mesh-x", grantFP('a'))
|
||||||
|
yKey, _ := sdkdelivery.ControllerGrantKey("mesh-y", grantFP('b'))
|
||||||
|
zKey, _ := sdkdelivery.ControllerGrantKey("mesh-z", grantFP('d'))
|
||||||
webKey, _ := sdkdelivery.ControllerGrantKey("web", grantFP('c'))
|
webKey, _ := sdkdelivery.ControllerGrantKey("web", grantFP('c'))
|
||||||
if len(got) != 2 {
|
if len(got) != 3 {
|
||||||
t.Fatalf("grants = %v, want mesh-x's and web's", got)
|
t.Fatalf("grants = %v, want mesh-x's, mesh-z's and web's", got)
|
||||||
|
}
|
||||||
|
if g := got[zKey]; len(g.Accounts) != 1 || g.Accounts[0] != zn.Username() {
|
||||||
|
t.Errorf("mesh-z's grant = %+v, want its sealed account alone", g)
|
||||||
|
}
|
||||||
|
if len(drop) != 1 || drop[0] != yKey {
|
||||||
|
t.Errorf("dropped %v, want mesh-y's grant, none of whose accounts is sealed", drop)
|
||||||
}
|
}
|
||||||
if g := got[xKey]; len(g.Accounts) != 2 || g.Accounts[0] != x.Username() && g.Accounts[1] != x.Username() {
|
if g := got[xKey]; len(g.Accounts) != 2 || g.Accounts[0] != x.Username() && g.Accounts[1] != x.Username() {
|
||||||
t.Errorf("mesh-x's grant = %+v, want both its accounts", g)
|
t.Errorf("mesh-x's grant = %+v, want both its accounts", g)
|
||||||
|
|||||||
@@ -1,72 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Every catalogue module that runs something long-lived declares how it is ready (novox/hq ADR 0240 rule
|
|
||||||
// 8): `module check` warns and counts the undeclared before the date, and refuses them from it.
|
|
||||||
func TestModuleCheckCountsTheUndeclaredAndRefusesThemFromTheDate(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
digest := "@sha256:" + strings.Repeat("a", 64)
|
|
||||||
path := filepath.Join(dir, "module.json")
|
|
||||||
os.WriteFile(path, []byte(`{"module":"web","listens":[{"name":"web","port":80,"from":"mesh"}],"resources":[
|
|
||||||
{"id":"server","type":"container","name":"web","image":"registry.example/web`+digest+`","ports":["80"],
|
|
||||||
"health":{"kind":"http","endpoint":"web"}},
|
|
||||||
{"id":"worker","type":"container","name":"web-worker","image":"registry.example/web`+digest+`"},
|
|
||||||
{"id":"seed","type":"container","name":"web-seed","image":"registry.example/web`+digest+`","run-once":true}]}`), 0o600)
|
|
||||||
defer func() { checkNow = time.Now }()
|
|
||||||
|
|
||||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom.Add(-time.Hour) }
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
|
||||||
t.Fatalf("refused before the date: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
for _, want := range []string{"ready: server by http / on web every 30s", "WARNING: worker stay(s) up",
|
|
||||||
catalogue.HealthRequiredFrom.Format("2006-01-02"), UndeclaredHealthLine + " 1"} {
|
|
||||||
if !strings.Contains(out.String(), want) {
|
|
||||||
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
checkNow = func() time.Time { return catalogue.HealthRequiredFrom }
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{path}, &out); err == nil {
|
|
||||||
t.Fatalf("a long-running resource without health passed after the date:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "web: worker stays up and does not say how it is ready") {
|
|
||||||
t.Errorf("the refusal does not name the resource:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A file that asks for a setting without saying whether it is trusted counts as trusted (novox/hq issue 339):
|
|
||||||
// `module check` lists and counts it, and never refuses it — there is nothing unsafe to refuse.
|
|
||||||
func TestModuleCheckListsUnmarkedFilesAndNeverRefusesThem(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "module.json")
|
|
||||||
os.WriteFile(path, []byte(`{"module":"power","resources":[
|
|
||||||
{"id":"logind","type":"file","path":"/etc/systemd/logind.conf.d/power.conf","mode":"0644","trusted":true,
|
|
||||||
"content":"HandleLidSwitch=${setting:lid}\n"},
|
|
||||||
{"id":"note","type":"file","path":"/var/lib/power/note","mode":"0644","content":"${setting:greeting}\n"}]}`), 0o600)
|
|
||||||
defer func() { checkNow = time.Now }()
|
|
||||||
for _, at := range []time.Time{time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), time.Date(2036, 1, 1, 0, 0, 0, 0, time.UTC)} {
|
|
||||||
checkNow = func() time.Time { return at }
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{path}, &out); err != nil {
|
|
||||||
t.Fatalf("refused at %v: %v\n%s", at, err, out.String())
|
|
||||||
}
|
|
||||||
for _, want := range []string{"note ask(s) for a setting and do(es) not say whether it is trusted, so it counts as trusted",
|
|
||||||
UnsaidTrustLine + " 1"} {
|
|
||||||
if !strings.Contains(out.String(), want) {
|
|
||||||
t.Errorf("the check does not say %q:\n%s", want, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
|
||||||
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
|
||||||
// one consumer's identity.
|
|
||||||
|
|
||||||
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
|
||||||
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
write := func(name, body string) string {
|
|
||||||
p := filepath.Join(dir, name+".json")
|
|
||||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
objects := write("objects", `{"module":"objects","version":"1",
|
|
||||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
|
||||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
|
||||||
resolver := write("resolver", `{"module":"resolver","version":"1",
|
|
||||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
|
||||||
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
|
||||||
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
|
||||||
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
out.Reset()
|
|
||||||
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
|
||||||
}
|
|
||||||
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
|
||||||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
|
||||||
strings.Contains(out.String(), "networkmanager wants") {
|
|
||||||
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
|
||||||
// resolver provision, and a second consumer there overflows an object store's access key. The
|
|
||||||
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
|
||||||
// by push and by `status`, and the keyless consumer is granted with its long name.
|
|
||||||
@@ -182,6 +182,9 @@ func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssues
|
|||||||
offer := catalogue.Offer{Name: "gitea-read", Scope: catalogue.ScopeMesh}
|
offer := catalogue.Offer{Name: "gitea-read", Scope: catalogue.ScopeMesh}
|
||||||
if issued {
|
if issued {
|
||||||
offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"}
|
offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"}
|
||||||
|
} else {
|
||||||
|
// A login of each consumer's own, said: an offer naming no form has no credential (mesh-sdk 6b8c694).
|
||||||
|
offer.Credential = &catalogue.OfferCredential{Form: catalogue.CredentialFormPair}
|
||||||
}
|
}
|
||||||
return catalogue.Manifest{Module: "gitea", Version: "1", Moved: moved, Provides: []catalogue.Offer{offer}}
|
return catalogue.Manifest{Module: "gitea", Version: "1", Moved: moved, Provides: []catalogue.Offer{offer}}
|
||||||
}
|
}
|
||||||
@@ -216,8 +219,10 @@ func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssues
|
|||||||
// marked moved; another module is not.
|
// marked moved; another module is not.
|
||||||
func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) {
|
func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) {
|
||||||
vault := catalogue.Manifest{Module: "mesh-vault", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-vault", Scope: catalogue.ScopeMesh}},
|
vault := catalogue.Manifest{Module: "mesh-vault", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-vault", Scope: catalogue.ScopeMesh}},
|
||||||
Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh}}}
|
Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh,
|
||||||
other := catalogue.Manifest{Module: "mesh-vault", Version: "1", Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh}}}
|
Credential: &catalogue.OfferCredential{Form: catalogue.CredentialFormPair}}}}
|
||||||
|
other := catalogue.Manifest{Module: "mesh-vault", Version: "1", Provides: []catalogue.Offer{{Name: "secret", Scope: catalogue.ScopeMesh,
|
||||||
|
Credential: &catalogue.OfferCredential{Form: catalogue.CredentialFormPair}}}}
|
||||||
if got := needOf(t, map[string]catalogue.Manifest{"mesh-vault": vault}, "mesh-vault", "gitea", "secret"); !got.IssuedToHolder || !got.ProviderMoved {
|
if got := needOf(t, map[string]catalogue.Manifest{"mesh-vault": vault}, "mesh-vault", "gitea", "secret"); !got.IssuedToHolder || !got.ProviderMoved {
|
||||||
t.Errorf("the vault's consumer is not answered by a moved provider: %+v", got)
|
t.Errorf("the vault's consumer is not answered by a moved provider: %+v", got)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,9 +2,15 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/inventory"
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,17 +21,19 @@ import (
|
|||||||
// controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole
|
// controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole
|
||||||
// value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so
|
// value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so
|
||||||
// the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and
|
// the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and
|
||||||
// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. An account
|
// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. Until issue 774 an
|
||||||
// not covered is minted and kept as it was and handed to nobody.
|
// account not covered was minted and kept as it was and handed to nobody.
|
||||||
|
//
|
||||||
|
// **Since issue 774 every account the controller mints is handed, covered or not**, once a vault is installed: until
|
||||||
|
// the grants part's take-over the controller still mints (the bus loads its list), and a vault that holds every value
|
||||||
|
// at once needs no node to report a hash for it to adopt (ADR 0547 clause 9). From the take-over the vault mints every
|
||||||
|
// account itself (issueAccount forwards to its issue-account), so this hand is reached only before it, and by
|
||||||
|
// `rollout mint`'s re-issue of a module's account onto a new bus. The vault never lets such a hand replace an account it
|
||||||
|
// minted.
|
||||||
|
|
||||||
// vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it.
|
// vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it.
|
||||||
var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) }
|
var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) }
|
||||||
|
|
||||||
// accountIsCovered says whether the bus accounts minted for m are the vault's to hold (ADR 0374 clause 1).
|
|
||||||
func accountIsCovered(m catalogue.Manifest) bool {
|
|
||||||
return catalogue.CarriesVaultSealed(m.Module) || m.Moved
|
|
||||||
}
|
|
||||||
|
|
||||||
// vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18,
|
// vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18,
|
||||||
// mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no
|
// mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no
|
||||||
// vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the
|
// vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the
|
||||||
@@ -44,13 +52,45 @@ func vaultInstalled(ctx context.Context, inv *inventory.Inventory) (bool, error)
|
|||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, when the module's
|
// grantsTookOver says whether the grants part took over the bus's user list: its key `user-list.sealed` says
|
||||||
// accounts are covered, and does nothing otherwise. Loud when the vault does not take it: the bus's user list
|
// FollowGrants (novox/hq ADR 0547 clauses 7 and 8), which the bus node's engine follows. No bucket or no key yet is no
|
||||||
// already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in.
|
// take-over; a bus that cannot be read is an error, never a guess. A test replaces it.
|
||||||
func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error {
|
var grantsTookOver = func(ctx context.Context) (bool, error) {
|
||||||
if !accountIsCovered(m) {
|
over := false
|
||||||
|
err := onTheBus(func(conn *nats.Conn) error {
|
||||||
|
js, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
kv, err := js.KeyValue(ctx, broker.GrantsUserListBucket)
|
||||||
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||||
|
return nil
|
||||||
|
} else if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, broker.GrantsUserListKey)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return nil
|
||||||
|
} else if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
g, err := sdkdelivery.DecodeGrantsUserList(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
over = g.Follow == sdkdelivery.FollowGrants
|
||||||
return nil
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("whether the grants part took over the bus's user list cannot be read: %w", err)
|
||||||
}
|
}
|
||||||
|
return over, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, once one is
|
||||||
|
// installed, covered or not (issue 774), and does nothing before. Loud when the vault does not take it: the bus's user
|
||||||
|
// list already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in.
|
||||||
|
func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error {
|
||||||
if installed, err := vaultInstalled(ctx, inv); err != nil {
|
if installed, err := vaultInstalled(ctx, inv); err != nil {
|
||||||
return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name,
|
return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name,
|
||||||
m.Module, err)
|
m.Module, err)
|
||||||
|
|||||||
@@ -8,21 +8,50 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below.
|
// grantsTookOverOnTheBus is the reader of the grants part's take-over a test reads on its own bus.
|
||||||
|
var grantsTookOverOnTheBus = grantsTookOver
|
||||||
|
|
||||||
|
// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below; and
|
||||||
|
// before the grants part's take-over, which no test bus says.
|
||||||
func init() {
|
func init() {
|
||||||
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask }
|
||||||
|
grantsTookOver = func(context.Context) (bool, error) { return false, nil }
|
||||||
}
|
}
|
||||||
|
|
||||||
// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed
|
// theGrantsPartTookOver says the grants part's take-over is on, or off, for the rest of the test.
|
||||||
// to the key the vault made for the call, with origin issued. A module that is neither core nor moved hands nothing.
|
func theGrantsPartTookOver(t *testing.T, over bool) {
|
||||||
func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) {
|
t.Helper()
|
||||||
|
was := grantsTookOver
|
||||||
|
grantsTookOver = func(context.Context) (bool, error) { return over, nil }
|
||||||
|
t.Cleanup(func() { grantsTookOver = was })
|
||||||
|
}
|
||||||
|
|
||||||
|
// An account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed to the
|
||||||
|
// key the vault made for the call, with origin issued. Since issue 774 every account is, covered or not, until the
|
||||||
|
// grants part's take-over, read on the bus as the bus node reads it; from the take-over nothing is minted here and the
|
||||||
|
// issue is forwarded to the vault's issue-account.
|
||||||
|
func TestModuleIssueHandsEveryAccountToTheVaultUntilTheTakeOver(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
theVaultIsInstalled(t, open)
|
theVaultIsInstalled(t, open)
|
||||||
|
js, err := broker.Dial(testbus.URL(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
servingBus.Store(js)
|
||||||
|
t.Cleanup(func() { servingBus.Store(nil) })
|
||||||
|
was := grantsTookOver
|
||||||
|
grantsTookOver = grantsTookOverOnTheBus
|
||||||
|
t.Cleanup(func() { grantsTookOver = was })
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
known := broker.Broker{Fingerprint: "ab"}
|
known := broker.Broker{Fingerprint: "ab"}
|
||||||
|
|
||||||
@@ -66,11 +95,11 @@ func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) {
|
|||||||
t.Errorf("the vault was told %s, without %s", said, want)
|
t.Errorf("the vault was told %s, without %s", said, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if n := strings.Count(said, `"origin":"issued"`); n != 3 {
|
if n := strings.Count(said, `"origin":"issued"`); n != 4 {
|
||||||
t.Errorf("%d accounts were handed with origin issued, want 3 (the core's two and the moved module's): %s", n, said)
|
t.Errorf("%d accounts were handed with origin issued, want 4 (the core's two, the moved and the plain module's): %s", n, said)
|
||||||
}
|
}
|
||||||
if strings.Contains(said, "kept-bot") {
|
if !strings.Contains(said, "kept-bot") {
|
||||||
t.Errorf("an account that is neither core nor moved was handed to the vault: %s", said)
|
t.Errorf("an account that is neither core nor moved was not handed to the vault: %s", said)
|
||||||
}
|
}
|
||||||
if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") {
|
if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") {
|
||||||
t.Errorf("a value crossed the bus in the clear: %s", said)
|
t.Errorf("a value crossed the bus in the clear: %s", said)
|
||||||
@@ -83,6 +112,38 @@ func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) {
|
|||||||
!strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) {
|
!strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) {
|
||||||
t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got)
|
t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The take-over: the grants part's key says the bus node follows its list. From then the vault mints.
|
||||||
|
api, err := jetstream.New(js.Conn())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := api.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.GrantsUserListBucket})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = api.DeleteKeyValue(context.Background(), broker.GrantsUserListBucket) })
|
||||||
|
client := broker.Principal{Kind: broker.KindProvisioner, Node: "laptop", Module: "nats"}.Username()
|
||||||
|
for _, follow := range []string{sdkdelivery.FollowController, sdkdelivery.FollowGrants} {
|
||||||
|
value, err := sdkdelivery.EncodeGrantsUserList(sdkdelivery.GrantsUserList{Follow: follow})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(ctx, broker.GrantsUserListKey, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
before, _, _ := inv.BusUserHash(ctx, client)
|
||||||
|
if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
after, _, _ := inv.BusUserHash(ctx, client)
|
||||||
|
if minted := before != after; minted != (follow == sdkdelivery.FollowController) {
|
||||||
|
t.Errorf("following %s's list, the controller minted %s: %v", follow, client, minted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if said := vault.said(); strings.Count(said, `"verb":"issue-account"`) != 1 || !strings.Contains(said, `"user":"`+client+`"`) {
|
||||||
|
t.Errorf("the vault was not asked once, after the take-over, to issue %s: %s", client, said)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already.
|
// A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already.
|
||||||
|
|||||||
@@ -0,0 +1,352 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A kept composition follows its inputs** (novox/hq ADR 0545, issue 1945). The controller composes a module only at
|
||||||
|
// the fingerprint it is registered at, and keeps the compositions of up to five fingerprints (ADR 0539 §4) so that
|
||||||
|
// going back is one want. A kept one was never composed again, so it carried the config of the moment it was made: on
|
||||||
|
// 2026-10-25 a want back to an earlier mesh-build-go on g14 applied its capacity of 0, the value from before it was
|
||||||
|
// set. Now a write of mesh-config's resolved values for a module on a node, heard from that bucket, composes the node
|
||||||
|
// and then every kept fingerprint of that module on that node again, each from its own build's manifest with today's
|
||||||
|
// inputs, writing only those whose composition differs (writeComposed). The writes are read from a durable consumer,
|
||||||
|
// acknowledged once handled, so a change made while no controller ran is delivered to the next one, and a start
|
||||||
|
// composes again only what was written since the last write handled, never every kept composition.
|
||||||
|
|
||||||
|
// configTouched is the node and module a key of mesh-config_resolved (`<node>.<module>.<key>`) names.
|
||||||
|
func configTouched(key string) (node, module string, ok bool) {
|
||||||
|
parts := strings.SplitN(key, ".", 3)
|
||||||
|
if len(parts) != 3 || parts[0] == "" || parts[1] == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return parts[0], parts[1], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptKey is one kept composition to compose again.
|
||||||
|
type keptKey struct{ module, fp string }
|
||||||
|
|
||||||
|
// keptToFollow is, of a node's composition keys, each of modules (every module when nil) at a fingerprint other than
|
||||||
|
// the one it is registered at, sorted: the kept compositions. The node's own composition and other nodes' are not.
|
||||||
|
func keptToFollow(keys []string, node string, modules map[string]bool, registered map[string]string) []keptKey {
|
||||||
|
var out []keptKey
|
||||||
|
for _, key := range keys {
|
||||||
|
n, module, fp, isNode, err := sdkdelivery.ParseComposedKey(key)
|
||||||
|
if err != nil || isNode || n != node || (modules != nil && !modules[module]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if fp == registered[module] || !strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, keptKey{module, fp})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].module != out[j].module {
|
||||||
|
return out[i].module < out[j].module
|
||||||
|
}
|
||||||
|
return out[i].fp < out[j].fp
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildAt is, of a module's builds newest first, the newest that worked at fp and kept its manifest.
|
||||||
|
func buildAt(builds []inventory.Build, fp string) (inventory.Build, bool) {
|
||||||
|
for _, b := range builds {
|
||||||
|
if b.Worked() && b.SourceFingerprint == fp && len(b.Manifest) > 0 {
|
||||||
|
return b, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.Build{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// followKept composes again, on node, every kept composition of modules (every module when nil) with today's inputs,
|
||||||
|
// and writes those that differ. It answers the keys written; a fingerprint whose build's manifest is gone is said and
|
||||||
|
// left as it is.
|
||||||
|
func followKept(ctx context.Context, open *stores, node string, modules map[string]bool) ([]string, error) {
|
||||||
|
if _, err := theLease.epoch(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
inv := open.inventory
|
||||||
|
registered, err := currentFingerprints(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
at, err := openComposedPlace(ctx, js, bus.Conn(), node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
lister, err := at.bucket.ListKeys(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||||
|
return nil, nil // empty-on-error: a node with no compositions keeps none to compose again
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
for k := range lister.Keys() {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
kept := keptToFollow(keys, node, modules, registered)
|
||||||
|
if len(kept) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
sign, err := controllerSigner(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
upgrades, err := inv.Upgrades(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
at.sign = sign
|
||||||
|
at.policy = func(module string) (string, string, string) {
|
||||||
|
u, ok := upgrades[module]
|
||||||
|
if !ok {
|
||||||
|
return "", "", ""
|
||||||
|
}
|
||||||
|
return u.Policy(), u.From, u.Why
|
||||||
|
}
|
||||||
|
in, err := inputFor(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var wrote []string
|
||||||
|
var errs []error
|
||||||
|
builds := map[string][]inventory.Build{}
|
||||||
|
for _, k := range kept {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return wrote, errors.Join(append(errs, ctx.Err())...) // stopped: the rest is the next controller's
|
||||||
|
}
|
||||||
|
if _, read := builds[k.module]; !read {
|
||||||
|
if builds[k.module], err = inv.Builds(ctx, k.module, 20); err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s's builds: %w", k.module, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b, found := buildAt(builds[k.module], k.fp)
|
||||||
|
if !found {
|
||||||
|
fmt.Printf("compositions: %s's kept %s on %s has no build whose manifest is kept, and is not composed again\n",
|
||||||
|
k.module, short(k.fp), node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var m catalogue.Manifest
|
||||||
|
if err := json.Unmarshal(b.Manifest, &m); err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("the manifest of %s's build %s: %w", k.module, b.ID, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c, err := keptComposition(in.Input, node, k.module, k.fp, m)
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, fmt.Errorf("%s at %s on %s: %w", k.module, short(k.fp), node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, _ := sdkdelivery.ComposedKey(node, k.module, k.fp)
|
||||||
|
// One write on the node's keys at a time in this process, held only for the write: composing in memory holds
|
||||||
|
// nothing, so a sweep never keeps a take-in from the node (the review of ADR 0545).
|
||||||
|
unlock := lockComposing(node)
|
||||||
|
w, err := writeComposed(ctx, at, key, c)
|
||||||
|
unlock()
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if w {
|
||||||
|
wrote = append(wrote, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(wrote) > 0 {
|
||||||
|
fmt.Printf("compositions: %s: kept compositions composed again with today's inputs: %s\n", node, strings.Join(wrote, ", "))
|
||||||
|
}
|
||||||
|
return wrote, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// configFollowGather is how long the config consumer waits after a write for more before it composes: mesh-config writes
|
||||||
|
// a module's keys one by one.
|
||||||
|
var configFollowGather = time.Second
|
||||||
|
|
||||||
|
// configFollowConsumer is the controller's durable consumer of mesh-config's resolved values (ADR 0545 §1, §3).
|
||||||
|
const configFollowConsumer = broker.ConfigFollowConsumer
|
||||||
|
|
||||||
|
// configFollowConfig is that consumer: each resolved key's last value once when it is made, then each write,
|
||||||
|
// acknowledged after the write is handled, so what a stop cut is delivered to the next controller. The first start of
|
||||||
|
// the release that makes it composes again the kept compositions of every module with config, through the same gather,
|
||||||
|
// lock and acknowledgement: mesh-config never writes an unchanged value again, so a value set before this release (g14's
|
||||||
|
// mesh-build-go capacity of 2, against kept compositions carrying 0) would otherwise never reach them. Its deliver
|
||||||
|
// policy is set before its first delivery; an existing durable's cannot be changed.
|
||||||
|
func configFollowConfig() jetstream.ConsumerConfig {
|
||||||
|
return jetstream.ConsumerConfig{Durable: configFollowConsumer, FilterSubject: "$KV." + broker.ConfigResolvedBucket + ".>",
|
||||||
|
DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy, AckPolicy: jetstream.AckExplicitPolicy, AckWait: takeInAckWait,
|
||||||
|
MaxAckPending: takeInPending, Description: "the controller's following of config into kept compositions (novox/hq ADR 0545)"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindConfigNotFollowed is the condition of a config consumer that cannot be read: a change of config composes nothing
|
||||||
|
// until it is.
|
||||||
|
const kindConfigNotFollowed = "config-not-followed"
|
||||||
|
|
||||||
|
func configNotFollowedObservation(err error) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeCore, ID: "config-follow", Token: kindConfigNotFollowed,
|
||||||
|
Kind: kindConfigNotFollowed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "config",
|
||||||
|
Summary: fmt.Sprintf("mesh-config's resolved values are not heard, so a change of config composes nothing: %v", err)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchConfigFollow reads the writes of mesh-config's resolved values from the controller's durable consumer and, for
|
||||||
|
// each node and module they touch, composes the node and the module's kept compositions there again, until ctx ends
|
||||||
|
// (ADR 0545). A consumer that cannot be made or read raises config-not-followed and is tried again a minute later; it is
|
||||||
|
// cleared once it is read.
|
||||||
|
func watchConfigFollow(ctx context.Context, open *stores) {
|
||||||
|
goServing(ctx, func() {
|
||||||
|
for {
|
||||||
|
err := followConfigOnce(ctx, open)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Printf("config: %v; tried again in a minute\n", err)
|
||||||
|
if keeper := conditionsFrom; keeper != nil {
|
||||||
|
_, _ = keeper.Observe(ctx, configNotFollowedObservation(err))
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(time.Minute):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// followConfigOnce reads the consumer until it fails or ctx ends.
|
||||||
|
func followConfigOnce(ctx context.Context, open *stores) error {
|
||||||
|
bus, err := aBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer bus.Close()
|
||||||
|
js, err := jetstream.New(bus.Conn())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, err := js.CreateOrUpdateConsumer(ctx, "KV_"+broker.ConfigResolvedBucket, configFollowConfig())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the consumer of %s: %w", broker.ConfigResolvedBucket, err)
|
||||||
|
}
|
||||||
|
it, err := c.Messages(jetstream.PullMaxMessages(startTakeIns))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the consumer of %s: %w", broker.ConfigResolvedBucket, err)
|
||||||
|
}
|
||||||
|
defer it.Stop()
|
||||||
|
if keeper := conditionsFrom; keeper != nil {
|
||||||
|
_, _ = keeper.Clear(ctx, configNotFollowedObservation(nil).Key(), "mesh-config's resolved values are heard")
|
||||||
|
}
|
||||||
|
type next struct {
|
||||||
|
m jetstream.Msg
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
got := make(chan next)
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
m, err := it.Next()
|
||||||
|
select {
|
||||||
|
case got <- next{m, err}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
if m != nil {
|
||||||
|
_ = m.Nak()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
var held []jetstream.Msg
|
||||||
|
touched := map[string]map[string]bool{}
|
||||||
|
var gather <-chan time.Time
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
for _, m := range held {
|
||||||
|
_ = m.Nak() // delivered at once to the next controller
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case n := <-got:
|
||||||
|
if n.err != nil {
|
||||||
|
for _, m := range held {
|
||||||
|
_ = m.Nak()
|
||||||
|
}
|
||||||
|
return fmt.Errorf("the consumer of %s stopped: %w", broker.ConfigResolvedBucket, n.err)
|
||||||
|
}
|
||||||
|
held = append(held, n.m)
|
||||||
|
key := strings.TrimPrefix(n.m.Subject(), "$KV."+broker.ConfigResolvedBucket+".")
|
||||||
|
if node, module, ok := configTouched(key); ok {
|
||||||
|
if touched[node] == nil {
|
||||||
|
touched[node] = map[string]bool{}
|
||||||
|
}
|
||||||
|
touched[node][module] = true
|
||||||
|
}
|
||||||
|
if gather == nil {
|
||||||
|
gather = time.After(configFollowGather)
|
||||||
|
}
|
||||||
|
case <-gather:
|
||||||
|
gather = nil
|
||||||
|
now, msgs := touched, held
|
||||||
|
touched, held = map[string]map[string]bool{}, nil
|
||||||
|
forgetResolved() // the next composition reads the values just written, not a read of the last 5 s
|
||||||
|
failed := false
|
||||||
|
for node, modules := range now {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
failed = true // stopped: the writes are refused below and delivered to the next controller
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err := composerFor(open).compose(ctx, []string{node}, false); err != nil {
|
||||||
|
fmt.Printf("config: %s is not composed after its config changed: %v\n", node, err)
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
if _, err := followKept(ctx, open, node, modules); err != nil {
|
||||||
|
fmt.Printf("config: %s's kept compositions are not composed again: %v\n", node, err)
|
||||||
|
failed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range msgs {
|
||||||
|
if failed && ctx.Err() != nil {
|
||||||
|
_ = m.Nak() // stopped: delivered at once to the next controller
|
||||||
|
} else if failed {
|
||||||
|
_ = m.NakWithDelay(doneKeyRetry) // tried again: the bus delivers the writes again
|
||||||
|
} else {
|
||||||
|
acking, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
|
if err := m.DoubleAck(acking); err != nil {
|
||||||
|
fmt.Printf("config: the bus did not take the acknowledgement of %s: %v\n", m.Subject(), err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgetResolved drops the last read of mesh-config's resolved values.
|
||||||
|
func forgetResolved() {
|
||||||
|
theResolved.mu.Lock()
|
||||||
|
theResolved.entries = nil
|
||||||
|
theResolved.mu.Unlock()
|
||||||
|
}
|
||||||
@@ -219,15 +219,3 @@ func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry,
|
|||||||
}
|
}
|
||||||
return entries, read, edges, nil
|
return entries, read, edges, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// refusedWord is the word a build's failure says the reader did not know; empty when it says none.
|
|
||||||
func refusedWord(failed string) string {
|
|
||||||
if m := unknownWord.FindStringSubmatch(failed); m != nil {
|
|
||||||
return m[1]
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
|
|
||||||
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
|
|
||||||
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
|
|
||||||
|
|||||||
@@ -39,6 +39,11 @@ import (
|
|||||||
// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure.
|
// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure.
|
||||||
var errKeyWaits = errors.New("waits")
|
var errKeyWaits = errors.New("waits")
|
||||||
|
|
||||||
|
// errWaitsForAWrite is a take-in that waits for the builder or the indexer to write again (the key not done, its
|
||||||
|
// artifacts not referenced, no fingerprint of the new form): that write wakes it, so nothing looks again before it
|
||||||
|
// (ADR 0544 §4).
|
||||||
|
var errWaitsForAWrite = fmt.Errorf("%w for the builder's or the indexer's next write", errKeyWaits)
|
||||||
|
|
||||||
// errKeyFailed says the module's key failed: the builder's own words follow.
|
// errKeyFailed says the module's key failed: the builder's own words follow.
|
||||||
var errKeyFailed = errors.New("the builder's key failed")
|
var errKeyFailed = errors.New("the builder's key failed")
|
||||||
|
|
||||||
@@ -46,6 +51,8 @@ var errKeyFailed = errors.New("the builder's key failed")
|
|||||||
type keyReads interface {
|
type keyReads interface {
|
||||||
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
||||||
Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error)
|
Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error)
|
||||||
|
// Verdict is the indexer's verdict on the manifest of digest, nil when it wrote none (taken as accepted).
|
||||||
|
Verdict(ctx context.Context, digest string) (*sdkdelivery.ManifestVerdict, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice
|
// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice
|
||||||
@@ -298,12 +305,15 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
|
|||||||
waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) {
|
waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) {
|
||||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...)
|
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...)
|
||||||
}
|
}
|
||||||
|
waitsForAWrite := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) {
|
||||||
|
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errWaitsForAWrite}, a...)...)
|
||||||
|
}
|
||||||
if t.Trunk.FP2 == "" && t.Hand == nil {
|
if t.Trunk.FP2 == "" && t.Hand == nil {
|
||||||
why := t.Trunk.None2
|
why := t.Trunk.None2
|
||||||
if why == "" {
|
if why == "" {
|
||||||
why = "the indexer wrote no fingerprint of the new form"
|
why = "the indexer wrote no fingerprint of the new form"
|
||||||
}
|
}
|
||||||
return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why)
|
return waitsForAWrite("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why)
|
||||||
}
|
}
|
||||||
k, err := keys.Build(ctx, t.Module, t.Trunk.FP2)
|
k, err := keys.Build(ctx, t.Module, t.Trunk.FP2)
|
||||||
if t.Hand != nil {
|
if t.Hand != nil {
|
||||||
@@ -319,24 +329,34 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
|
|||||||
case k.Failed():
|
case k.Failed():
|
||||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says())
|
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says())
|
||||||
case !k.Done():
|
case !k.Done():
|
||||||
return waits("%s", k.Says())
|
return waitsForAWrite("%s", k.Says())
|
||||||
case len(k.Unreferenced()) > 0:
|
case len(k.Unreferenced()) > 0:
|
||||||
return waits("%s", k.Says())
|
return waitsForAWrite("%s", k.Says())
|
||||||
}
|
}
|
||||||
src, err := keys.Manifest(ctx, k.Build.Manifest)
|
src, err := keys.Manifest(ctx, k.Build.Manifest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return waits("%s's manifest: %v", t.Module, err)
|
return waits("%s's manifest: %v", t.Module, err)
|
||||||
}
|
}
|
||||||
manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list)
|
// **The controller judges no manifest; the indexer did, once** (ADR 0578 §3, §6): the manifest is read leniently, and
|
||||||
|
// the indexer's verdict on its digest is taken. A refused one is held and named, never registered; a word this
|
||||||
|
// controller does not read is set aside and named, and the rest is registered with no second build.
|
||||||
|
manifest, err := catalogue.Decode([]byte(src.JSON))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the
|
|
||||||
// key stays done, and the controller that reads the word registers it with no second build.
|
|
||||||
if word := refusedWord(err.Error()); word != "" {
|
|
||||||
return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+
|
|
||||||
"when a controller that reads it runs", t.Module, word)
|
|
||||||
}
|
|
||||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+
|
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+
|
||||||
"refused: %w", t.Module, err)
|
"not read: %w", t.Module, err)
|
||||||
|
}
|
||||||
|
verdict, err := keys.Verdict(ctx, k.Build.Manifest)
|
||||||
|
if err != nil {
|
||||||
|
return waits("%s's verdict: %v", t.Module, err)
|
||||||
|
}
|
||||||
|
if refused, _ := sdkdelivery.TakeManifestVerdict(verdict); refused {
|
||||||
|
return waits("%s is held: %s", t.Module, verdict.Why())
|
||||||
|
}
|
||||||
|
if manifest.ReadNoFurther() {
|
||||||
|
return waits("%s is held: %s", t.Module, manifest.SetAside())
|
||||||
|
}
|
||||||
|
if aside := manifest.SetAside(); aside != "" {
|
||||||
|
fmt.Printf("%s's manifest has a field this controller does not read, set aside: %s\n", t.Module, aside)
|
||||||
}
|
}
|
||||||
if manifest.Module != t.Module {
|
if manifest.Module != t.Module {
|
||||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s",
|
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s",
|
||||||
|
|||||||
@@ -23,6 +23,14 @@ import (
|
|||||||
type keyBuckets struct {
|
type keyBuckets struct {
|
||||||
keys map[string]builderkeys.Key
|
keys map[string]builderkeys.Key
|
||||||
manifests map[string]string
|
manifests map[string]string
|
||||||
|
verdicts map[string]sdkdelivery.ManifestVerdict // by manifest digest; none is accepted
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k keyBuckets) Verdict(_ context.Context, digest string) (*sdkdelivery.ManifestVerdict, error) {
|
||||||
|
if v, ok := k.verdicts[digest]; ok {
|
||||||
|
return &v, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (k keyBuckets) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
|
func (k keyBuckets) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
|
||||||
@@ -148,15 +156,38 @@ func TestAKeyNotReadyWaitsAndAFailedKeyFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ADR 0501 §7: a manifest word this controller does not read waits for the controller that reads it; never fails.
|
// ADR 0578 §3, §6 (changing ADR 0501 §7): the controller judges no manifest. A word it does not read is set aside and the
|
||||||
func TestAManifestWordTheControllerDoesNotReadWaits(t *testing.T) {
|
// module registered with no second build; the module the indexer's verdict refuses is held and named, never failed.
|
||||||
raw := `{"module":"x","version":"1","not-a-word-yet":true,"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},
|
func TestAKeyIsTakenInOnTheIndexersVerdictAndNeverJudgedHere(t *testing.T) {
|
||||||
|
const tail = `"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},
|
||||||
"resources":[{"id":"a","type":"archive","path":"/opt/x","artifact":"a"}]}`
|
"resources":[{"id":"a","type":"archive","path":"/opt/x","artifact":"a"}]}`
|
||||||
keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "a", Kind: "archive", Digest: shaOf("a"),
|
refusal := func(digest string) map[string]sdkdelivery.ManifestVerdict {
|
||||||
Reference: "artifact-store://x/a/blobs/" + shaOf("a")})
|
return map[string]sdkdelivery.ManifestVerdict{digest: {Module: "x", State: sdkdelivery.VerdictRefused,
|
||||||
_, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{})
|
Problems: []string{"x names a seat nothing declares"}}}
|
||||||
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), `"not-a-word-yet"`) {
|
}
|
||||||
t.Fatalf("an unread word: %v", err)
|
for _, row := range []struct {
|
||||||
|
name, raw string
|
||||||
|
refused bool
|
||||||
|
holds string
|
||||||
|
}{
|
||||||
|
{"an unread word", `{"module":"x","version":"1","not-a-word-yet":true,` + tail, false, ""},
|
||||||
|
{"a fallback to a predefined key, no list at hand", `{"module":"x","version":"1","config":{"font-size":{"type":"number",
|
||||||
|
"description":"size","default":10,"predefined":"font-size"}},` + tail, false, ""},
|
||||||
|
{"no verdict is accepted", `{"module":"x","version":"1",` + tail, false, ""},
|
||||||
|
{"a refusal is held and named", `{"module":"x","version":"1",` + tail, true, "names a seat nothing declares"},
|
||||||
|
} {
|
||||||
|
keys, trunk := aKeyOf(row.raw, 1, sdkdelivery.BuiltArtifact{Name: "a", Kind: "archive", Digest: shaOf("a"),
|
||||||
|
Reference: "artifact-store://x/a/blobs/" + shaOf("a")})
|
||||||
|
if row.refused {
|
||||||
|
keys.verdicts = refusal(sdkdelivery.ManifestDigest([]byte(row.raw)))
|
||||||
|
}
|
||||||
|
_, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{})
|
||||||
|
switch {
|
||||||
|
case row.holds == "" && err != nil:
|
||||||
|
t.Errorf("%s: %v", row.name, err)
|
||||||
|
case row.holds != "" && (!errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), row.holds)):
|
||||||
|
t.Errorf("%s: %v", row.name, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -215,7 +246,7 @@ func TestATemplatedModulesTextsAreReadFromItsArchive(t *testing.T) {
|
|||||||
// The launchers a key's bundle is resolved with are mesh-sdk's one rule (buildstep.Launchers), the names
|
// The launchers a key's bundle is resolved with are mesh-sdk's one rule (buildstep.Launchers), the names
|
||||||
// mesh-build-node writes: a TypeScript bundle's entrypoint x.js runs as x.serve.mjs, a Go bundle has none.
|
// mesh-build-node writes: a TypeScript bundle's entrypoint x.js runs as x.serve.mjs, a Go bundle has none.
|
||||||
func TestALaunchersRuleIsTheBuildModules(t *testing.T) {
|
func TestALaunchersRuleIsTheBuildModules(t *testing.T) {
|
||||||
m, err := catalogue.ParseManifest([]byte(tsBundle))
|
m, err := catalogue.Decode([]byte(tsBundle))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,15 +89,28 @@ func TestTwoControllersOneActs(t *testing.T) {
|
|||||||
t.Fatalf("B, waiting, may act: %v", err)
|
t.Fatalf("B, waiting, may act: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch.
|
// A is stopped, as its service manager stops a controller being replaced (novox/hq issue 906): the stop
|
||||||
|
// alone gives the lease back, before anything else of A is taken down, and B takes it within a second, at
|
||||||
|
// a higher epoch — no wait for the key's age, and no condition said.
|
||||||
stopA()
|
stopA()
|
||||||
a.release()
|
stopped := time.Now()
|
||||||
var lostB <-chan struct{}
|
var lostB <-chan struct{}
|
||||||
select {
|
select {
|
||||||
case lostB = <-tookB:
|
case lostB = <-tookB:
|
||||||
case <-time.After(10 * time.Second):
|
case <-time.After(10 * time.Second):
|
||||||
t.Fatal("B did not take the lease A gave back")
|
t.Fatal("B did not take the lease A gave back")
|
||||||
}
|
}
|
||||||
|
if waited := time.Since(stopped); waited > time.Second {
|
||||||
|
t.Fatalf("B took the lease %s after A was stopped: the stop did not give it back first", waited)
|
||||||
|
}
|
||||||
|
a.release() // what serve's own end does after: waits for the release under way, gives nothing twice
|
||||||
|
conditionsOfB := conditions.NewInMemory()
|
||||||
|
keeperOfB := conditions.NewKeeper(ctx, conditions.Options{Store: conditionsOfB, History: conditionsOfB})
|
||||||
|
b.watchTakeovers(ctx, inv, keeperOfB)
|
||||||
|
keeperOfB.Close(ctx)
|
||||||
|
if said, err := conditionsOfB.Since(ctx, stopped.Add(-time.Minute)); err != nil || len(said) != 0 {
|
||||||
|
t.Fatalf("a lease given back is said as a condition: %+v, %v", said, err)
|
||||||
|
}
|
||||||
select {
|
select {
|
||||||
case <-lostA:
|
case <-lostA:
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func leftOutNFS(t *testing.T) catalogue.Resolution {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
m, err := catalogue.ParseManifest(raw)
|
m, err := catalogue.Decode(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -73,10 +73,8 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
|||||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||||
// The module name comes from the manifest, which only exists when the build got that far.
|
// The module name comes from the manifest, which only exists when the build got that far.
|
||||||
if len(kept.Manifest) > 0 {
|
if len(kept.Manifest) > 0 {
|
||||||
if list, err := judgedWith(result); err == nil {
|
if m, err := catalogue.Decode(kept.Manifest); err == nil {
|
||||||
if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
|
kept.Module = m.Module
|
||||||
kept.Module = m.Module
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return kept
|
return kept
|
||||||
@@ -130,15 +128,7 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
|||||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
||||||
result.On, result.Repository, result.Failed)
|
result.On, result.Repository, result.Failed)
|
||||||
}
|
}
|
||||||
// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
|
manifest, err := catalogue.Decode(kept.Manifest)
|
||||||
// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
|
|
||||||
// mesh publishes, before it is registered.
|
|
||||||
list, err := judgedWith(result)
|
|
||||||
if err != nil {
|
|
||||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
|
||||||
result.Repository, short(result.Commit), err)
|
|
||||||
}
|
|
||||||
manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||||
result.On, result.Repository, err)
|
result.On, result.Repository, err)
|
||||||
@@ -268,16 +258,6 @@ func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) {
|
|||||||
return list, nil
|
return list, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the
|
|
||||||
// commit built, which its build judged it against, where it carried one; the list kept otherwise.
|
|
||||||
func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) {
|
|
||||||
list, err := carriedList(result)
|
|
||||||
if err != nil || list != nil {
|
|
||||||
return list, err
|
|
||||||
}
|
|
||||||
return catalogue.PublishedPredefined(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the
|
// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the
|
||||||
// store keeps then — this build's, or a newer one another controller process kept.
|
// store keeps then — this build's, or a newer one another controller process kept.
|
||||||
func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult,
|
func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult,
|
||||||
|
|||||||
@@ -223,7 +223,6 @@ var usageText = `mesh-controller — the control plane
|
|||||||
overlay place <node> [flags] say where a node is and how it is reached
|
overlay place <node> [flags] say where a node is and how it is reached
|
||||||
overlay show the private network, as the mesh computes it
|
overlay show the private network, as the mesh computes it
|
||||||
module add <file> register a module from its manifest
|
module add <file> register a module from its manifest
|
||||||
module check <file|dir>... judge manifests where they are written, with no mesh (exit 1 on any problem)
|
|
||||||
module list what modules this mesh knows about
|
module list what modules this mesh knows about
|
||||||
module moved <name> <commit> the source has a newer commit than the mesh built
|
module moved <name> <commit> the source has a newer commit than the mesh built
|
||||||
module forget <name> remove one, unless a node runs it or the mesh holds things for it
|
module forget <name> remove one, unless a node runs it or the mesh holds things for it
|
||||||
|
|||||||
+253
-119
@@ -8,9 +8,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
@@ -62,35 +63,7 @@ var provided = providedModules()
|
|||||||
|
|
||||||
func moduleCommand(ctx context.Context, args []string) error {
|
func moduleCommand(ctx context.Context, args []string) error {
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return errors.New("module add <file>, module check <file>..., module list, or module forget <name>")
|
return errors.New("module add <file>, module list, or module forget <name>")
|
||||||
}
|
|
||||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
|
||||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
|
||||||
if args[0] == "check" {
|
|
||||||
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
|
||||||
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
|
||||||
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
|
||||||
"judge each module's identity on a machine name this many characters long")
|
|
||||||
given, err := parseAround(set, args[1:])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if *longest < 1 {
|
|
||||||
return errors.New("--longest-machine-name is a length, at least 1")
|
|
||||||
}
|
|
||||||
var paths []string
|
|
||||||
for _, a := range given {
|
|
||||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
|
||||||
under, err := manifestsUnder(a)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
paths = append(paths, under...)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
paths = append(paths, a)
|
|
||||||
}
|
|
||||||
return moduleCheckFor(paths, *longest, os.Stdout)
|
|
||||||
}
|
}
|
||||||
open, err := openStores(ctx)
|
open, err := openStores(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -124,21 +97,19 @@ func moduleCommand(ctx context.Context, args []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
m, err := catalogue.ParseManifest(raw)
|
m, err := catalogue.Decode(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2).
|
// A manifest handed over by hand is not judged here (ADR 0578 §3): a templated one has no build to read its
|
||||||
if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil {
|
// texts from, so it is refused by name.
|
||||||
return err
|
if p := catalogue.HeldTemplateProblem(m); p != "" {
|
||||||
|
return errors.New(p)
|
||||||
}
|
}
|
||||||
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
|
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := namesNoInstallation(m); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := inv.RegisterModule(ctx, m, from); err != nil {
|
if err := inv.RegisterModule(ctx, m, from); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -965,6 +936,175 @@ func processPrincipalFor(m catalogue.Manifest, processID string) (broker.Process
|
|||||||
return broker.ProcessPrincipal{}, false
|
return broker.ProcessPrincipal{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// mintOnlyIfAbsent marks a context whose issue mints only accounts that are not issued (onlyIfAbsent).
|
||||||
|
type mintOnlyIfAbsent struct{}
|
||||||
|
|
||||||
|
// onlyIfAbsent is ctx for an issue that never replaces an issued account: the assign act's, the assignments' watch's
|
||||||
|
// and a registration's, which may run at once for one account (novox/hq ADR 0543, issue 1945).
|
||||||
|
func onlyIfAbsent(ctx context.Context) context.Context {
|
||||||
|
return context.WithValue(ctx, mintOnlyIfAbsent{}, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// issuedNow collects the accounts an issue minted and sealed, for the line that says it (withIssued).
|
||||||
|
type issuedNow struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
users []string
|
||||||
|
}
|
||||||
|
|
||||||
|
type issuedNowKey struct{}
|
||||||
|
|
||||||
|
// withIssued is ctx collecting the accounts its issues mint and seal.
|
||||||
|
func withIssued(ctx context.Context) (context.Context, *issuedNow) {
|
||||||
|
got := &issuedNow{}
|
||||||
|
return context.WithValue(ctx, issuedNowKey{}, got), got
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *issuedNow) list() []string {
|
||||||
|
if g == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
return slices.Clone(g.users)
|
||||||
|
}
|
||||||
|
|
||||||
|
// accountIssuer is what issuing one account needs of the store.
|
||||||
|
type accountIssuer interface {
|
||||||
|
LockBusAccount(ctx context.Context, username string) (func(), error)
|
||||||
|
BusAccountIssued(ctx context.Context, username string) (minted, sealed bool, err error)
|
||||||
|
MintBusPassword(ctx context.Context, u inventory.BusUser) (string, error)
|
||||||
|
MintBusPasswordUnsealed(ctx context.Context, u inventory.BusUser) (string, error)
|
||||||
|
MarkBusAccountSealed(ctx context.Context, username string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// issueLockedFor is the longest an issue holds an account's lock: a seal cut by it leaves the account unsealed, issued
|
||||||
|
// again by the next issue.
|
||||||
|
const issueLockedFor = 2 * time.Minute
|
||||||
|
|
||||||
|
// issueAccount mints u's password and seals it to its node with seal. By default it replaces whatever stands (a
|
||||||
|
// rotation: `module issue`). Under onlyIfAbsent it issues only an account that is not issued, a row with no sealed
|
||||||
|
// credential counting as none (issue 1945, live on 2026-10-25 at 20:04: a row whose credential the node no longer held
|
||||||
|
// left ace's mesh-build-go with no credential), under the store's lock on the account, so two issuers never interleave
|
||||||
|
// their look, mint and seal: the hash the bus holds is always the password sealed. A seal that fails leaves the
|
||||||
|
// account unsealed, issued again by the next issue.
|
||||||
|
//
|
||||||
|
// **From the grants part's take-over the controller mints nothing** (issue 774, research 072's gap "vault mints after
|
||||||
|
// the lease"; the controller is deprecated and hands its duties over): the bus then loads the grants part's list, whose
|
||||||
|
// hashes are the vault's, so the issue is forwarded to the vault's issue-account and only after runs (what the account
|
||||||
|
// needs beside its value: its durable consumers). **Before it** the bus loads the controller's own list, whose hashes
|
||||||
|
// come from its store alone, so an account the vault minted would be refused: the controller mints as before and hands
|
||||||
|
// every account it mints to an installed vault at once (handIssuedToVault), so none waits on a node reporting its hash.
|
||||||
|
func issueAccount(ctx context.Context, accounts accountIssuer, u inventory.BusUser, seal func(password string) error,
|
||||||
|
after func() error) error {
|
||||||
|
only, _ := ctx.Value(mintOnlyIfAbsent{}).(bool)
|
||||||
|
if ask, installed, err := vaultIssuer(ctx, accounts); err != nil {
|
||||||
|
return fmt.Errorf("whether the vault issues bus accounts yet cannot be read, so %s is not issued: %w", u.Username, err)
|
||||||
|
} else if installed {
|
||||||
|
return forwardToTheVault(ctx, ask, u.Username, only, after)
|
||||||
|
}
|
||||||
|
if !only {
|
||||||
|
password, err := accounts.MintBusPassword(ctx, u)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := seal(password); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// A deadline on the locked section: it holds a connection of the store's pool across the vault and the bus.
|
||||||
|
var cancel context.CancelFunc
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, issueLockedFor)
|
||||||
|
defer cancel()
|
||||||
|
release, err := accounts.LockBusAccount(ctx, u.Username)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
if minted, sealed, err := accounts.BusAccountIssued(ctx, u.Username); err != nil {
|
||||||
|
return err
|
||||||
|
} else if minted && sealed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
password, err := accounts.MintBusPasswordUnsealed(ctx, u)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := seal(password); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := accounts.MarkBusAccountSealed(ctx, u.Username); err != nil {
|
||||||
|
return fmt.Errorf("%s is sealed and not marked so, and is issued again by the next issue: %w", u.Username, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got, _ := ctx.Value(issuedNowKey{}).(*issuedNow); got != nil {
|
||||||
|
got.mu.Lock()
|
||||||
|
got.users = append(got.users, u.Username)
|
||||||
|
got.mu.Unlock()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// vaultIssuer answers the vault to forward an issue to, and whether the issue is the vault's: one is installed
|
||||||
|
// (vaultInstalled) and the grants part took over the bus's user list (grantsTookOver). A store that is not the
|
||||||
|
// controller's inventory (a test's) has none. A test replaces it.
|
||||||
|
var vaultIssuer = func(ctx context.Context, accounts accountIssuer) (vaultAsker, bool, error) {
|
||||||
|
inv, ok := accounts.(*inventory.Inventory)
|
||||||
|
if !ok {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
installed, err := vaultInstalled(ctx, inv)
|
||||||
|
if err != nil || !installed {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if over, err := grantsTookOver(ctx); err != nil || !over {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return vaultAskerForIssue(ctx), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// issuedByTheVault is the vault's answer to issue-account (mesh-vault heldBackOrMinted).
|
||||||
|
type issuedByTheVault struct {
|
||||||
|
Minted bool `json:"minted"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
Account struct {
|
||||||
|
Secret string `json:"secret"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
} `json:"account"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// forwardToTheVault asks the vault to issue user (its verb issue-account) and then runs after. The vault mints only
|
||||||
|
// what the grants part wants and never anew a value it holds, so asking twice, or with the want's own mint, is harmless.
|
||||||
|
// An issue on an assignment (only) is not failed by a refusal: the vault mints on the want itself when it is heard, and
|
||||||
|
// the refusal is said; `module issue` fails with it.
|
||||||
|
func forwardToTheVault(ctx context.Context, ask vaultAsker, user string, only bool, after func() error) error {
|
||||||
|
raw, err := ask("issue-account", map[string]any{"user": user})
|
||||||
|
if err != nil {
|
||||||
|
if !only {
|
||||||
|
return fmt.Errorf("the vault is the one issuer of bus accounts and did not issue %s: %w", user, err)
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s is the vault's to mint, on the grants part's want of it; issue-account said: %v\n", user, err)
|
||||||
|
} else {
|
||||||
|
var got issuedByTheVault
|
||||||
|
if err := json.Unmarshal(raw, &got); err != nil {
|
||||||
|
return fmt.Errorf("the vault's answer to issue-account %s does not read: %w", user, err)
|
||||||
|
}
|
||||||
|
if got.Minted {
|
||||||
|
fmt.Printf("bus user %s minted by the vault as %s (generation %d)\n", user, got.Account.Secret, got.Account.Generation)
|
||||||
|
if g, _ := ctx.Value(issuedNowKey{}).(*issuedNow); g != nil {
|
||||||
|
g.mu.Lock()
|
||||||
|
g.users = append(g.users, user)
|
||||||
|
g.mu.Unlock()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Printf("bus user %s not minted anew by the vault: %s\n", user, got.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if after == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return after()
|
||||||
|
}
|
||||||
|
|
||||||
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||||
//
|
//
|
||||||
// **Three things differ from a management call, and each is the point of the move.** The credential
|
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||||
@@ -990,13 +1130,10 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
|||||||
}
|
}
|
||||||
if _, reads := m.OwnSecrets["broker"]; reads {
|
if _, reads := m.OwnSecrets["broker"]; reads {
|
||||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
if err := issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: busKindOf(m.Module), Node: node,
|
||||||
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
Module: m.Module}, func(password string) error {
|
||||||
})
|
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||||
if err != nil {
|
}, func() error { return ensureModuleConsumer(m, node, busAddress) }); err != nil {
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1021,12 +1158,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
|||||||
func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
|
func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
|
||||||
node, busAddress string, known broker.Broker, reachable string) error {
|
node, busAddress string, known broker.Broker, reachable string) error {
|
||||||
user := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}.Username()
|
user := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}.Username()
|
||||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
return issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module},
|
||||||
Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module,
|
func(password string) error {
|
||||||
})
|
return sealProcess(ctx, inv, m, pr, node, busAddress, known, reachable, user, password)
|
||||||
if err != nil {
|
}, func() error { return ensureProcessQueues(m, pr, node, busAddress) })
|
||||||
return err
|
}
|
||||||
}
|
|
||||||
|
// sealProcess seals a process's minted account to its node, hands it to the vault and makes its work-queue consumers.
|
||||||
|
func sealProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
|
||||||
|
node, busAddress string, known broker.Broker, reachable, user, password string) error {
|
||||||
name := processSecretName(pr.Process)
|
name := processSecretName(pr.Process)
|
||||||
held, err := heldProcessAccount(known, reachable, node, m.Module, user, password)
|
held, err := heldProcessAccount(known, reachable, node, m.Module, user, password)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1038,30 +1178,40 @@ func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Man
|
|||||||
if err := handIssuedToVault(ctx, inv, m, node, name, string(held)); err != nil {
|
if err := handIssuedToVault(ctx, inv, m, node, name, string(held)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(pr.Queues) > 0 {
|
if err := ensureProcessQueues(m, pr, node, busAddress); err != nil {
|
||||||
if busAddress == "" {
|
return err
|
||||||
fmt.Printf(" %s's process %s consumes its work queue; its consumer is created when the bus is "+
|
|
||||||
"reachable (`push --behind`, then `rollout mint` again is harmless)\n", m.Module, pr.Process)
|
|
||||||
} else {
|
|
||||||
js, err := broker.Dial(busAddress)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
|
||||||
"%s's process %s's work-queue consumer: %w", m.Module, pr.Process, err)
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
who := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}
|
|
||||||
for _, q := range pr.Queues {
|
|
||||||
if err := js.EnsureConsumer(broker.QueueConsumer(who, q)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
fmt.Printf("bus user %s minted for %s's process %s, scoped to its bus declaration\n", user, m.Module, pr.Process)
|
fmt.Printf("bus user %s minted for %s's process %s, scoped to its bus declaration\n", user, m.Module, pr.Process)
|
||||||
fmt.Printf(" sealed to %s as %s. It arrives with the next push — `push %s` to send it\n", node, name, node)
|
fmt.Printf(" sealed to %s as %s. It arrives with the next push — `push %s` to send it\n", node, name, node)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ensureProcessQueues makes a process's work-queue consumers where the bus can be reached: what its account needs beside
|
||||||
|
// its value, whoever minted it.
|
||||||
|
func ensureProcessQueues(m catalogue.Manifest, pr broker.ProcessPrincipal, node, busAddress string) error {
|
||||||
|
if len(pr.Queues) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if busAddress == "" {
|
||||||
|
fmt.Printf(" %s's process %s consumes its work queue; its consumer is created when the bus is "+
|
||||||
|
"reachable (`push --behind`, then `rollout mint` again is harmless)\n", m.Module, pr.Process)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(busAddress)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the mesh cannot reach the bus to create %s's process %s's work-queue consumer: %w",
|
||||||
|
m.Module, pr.Process, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
who := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}
|
||||||
|
for _, q := range pr.Queues {
|
||||||
|
if err := js.EnsureConsumer(broker.QueueConsumer(who, q)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// heldProcessAccount is a process's account, in the provisioner client's form (novox/hq ADR 0436 §2):
|
// heldProcessAccount is a process's account, in the provisioner client's form (novox/hq ADR 0436 §2):
|
||||||
// {url, fingerprint, node, module, user, password} — never claims, since a process holds no seat (§3).
|
// {url, fingerprint, node, module, user, password} — never claims, since a process holds no seat (§3).
|
||||||
func heldProcessAccount(known broker.Broker, reachable, node, module, user, password string) ([]byte, error) {
|
func heldProcessAccount(known broker.Broker, reachable, node, module, user, password string) ([]byte, error) {
|
||||||
@@ -1091,12 +1241,15 @@ func runsTheProvisionerClient(m catalogue.Manifest) bool {
|
|||||||
func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
|
func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
|
||||||
known broker.Broker, reachable string) error {
|
known broker.Broker, reachable string) error {
|
||||||
user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()
|
user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()
|
||||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
return issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: inventory.BusProvisioner, Node: node,
|
||||||
Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module,
|
Module: m.Module}, func(password string) error {
|
||||||
})
|
return sealProvisionerClient(ctx, inv, m, node, known, reachable, user, password)
|
||||||
if err != nil {
|
}, nil)
|
||||||
return err
|
}
|
||||||
}
|
|
||||||
|
// sealProvisionerClient seals the provisioner client's minted account to its machine and hands it to the vault.
|
||||||
|
func sealProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
|
||||||
|
known broker.Broker, reachable, user, password string) error {
|
||||||
held, err := json.Marshal(struct {
|
held, err := json.Marshal(struct {
|
||||||
URL string `json:"url"`
|
URL string `json:"url"`
|
||||||
Fingerprint string `json:"fingerprint,omitempty"`
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
@@ -1163,28 +1316,8 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
|||||||
if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil {
|
if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := ensureModuleConsumer(m, node, busAddress); err != nil {
|
||||||
// And how it hears what it consumes. Derived from its declaration, and only when it declared
|
return err
|
||||||
// something: a module that consumes nothing needs no consumer, and creating one would be a
|
|
||||||
// durable subscription nobody reads.
|
|
||||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
|
||||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
|
||||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
|
||||||
}); needed {
|
|
||||||
if busAddress == "" {
|
|
||||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+
|
|
||||||
"`rollout mint` again is harmless)\n", m.Module)
|
|
||||||
} else {
|
|
||||||
js, err := broker.Dial(busAddress)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
|
||||||
"how %s hears what it consumes: %w", m.Module, err)
|
|
||||||
}
|
|
||||||
defer js.Close()
|
|
||||||
if err := js.EnsureConsumer(consumer); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
|
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
|
||||||
@@ -1194,6 +1327,30 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ensureModuleConsumer makes how a module hears what it consumes, where the bus can be reached: what its account needs
|
||||||
|
// beside its value, whoever minted it. Derived from its declaration, and only when it declared something: a module that
|
||||||
|
// consumes nothing needs no consumer, and creating one would be a durable subscription nobody reads.
|
||||||
|
func ensureModuleConsumer(m catalogue.Manifest, node, busAddress string) error {
|
||||||
|
consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||||
|
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||||
|
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||||
|
})
|
||||||
|
if !needed {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if busAddress == "" {
|
||||||
|
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push --behind`, then "+
|
||||||
|
"`rollout mint` again is harmless)\n", m.Module)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(busAddress)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the mesh cannot reach the bus to create how %s hears what it consumes: %w", m.Module, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
return js.EnsureConsumer(consumer)
|
||||||
|
}
|
||||||
|
|
||||||
// whereItComesFrom is the provenance a module handed over by hand records, and what a record must
|
// whereItComesFrom is the provenance a module handed over by hand records, and what a record must
|
||||||
// say to be worth anything later.
|
// say to be worth anything later.
|
||||||
//
|
//
|
||||||
@@ -1228,29 +1385,6 @@ func whereItComesFrom(repository, ref, commit, path string, self bool) (inventor
|
|||||||
return from, nil
|
return from, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// namesNoInstallation is the mesh refusing a definition that names an installation, at the moment
|
|
||||||
// it would enter the catalogue (novox/hq ADR 0112, ADR 0155). `module check` says the same thing
|
|
||||||
// earlier, where the author is; this is the last moment the mesh can still say no, and a
|
|
||||||
// definition that got past the check — written elsewhere, or checked by nobody — is refused here
|
|
||||||
// in the same words. A name meant on purpose is declared with its reason and passes.
|
|
||||||
func namesNoInstallation(m catalogue.Manifest) error {
|
|
||||||
if problems := catalogue.TrustProblems(m); len(problems) > 0 {
|
|
||||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
||||||
}
|
|
||||||
// And a module that declares a setting names the keys of its mergeable files (novox/hq ADR 0326
|
|
||||||
// §3, issue 574): judged where a definition is written, never where a stored build is read again.
|
|
||||||
if problems := catalogue.WrittenSettingsProblems(m); len(problems) > 0 {
|
|
||||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
|
||||||
}
|
|
||||||
named := catalogue.InstallationProblems(m)
|
|
||||||
if len(named) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return fmt.Errorf("%s names an installation, and a definition names none — declare a name meant "+
|
|
||||||
"on purpose under %s with its reason, or take it out:\n - %s",
|
|
||||||
m.Module, catalogue.NamesOnPurpose, strings.Join(named, "\n - "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// seatClaimed is one seat a module claims, as its runtime needs it: the name, the scope (a
|
// seatClaimed is one seat a module claims, as its runtime needs it: the name, the scope (a
|
||||||
// node-scoped seat's verb carries the machine, design 33 §4) and the verbs the seat promises.
|
// node-scoped seat's verb carries the machine, design 33 §4) and the verbs the seat promises.
|
||||||
type seatClaimed struct {
|
type seatClaimed struct {
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
// one module's directory and touches nothing else the mesh holds: nothing asks its first build, it opens no plan, and
|
// one module's directory and touches nothing else the mesh holds: nothing asks its first build, it opens no plan, and
|
||||||
// the module is registered from the builder's first done key of it, assigned nowhere, with no build request written.
|
// the module is registered from the builder's first done key of it, assigned nowhere, with no build request written.
|
||||||
func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T) {
|
func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T) {
|
||||||
|
withNoNewModuleWaits(t)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -66,9 +67,11 @@ func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ADR 0501 §7: a new module whose manifest uses a word this controller does not read is named waiting at once, never
|
// ADR 0578 §6 (which changes ADR 0501 §7): a new module whose manifest uses a word this controller does not read is
|
||||||
// failed, and registered from the same key, with no second build, when a controller that reads it looks.
|
// registered at once, the word set aside, never failed and with no second build; one whose manifest the indexer's judge
|
||||||
func TestANewModuleWithAnUnreadWordWaitsForTheControllerThatReadsIt(t *testing.T) {
|
// refused is held, named in the indexer's words, and never registered (ADR 0578 §3).
|
||||||
|
func TestANewModuleWithAnUnreadWordIsRegisteredAndARefusedOneIsHeld(t *testing.T) {
|
||||||
|
withNoNewModuleWaits(t)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
@@ -94,12 +97,38 @@ func TestANewModuleWithAnUnreadWordWaitsForTheControllerThatReadsIt(t *testing.T
|
|||||||
}
|
}
|
||||||
b.built(t, "watchman", "1", fp, digest, sdkdelivery.BuildDone)
|
b.built(t, "watchman", "1", fp, digest, sdkdelivery.BuildDone)
|
||||||
takeNewModules(ctx, open, "watchman")
|
takeNewModules(ctx, open, "watchman")
|
||||||
if _, known := inventoryHolds(t, inv, "watchman"); known {
|
if _, known := inventoryHolds(t, inv, "watchman"); !known {
|
||||||
t.Fatal("0501 §7: a manifest with an unread word was registered")
|
t.Fatalf("0578 §6: a manifest with a word set aside is not registered: %+v", newModuleObservations(time.Now()))
|
||||||
}
|
}
|
||||||
obs := newModuleObservations(time.Now())
|
// The indexer refused another: held, named, not registered.
|
||||||
if len(obs) != 1 || obs[0].Kind != kindNewModuleWaits || !strings.Contains(obs[0].Summary, `"not-a-word-yet"`) {
|
refusedFP := fpOf('9')
|
||||||
t.Fatalf("0501 §7: the module waiting for a reader of its word is named by %+v", obs)
|
refusedRaw := `{"module":"refusedmod","version":"1","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` +
|
||||||
|
`"resources":[{"id":"a","type":"archive","path":"/opt/refusedmod","artifact":"a"}]}`
|
||||||
|
refusedDigest := sdkdelivery.ManifestDigest([]byte(refusedRaw))
|
||||||
|
value, _ = sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "refusedmod", JSON: refusedRaw})
|
||||||
|
if _, err := b.manifests.Put(ctx, refusedDigest, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
verdict, _ := sdkdelivery.EncodeManifestVerdict(sdkdelivery.ManifestVerdict{Module: "refusedmod", State: sdkdelivery.VerdictRefused,
|
||||||
|
Problems: []string{"it breaks a rule"}, Judge: "exe:1", At: time.Now()})
|
||||||
|
if _, err := b.verdicts.Put(ctx, refusedDigest, verdict); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
trunk, _ = sdkdelivery.EncodeModule(sdkdelivery.Module{FP: "src1:" + refusedFP[5:], FP2: refusedFP, Manifest: refusedDigest,
|
||||||
|
Repository: "novox/mesh-catalog", Commit: "5555555500000000", Path: "modules/refusedmod"})
|
||||||
|
if _, err := b.modules.Put(ctx, "refusedmod", trunk); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.built(t, "refusedmod", "1", refusedFP, refusedDigest, sdkdelivery.BuildDone)
|
||||||
|
takeNewModules(ctx, open, "refusedmod")
|
||||||
|
if _, known := inventoryHolds(t, inv, "refusedmod"); known {
|
||||||
|
t.Fatal("0578 §3: a manifest the indexer refused was registered")
|
||||||
|
}
|
||||||
|
// Said at once in the controller's journal; a condition once it waits past the bound (the watchman raises the
|
||||||
|
// indexer's manifest-refused at once).
|
||||||
|
obs := newModuleObservations(time.Now().Add(newModuleWaitBound + time.Minute))
|
||||||
|
if len(obs) != 1 || obs[0].Kind != kindNewModuleWaits || !strings.Contains(obs[0].Summary, "refused the manifest of refusedmod: it breaks a rule") {
|
||||||
|
t.Fatalf("0578 §3: the refused module is not held and named: %+v", obs)
|
||||||
}
|
}
|
||||||
builds, _ := inv.Builds(ctx, "watchman", 5)
|
builds, _ := inv.Builds(ctx, "watchman", 5)
|
||||||
for _, b := range builds {
|
for _, b := range builds {
|
||||||
|
|||||||
@@ -200,14 +200,20 @@ func newModuleObservations(now time.Time) []conditions.Observation {
|
|||||||
// watchNewModules wakes the take-in of new modules for each module the builder or the indexer writes, until ctx ends, and
|
// watchNewModules wakes the take-in of new modules for each module the builder or the indexer writes, until ctx ends, and
|
||||||
// looks at every module once now. A bucket not there yet is said: the look at start and a first merge's word still work.
|
// looks at every module once now. A bucket not there yet is said: the look at start and a first merge's word still work.
|
||||||
func watchNewModules(ctx context.Context, open *stores) {
|
func watchNewModules(ctx context.Context, open *stores) {
|
||||||
inBackground(func() {
|
inBackground(ctx, func() {
|
||||||
takeNewModules(ctx, open)
|
takeNewModules(ctx, open)
|
||||||
})
|
})
|
||||||
// And, beside it, the start's pass of release 6 and the take-in of each registered module's new done key (novox/hq
|
// And, beside it, the start's pass of release 6 and the take-in of each registered module's new done key (novox/hq
|
||||||
// ADR 0539 clauses 5, 7, 10 and 13; composed_start.go, composed_keys.go): the plans' buckets deleted, every node
|
// ADR 0539 clauses 5, 7, 10 and 13; composed_start.go, composed_keys.go): the plans' buckets deleted, every node
|
||||||
// composed, the replaced controller composed, and from now on each new fingerprint registered, composed and granted.
|
// composed, the replaced controller composed, and from now on each new fingerprint registered, composed and granted.
|
||||||
inBackground(func() { composeAtStart(ctx, open) })
|
inBackground(ctx, func() { composeAtStart(ctx, open) })
|
||||||
watchDoneKeys(ctx, open)
|
watchDoneKeys(ctx, open)
|
||||||
|
// And a change of config composes the node and the touched modules' kept compositions again, heard from
|
||||||
|
// mesh-config's resolved values (ADR 0545, kept_compositions.go).
|
||||||
|
watchConfigFollow(ctx, open)
|
||||||
|
// And each assignment that comes to stand issues its node's bus accounts and the module's grant (ADR 0543): a module
|
||||||
|
// built long before is assigned to one more node with no build done again (issue 1945, assignment_accounts.go).
|
||||||
|
watchAssignmentAccounts(ctx, open)
|
||||||
watchHavesForJudgement(ctx, open)
|
watchHavesForJudgement(ctx, open)
|
||||||
// And the keep set keeps the builds every want names (ADR 0539 clause 1), with no plan left to name them.
|
// And the keep set keeps the builds every want names (ADR 0539 clause 1), with no plan left to name them.
|
||||||
open.inventory.SetWanted(wantedOnTheBus)
|
open.inventory.SetWanted(wantedOnTheBus)
|
||||||
@@ -223,16 +229,16 @@ func watchNewModules(ctx context.Context, open *stores) {
|
|||||||
fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
|
fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
go func() {
|
goServing(ctx, func() {
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
closeKeys()
|
closeKeys()
|
||||||
}()
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// wakeNewModule takes module in when the catalogue does not hold it: never on the watch's own goroutine, which must keep
|
// wakeNewModule takes module in when the catalogue does not hold it: never on the watch's own goroutine, which must keep
|
||||||
// draining.
|
// draining.
|
||||||
func wakeNewModule(ctx context.Context, open *stores, module string) {
|
func wakeNewModule(ctx context.Context, open *stores, module string) {
|
||||||
inBackground(func() {
|
inBackground(ctx, func() {
|
||||||
entries, err := open.inventory.Catalogued(ctx)
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
@@ -250,14 +256,17 @@ func wakeNewModule(ctx context.Context, open *stores, module string) {
|
|||||||
// ends only once the work its mesh started has.
|
// ends only once the work its mesh started has.
|
||||||
var background sync.WaitGroup
|
var background sync.WaitGroup
|
||||||
|
|
||||||
// inBackground runs work beside the caller, counted in background. A variable so a test of what a walk does around a
|
// inBackground runs work beside the caller, counted in background and, under a serving controller, in its serving group,
|
||||||
// take-in may run it in the caller's turn and need not wait for it.
|
// so serve's stop waits for it and a serve that is stopping starts none (serving_group.go). A variable so a test of what a
|
||||||
var inBackground = func(work func()) {
|
// walk does around a take-in may run it in the caller's turn and need not wait for it.
|
||||||
|
var inBackground = func(ctx context.Context, work func()) {
|
||||||
background.Add(1)
|
background.Add(1)
|
||||||
go func() {
|
if !goServing(ctx, func() {
|
||||||
defer background.Done()
|
defer background.Done()
|
||||||
work()
|
work()
|
||||||
}()
|
}) {
|
||||||
|
background.Done()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// newModulesOf are the modules the indexer holds in a repository: what a first merge's word wakes.
|
// newModulesOf are the modules the indexer holds in a repository: what a first merge's word wakes.
|
||||||
|
|||||||
@@ -9,11 +9,7 @@ import (
|
|||||||
// ADR 0501 §6, §7 (issue 1851, A5b): the condition of a new module reads its key's take-in: one waiting for a controller
|
// ADR 0501 §6, §7 (issue 1851, A5b): the condition of a new module reads its key's take-in: one waiting for a controller
|
||||||
// that reads its manifest's word is named at once, one waiting for its key only past the bound, each reason said once.
|
// that reads its manifest's word is named at once, one waiting for its key only past the bound, each reason said once.
|
||||||
func TestANewModuleWaitingIsNamedByWhatItWaitsFor(t *testing.T) {
|
func TestANewModuleWaitingIsNamedByWhatItWaitsFor(t *testing.T) {
|
||||||
t.Cleanup(func() {
|
withNoNewModuleWaits(t)
|
||||||
newModulesWaiting.Lock()
|
|
||||||
newModulesWaiting.by = map[string]newModuleWait{}
|
|
||||||
newModulesWaiting.Unlock()
|
|
||||||
})
|
|
||||||
word := `waits: watchman's manifest uses the word "key-ttl", which this controller does not read: it is registered when a controller that reads it runs`
|
word := `waits: watchman's manifest uses the word "key-ttl", which this controller does not read: it is registered when a controller that reads it runs`
|
||||||
sayNewModuleWait(newModuleWait{Module: "watchman", Commit: "c1", Why: word, Word: wordIn(word)})
|
sayNewModuleWait(newModuleWait{Module: "watchman", Commit: "c1", Why: word, Word: wordIn(word)})
|
||||||
sayNewModuleWait(newModuleWait{Module: "view", Commit: "c1", Why: "waits: the build of view at aaaa is building on ace"})
|
sayNewModuleWait(newModuleWait{Module: "view", Commit: "c1", Why: "waits: the build of view at aaaa is building on ace"})
|
||||||
@@ -31,3 +27,15 @@ func TestANewModuleWaitingIsNamedByWhatItWaitsFor(t *testing.T) {
|
|||||||
t.Fatalf("registered modules are still named: %+v", obs)
|
t.Fatalf("registered modules are still named: %+v", obs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withNoNewModuleWaits empties the controller's list of new modules waiting, for a test, at its start and at its end: the
|
||||||
|
// list is the process's, so a test that leaves a module in it is read by the next.
|
||||||
|
func withNoNewModuleWaits(t *testing.T) {
|
||||||
|
reset := func() {
|
||||||
|
newModulesWaiting.Lock()
|
||||||
|
newModulesWaiting.by = map[string]newModuleWait{}
|
||||||
|
newModulesWaiting.Unlock()
|
||||||
|
}
|
||||||
|
reset()
|
||||||
|
t.Cleanup(reset)
|
||||||
|
}
|
||||||
|
|||||||
@@ -133,9 +133,14 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
|
// refuse (novox/hq issue 358). Read-only on the node and here, so a verb may run it too.
|
||||||
return nodeJudgeDirectories(ctx, open, args[1:])
|
return nodeJudgeDirectories(ctx, open, args[1:])
|
||||||
|
|
||||||
|
case "placement-export":
|
||||||
|
// Each node's placement as the catalogue's placement file says it (novox/hq ADR 0555 §6): read-only, so a verb
|
||||||
|
// may run it.
|
||||||
|
return placementExport(ctx, open, args[1:])
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
|
return fmt.Errorf("node has no %q; it has add, list, show, public-domain, account, agent-account, hand-over, "+
|
||||||
"setuid-search and judge-directories", args[0])
|
"setuid-search, judge-directories and placement-export", args[0])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
// paths and its manifest alone. mesh-sdk's TypeScript package at its repository's root: a change to its go/ touches no
|
// paths and its manifest alone. mesh-sdk's TypeScript package at its repository's root: a change to its go/ touches no
|
||||||
// module, one to its src/ or its package.json touches it. A module with an artifact naming no paths reads its directory.
|
// module, one to its src/ or its package.json touches it. A module with an artifact naming no paths reads its directory.
|
||||||
func TestAModuleNamingItsPathsIsReadFromThemAlone(t *testing.T) {
|
func TestAModuleNamingItsPathsIsReadFromThemAlone(t *testing.T) {
|
||||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"lib",
|
m, err := catalogue.Decode([]byte(`{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"lib",
|
||||||
"kind":"package","language":"typescript","paths":["src","package.json","package-lock.json","tsconfig.json"]}]},
|
"kind":"package","language":"typescript","paths":["src","package.json","package-lock.json","tsconfig.json"]}]},
|
||||||
"resources":[]}`))
|
"resources":[]}`))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// placementExport prints each node's placement as the catalogue's placement file says it (novox/hq ADR 0555 §6, issue
|
||||||
|
// 1126): what a person decided of the node and where its modules run, read by this controller's own code from its own
|
||||||
|
// store, as JSON keyed by node. The one verb the controller gains for its exit (the operator, 2026-10-26 19:06:40, ADR
|
||||||
|
// 0547 §1's one exception besides §5's grant): read once by the seed of the placement files, whose pull request merges
|
||||||
|
// only when its composition check composes every node equal to what it runs. It reads the same Input every composition
|
||||||
|
// of this controller reads (readMeshAndNodes, readSettings) and writes nothing.
|
||||||
|
//
|
||||||
|
// It moves a piece of the research "The controller, NATS-native": the controller's last node data (node rows, placement,
|
||||||
|
// ports, pins, bindings, config placement) leaves it for the catalogue, before its exit at ADR 0547's third milestone.
|
||||||
|
func placementExport(ctx context.Context, open *stores, args []string) error {
|
||||||
|
if len(args) > 1 {
|
||||||
|
return errors.New("node placement-export [<name>]")
|
||||||
|
}
|
||||||
|
in, _, err := readMeshAndNodes(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := readSettings(ctx, open, &in); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
if len(args) == 1 {
|
||||||
|
names = []string{args[0]}
|
||||||
|
} else {
|
||||||
|
for _, n := range in.Nodes {
|
||||||
|
names = append(names, n.Name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
}
|
||||||
|
out := make(map[string]delivery.Placement, len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
p, err := frombus.Export(in, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
out[name] = p
|
||||||
|
}
|
||||||
|
return printJSON(out)
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Issue 1126 (novox/hq ADR 0555 §6, the operator's one export verb of 2026-10-26 19:06:40): `node placement-export`
|
||||||
|
// answers, from the real store, each node's placement as the catalogue's placement file says it: what a person decided
|
||||||
|
// of the node (its endpoint, site, hub and address on the private network) and its modules in the order the composition
|
||||||
|
// reads them, exactly frombus.Export of the input this controller composes from; one node when named; a node the store
|
||||||
|
// lacks is refused. Checked within TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker, over the same fresh
|
||||||
|
// mesh, so the export adds no integration test of its own (the test budget).
|
||||||
|
func placementExportIsEachNodesPlacementFromTheStore(t *testing.T, open *stores) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
in, _, err := readMeshAndNodes(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := readSettings(ctx, open, &in); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var all map[string]sdkdelivery.Placement
|
||||||
|
if err := json.Unmarshal([]byte(stdoutOf(t, func() error { return placementExport(ctx, open, nil) })), &all); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(all) != 2 {
|
||||||
|
t.Fatalf("exported %d nodes, want anchor and laptop", len(all))
|
||||||
|
}
|
||||||
|
for i, name := range []string{"anchor", "laptop"} {
|
||||||
|
got := all[name]
|
||||||
|
d := got.Decided
|
||||||
|
if d.Endpoint != name+".example:51820" || d.Site != "here" || d.Hub != (name == "anchor") || d.Address != []string{"10.77.0.1", "10.77.0.2"}[i] {
|
||||||
|
t.Errorf("%s's decided facts are %+v", name, d)
|
||||||
|
}
|
||||||
|
if a := got.Assigned(); len(a) == 0 || !strings.Contains(strings.Join(a, ","), overlay.Name) {
|
||||||
|
t.Errorf("%s's modules are %v, want the private network among them", name, a)
|
||||||
|
}
|
||||||
|
// What it prints is the file the seed writes: at the placement file's version, read back by its reader.
|
||||||
|
if raw, _ := json.Marshal(got); got.V != sdkdelivery.Version {
|
||||||
|
t.Errorf("%s is exported at version %d: %s", name, got.V, raw)
|
||||||
|
} else if _, err := sdkdelivery.DecodePlacement(raw); err != nil {
|
||||||
|
t.Errorf("%s's export does not read as a placement file: %v", name, err)
|
||||||
|
}
|
||||||
|
want, err := frombus.Export(in, name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !sdkdelivery.SamePlacement(got, want) {
|
||||||
|
t.Errorf("%s's export differs from frombus.Export of the controller's own input:\n%+v\n%+v", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var one map[string]sdkdelivery.Placement
|
||||||
|
if err := json.Unmarshal([]byte(stdoutOf(t, func() error { return placementExport(ctx, open, []string{"laptop"}) })), &one); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, ok := one["laptop"]; !ok || len(one) != 1 {
|
||||||
|
t.Errorf("one node named exported %v", one)
|
||||||
|
}
|
||||||
|
if err := placementExport(ctx, open, []string{"stranger"}); err == nil {
|
||||||
|
t.Error("a node the store lacks was exported")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -21,7 +21,7 @@ func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) {
|
|||||||
register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"),
|
register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"),
|
||||||
Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}})
|
Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}})
|
||||||
for _, text := range w.Modules {
|
for _, text := range w.Modules {
|
||||||
m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image)))
|
m, err := catalogue.Decode([]byte(strings.ReplaceAll(string(text), "IMAGE", image)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,6 +59,38 @@ func dataWords(o conditions.Observation) (what, module string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var plainWordings = map[string]func(conditions.Observation) words{
|
var plainWordings = map[string]func(conditions.Observation) words{
|
||||||
|
// A module's new build whose compositions or grant the controller could not write (composed_keys.go).
|
||||||
|
kindNotComposed: worded(func(o conditions.Observation) words {
|
||||||
|
m := "a module"
|
||||||
|
if p := idPart(o, 0); o.Scope == conditions.ScopeBuild && p != "" {
|
||||||
|
m = p // the module: raised in the build's scope, its id the module's name
|
||||||
|
}
|
||||||
|
return words{Headline: "A new build of " + m + " cannot be delivered",
|
||||||
|
Explanation: fmt.Sprintf("A new build of %s was made, and the mesh could not prepare it for the machines, so "+
|
||||||
|
"none can be given it yet. It is tried again every half minute; the details say what fails.", m),
|
||||||
|
Resolved: "the new build of " + m + " can be delivered"}
|
||||||
|
}),
|
||||||
|
// The take-in of done keys whose backlog stood still (take_in_consumer.go, novox/hq ADR 0544 §6).
|
||||||
|
kindTakeInStalled: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "New builds are not being prepared for the machines",
|
||||||
|
Explanation: "The mesh prepares each new build for the machines as it is made, and has prepared none for " +
|
||||||
|
"five minutes while builds wait. Until it moves, no new build reaches a machine; the details say how many wait.",
|
||||||
|
Resolved: "new builds are being prepared again"}
|
||||||
|
}),
|
||||||
|
// A bucket of done keys whose take-in consumer could not be made (take_in_consumer.go, novox/hq ADR 0544 §6).
|
||||||
|
kindTakeInNotConsumed: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "Some new builds are not being prepared",
|
||||||
|
Explanation: "The mesh could not start reading one of the two places new builds are announced in, so the " +
|
||||||
|
"builds announced there wait until the controller starts again. The details say which and why.",
|
||||||
|
Resolved: "the mesh reads where new builds are announced again"}
|
||||||
|
}),
|
||||||
|
// Config not followed into the compositions (kept_compositions.go, novox/hq ADR 0545).
|
||||||
|
kindConfigNotFollowed: worded(func(o conditions.Observation) words {
|
||||||
|
return words{Headline: "Changes of settings are not reaching the machines",
|
||||||
|
Explanation: "The mesh could not read the settings as they change, so a changed setting is not prepared for " +
|
||||||
|
"the machines until it can. It tries again every minute; the details say why.",
|
||||||
|
Resolved: "changes of settings reach the machines again"}
|
||||||
|
}),
|
||||||
// Machines and their node-engine.
|
// Machines and their node-engine.
|
||||||
"silent": worded(func(o conditions.Observation) words {
|
"silent": worded(func(o conditions.Observation) words {
|
||||||
m := machineOr(o, "a machine")
|
m := machineOr(o, "a machine")
|
||||||
|
|||||||
@@ -10,7 +10,10 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) {
|
// Registration publishes the newest trunk's list of predefined keys. It judges no manifest against it (ADR 0578 §3): the
|
||||||
|
// indexer's judge does, with the list it keeps, and a fallback to a key not on it, or of another type, is refused there
|
||||||
|
// (mesh-delivery-indexer judge's tests).
|
||||||
|
func TestRegistrationPublishesTheNewestTrunkList(t *testing.T) {
|
||||||
withoutAPublishedList(t)
|
withoutAPublishedList(t)
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -28,22 +31,6 @@ func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t
|
|||||||
t.Fatalf("the controller judges against %v, not the list published", got)
|
t.Fatalf("the controller judges against %v, not the list published", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// One naming a key not on the list, or of another type, is refused and never enters the catalogue.
|
|
||||||
for i, c := range []struct{ manifest, says string }{
|
|
||||||
{notifierFallingBackTo("font-scale", "number"), "not on the list"},
|
|
||||||
{notifierFallingBackTo("font-size", "string"), "which is a number"},
|
|
||||||
} {
|
|
||||||
id := []string{"b-2", "b-3"}[i]
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil ||
|
|
||||||
!strings.Contains(err.Error(), c.says) {
|
|
||||||
t.Errorf("wanted the registration refused saying %q, got %v", c.says, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
src, err := open.inventory.SourceOf(ctx, "notifier")
|
|
||||||
if err != nil || src.BuiltFrom != strings.Repeat("1", 12) {
|
|
||||||
t.Fatalf("a refused build replaced the registered one: %+v %v", src, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A newer build's list, which drops agent-managed-settings, replaces the published one.
|
// A newer build's list, which drops agent-managed-settings, replaces the published one.
|
||||||
smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}`
|
smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}`
|
||||||
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4",
|
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4",
|
||||||
@@ -60,7 +47,7 @@ func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t
|
|||||||
t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept)
|
t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A build of another repository publishes nothing, and its manifest is judged against the published list.
|
// A build of another repository publishes nothing.
|
||||||
other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99)
|
other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99)
|
||||||
other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier"
|
other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier"
|
||||||
other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`,
|
other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`,
|
||||||
@@ -71,9 +58,4 @@ func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t
|
|||||||
if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" {
|
if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" {
|
||||||
t.Fatalf("a build of another repository published its list: %+v", kept)
|
t.Fatalf("a build of another repository published its list: %+v", kept)
|
||||||
}
|
}
|
||||||
other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings",
|
|
||||||
"number"), `"notifier"`, `"chimes"`, 1))
|
|
||||||
if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") {
|
|
||||||
t.Fatalf("a key only another repository's list holds was registered: %v", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -12,8 +9,8 @@ import (
|
|||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): `module check` and the
|
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): the
|
||||||
// merge gate judge each manifest against the list at the root of its tree; registration publishes the newest
|
// merge gate judges each manifest against the list at the root of its tree; registration publishes the newest
|
||||||
// trunk build's list of the catalogue and judges every module against the list the mesh publishes.
|
// trunk build's list of the catalogue and judges every module against the list the mesh publishes.
|
||||||
|
|
||||||
const theFirstKeys = `{
|
const theFirstKeys = `{
|
||||||
@@ -41,60 +38,6 @@ func withoutAPublishedList(t *testing.T) {
|
|||||||
t.Cleanup(func() { catalogue.ReplacePublished(was) })
|
t.Cleanup(func() { catalogue.ReplacePublished(was) })
|
||||||
}
|
}
|
||||||
|
|
||||||
// aRepositoryTree is a repository's checkout holding one module at modules/notifier, and the list when given.
|
|
||||||
func aRepositoryTree(t *testing.T, manifest string, list *string) string {
|
|
||||||
t.Helper()
|
|
||||||
root := t.TempDir()
|
|
||||||
if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if list != nil {
|
|
||||||
if err := os.WriteFile(filepath.Join(root, catalogue.PredefinedFile), []byte(*list), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return aTemplatedModule(t, filepath.Join(root, "modules", "notifier"), manifest, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestModuleCheckJudgesAManifestAgainstTheListAtItsRepositoryRoot(t *testing.T) {
|
|
||||||
withoutAPublishedList(t)
|
|
||||||
list := theFirstKeys
|
|
||||||
check := func(path string) (string, error) {
|
|
||||||
var out bytes.Buffer
|
|
||||||
err := moduleCheck([]string{path}, &out)
|
|
||||||
return out.String(), err
|
|
||||||
}
|
|
||||||
if out, err := check(aRepositoryTree(t, notifierFallingBackTo("font-size", "number"), &list)); err != nil ||
|
|
||||||
!strings.Contains(out, "notifier: ok") {
|
|
||||||
t.Fatalf("a module naming font-size was refused: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
for _, c := range []struct {
|
|
||||||
manifest string
|
|
||||||
list *string
|
|
||||||
says string
|
|
||||||
}{
|
|
||||||
{notifierFallingBackTo("font-scale", "number"), &list, `"font-scale", which is not on the list`},
|
|
||||||
{notifierFallingBackTo("font-size", "string"), &list, "which is a number, and this value is a string"},
|
|
||||||
{notifierFallingBackTo("font-size", "number"), nil, "is not present"},
|
|
||||||
{notifierFallingBackTo("font-size", "number"), ptr(`{"font-size": {"type": "number"}}`), "no description"},
|
|
||||||
} {
|
|
||||||
if out, err := check(aRepositoryTree(t, c.manifest, c.list)); err == nil || !strings.Contains(out, c.says) {
|
|
||||||
t.Errorf("wanted the check refused saying %q, got %v:\n%s", c.says, err, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func ptr(s string) *string { return &s }
|
|
||||||
|
|
||||||
// writeTheList puts the list at the root of a tree the gate judges.
|
|
||||||
func writeTheList(t *testing.T, tree, list string) string {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.WriteFile(filepath.Join(tree, catalogue.PredefinedFile), []byte(list), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return tree
|
|
||||||
}
|
|
||||||
|
|
||||||
// catalogueBuild is a trunk build of the catalogue carrying the list at a generation.
|
// catalogueBuild is a trunk build of the catalogue carrying the list at a generation.
|
||||||
func catalogueBuild(id, manifest, list string, generation int) link.BuildResult {
|
func catalogueBuild(id, manifest, list string, generation int) link.BuildResult {
|
||||||
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||||
|
|||||||
@@ -72,6 +72,8 @@ func issueMissingProcessAccounts(ctx context.Context, inv *inventory.Inventory,
|
|||||||
if len(m.ProcessBuses()) == 0 || mayIssue(m) != nil {
|
if len(m.ProcessBuses()) == 0 || mayIssue(m) != nil {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
// Only the missing: never a second password over one another caller minted between this look and the mint.
|
||||||
|
ctx = onlyIfAbsent(ctx)
|
||||||
minted, err := inv.BusMinted(ctx)
|
minted, err := inv.BusMinted(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@@ -92,15 +94,18 @@ func issueMissingProcessAccounts(ctx context.Context, inv *inventory.Inventory,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
busAddress = ""
|
busAddress = ""
|
||||||
}
|
}
|
||||||
var ids []string
|
ctx, got := withIssued(ctx)
|
||||||
for _, pr := range missing {
|
for _, pr := range missing {
|
||||||
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
|
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
ids = append(ids, pr.Process)
|
}
|
||||||
|
issued := got.list()
|
||||||
|
if len(issued) == 0 {
|
||||||
|
return "", nil // another issuer sealed them meanwhile
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push",
|
return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push",
|
||||||
strings.Join(ids, ", "), node), nil
|
strings.Join(issued, ", "), node), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the
|
// issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import (
|
|||||||
|
|
||||||
func gainedAndPlain(t *testing.T) (plain, gained catalogue.Manifest) {
|
func gainedAndPlain(t *testing.T) (plain, gained catalogue.Manifest) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
plain, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
|
plain, err := catalogue.Decode([]byte(`{"module":"mesh-mail","version":"1",
|
||||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
|
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -21,7 +21,9 @@ import (
|
|||||||
// account against the store — no bus to dial, busAddress "" — for mesh-mail's real manifest shape (review
|
// account against the store — no bus to dial, busAddress "" — for mesh-mail's real manifest shape (review
|
||||||
// round 1 finding 1: no own secret `broker` at all, its tools served by the tool runner), and issuing it
|
// round 1 finding 1: no own secret `broker` at all, its tools served by the tool runner), and issuing it
|
||||||
// again is the path every bus account takes (`rollout mint`'s KindProcess case, and a re-run of `module
|
// again is the path every bus account takes (`rollout mint`'s KindProcess case, and a re-run of `module
|
||||||
// issue`): it rotates the password, and the old hash no longer matches.
|
// issue`): it rotates the password, and the old hash no longer matches. Issue 774: every account minted is handed to the
|
||||||
|
// vault, covered or not, until the grants part's take-over; from it the issue is forwarded to the vault's issue-account,
|
||||||
|
// and the controller's hash stays as it was.
|
||||||
func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) {
|
func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
@@ -29,7 +31,6 @@ func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) {
|
|||||||
theVaultIsInstalled(t, open)
|
theVaultIsInstalled(t, open)
|
||||||
|
|
||||||
m := mailSenderWithProcess()
|
m := mailSenderWithProcess()
|
||||||
m.Moved = true // so handIssuedToVault's accountIsCovered is true, and the hand-off is exercised whole
|
|
||||||
register(t, open, m)
|
register(t, open, m)
|
||||||
|
|
||||||
vault := newStandInVault(t)
|
vault := newStandInVault(t)
|
||||||
@@ -84,4 +85,13 @@ func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) {
|
|||||||
if hash1 == hash2 {
|
if hash1 == hash2 {
|
||||||
t.Fatal("issuing the process's account again did not rotate its password")
|
t.Fatal("issuing the process's account again did not rotate its password")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
theGrantsPartTookOver(t, true)
|
||||||
|
issue()
|
||||||
|
if hash3, held, err := inv.BusUserHash(ctx, user); err != nil || !held || hash3 != hash2 {
|
||||||
|
t.Fatalf("the controller minted again after the grants part's take-over (held %v, %v)", held, err)
|
||||||
|
}
|
||||||
|
if said := vault.said(); !strings.Contains(said, `"verb":"issue-account"`) || !strings.Contains(said, `"user":"`+user+`"`) {
|
||||||
|
t.Errorf("the vault was not asked to issue %s after the take-over: %s", user, said)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,9 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// mailSenderWithProcess is mesh-mail's real manifest shape (novox/hq ADR 0436 §7, §8, the lead's fixture):
|
// mailSenderWithProcess is mesh-mail's real manifest shape (novox/hq ADR 0436 §7, §8, the lead's fixture):
|
||||||
@@ -14,7 +11,7 @@ import (
|
|||||||
// declares bus, whose own secret broker-send is where module issue delivers its account (ADR 0436 §2),
|
// declares bus, whose own secret broker-send is where module issue delivers its account (ADR 0436 §2),
|
||||||
// owned by mesh-mail-send, the user resource the process declares, at mode 0600 as every own secret is.
|
// owned by mesh-mail-send, the user resource the process declares, at mode 0600 as every own secret is.
|
||||||
func mailSenderWithProcess() catalogue.Manifest {
|
func mailSenderWithProcess() catalogue.Manifest {
|
||||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1","emits":["mail.send.sent"],
|
m, err := catalogue.Decode([]byte(`{"module":"mesh-mail","version":"1","emits":["mail.send.sent"],
|
||||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
||||||
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
||||||
@@ -25,61 +22,6 @@ func mailSenderWithProcess() catalogue.Manifest {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// review round 1, finding 1 (1181): a module declaring no `broker` of its own at all — mesh-mail's real
|
|
||||||
// shape, tools served by the tool runner — still passes mayIssue on its process's bus declaration alone.
|
|
||||||
func TestAModuleWithNoBrokerAndOnlyAProcessMayBeIssued(t *testing.T) {
|
|
||||||
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
|
||||||
t.Fatalf("a module with no broker of its own, whose process declares bus, was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1181/AC1: a process with a bus declaration and no own secret broker-<process id> cannot be issued — the
|
|
||||||
// same refusal mayIssue already gives a module with no `broker` (novox/hq ADR 0436 §2).
|
|
||||||
func TestAProcessWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
|
||||||
m := mailSenderWithProcess()
|
|
||||||
delete(m.OwnSecrets, "broker-send")
|
|
||||||
err := mayIssue(m)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("a process with no broker-<id> own secret was issued an account")
|
|
||||||
}
|
|
||||||
for _, want := range []string{"mesh-mail", "send", "broker-send", "own-secrets"} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
|
||||||
t.Errorf("a process declaring its own broker-<id> secret was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1181/AC1: the account held for a process is the provisioner client's form — {url, fingerprint, node,
|
|
||||||
// module, user, password} — named for the process, never the module's own account.
|
|
||||||
func TestAProcesssHeldAccountIsTheProvisionersForm(t *testing.T) {
|
|
||||||
user := broker.Principal{Kind: broker.KindProcess, Node: "novox", Module: "mesh-mail", Process: "send"}.Username()
|
|
||||||
if user != "novox.mesh-mail.send" {
|
|
||||||
t.Fatalf("the process's username: %q", user)
|
|
||||||
}
|
|
||||||
held, err := heldProcessAccount(broker.Broker{Fingerprint: "abc"}, "novox.example:4222", "novox", "mesh-mail",
|
|
||||||
user, "a-password")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var got map[string]any
|
|
||||||
if err := json.Unmarshal(held, &got); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := map[string]any{"url": "nats://novox.example:4222", "fingerprint": "abc", "node": "novox",
|
|
||||||
"module": "mesh-mail", "user": "novox.mesh-mail.send", "password": "a-password"}
|
|
||||||
for k, v := range want {
|
|
||||||
if got[k] != v {
|
|
||||||
t.Errorf("%s: %v, want %v", k, got[k], v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(got) != len(want) {
|
|
||||||
t.Errorf("the held account carries %v; want exactly %v (never claims — a process holds no seat)", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1181/AC1: composed, the process's own secret belongs to its own account at 0600 (novox/hq ADR 0436 §1, §2;
|
// 1181/AC1: composed, the process's own secret belongs to its own account at 0600 (novox/hq ADR 0436 §1, §2;
|
||||||
// the owner mechanism is issue 1170's, proven generically there — this is mesh-mail's own shape).
|
// the owner mechanism is issue 1170's, proven generically there — this is mesh-mail's own shape).
|
||||||
func TestAProcessesOwnSecretIsOwnedByItsOwnAccount(t *testing.T) {
|
func TestAProcessesOwnSecretIsOwnedByItsOwnAccount(t *testing.T) {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import (
|
|||||||
// invoked seat verb), so the account module issue delivers for it can be put through every grant at once.
|
// invoked seat verb), so the account module issue delivers for it can be put through every grant at once.
|
||||||
func mailSenderLiveManifest(t *testing.T) catalogue.Manifest {
|
func mailSenderLiveManifest(t *testing.T) catalogue.Manifest {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
|
m, err := catalogue.Decode([]byte(`{"module":"mesh-mail","version":"1",
|
||||||
"emits":["mail.send.sent","mail.send.failed"],"state":["sends"],
|
"emits":["mail.send.sent","mail.send.failed"],"state":["sends"],
|
||||||
"streams":[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":8,
|
"streams":[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":8,
|
||||||
"retention":"work-queue","max-deliveries":5}],
|
"retention":"work-queue","max-deliveries":5}],
|
||||||
|
|||||||
@@ -27,9 +27,10 @@ func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *te
|
|||||||
Username: "laptop.mesh-provisioner", Kind: inventory.BusModule, Node: "laptop", Module: "mesh-provisioner"}); err != nil {
|
Username: "laptop.mesh-provisioner", Kind: inventory.BusModule, Node: "laptop", Module: "mesh-provisioner"}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// The composition refuses it in the send's words (mesh-sdk's go/compose: TestTheSendsRefusalsAreRefusedInTheSendsWords).
|
// The composition holds the module out in the send's words, and composes the rest of the machine.
|
||||||
if _, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false); err == nil || !strings.Contains(err.Error(), "provisioner.laptop") {
|
if held, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false); err != nil ||
|
||||||
t.Fatalf("a push would seal a placeholder for the client's account before it was issued: %v", err)
|
!strings.Contains(held.leftOutWhy[m.Module], "provisioner.laptop") {
|
||||||
|
t.Fatalf("a push would seal a placeholder for the client's account before it was issued: %v %v", err, held.leftOutWhy)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
|
if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
|
||||||
@@ -45,8 +46,9 @@ func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *te
|
|||||||
if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) {
|
if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) {
|
||||||
t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states)
|
t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states)
|
||||||
}
|
}
|
||||||
if _, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false); err != nil && strings.Contains(err.Error(), "provisioner.laptop") {
|
if held, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false); err != nil ||
|
||||||
t.Fatalf("an issued client account is refused: %v", err)
|
slices.Contains(held.heldForCredential, m.Module) {
|
||||||
|
t.Fatalf("an issued client account is held: %v %v", err, held.leftOutWhy)
|
||||||
}
|
}
|
||||||
if !runsTheProvisionerClient(m) {
|
if !runsTheProvisionerClient(m) {
|
||||||
t.Fatal("the module is not read as running the client")
|
t.Fatal("the module is not read as running the client")
|
||||||
|
|||||||
+42
-16
@@ -59,6 +59,9 @@ func serve(ctx context.Context) (err error) {
|
|||||||
}
|
}
|
||||||
defer open.Close()
|
defer open.Close()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
// Lent to every reader in this process that holds only the inventory, so none opens pools of its own per call
|
||||||
|
// (issues 1945 and 1277: about 120 take-ins at once filled novox's store at 16:08 on 2026-10-25).
|
||||||
|
defer shareStores(open)()
|
||||||
// **What this build reads of the store's schema, on record** (novox/hq issue 352): the highest migration
|
// **What this build reads of the store's schema, on record** (novox/hq issue 352): the highest migration
|
||||||
// it carries, by its version, so a gate that would put this build back later knows it reads the store
|
// it carries, by its version, so a gate that would put this build back later knows it reads the store
|
||||||
// as it is then. A build that does not know its version records nothing, and is never put back.
|
// as it is then. A build that does not know its version records nothing, and is never put back.
|
||||||
@@ -75,11 +78,11 @@ func serve(ctx context.Context) (err error) {
|
|||||||
"recorded, and a gate will never put it back\n", RunningBuildVar)
|
"recorded, and a gate will never put it back\n", RunningBuildVar)
|
||||||
}
|
}
|
||||||
|
|
||||||
ident, err := openIdentity(ctx)
|
// The controller's own identity store, the one its readers share (closed with open), not a second pool.
|
||||||
|
ident, err := open.Identity(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer ident.Close()
|
|
||||||
|
|
||||||
// Established at start rather than on first use. A control plane that cannot sign is one
|
// Established at start rather than on first use. A control plane that cannot sign is one
|
||||||
// whose declarations every node correctly refuses, and that should be a startup failure
|
// whose declarations every node correctly refuses, and that should be a startup failure
|
||||||
@@ -122,7 +125,15 @@ func serve(ctx context.Context) (err error) {
|
|||||||
defer theLease.release()
|
defer theLease.release()
|
||||||
ctx, stopActing := context.WithCancel(ctx)
|
ctx, stopActing := context.WithCancel(ctx)
|
||||||
defer stopActing()
|
defer stopActing()
|
||||||
go func() {
|
// Every goroutine started from here is counted, and ended and waited for before serve returns (serving_group.go):
|
||||||
|
// on the way out by error here, and at the end before anything else is undone.
|
||||||
|
ctx, running := withServingGroup(ctx)
|
||||||
|
stopRunning := func() {
|
||||||
|
stopActing()
|
||||||
|
running.stop()
|
||||||
|
}
|
||||||
|
defer stopRunning()
|
||||||
|
goServing(ctx, func() {
|
||||||
select {
|
select {
|
||||||
case <-lost:
|
case <-lost:
|
||||||
stopActing()
|
stopActing()
|
||||||
@@ -132,7 +143,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// asks the controller after this one rather than have a command started and killed with this
|
// asks the controller after this one rather than have a command started and killed with this
|
||||||
// process (novox/hq issue 289).
|
// process (novox/hq issue 289).
|
||||||
handingOver.Store(true)
|
handingOver.Store(true)
|
||||||
}()
|
})
|
||||||
defer func() {
|
defer func() {
|
||||||
select {
|
select {
|
||||||
case <-lost:
|
case <-lost:
|
||||||
@@ -159,6 +170,20 @@ func serve(ctx context.Context) (err error) {
|
|||||||
if err := attachTheChain(ctx, inv, server); err != nil {
|
if err := attachTheChain(ctx, inv, server); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
bus, isNATS := server.Bus().(link.OverNATS)
|
||||||
|
if !isNATS {
|
||||||
|
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||||
|
}
|
||||||
|
// The condition store, opened before anything that raises or reads a condition starts beside the serving, so none
|
||||||
|
// of them meets conditionsFrom being written (issue 1945); closed after they all ended.
|
||||||
|
keeper := openServingConditions(ctx, open, bus)
|
||||||
|
if keeper != nil {
|
||||||
|
defer closeConditions(keeper)
|
||||||
|
defer stopRunning() // on an error return, the goroutines end before the keeper closes
|
||||||
|
// The takeover of the lease said as it happened, raised and cleared (novox/hq issue 906).
|
||||||
|
theLease.watchTakeovers(ctx, inv, keeper)
|
||||||
|
defer theLease.watchTakeovers(ctx, nil, nil) // the keeper closes after this: nothing said into it then
|
||||||
|
}
|
||||||
|
|
||||||
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
|
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
|
||||||
// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
|
// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
|
||||||
@@ -173,7 +198,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
// And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module
|
||||||
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
// is registered, ended with why when its build will not register it, raised and cleared as conditions.
|
||||||
// Never by a read.
|
// Never by a read.
|
||||||
go settlingPending(ctx, open)
|
goServing(ctx, func() { settlingPending(ctx, open) })
|
||||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||||
// above; this is the other half — the control plane is the only one of the three that knows
|
// above; this is the other half — the control plane is the only one of the three that knows
|
||||||
@@ -221,19 +246,17 @@ func serve(ctx context.Context) (err error) {
|
|||||||
"Those answer the reason when called; everything else is served as usual\n",
|
"Those answer the reason when called; everything else is served as usual\n",
|
||||||
catalogue.ControllerSeatName, strings.Join(behind, ", "))
|
catalogue.ControllerSeatName, strings.Join(behind, ", "))
|
||||||
}
|
}
|
||||||
bus, isNATS := server.Bus().(link.OverNATS)
|
|
||||||
if !isNATS {
|
|
||||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
|
||||||
}
|
|
||||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute; no longer serving, the
|
// The hand-act log is counted for `status` on this connection rather than a new one a minute; no longer serving, the
|
||||||
// connection is closed, and a verb run after in this process dials its own (a served test's leftover failed the
|
// connection is closed, and a verb run after in this process dials its own (a served test's leftover failed the
|
||||||
// next test's `conditions` with "connection closed", issue 1945).
|
// next test's `conditions` with "connection closed", issue 1945).
|
||||||
handActConn = bus.Conn
|
handActConn = bus.Conn
|
||||||
defer func() { handActConn = nil }()
|
defer func() { handActConn = nil }()
|
||||||
|
defer stopRunning() // on an error return, the goroutines end before handActConn is put back
|
||||||
// And everything else this controller does on the bus for a moment (novox/hq issue 327).
|
// And everything else this controller does on the bus for a moment (novox/hq issue 327).
|
||||||
servingBus.Store(server.JetStream())
|
servingBus.Store(server.JetStream())
|
||||||
// No longer serving: nothing is lent, and dead-letters says it is not read here (novox/hq issue 330).
|
// No longer serving: nothing is lent, and dead-letters says it is not read here (novox/hq issue 330).
|
||||||
defer servingBus.Store(nil)
|
defer servingBus.Store(nil)
|
||||||
|
defer stopRunning() // on an error return, the goroutines end before the bus is taken back
|
||||||
// And says when it replaced a value given by hand (novox/hq ADR 0228).
|
// And says when it replaced a value given by hand (novox/hq ADR 0228).
|
||||||
givenEvents = bus
|
givenEvents = bus
|
||||||
// And a pull request's merge check, asked when the forge announces its head and said when judged
|
// And a pull request's merge check, asked when the forge announces its head and said when judged
|
||||||
@@ -244,18 +267,18 @@ func serve(ctx context.Context) (err error) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Composed now and kept current, before the verb that answers from it is served.
|
// Composed now and kept current, before the verb that answers from it is served.
|
||||||
go statusFrom.keep(ctx)
|
goServing(ctx, func() { statusFrom.keep(ctx) })
|
||||||
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
|
// The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this
|
||||||
// controller holds the lease, read by the build seat from the artifact store.
|
// controller holds the lease, read by the build seat from the artifact store.
|
||||||
go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion)
|
goServing(ctx, func() { exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion) })
|
||||||
// The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's
|
// The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's
|
||||||
// declared config on the bus, written from the store only when a record changed, by the lease holder.
|
// declared config on the bus, written from the store only when a record changed, by the lease holder.
|
||||||
// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
|
// The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's
|
||||||
// assignment.
|
// assignment.
|
||||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||||
go mirroringRecords(ctx, open, inv, bus.Conn, bus)
|
goServing(ctx, func() { mirroringRecords(ctx, open, inv, bus.Conn, bus) })
|
||||||
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
|
// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
|
||||||
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
|
goServing(ctx, func() { sweepingDeclarationFiles(ctx, inv, bus.Conn) })
|
||||||
// And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by
|
// And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by
|
||||||
// the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build.
|
// the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build.
|
||||||
watchNewModules(ctx, open)
|
watchNewModules(ctx, open)
|
||||||
@@ -287,7 +310,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
|
// signals table, what the bus says about itself, and the self-check. A store that cannot be opened
|
||||||
// is said and the controller serves on: status then says the conditions cannot be read, and is
|
// is said and the controller serves on: status then says the conditions cannot be read, and is
|
||||||
// not well — louder than not serving at all, and the push that repairs the bus still runs.
|
// not well — louder than not serving at all, and the push that repairs the bus still runs.
|
||||||
if stopWatching := watchTheMesh(ctx, open, server, bus); stopWatching != nil {
|
if stopWatching := watchTheMesh(ctx, open, server, bus, keeper); stopWatching != nil {
|
||||||
defer stopWatching()
|
defer stopWatching()
|
||||||
}
|
}
|
||||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||||
@@ -302,7 +325,7 @@ func serve(ctx context.Context) (err error) {
|
|||||||
}
|
}
|
||||||
defer stopCLI()
|
defer stopCLI()
|
||||||
// Its read calls write no line each; how many each caller made is said once an hour (novox/hq issue 762).
|
// Its read calls write no line each; how many each caller made is said once an hour (novox/hq issue 762).
|
||||||
go sayReadCalls(ctx)
|
goServing(ctx, func() { sayReadCalls(ctx) })
|
||||||
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
|
// And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks.
|
||||||
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
|
stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -310,7 +333,10 @@ func serve(ctx context.Context) (err error) {
|
|||||||
}
|
}
|
||||||
defer stopAnnouncing()
|
defer stopAnnouncing()
|
||||||
|
|
||||||
return server.Serve(ctx)
|
err = server.Serve(ctx)
|
||||||
|
// Ended and waited for before the verbs stop being served and the globals are put back, not after.
|
||||||
|
stopRunning()
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// OverlayCIDRVar is the range the mesh allocates node addresses from.
|
// OverlayCIDRVar is the range the mesh allocates node addresses from.
|
||||||
|
|||||||
@@ -111,6 +111,9 @@ type meshStatus struct {
|
|||||||
// why they could not be read.
|
// why they could not be read.
|
||||||
Pending []pendingStatus `json:"pending,omitempty"`
|
Pending []pendingStatus `json:"pending,omitempty"`
|
||||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||||
|
// TakeIn is the take-in's backlog of done keys: keys not delivered yet and keys being taken in (novox/hq ADR 0544
|
||||||
|
// §6). Absent where it cannot be read.
|
||||||
|
TakeIn *takeInBacklog `json:"take-in,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// pendingStatus is one pending assignment as status says it.
|
// pendingStatus is one pending assignment as status says it.
|
||||||
@@ -266,6 +269,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out.Failing = providerStandings(asked.conditions)
|
out.Failing = providerStandings(asked.conditions)
|
||||||
out.Overflowing = asked.overflowing
|
out.Overflowing = asked.overflowing
|
||||||
out.PendingUnread = asked.pendingUnread
|
out.PendingUnread = asked.pendingUnread
|
||||||
|
out.TakeIn = asked.takeIn
|
||||||
for _, p := range asked.pending {
|
for _, p := range asked.pending {
|
||||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||||
|
|||||||
@@ -2,12 +2,15 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
@@ -24,7 +27,8 @@ import (
|
|||||||
// so no one call site sees every change; what every change has in common is that it is committed to the
|
// so no one call site sees every change; what every change has in common is that it is committed to the
|
||||||
// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what
|
// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what
|
||||||
// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A
|
// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A
|
||||||
// mirror is therefore at most mirrorEvery behind its store.
|
// mirror is therefore at most mirrorEvery behind its store. The declared-packages mirror is the exception: it composes
|
||||||
|
// every node, so it passes on a change of what it composes from, not on this tick (keepingDeclaredPackages, issue 1945).
|
||||||
|
|
||||||
// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store.
|
// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store.
|
||||||
const mirrorEvery = 10 * time.Second
|
const mirrorEvery = 10 * time.Second
|
||||||
@@ -45,21 +49,31 @@ func mirroringRecords(ctx context.Context, open *stores, inv *inventory.Inventor
|
|||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||||
recordmirror.DeclaredPackages.Name: declaredPackagesMirror(open),
|
|
||||||
}
|
}
|
||||||
|
packages := keepingDeclaredPackages(ctx, open, api, announce, epoch, holds)
|
||||||
|
held := false
|
||||||
writers := map[string]*recordmirror.Writer{}
|
writers := map[string]*recordmirror.Writer{}
|
||||||
failing := map[string]string{}
|
failing := map[string]string{}
|
||||||
tick := time.NewTicker(mirrorEvery)
|
tick := time.NewTicker(mirrorEvery)
|
||||||
defer tick.Stop()
|
defer tick.Stop()
|
||||||
events := &assignmentEvents{}
|
events := &assignmentEvents{}
|
||||||
for {
|
for {
|
||||||
if holds() {
|
now := holds()
|
||||||
|
if now && !held {
|
||||||
|
// The lease taken: the declared-packages mirror is brought to the stores at once, not at its next change.
|
||||||
|
packages.poke()
|
||||||
|
}
|
||||||
|
held = now
|
||||||
|
if now {
|
||||||
// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
|
// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
|
||||||
// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
|
// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
|
||||||
keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
||||||
for _, m := range recordmirror.Mirrors() {
|
for _, m := range recordmirror.Mirrors() {
|
||||||
|
if sources[m.Name] == nil {
|
||||||
|
continue // the declared packages, kept on a change (keepingDeclaredPackages)
|
||||||
|
}
|
||||||
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||||
if why != failing[m.Name] {
|
if why != failing[m.Name] {
|
||||||
if why != "" {
|
if why != "" {
|
||||||
@@ -179,35 +193,263 @@ func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce
|
|||||||
events.known = now
|
events.known = now
|
||||||
}
|
}
|
||||||
|
|
||||||
// declaredPackagesMirror is the source of the declared-packages mirror (ADR 0507 §5): each node's composition, asked as a
|
// declaredPackagesMirror is the source of the declared-packages mirror (ADR 0507 §5) over a fresh reading of the stores.
|
||||||
// question that makes nothing, and the packages it declares with the modules that declare them. A node whose own set does
|
|
||||||
// not compose has no key, and any other failure to read is the pass's error, so a key is never purged for a store that
|
|
||||||
// could not be read.
|
|
||||||
func declaredPackagesMirror(open *stores) func(context.Context) (map[string][]byte, error) {
|
func declaredPackagesMirror(open *stores) func(context.Context) (map[string][]byte, error) {
|
||||||
return func(ctx context.Context) (map[string][]byte, error) {
|
return func(ctx context.Context) (map[string][]byte, error) {
|
||||||
in, err := inputFor(ctx, open)
|
in, err := inputFor(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Which module owns each resource is carried only for a node-engine that reads it, so every node is asked as one that does.
|
return declaredPackagesFrom(ctx, in)
|
||||||
asked := in.Input
|
}
|
||||||
asked.Nodes = append([]compose.Node(nil), in.Nodes...)
|
}
|
||||||
for i := range asked.Nodes {
|
|
||||||
asked.Nodes[i].ReadsModules = true
|
// declaredPackagesFrom is the declared-packages mirror over an Input: each node's composition, asked as a question that makes
|
||||||
}
|
// nothing, and the packages it declares with the modules that declare them. A node whose own set does not compose has no key,
|
||||||
out := map[string][]byte{}
|
// and any other failure to read is the pass's error, so a key is never purged for a store that could not be read.
|
||||||
for _, n := range in.sortedNodes() {
|
func declaredPackagesFrom(_ context.Context, in *meshInput) (map[string][]byte, error) {
|
||||||
d, _, err := compose.ComposeIn(asked, n.Name, compose.Foreseeing)
|
// Which module owns each resource is carried only for a node-engine that reads it, so every node is asked as one that does.
|
||||||
if compose.IsUnresolvable(err) {
|
asked := in.Input
|
||||||
continue
|
asked.Nodes = append([]compose.Node(nil), in.Nodes...)
|
||||||
|
for i := range asked.Nodes {
|
||||||
|
asked.Nodes[i].ReadsModules = true
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for _, n := range in.sortedNodes() {
|
||||||
|
d, _, err := compose.ComposeIn(asked, n.Name, compose.Foreseeing)
|
||||||
|
if compose.IsUnresolvable(err) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the packages %s declares cannot be read: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
if out[n.Name], err = inventory.DeclaredPackagesValue(n.Name, d.Resources, d.ModuleOf); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The declared-packages mirror passes on a change, not on a tick** (issue 1945 note 22). Its source composes every node in
|
||||||
|
// full, about 45 s on the control node; on the 10 s tick it never rested and held the controller at 130-175 % CPU. Every
|
||||||
|
// change that can move a node's packages is a change of its composition, which the controller writes into the bucket of the
|
||||||
|
// compositions' digests, and of the assignments, which the verbs write into theirs: a watch of those two buckets wakes the
|
||||||
|
// keeper. A pass reads the stores' Input once and composes only when the Input's digest differs from the one it last
|
||||||
|
// mirrored, so a wake that changed nothing it composes from costs one reading. One pass at a time; the wakes during a pass
|
||||||
|
// are one pass after it, at least packagesGap after the last began. A rare safety pass (packagesSafetyEvery) covers a
|
||||||
|
// change no watched key says, and with an unchanged digest composes nothing either.
|
||||||
|
const (
|
||||||
|
packagesSafetyEvery = 10 * time.Minute
|
||||||
|
packagesGap = 30 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// packagesTiming is when the declared-packages mirror passes: the safety pass's period and the least time between the
|
||||||
|
// starts of two passes.
|
||||||
|
type packagesTiming struct{ safety, gap time.Duration }
|
||||||
|
|
||||||
|
// packagesKeeper keeps the declared-packages mirror: read is the Input, compose the mirror's values from it, sync writes
|
||||||
|
// those that differ from the bucket, holds says this controller holds the lease. Each a seam a test stands in for.
|
||||||
|
type packagesKeeper struct {
|
||||||
|
read func(ctx context.Context) (*meshInput, error)
|
||||||
|
compose func(ctx context.Context, in *meshInput) (map[string][]byte, error)
|
||||||
|
sync func(ctx context.Context, want map[string][]byte) error
|
||||||
|
holds func() bool
|
||||||
|
wake chan struct{}
|
||||||
|
// last is the digest of the Input last mirrored whole; empty after a failure or while the lease is not held, so the
|
||||||
|
// next pass composes.
|
||||||
|
last string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPackagesKeeper(read func(context.Context) (*meshInput, error),
|
||||||
|
compose func(context.Context, *meshInput) (map[string][]byte, error),
|
||||||
|
sync func(context.Context, map[string][]byte) error, holds func() bool) *packagesKeeper {
|
||||||
|
return &packagesKeeper{read: read, compose: compose, sync: sync, holds: holds, wake: make(chan struct{}, 1)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// poke says something the mirror composes from may have changed; never blocks, and pokes before a pass are one.
|
||||||
|
func (k *packagesKeeper) poke() {
|
||||||
|
select {
|
||||||
|
case k.wake <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pass brings the mirror to the Input, composing only when the Input differs from the one last mirrored.
|
||||||
|
func (k *packagesKeeper) pass(ctx context.Context) error {
|
||||||
|
if !k.holds() {
|
||||||
|
k.last = ""
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
in, err := k.read(ctx)
|
||||||
|
if err != nil {
|
||||||
|
k.last = ""
|
||||||
|
return fmt.Errorf("the store cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
digest := inputDigest(in)
|
||||||
|
if digest != "" && digest == k.last {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
want, err := k.compose(ctx, in)
|
||||||
|
if err != nil {
|
||||||
|
k.last = ""
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := k.sync(ctx, want); err != nil {
|
||||||
|
k.last = ""
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
k.last = digest
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// run passes once at its start, then on each wake and each safety period, until ctx ends. said hears why a pass failed,
|
||||||
|
// and an empty line when one succeeds again.
|
||||||
|
func (k *packagesKeeper) run(ctx context.Context, at packagesTiming, said func(why string)) {
|
||||||
|
safety := time.NewTicker(at.safety)
|
||||||
|
defer safety.Stop()
|
||||||
|
failing := ""
|
||||||
|
var began time.Time
|
||||||
|
for {
|
||||||
|
// The wakes so far are this pass's: drained before it reads, so a change after is a pass of its own.
|
||||||
|
select {
|
||||||
|
case <-k.wake:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
began = time.Now()
|
||||||
|
why := ""
|
||||||
|
if err := k.pass(ctx); err != nil {
|
||||||
|
why = err.Error()
|
||||||
|
}
|
||||||
|
if why != failing {
|
||||||
|
said(why)
|
||||||
|
failing = why
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-k.wake:
|
||||||
|
case <-safety.C:
|
||||||
|
}
|
||||||
|
if rest := at.gap - time.Since(began); rest > 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(rest):
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// inputDigest is a digest of everything the composition reads (the Input, and why a node or the bus is refused): the same
|
||||||
|
// for the same Input. Empty when it cannot be made, and then the pass composes.
|
||||||
|
func inputDigest(in *meshInput) string {
|
||||||
|
refused := map[string]string{}
|
||||||
|
for node, err := range in.refused {
|
||||||
|
if err != nil {
|
||||||
|
refused[node] = err.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bus := ""
|
||||||
|
if in.busRefused != nil {
|
||||||
|
bus = in.busRefused.Error()
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(struct {
|
||||||
|
Input compose.Input
|
||||||
|
Refused map[string]string
|
||||||
|
BusRefused string
|
||||||
|
}{in.Input, refused, bus})
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepingDeclaredPackages keeps the declared-packages mirror for as long as ctx: woken by the watch of the compositions'
|
||||||
|
// digests and of the assignments. It answers the keeper, whose poke a caller may also wake it with.
|
||||||
|
func keepingDeclaredPackages(ctx context.Context, open *stores, api jetstream.JetStream, announce recordmirror.Announcer,
|
||||||
|
epoch func() uint64, holds func() bool) *packagesKeeper {
|
||||||
|
m := recordmirror.DeclaredPackages
|
||||||
|
var w *recordmirror.Writer
|
||||||
|
read := func(ctx context.Context) (*meshInput, error) {
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
return readComposeInput(reading, open)
|
||||||
|
}
|
||||||
|
sync := func(ctx context.Context, want map[string][]byte) error {
|
||||||
|
writing, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if w == nil {
|
||||||
|
kv, err := api.KeyValue(writing, m.Bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the packages %s declares cannot be read: %w", n.Name, err)
|
return fmt.Errorf("the bucket %s cannot be read: %v", m.Bucket, err)
|
||||||
|
}
|
||||||
|
w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds}
|
||||||
|
}
|
||||||
|
lost := w.LostEvents
|
||||||
|
r, err := w.Sync(writing, want)
|
||||||
|
if r.Written+r.Removed > 0 {
|
||||||
|
fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed)
|
||||||
|
}
|
||||||
|
if w.LostEvents > lost {
|
||||||
|
fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n",
|
||||||
|
m.Name, w.LostEvents-lost)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
k := newPackagesKeeper(read, declaredPackagesFrom, sync, holds)
|
||||||
|
for _, bucket := range []string{sdkdelivery.ControllerComposedBucket, broker.AssignmentsBucket} {
|
||||||
|
goServing(ctx, func() { wakeOnBucket(ctx, api, bucket, k.poke) })
|
||||||
|
}
|
||||||
|
goServing(ctx, func() {
|
||||||
|
k.run(ctx, packagesTiming{safety: packagesSafetyEvery, gap: packagesGap}, func(why string) {
|
||||||
|
if why != "" {
|
||||||
|
fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("the %s mirror is kept again\n", m.Name)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
return k
|
||||||
|
}
|
||||||
|
|
||||||
|
// wakeOnBucket calls wake on every write or delete of a key in the bucket, for as long as ctx: only the keys' metadata is
|
||||||
|
// watched. A bucket not there yet, or a watch that ends, is watched again a minute later; the safety pass covers the gap.
|
||||||
|
func wakeOnBucket(ctx context.Context, api jetstream.JetStream, bucket string, wake func()) {
|
||||||
|
for {
|
||||||
|
if err := watchBucketOnce(ctx, api, bucket, wake); err != nil && ctx.Err() == nil {
|
||||||
|
fmt.Printf("the declared-packages mirror does not hear %s: %v\n", bucket, err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(time.Minute):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchBucketOnce watches the bucket until its watch ends or ctx does.
|
||||||
|
func watchBucketOnce(ctx context.Context, api jetstream.JetStream, bucket string, wake func()) error {
|
||||||
|
kv, err := api.KeyValue(ctx, bucket)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
watch, err := kv.WatchAll(ctx, jetstream.UpdatesOnly(), jetstream.MetaOnly())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = watch.Stop() }()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case e, ok := <-watch.Updates():
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("the watch ended")
|
||||||
}
|
}
|
||||||
if out[n.Name], err = inventory.DeclaredPackagesValue(n.Name, d.Resources, d.ModuleOf); err != nil {
|
if e != nil {
|
||||||
return nil, err
|
wake()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The declared-packages mirror composes only when what it composes from changed (issue 1945 note 22: it composed every node
|
||||||
|
// in full every 10 s, a pass of about 45 s, so the controller on the control node never rested at 130-175 % CPU).
|
||||||
|
|
||||||
|
// scaled is a cadence of the mirror with a second taken as a millisecond, so a test's minute is 60 ms.
|
||||||
|
func scaled() packagesTiming {
|
||||||
|
return packagesTiming{safety: packagesSafetyEvery / 1000, gap: packagesGap / 1000}
|
||||||
|
}
|
||||||
|
|
||||||
|
const scaledMinute = 60 * time.Millisecond
|
||||||
|
|
||||||
|
// fakeMesh is the keeper's input as a test changes it: each node's assigned modules, and the compositions counted by node.
|
||||||
|
type fakeMesh struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
assigned map[string][]string
|
||||||
|
composed atomic.Int64 // node compositions
|
||||||
|
synced atomic.Int64
|
||||||
|
lastSync map[string][]byte
|
||||||
|
// block, when set, holds the next composition until it is closed; entered is closed when that composition began.
|
||||||
|
block chan struct{}
|
||||||
|
entered chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeMesh(nodes int) *fakeMesh {
|
||||||
|
f := &fakeMesh{assigned: map[string][]string{}}
|
||||||
|
for i := 0; i < nodes; i++ {
|
||||||
|
f.assigned[fmt.Sprintf("node%d", i)] = []string{"base"}
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeMesh) assign(node, module string) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.assigned[node] = append(append([]string(nil), f.assigned[node]...), module)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeMesh) keeper() *packagesKeeper {
|
||||||
|
read := func(context.Context) (*meshInput, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
in := compose.Input{Catalogue: map[string]catalogue.Manifest{"base": {Module: "base", Version: "1"}}}
|
||||||
|
for _, n := range nodesOfFake(f.assigned) {
|
||||||
|
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: append([]string(nil), f.assigned[n]...)})
|
||||||
|
}
|
||||||
|
return &meshInput{Input: in}, nil
|
||||||
|
}
|
||||||
|
composeAll := func(_ context.Context, in *meshInput) (map[string][]byte, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
block, entered := f.block, f.entered
|
||||||
|
f.block = nil
|
||||||
|
f.mu.Unlock()
|
||||||
|
if block != nil {
|
||||||
|
close(entered)
|
||||||
|
<-block
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for _, n := range in.Nodes {
|
||||||
|
f.composed.Add(1)
|
||||||
|
out[n.Name] = []byte(fmt.Sprint(n.Assigned))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
syncAll := func(_ context.Context, want map[string][]byte) error {
|
||||||
|
f.mu.Lock()
|
||||||
|
f.lastSync = want
|
||||||
|
f.mu.Unlock()
|
||||||
|
f.synced.Add(1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return newPackagesKeeper(read, composeAll, syncAll, func() bool { return true })
|
||||||
|
}
|
||||||
|
|
||||||
|
func nodesOfFake(m map[string][]string) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
for j := i + 1; j < len(out); j++ {
|
||||||
|
if out[j] < out[i] {
|
||||||
|
out[i], out[j] = out[j], out[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func runKeeper(t *testing.T, k *packagesKeeper) context.CancelFunc {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
k.run(ctx, scaled(), func(string) {})
|
||||||
|
}()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
cancel()
|
||||||
|
<-done
|
||||||
|
})
|
||||||
|
return cancel
|
||||||
|
}
|
||||||
|
|
||||||
|
func eventually(t *testing.T, what string, cond func() bool) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(time.Second)
|
||||||
|
for !cond() {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("not within a second: %s", what)
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC-mirror-1: given a mesh whose input does not change, when a minute passes after the mirror's first pass, then it
|
||||||
|
// composes no node (before: a pass every 10 s, each composing every node). The count per minute is said.
|
||||||
|
func TestTheDeclaredPackagesMirrorComposesNothingWithoutAChange(t *testing.T) {
|
||||||
|
const nodes = 5
|
||||||
|
f := newFakeMesh(nodes)
|
||||||
|
runKeeper(t, f.keeper())
|
||||||
|
eventually(t, "the first pass", func() bool { return f.synced.Load() >= 1 })
|
||||||
|
start := f.composed.Load()
|
||||||
|
time.Sleep(scaledMinute)
|
||||||
|
perMinute := f.composed.Load() - start
|
||||||
|
t.Logf("node compositions in a minute of an unchanged mesh of %d nodes: %d", nodes, perMinute)
|
||||||
|
if perMinute != 0 {
|
||||||
|
t.Errorf("an unchanged mesh of %d nodes was composed %d times in a minute; want 0", nodes, perMinute)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC-mirror-2: given a pass that is running, when fifty changes arrive during it, then exactly one more pass follows,
|
||||||
|
// never two at once and never one per change.
|
||||||
|
func TestTheDeclaredPackagesMirrorPassesOncePerBurst(t *testing.T) {
|
||||||
|
f := newFakeMesh(1)
|
||||||
|
release := make(chan struct{})
|
||||||
|
f.block, f.entered = release, make(chan struct{})
|
||||||
|
k := f.keeper()
|
||||||
|
runKeeper(t, k)
|
||||||
|
<-f.entered
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
f.assign("node0", fmt.Sprintf("m%d", i))
|
||||||
|
k.poke()
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
eventually(t, "the pass after the burst", func() bool { return f.synced.Load() >= 2 })
|
||||||
|
time.Sleep(2*scaled().gap + 20*time.Millisecond)
|
||||||
|
if got := f.synced.Load(); got != 2 {
|
||||||
|
t.Errorf("a burst of 50 changes during a pass gave %d passes in all; want 2 (the running one and one after)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC-mirror-3: given the mirror at rest, when an input changes and its event arrives, then the change is mirrored.
|
||||||
|
func TestTheDeclaredPackagesMirrorMirrorsAChange(t *testing.T) {
|
||||||
|
f := newFakeMesh(2)
|
||||||
|
k := f.keeper()
|
||||||
|
runKeeper(t, k)
|
||||||
|
eventually(t, "the first pass", func() bool { return f.synced.Load() >= 1 })
|
||||||
|
f.assign("node1", "viewer")
|
||||||
|
k.poke()
|
||||||
|
eventually(t, "the change mirrored", func() bool {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return string(f.lastSync["node1"]) == "[base viewer]"
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC-mirror-4: the digest of the input is the same for the same input and differs when an assignment differs.
|
||||||
|
func TestTheInputDigestFollowsTheInput(t *testing.T) {
|
||||||
|
of := func(assigned ...string) *meshInput {
|
||||||
|
return &meshInput{Input: compose.Input{Nodes: []compose.Node{{Name: "anchor", Assigned: assigned}}}}
|
||||||
|
}
|
||||||
|
a, b, c := inputDigest(of("base")), inputDigest(of("base")), inputDigest(of("base", "viewer"))
|
||||||
|
if a == "" || a != b {
|
||||||
|
t.Errorf("the same input gave the digests %q and %q", a, b)
|
||||||
|
}
|
||||||
|
if a == c {
|
||||||
|
t.Errorf("an added assignment left the digest at %q", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -207,8 +207,13 @@ func giveVerifiedSendersBack(ctx context.Context, d *doctor) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// reissueVerifiedSendersOn is reissueVerifiedSenders on the bus: memberships read from the AssignmentsStream and
|
// reissueVerifiedSendersOn is reissueVerifiedSenders on the bus: memberships read from the AssignmentsStream and
|
||||||
// published to it.
|
// published to it. Once the stream is gone (the bus module's tools delete it at the lease write, research 072 row 33)
|
||||||
|
// the memberships are the grants part's alone, and there is nothing here to issue again.
|
||||||
func reissueVerifiedSendersOn(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements) int {
|
func reissueVerifiedSendersOn(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements) int {
|
||||||
|
if _, err := bus.JS.StreamInfo(broker.AssignmentsStream, nats.Context(ctx)); errors.Is(err, nats.ErrStreamNotFound) {
|
||||||
|
fmt.Printf(" the assignments stream is gone: the grants part issues the memberships, none is given a verified sender here\n")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
return reissueVerifiedSenders(records, where,
|
return reissueVerifiedSenders(records, where,
|
||||||
func(node, module string) ([]byte, uint64, error) {
|
func(node, module string) ([]byte, uint64, error) {
|
||||||
m, err := bus.JS.GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject(node, module))
|
m, err := bus.JS.GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject(node, module))
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/broker"
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
"github.com/novox/mesh-controller/internal/link"
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
"github.com/novox/mesh-controller/internal/testbus"
|
"github.com/novox/mesh-controller/internal/testbus"
|
||||||
@@ -17,7 +19,10 @@ func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgem
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
t.Cleanup(js.Close)
|
t.Cleanup(js.Close)
|
||||||
if err := broker.AssertMeshStreams(js); err != nil {
|
// The assignments stream is no longer the mesh's (research 072 row 33): it is asserted by nothing, and stays on a
|
||||||
|
// bus only until the lease write deletes it. Made here as it is left there.
|
||||||
|
if _, err := js.Context().AddStream(&nats.StreamConfig{Name: broker.AssignmentsStream,
|
||||||
|
Subjects: []string{"mesh.assignment.*.*"}, MaxMsgsPerSubject: 1, AllowDirect: true}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()}
|
bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()}
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ func TestReplay239AModuleNameIsNotTakenOverByAnotherRepository(t *testing.T) {
|
|||||||
|
|
||||||
register := func(raw, repository, path string) {
|
register := func(raw, repository, path string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
m, err := catalogue.ParseManifest([]byte(raw))
|
m, err := catalogue.Decode([]byte(raw))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -349,6 +349,11 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return []string{"node", "judge-directories", str("node")}, nil
|
return []string{"node", "judge-directories", str("node")}, nil
|
||||||
|
case "placement-export":
|
||||||
|
if node := str("node"); node != "" {
|
||||||
|
return []string{"node", "placement-export", node}, nil
|
||||||
|
}
|
||||||
|
return []string{"node", "placement-export"}, nil
|
||||||
case "modules":
|
case "modules":
|
||||||
return []string{"module", "list", "--json"}, nil
|
return []string{"module", "list", "--json"}, nil
|
||||||
case "seats":
|
case "seats":
|
||||||
@@ -1400,7 +1405,9 @@ func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
|||||||
// nodeReads are the `node` subcommands a verb may run: the ones that only read.
|
// nodeReads are the `node` subcommands a verb may run: the ones that only read.
|
||||||
var nodeReads = map[string]bool{"list": true, "show": true,
|
var nodeReads = map[string]bool{"list": true, "show": true,
|
||||||
// Read-only on the node too: the engine judges its directories and changes nothing (novox/hq issue 358).
|
// Read-only on the node too: the engine judges its directories and changes nothing (novox/hq issue 358).
|
||||||
"judge-directories": true}
|
"judge-directories": true,
|
||||||
|
// Each node's placement as the catalogue's placement file says it, read from the store (novox/hq ADR 0555 §6).
|
||||||
|
"placement-export": true}
|
||||||
|
|
||||||
// flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
|
// flagsOnly says a command line's rest names no subcommand: empty, or beginning with a flag. For a command
|
||||||
// with no subcommands every word is a flag, its value or a name it reads.
|
// with no subcommands every word is a flag, its value or a name it reads.
|
||||||
|
|||||||
@@ -73,6 +73,9 @@ type sendable struct {
|
|||||||
LeftOut []string
|
LeftOut []string
|
||||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||||
leftOutWhy map[string]string
|
leftOutWhy map[string]string
|
||||||
|
// heldForCredential are the modules held out because their bus account was issued by nobody
|
||||||
|
// (holdBackUnissuedBusAccounts), sorted; never on the wire.
|
||||||
|
heldForCredential []string
|
||||||
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
||||||
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
||||||
withheld []catalogue.Overflow
|
withheld []catalogue.Overflow
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
func TestAProviderElsewhereServesItsOwnComposedName(t *testing.T) {
|
func TestAProviderElsewhereServesItsOwnComposedName(t *testing.T) {
|
||||||
open := aMesh(t)
|
open := aMesh(t)
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
forge, err := catalogue.ParseManifest([]byte(`{"module":"forge","version":"1","requires":["route"],
|
forge, err := catalogue.Decode([]byte(`{"module":"forge","version":"1","requires":["route"],
|
||||||
"provides":[{"name":"npm-package-registry","scope":"mesh"}],
|
"provides":[{"name":"npm-package-registry","scope":"mesh"}],
|
||||||
"contributes":{"route":{"web":{"label":"git","port":3000}}},
|
"contributes":{"route":{"web":{"label":"git","port":3000}}},
|
||||||
"serves":{"npm-package-registry":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/",
|
"serves":{"npm-package-registry":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/",
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **Every goroutine the serving controller starts ends before serve returns** (issue 1945). A loop or a watch started
|
||||||
|
// beside the serving runs until serve's context ends, and serve's stop waits for it before it closes the stores, the
|
||||||
|
// bus and the condition keeper those loops read: a loop that outlived its serve met the next serve's globals, its
|
||||||
|
// closed stores and its bus, and in the test process that is the next test (a race on conditionsFrom, a package that
|
||||||
|
// passed every test and still exited failing). Where it is checked: the tests that run serve whole, under -race.
|
||||||
|
|
||||||
|
// servingGroup counts the goroutines one serve started. Once its stop began it starts nothing more: what would start is
|
||||||
|
// work for a controller that is stopping.
|
||||||
|
type servingGroup struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
stopped bool
|
||||||
|
wg sync.WaitGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
type servingGroupKey struct{}
|
||||||
|
|
||||||
|
// withServingGroup is ctx carrying a new group, which goServing under it counts.
|
||||||
|
func withServingGroup(ctx context.Context) (context.Context, *servingGroup) {
|
||||||
|
g := &servingGroup{}
|
||||||
|
return context.WithValue(ctx, servingGroupKey{}, g), g
|
||||||
|
}
|
||||||
|
|
||||||
|
// goServing runs work beside the caller, counted in the group ctx carries, so serve's stop waits for it. Under a group
|
||||||
|
// whose stop began it runs nothing; under a context with no group (a part run alone, by a test or a verb) it is a plain
|
||||||
|
// goroutine, as before. It says whether it started work.
|
||||||
|
func goServing(ctx context.Context, work func()) bool {
|
||||||
|
g, _ := ctx.Value(servingGroupKey{}).(*servingGroup)
|
||||||
|
if g == nil {
|
||||||
|
go work()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
g.mu.Lock()
|
||||||
|
if g.stopped {
|
||||||
|
g.mu.Unlock()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
g.wg.Add(1)
|
||||||
|
g.mu.Unlock()
|
||||||
|
go func() {
|
||||||
|
defer g.wg.Done()
|
||||||
|
work()
|
||||||
|
}()
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// stop starts nothing more and waits for what was started; the caller ends their context first. A second stop waits
|
||||||
|
// for nothing.
|
||||||
|
func (g *servingGroup) stop() {
|
||||||
|
g.mu.Lock()
|
||||||
|
g.stopped = true
|
||||||
|
g.mu.Unlock()
|
||||||
|
g.wg.Wait()
|
||||||
|
}
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The serving controller's reads of the rest of the mesh open no pool per call: the licences and identity stores are the
|
||||||
|
// controller's own, opened once and shared. On 2026-10-25 at 16:08 each take-in opened two pools of its own
|
||||||
|
// (restOfTheMeshOf), about 120 at once, and novox's store refused every new connection (issues 1945 and 1277).
|
||||||
|
func TestRestOfTheMeshOfOpensNoPoolPerCall(t *testing.T) {
|
||||||
|
wasIdent, wasLic, wasRest := identityOpener, licencesOpener, restOfTheMeshOn
|
||||||
|
t.Cleanup(func() { identityOpener, licencesOpener, restOfTheMeshOn = wasIdent, wasLic, wasRest })
|
||||||
|
var identities, licencesOpened atomic.Int64
|
||||||
|
identityOpener = func(context.Context) (*identity.Identity, error) {
|
||||||
|
identities.Add(1)
|
||||||
|
return &identity.Identity{}, nil
|
||||||
|
}
|
||||||
|
licencesOpener = func(context.Context) (*licences.Licences, error) {
|
||||||
|
licencesOpened.Add(1)
|
||||||
|
return &licences.Licences{}, nil
|
||||||
|
}
|
||||||
|
// The read of the Input reaches both stores, as readComposeInput does.
|
||||||
|
restOfTheMeshOn = func(ctx context.Context, open *stores, _ map[string]catalogue.Manifest, _ string) (catalogue.World, error) {
|
||||||
|
if _, err := open.Licences(ctx); err != nil {
|
||||||
|
return catalogue.World{}, err
|
||||||
|
}
|
||||||
|
if _, err := open.Identity(ctx); err != nil {
|
||||||
|
return catalogue.World{}, err
|
||||||
|
}
|
||||||
|
return catalogue.World{}, nil
|
||||||
|
}
|
||||||
|
inv := &inventory.Inventory{}
|
||||||
|
serving := &stores{inventory: inv}
|
||||||
|
unshare := shareStores(serving)
|
||||||
|
t.Cleanup(unshare)
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range 120 {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
if _, err := restOfTheMeshOf(context.Background(), inv, nil, ""); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if n := licencesOpened.Load(); n != 1 {
|
||||||
|
t.Fatalf("opened the licences store %d times for 120 reads, want once", n)
|
||||||
|
}
|
||||||
|
if n := identities.Load(); n != 1 {
|
||||||
|
t.Fatalf("opened the identity store %d times for 120 reads, want once", n)
|
||||||
|
}
|
||||||
|
if serving.licences == nil || serving.identity == nil {
|
||||||
|
t.Fatal("the reads did not use the serving controller's stores")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A process that shares no stores (a verb run as its own short process) still opens them for the call and lets go.
|
||||||
|
func TestRestOfTheMeshOfOpensItsOwnWhereNoneAreShared(t *testing.T) {
|
||||||
|
wasIdent, wasLic, wasRest := identityOpener, licencesOpener, restOfTheMeshOn
|
||||||
|
t.Cleanup(func() { identityOpener, licencesOpener, restOfTheMeshOn = wasIdent, wasLic, wasRest })
|
||||||
|
var opened atomic.Int64
|
||||||
|
licencesOpener = func(context.Context) (*licences.Licences, error) { opened.Add(1); return &licences.Licences{}, nil }
|
||||||
|
restOfTheMeshOn = func(ctx context.Context, open *stores, _ map[string]catalogue.Manifest, _ string) (catalogue.World, error) {
|
||||||
|
_, err := open.Licences(ctx)
|
||||||
|
return catalogue.World{}, err
|
||||||
|
}
|
||||||
|
inv := &inventory.Inventory{}
|
||||||
|
for range 2 {
|
||||||
|
if _, err := restOfTheMeshOf(context.Background(), inv, nil, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := opened.Load(); n != 2 {
|
||||||
|
t.Fatalf("opened %d times, want once per call", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,14 +1,10 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"crypto/ecdh"
|
"crypto/ecdh"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
@@ -41,7 +37,7 @@ func TestTheProvidersMachineReceivesTheSharesOfAnS3BucketRequirement(t *testing.
|
|||||||
if err := json.Unmarshal(raw, &overlayModule); err != nil {
|
if err := json.Unmarshal(raw, &overlayModule); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
consumer, err := catalogue.ParseManifest([]byte(filesRequiringShares))
|
consumer, err := catalogue.Decode([]byte(filesRequiringShares))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -79,26 +75,3 @@ func TestTheProvidersMachineReceivesTheSharesOfAnS3BucketRequirement(t *testing.
|
|||||||
t.Fatalf("the provider's grant for the consumer carries shares %s, want %s", got, want)
|
t.Fatalf("the provider's grant for the consumer carries shares %s, want %s", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1593/AC2 unit: the module check accepts a manifest with shares and refuses an unknown method, citing ADR 0468 §8.
|
|
||||||
func TestModuleCheckAcceptsSharesAndRefusesAnUnknownMethod(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
write := func(name, body string) string {
|
|
||||||
p := filepath.Join(dir, name+".json")
|
|
||||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheckFor([]string{write("files", filesRequiringShares)}, 6, &out); err != nil {
|
|
||||||
t.Fatalf("a manifest with shares was refused: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
out.Reset()
|
|
||||||
bad := write("bad", `{"module":"bad","version":"1","requires":[
|
|
||||||
{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["delete"]}]}]}`)
|
|
||||||
err := moduleCheckFor([]string{bad}, 6, &out)
|
|
||||||
if err == nil || !strings.Contains(out.String(), "ADR 0468 §8") {
|
|
||||||
t.Fatalf("an unknown method passed or the refusal does not cite the ADR: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+113
-15
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -147,9 +148,10 @@ var signalsTable = []signalRow{
|
|||||||
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
|
return newestOf(f.machines, func(m machineFacts) time.Time { return m.toolsHeard })
|
||||||
}},
|
}},
|
||||||
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
|
{Row: "S12", Signal: "the controller lease renewed", Emitter: "controller", Trigger: "every 5 s",
|
||||||
Bound: "15 s (the key's age); a holder that lost the lease, or stopped renewing and was taken over, and a " +
|
Bound: "15 s (the key's age); a holder that lost the lease or stopped renewing it is said once by the " +
|
||||||
"lease bucket found raised again from nothing, are said for an hour after; a controller serving without " +
|
"controller that took it over, raised and cleared as it takes it, and for an hour after while none " +
|
||||||
"the lease, for as long as it does",
|
"holds it; a lease given back is never said; a lease bucket found raised again from nothing is said " +
|
||||||
|
"for an hour after; a controller serving without the lease, for as long as it does",
|
||||||
Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
|
Kind: "lease-lost", Severity: conditions.Urgent, Phase: 2,
|
||||||
needs: func(f *signalFacts) error { return f.leaseErr }, watch: watchLease,
|
needs: func(f *signalFacts) error { return f.leaseErr }, watch: watchLease,
|
||||||
newest: func(f *signalFacts) time.Time { return f.lease.renewed }},
|
newest: func(f *signalFacts) time.Time { return f.lease.renewed }},
|
||||||
@@ -644,10 +646,29 @@ func watchLease(f *signalFacts) []conditions.Observation {
|
|||||||
"does not know it lost it",
|
"does not know it lost it",
|
||||||
Said: l.resetSaid})
|
Said: l.resetSaid})
|
||||||
}
|
}
|
||||||
|
// An epoch below the one this controller holds was taken over by it: said once by the takeover itself,
|
||||||
|
// raised and cleared (sayTakeover), not observed for an hour after (novox/hq issue 906). One that ended
|
||||||
|
// while this controller does not hold the lease — serving without it — is still said for the hour.
|
||||||
|
var ended []inventory.Epoch
|
||||||
|
for _, e := range l.ended {
|
||||||
|
if l.held && e.Epoch < l.epoch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ended = append(ended, e)
|
||||||
|
}
|
||||||
|
if o, ok := lostObservation(ended, l.epoch, f.now); ok {
|
||||||
|
out = append(out, o)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// lostObservation is the lease-lost condition of the epochs that lost the lease or let it expire within the
|
||||||
|
// hour, named by the newest; false when none did. An epoch given back is no loss.
|
||||||
|
func lostObservation(ended []inventory.Epoch, acting uint64, now time.Time) (conditions.Observation, bool) {
|
||||||
var lost []string
|
var lost []string
|
||||||
newest := inventory.Epoch{}
|
newest := inventory.Epoch{}
|
||||||
for _, e := range l.ended {
|
for _, e := range ended {
|
||||||
if e.Ended == nil || e.How == inventory.EpochReleased || f.now.Sub(*e.Ended) > advisoryQuiet {
|
if e.Ended == nil || e.How == inventory.EpochReleased || now.Sub(*e.Ended) > advisoryQuiet {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
lost = append(lost, fmt.Sprintf("epoch %d (%s) %s at %s", e.Epoch, e.Instance, e.How,
|
lost = append(lost, fmt.Sprintf("epoch %d (%s) %s at %s", e.Epoch, e.Instance, e.How,
|
||||||
@@ -656,18 +677,95 @@ func watchLease(f *signalFacts) []conditions.Observation {
|
|||||||
newest = e
|
newest = e
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(lost) > 0 {
|
if len(lost) == 0 {
|
||||||
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
|
return conditions.Observation{}, false
|
||||||
if newest.How == inventory.EpochExpired {
|
}
|
||||||
how = "stopped renewing it without giving it back, and was taken over"
|
how := "lost it: another holder wrote its key, or no renewal was made within the key's age"
|
||||||
|
if newest.How == inventory.EpochExpired {
|
||||||
|
how = "stopped renewing it without giving it back, and was taken over"
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
|
||||||
|
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent, Source: "S12",
|
||||||
|
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
|
||||||
|
newest.Instance, how, acting),
|
||||||
|
Said: strings.Join(lost, "; ")}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeover is the condition a takeover says: the epoch just before the one taken, when it lost the lease or
|
||||||
|
// let it expire; false when it gave it back, or none is recorded.
|
||||||
|
func takeover(ended []inventory.Epoch, taken uint64, now time.Time) (conditions.Observation, bool) {
|
||||||
|
var before *inventory.Epoch
|
||||||
|
for i, e := range ended {
|
||||||
|
if e.Epoch < taken && (before == nil || e.Epoch > before.Epoch) {
|
||||||
|
before = &ended[i]
|
||||||
}
|
}
|
||||||
out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
|
}
|
||||||
Kind: "lease-lost", Machine: newest.Host, Severity: conditions.Urgent,
|
if before == nil {
|
||||||
Summary: fmt.Sprintf("the controller of epoch %d (%s) %s; the controller of epoch %d acts now", newest.Epoch,
|
return conditions.Observation{}, false
|
||||||
newest.Instance, how, l.epoch),
|
}
|
||||||
Said: strings.Join(lost, "; ")})
|
return lostObservation([]inventory.Epoch{*before}, taken, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayTakeover is the takeover said as an event (novox/hq issue 906): when the controller before this one lost
|
||||||
|
// the lease or let it expire, the lease-lost condition is raised and cleared at once by this controller, which
|
||||||
|
// holds the lease — the operator reads that it happened and that it is over, not an urgent alarm for an hour.
|
||||||
|
// A lease given back says nothing. Said once the condition store is open; a take before that is said then.
|
||||||
|
func (a *actor) sayTakeover(ctx context.Context, epoch uint64) {
|
||||||
|
a.mu.Lock()
|
||||||
|
keeper, inv := a.keeper, a.inv
|
||||||
|
a.mu.Unlock()
|
||||||
|
if keeper == nil || inv == nil || epoch == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
reading, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
ended, err := inv.EpochsSince(reading, now.Add(-advisoryQuiet))
|
||||||
|
if err != nil {
|
||||||
|
// Failure is loud: S12 no longer observes an epoch below the one held, so an unread takeover is said as
|
||||||
|
// one that was not a handover — raised, and left for S12 to clear (ADR 0558 §3).
|
||||||
|
fmt.Printf("how the epoch before %d ended cannot be read, so its takeover is said as an expiry: %v\n", epoch, err)
|
||||||
|
o := conditions.Observation{Scope: conditions.ScopeCore, ID: "controller.lease", Token: "lost",
|
||||||
|
Kind: "lease-lost", Severity: conditions.Urgent, Source: "S12",
|
||||||
|
Summary: fmt.Sprintf("the controller of epoch %d took the lease, and how the controller before it ended "+
|
||||||
|
"cannot be read: it is said as one that stopped renewing without giving it back", epoch),
|
||||||
|
Said: err.Error()}
|
||||||
|
if _, err := keeper.Observe(reading, o); err != nil {
|
||||||
|
fmt.Printf("the takeover at epoch %d could not be said: %v\n", epoch, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sayTakeoverOf(reading, keeper, ended, epoch, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayTakeoverOf raises and clears the takeover's condition over the epochs read.
|
||||||
|
func sayTakeoverOf(ctx context.Context, keeper *conditions.Keeper, ended []inventory.Epoch, epoch uint64, now time.Time) {
|
||||||
|
o, ok := takeover(ended, epoch, now)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := keeper.Observe(ctx, o); err != nil {
|
||||||
|
fmt.Printf("the takeover at epoch %d could not be said: %v\n", epoch, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := keeper.Clear(ctx, o.Key(), fmt.Sprintf("taken over: the controller of epoch %d holds the lease", epoch)); err != nil {
|
||||||
|
fmt.Printf("the takeover at epoch %d was said and could not be cleared: %v\n", epoch, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchTakeovers gives the serving controller's condition store to the lease, and says the takeover of the
|
||||||
|
// epoch it holds, if it took one over before the store opened.
|
||||||
|
func (a *actor) watchTakeovers(ctx context.Context, inv *inventory.Inventory, keeper *conditions.Keeper) {
|
||||||
|
a.mu.Lock()
|
||||||
|
a.keeper, a.inv = keeper, inv
|
||||||
|
held := a.held
|
||||||
|
a.mu.Unlock()
|
||||||
|
if held == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if epoch, err := held.Epoch(); err == nil {
|
||||||
|
a.sayTakeover(ctx, epoch)
|
||||||
}
|
}
|
||||||
return out
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handActsWithin is how far back a repeated cause counts (S15).
|
// handActsWithin is how far back a repeated cause counts (S15).
|
||||||
|
|||||||
@@ -491,13 +491,41 @@ func TestAStaleWriterIsNamedByItsEpoch(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **The lease lost is said for an hour, and serving without it for as long as it lasts** (S12).
|
// **The lease lost is said for an hour while nobody took it over, a takeover is said once as it happens —
|
||||||
|
// raised and cleared by the controller that took it — a lease given back is never said, and serving without
|
||||||
|
// it for as long as it lasts** (S12, novox/hq issue 906).
|
||||||
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
f := calm(now)
|
f := calm(now)
|
||||||
expired := now.Add(-59 * time.Minute)
|
expired := now.Add(-59 * time.Minute)
|
||||||
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
|
f.lease.ended = []inventory.Epoch{{Epoch: 41, Instance: "controller@anchor pid 7", Host: "anchor",
|
||||||
Ended: &expired, How: inventory.EpochExpired}}
|
Ended: &expired, How: inventory.EpochExpired}}
|
||||||
|
// Held at epoch 57, above it: the takeover said it, and no hour of observation follows.
|
||||||
|
if got := watchLease(f); len(got) != 0 {
|
||||||
|
t.Fatalf("an epoch this controller took over is observed after the takeover: %+v", got)
|
||||||
|
}
|
||||||
|
store := conditions.NewInMemory()
|
||||||
|
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Now: func() time.Time { return now }})
|
||||||
|
sayTakeoverOf(t.Context(), k, f.lease.ended, 57, now)
|
||||||
|
if open, err := k.Open(t.Context()); err != nil || len(open) != 0 {
|
||||||
|
t.Fatalf("the takeover is left open: %+v, %v", open, err)
|
||||||
|
}
|
||||||
|
k.Close(t.Context()) // its transitions written to the history
|
||||||
|
history, err := store.Since(t.Context(), now.Add(-time.Minute))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var changes []string
|
||||||
|
for _, e := range history {
|
||||||
|
if e.Key == "core.controller.lease.lost" {
|
||||||
|
changes = append(changes, e.Change)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !slices.Equal(changes, []string{"raised", "cleared"}) {
|
||||||
|
t.Fatalf("the takeover of an expired lease is not raised and cleared: %v", changes)
|
||||||
|
}
|
||||||
|
// Not holding the lease, the loss is said for the hour.
|
||||||
|
f.lease.held = false
|
||||||
got := watchLease(f)
|
got := watchLease(f)
|
||||||
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
|
if len(got) != 1 || got[0].Key() != "core.controller.lease.lost" || !strings.Contains(got[0].Summary, "epoch 41") ||
|
||||||
got[0].Severity != conditions.Urgent {
|
got[0].Severity != conditions.Urgent {
|
||||||
@@ -512,6 +540,9 @@ func TestALeaseLostOrMissingIsSaid(t *testing.T) {
|
|||||||
if got := watchLease(f); len(got) != 0 {
|
if got := watchLease(f); len(got) != 0 {
|
||||||
t.Fatalf("a lease given back is said as lost: %+v", got)
|
t.Fatalf("a lease given back is said as lost: %+v", got)
|
||||||
}
|
}
|
||||||
|
if _, said := takeover(f.lease.ended, 57, now); said {
|
||||||
|
t.Fatal("the takeover of a lease given back is said")
|
||||||
|
}
|
||||||
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
|
f.lease = leaseFacts{held: true, epoch: 501, renewed: now, reset: now.Add(-time.Minute), resetSaid: "moved past 500"}
|
||||||
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
|
if got := watchLease(f); len(got) != 1 || got[0].Key() != "core.controller.lease.reset" {
|
||||||
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
|
t.Fatalf("a lease bucket raised again from nothing was not said: %+v", got)
|
||||||
|
|||||||
@@ -67,16 +67,16 @@ func keepAnnouncements(ctx context.Context, js *broker.JetStream, open *stores)
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if changed {
|
if changed {
|
||||||
inBackground(func() { composeAnnouncedReads(context.WithoutCancel(ctx), open) })
|
inBackground(ctx, func() { composeAnnouncedReads(context.WithoutCancel(ctx), open) })
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("following the states announced: %w", err)
|
return fmt.Errorf("following the states announced: %w", err)
|
||||||
}
|
}
|
||||||
go func() {
|
goServing(ctx, func() {
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
consuming.Stop()
|
consuming.Stop()
|
||||||
}()
|
})
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -151,6 +151,10 @@ func printStatus(asked answers) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
printPending(asked.pending, asked.pendingUnread)
|
printPending(asked.pending, asked.pendingUnread)
|
||||||
|
if b := asked.takeIn; b != nil && b.Total() > 0 {
|
||||||
|
fmt.Printf("the take-in of done keys has %d key(s) not taken in yet: %d waiting to be delivered, %d being taken in\n\n",
|
||||||
|
b.Total(), b.Pending, b.Taking)
|
||||||
|
}
|
||||||
|
|
||||||
if len(behind) > 0 {
|
if len(behind) > 0 {
|
||||||
var names []string
|
var names []string
|
||||||
@@ -314,7 +318,7 @@ func printStatus(asked answers) error {
|
|||||||
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
||||||
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
||||||
}
|
}
|
||||||
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
fmt.Printf("\n a shorter `slug` in the module's definition fits it\n\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||||
@@ -471,6 +475,13 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
|||||||
out.pendingUnread = err.Error()
|
out.pendingUnread = err.Error()
|
||||||
err = nil
|
err = nil
|
||||||
}
|
}
|
||||||
|
// And the take-in's backlog of done keys, read from its consumers on the bus (novox/hq ADR 0544 §6): where there is
|
||||||
|
// a bus; a stall is a condition of its own, so this is said and judges nothing.
|
||||||
|
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||||
|
if b, berr := readTakeInBacklog(ctx); berr == nil {
|
||||||
|
out.takeIn = &b
|
||||||
|
}
|
||||||
|
}
|
||||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||||
// to read the log from; a process with none has no log to count.
|
// to read the log from; a process with none has no log to count.
|
||||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||||
|
|||||||
@@ -210,7 +210,7 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
|||||||
// And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3,
|
// And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3,
|
||||||
// issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path.
|
// issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path.
|
||||||
if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
||||||
go moveWhenHeard(context.WithoutCancel(ctx), l.open, moveHeardOn(), report.Node)
|
goServing(ctx, func() { moveWhenHeard(context.WithoutCancel(ctx), l.open, moveHeardOn(), report.Node) })
|
||||||
}
|
}
|
||||||
return news, err
|
return news, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
@@ -163,6 +164,13 @@ func openLicences(ctx context.Context) (*licences.Licences, error) {
|
|||||||
return held, nil
|
return held, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// identityOpener and licencesOpener open a store the first time a set of stores needs it; variables so a test counts
|
||||||
|
// the pools opened.
|
||||||
|
var (
|
||||||
|
identityOpener = openIdentity
|
||||||
|
licencesOpener = openLicences
|
||||||
|
)
|
||||||
|
|
||||||
// stores is what one command has open.
|
// stores is what one command has open.
|
||||||
//
|
//
|
||||||
// **Opened once, not once per machine.** Working out what a machine should be reaches the identity
|
// **Opened once, not once per machine.** Working out what a machine should be reaches the identity
|
||||||
@@ -176,6 +184,9 @@ type stores struct {
|
|||||||
inventory *inventory.Inventory
|
inventory *inventory.Inventory
|
||||||
identity *identity.Identity
|
identity *identity.Identity
|
||||||
licences *licences.Licences
|
licences *licences.Licences
|
||||||
|
// mu keeps one opening of each store at a time: a serving controller's stores are shared by readers that run side
|
||||||
|
// by side (sharedStores).
|
||||||
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// open connects to the inventory, which every command that touches the mesh needs.
|
// open connects to the inventory, which every command that touches the mesh needs.
|
||||||
@@ -189,10 +200,12 @@ func openStores(ctx context.Context) (*stores, error) {
|
|||||||
|
|
||||||
// Identity is this control plane's own identity context, opened if it has not been.
|
// Identity is this control plane's own identity context, opened if it has not been.
|
||||||
func (h *stores) Identity(ctx context.Context) (*identity.Identity, error) {
|
func (h *stores) Identity(ctx context.Context) (*identity.Identity, error) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
if h.identity != nil {
|
if h.identity != nil {
|
||||||
return h.identity, nil
|
return h.identity, nil
|
||||||
}
|
}
|
||||||
opened, err := openIdentity(ctx)
|
opened, err := identityOpener(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -202,10 +215,12 @@ func (h *stores) Identity(ctx context.Context) (*identity.Identity, error) {
|
|||||||
|
|
||||||
// Licences is the context holding model access, opened if it has not been.
|
// Licences is the context holding model access, opened if it has not been.
|
||||||
func (h *stores) Licences(ctx context.Context) (*licences.Licences, error) {
|
func (h *stores) Licences(ctx context.Context) (*licences.Licences, error) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
if h.licences != nil {
|
if h.licences != nil {
|
||||||
return h.licences, nil
|
return h.licences, nil
|
||||||
}
|
}
|
||||||
opened, err := openLicences(ctx)
|
opened, err := licencesOpener(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -226,3 +241,22 @@ func (h *stores) Close() {
|
|||||||
h.inventory.Close()
|
h.inventory.Close()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sharedStores are the stores a serving controller lends, by its inventory, to every reader in the process that holds
|
||||||
|
// only the inventory (restOfTheMeshOf): one pool of each store for the controller, never one per call (issues 1945 and
|
||||||
|
// 1277, 2026-10-25 16:08: about 120 take-ins opened two pools each and filled novox's store).
|
||||||
|
var sharedStores sync.Map // *inventory.Inventory -> *stores
|
||||||
|
|
||||||
|
// shareStores lends open by its inventory until the function returned is called.
|
||||||
|
func shareStores(open *stores) func() {
|
||||||
|
sharedStores.Store(open.inventory, open)
|
||||||
|
return func() { sharedStores.CompareAndDelete(open.inventory, open) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// sharedStoresOf are the stores lent for inv, if any are.
|
||||||
|
func sharedStoresOf(inv *inventory.Inventory) (*stores, bool) {
|
||||||
|
if open, ok := sharedStores.Load(inv); ok {
|
||||||
|
return open.(*stores), true
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,452 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/conditions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The take-in's two durable pull consumers (novox/hq ADR 0544; research 072, milestones 2 and 4). On the indexer's
|
||||||
|
// modules bucket the consumer delivers each module's last trunk value when it is made, then each write: the start's look
|
||||||
|
// of a controller whose consumer is new, and nothing at a restart but what the controller before it left
|
||||||
|
// unacknowledged. On the builder's builds bucket it delivers each write from when it is made. A key is acknowledged once
|
||||||
|
// its module is taken in, has nothing new, or waits for a write that will be delivered; any other wait or failure is
|
||||||
|
// refused with a delay that grows with each refusal, and the bus delivers it again.
|
||||||
|
//
|
||||||
|
// **The bound on the store is the one take-in loop**, a batch of at most startTakeIns keys at a time (issues 1945 and
|
||||||
|
// 1277: a burst of take-ins once filled the store until it refused every connection). The consumers' own limit on
|
||||||
|
// unacknowledged keys is far above it, because a refused key stays unacknowledged until its delay ends: with a limit of
|
||||||
|
// a batch, a handful of keys refused for long (a manifest word this controller does not read yet) would stop a consumer
|
||||||
|
// for good, the build that reads the word among what it holds back (the review of ADR 0544).
|
||||||
|
|
||||||
|
// startTakeIns is the most a batch takes: the take-ins that may wait on the store at once.
|
||||||
|
const startTakeIns = 16
|
||||||
|
|
||||||
|
// takeInPending is how many keys each consumer may leave unacknowledged: delivered and being taken in, or refused and
|
||||||
|
// waiting for their delay. Far above a batch, so refused keys never stop the others.
|
||||||
|
const takeInPending = 1000
|
||||||
|
|
||||||
|
// takeInBuffered is how many delivered keys each consumer's reader holds ahead of the loop.
|
||||||
|
const takeInBuffered = startTakeIns / 2
|
||||||
|
|
||||||
|
// takeInAckWait is how long a delivered key may stay unacknowledged before the bus delivers it again: longer than a batch
|
||||||
|
// takes, its two composer passes among it.
|
||||||
|
const takeInAckWait = 10 * time.Minute
|
||||||
|
|
||||||
|
// takeInGather is how long a batch waits for more keys after its first, so a burst is one batch.
|
||||||
|
var takeInGather = 200 * time.Millisecond
|
||||||
|
|
||||||
|
// takeInStall is how long a backlog may stand without a key delivered before it is a condition (ADR 0544 §6).
|
||||||
|
var takeInStall = 5 * time.Minute
|
||||||
|
|
||||||
|
// takeInMsg is one delivered key, as the take-in uses it.
|
||||||
|
type takeInMsg struct {
|
||||||
|
delivered
|
||||||
|
ack func()
|
||||||
|
nak func(delay time.Duration)
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInKeyOf is the module a delivered key names, false for a delete or a purge, which takes nothing in.
|
||||||
|
func takeInKeyOf(bucket string, m jetstream.Msg) (delivered, bool) {
|
||||||
|
if op := m.Headers().Get("KV-Operation"); op == "DEL" || op == "PURGE" {
|
||||||
|
return delivered{}, false
|
||||||
|
}
|
||||||
|
key := strings.TrimPrefix(m.Subject(), "$KV."+bucket+".")
|
||||||
|
module, _, _ := strings.Cut(key, ".")
|
||||||
|
if module == "" {
|
||||||
|
return delivered{}, false
|
||||||
|
}
|
||||||
|
var deliveries uint64 = 1
|
||||||
|
if meta, err := m.Metadata(); err == nil {
|
||||||
|
deliveries = meta.NumDelivered
|
||||||
|
}
|
||||||
|
return delivered{module: module, again: deliveries > 1, deliveries: deliveries}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInConsumer is one bucket's consumer of the take-in.
|
||||||
|
type takeInConsumer struct {
|
||||||
|
bucket string
|
||||||
|
consumer jetstream.Consumer
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInConfig is the take-in's consumer on bucket's stream.
|
||||||
|
func takeInConfig(bucket string) jetstream.ConsumerConfig {
|
||||||
|
deliver := jetstream.DeliverNewPolicy
|
||||||
|
if bucket == sdkdelivery.ModulesBucket {
|
||||||
|
deliver = jetstream.DeliverLastPerSubjectPolicy // the start's look: every module's trunk value once
|
||||||
|
}
|
||||||
|
return jetstream.ConsumerConfig{
|
||||||
|
Durable: broker.TakeInConsumer, FilterSubject: "$KV." + bucket + ".>", DeliverPolicy: deliver,
|
||||||
|
AckPolicy: jetstream.AckExplicitPolicy, AckWait: takeInAckWait, MaxAckPending: takeInPending,
|
||||||
|
Description: "the controller's take-in of done keys (novox/hq ADR 0544)",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInConsumers makes or finds the take-in's consumer on each bucket's stream; one that cannot be made is said, and
|
||||||
|
// the other is still read.
|
||||||
|
func takeInConsumers(ctx context.Context, js jetstream.JetStream) []takeInConsumer {
|
||||||
|
var out []takeInConsumer
|
||||||
|
for _, bucket := range broker.TakeInBuckets {
|
||||||
|
c, err := js.CreateOrUpdateConsumer(ctx, "KV_"+bucket, takeInConfig(bucket))
|
||||||
|
o := takeInNotConsumedObservation(bucket, err)
|
||||||
|
if err != nil {
|
||||||
|
// Loud: a consumer whose deliver policy differs cannot be updated, and nothing of its bucket is taken in.
|
||||||
|
doneKeysSay("done keys: " + o.Summary)
|
||||||
|
if keeper := conditionsFrom; keeper != nil {
|
||||||
|
_, _ = keeper.Observe(ctx, o)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Made: a condition a controller before this one raised for it is cleared (a restart that works clears it).
|
||||||
|
if keeper := conditionsFrom; keeper != nil {
|
||||||
|
_, _ = keeper.Clear(ctx, o.Key(), "the take-in's consumer on "+bucket+" is made")
|
||||||
|
}
|
||||||
|
out = append(out, takeInConsumer{bucket: bucket, consumer: c})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchDoneKeys takes in the done keys the two consumers deliver, batch after batch, until ctx ends, and watches their
|
||||||
|
// backlog. A consumer that cannot be made is said: its bucket is then not taken in until the next start.
|
||||||
|
func watchDoneKeys(ctx context.Context, open *stores) {
|
||||||
|
keys, closeKeys, err := openBuilderKeys()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("done keys: the builder's keys are not taken in: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
consumers := takeInConsumers(ctx, keys.JS)
|
||||||
|
if len(consumers) == 0 {
|
||||||
|
closeKeys()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
in := make(chan takeInMsg)
|
||||||
|
// moving counts the keys handed to the loop and the keys it settled (acknowledged or refused, each as the bus took
|
||||||
|
// it): what tells a take-in that moves from one that holds work and does nothing (the stall watch, ADR 0544 §6).
|
||||||
|
moving := &takeInMoving{}
|
||||||
|
read := make([]jetstream.Consumer, 0, len(consumers))
|
||||||
|
// The connection closes only once the readers and the loop have ended, so the refusals they send at the stop reach
|
||||||
|
// the bus (the third review of ADR 0544).
|
||||||
|
var users sync.WaitGroup
|
||||||
|
for _, c := range consumers {
|
||||||
|
bucket := c.bucket
|
||||||
|
read = append(read, c.consumer)
|
||||||
|
it, err := c.consumer.Messages(jetstream.PullMaxMessages(takeInBuffered))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("done keys: the take-in's consumer on %s is not read: %v\n", bucket, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
goServing(ctx, func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
it.Drain() // the keys it holds are handed out and refused at once below, not lost to their ack wait
|
||||||
|
})
|
||||||
|
users.Add(1)
|
||||||
|
goServing(ctx, func() {
|
||||||
|
defer users.Done()
|
||||||
|
for {
|
||||||
|
m, err := it.Next()
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() == nil && !errors.Is(err, jetstream.ErrMsgIteratorClosed) {
|
||||||
|
fmt.Printf("done keys: the take-in's consumer on %s stopped: %v\n", bucket, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
d, ok := takeInKeyOf(bucket, m)
|
||||||
|
if !ok {
|
||||||
|
_ = m.Ack()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Settled is the loop's attempt, whatever the bus answers: an acknowledgement it refuses (at a first start,
|
||||||
|
// until the bus node is composed) is delivered again and settled again, and never counts as work held
|
||||||
|
// (the third review of ADR 0544). Acknowledged synchronously and past the stop, so a refusal is said.
|
||||||
|
msg := takeInMsg{delivered: d, ack: func() {
|
||||||
|
acking, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := m.DoubleAck(acking); err != nil {
|
||||||
|
doneKeysSay(fmt.Sprintf("done keys: the bus did not take the acknowledgement of %s: %v", d.module, err))
|
||||||
|
}
|
||||||
|
moving.settled.Add(1)
|
||||||
|
},
|
||||||
|
nak: func(delay time.Duration) {
|
||||||
|
_ = m.NakWithDelay(delay)
|
||||||
|
moving.settled.Add(1)
|
||||||
|
}}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
_ = m.Nak()
|
||||||
|
continue // drained: each key it held, refused at once
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case in <- msg:
|
||||||
|
moving.handed.Add(1)
|
||||||
|
case <-ctx.Done():
|
||||||
|
_ = m.Nak() // delivered again at once to the controller after this one, not after the ack wait
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
backlog := func(ctx context.Context) (takeInBacklog, error) { return backlogOf(ctx, read) }
|
||||||
|
steps := liveTakeInSteps(open, keys, keys.JS)
|
||||||
|
users.Add(1)
|
||||||
|
goServing(ctx, func() {
|
||||||
|
defer users.Done()
|
||||||
|
takeInLoop(ctx, in, steps, backlog)
|
||||||
|
})
|
||||||
|
goServing(ctx, func() { watchTakeInStall(ctx, backlog, moving) })
|
||||||
|
goServing(ctx, func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
users.Wait()
|
||||||
|
closeKeys()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInLoop gathers delivered keys into batches and takes each batch in, until ctx ends.
|
||||||
|
func takeInLoop(ctx context.Context, in <-chan takeInMsg, steps takeInSteps,
|
||||||
|
backlog func(context.Context) (takeInBacklog, error)) {
|
||||||
|
t := &takeInState{memo: map[string]string{}, raised: map[string]bool{}, saidWait: map[string]string{}}
|
||||||
|
for {
|
||||||
|
var batch []takeInMsg
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case m := <-in:
|
||||||
|
batch = append(batch, m)
|
||||||
|
}
|
||||||
|
gather := time.NewTimer(takeInGather)
|
||||||
|
more:
|
||||||
|
for len(batch) < startTakeIns {
|
||||||
|
select {
|
||||||
|
case m := <-in:
|
||||||
|
batch = append(batch, m)
|
||||||
|
case <-gather.C:
|
||||||
|
break more
|
||||||
|
case <-ctx.Done():
|
||||||
|
gather.Stop()
|
||||||
|
for _, m := range batch {
|
||||||
|
m.nak(0)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gather.Stop()
|
||||||
|
said := t.take(ctx, steps, batch)
|
||||||
|
if b, err := backlog(ctx); err == nil {
|
||||||
|
said += fmt.Sprintf("; backlog %d", b.Total())
|
||||||
|
}
|
||||||
|
doneKeysSay("done keys: " + said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInState is what the take-in keeps between batches in this process: the trunk each module was last taken in at,
|
||||||
|
// the modules whose condition it raised, and the wait last said of each.
|
||||||
|
type takeInState struct {
|
||||||
|
memo map[string]string
|
||||||
|
raised map[string]bool
|
||||||
|
saidWait map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// take takes one batch in and acknowledges or refuses each key, raising and clearing the modules' conditions. It answers
|
||||||
|
// the batch in a line.
|
||||||
|
func (t *takeInState) take(ctx context.Context, steps takeInSteps, batch []takeInMsg) string {
|
||||||
|
keys := make([]delivered, len(batch))
|
||||||
|
for i, m := range batch {
|
||||||
|
keys[i] = m.delivered
|
||||||
|
}
|
||||||
|
settled, passes := steps.batch(ctx, keys, t.memo)
|
||||||
|
verdicts := make(map[string]verdict, len(settled))
|
||||||
|
var taken, nothing, waiting, failed int
|
||||||
|
for _, r := range settled {
|
||||||
|
verdicts[r.module] = r.verdict
|
||||||
|
o := doneKeyObservation(r.module)
|
||||||
|
switch {
|
||||||
|
case r.err != nil:
|
||||||
|
failed++
|
||||||
|
doneKeysSay(fmt.Sprintf("done keys: %s: %v", r.module, r.err))
|
||||||
|
if keeper := conditionsFrom; keeper != nil && !t.raised[r.module] && ctx.Err() == nil &&
|
||||||
|
!errors.Is(r.err, context.Canceled) {
|
||||||
|
o.Summary += r.err.Error()
|
||||||
|
if _, err := keeper.Observe(ctx, o); err == nil {
|
||||||
|
t.raised[r.module] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case r.waits != "":
|
||||||
|
waiting++
|
||||||
|
if t.saidWait[r.module] != r.waits {
|
||||||
|
doneKeysSay(fmt.Sprintf("done keys: %s %s", r.module, r.waits))
|
||||||
|
t.saidWait[r.module] = r.waits
|
||||||
|
}
|
||||||
|
case r.taken:
|
||||||
|
taken++
|
||||||
|
delete(t.saidWait, r.module)
|
||||||
|
if r.said != "" {
|
||||||
|
doneKeysSay("done keys: " + r.said)
|
||||||
|
}
|
||||||
|
// Cleared on every take-in, whoever raised it: a controller before a restart, whose key was delivered again.
|
||||||
|
if keeper := conditionsFrom; keeper != nil {
|
||||||
|
if _, err := keeper.Clear(ctx, o.Key(), "composed and granted"); err == nil {
|
||||||
|
delete(t.raised, r.module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case r.nothingNew:
|
||||||
|
nothing++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range batch {
|
||||||
|
if verdicts[m.module] == refused {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
m.nak(0) // cut by the stop: the next controller takes it at once
|
||||||
|
} else {
|
||||||
|
m.nak(refusalDelay(m.deliveries))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
m.ack()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d key(s) of %d module(s): %d taken in, %d with nothing new, %d waiting, %d failed; %d composer pass(es)",
|
||||||
|
len(batch), len(settled), taken, nothing, waiting, failed, passes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInBacklog is the take-in's backlog on the bus: keys not delivered yet, and keys delivered and not acknowledged
|
||||||
|
// (being taken in, or refused and waiting for their delay).
|
||||||
|
type takeInBacklog struct {
|
||||||
|
Pending uint64 `json:"pending"`
|
||||||
|
Taking uint64 `json:"taking"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Total is every key the take-in has not acknowledged.
|
||||||
|
func (b takeInBacklog) Total() uint64 { return b.Pending + b.Taking }
|
||||||
|
|
||||||
|
// backlogOf reads the consumers' own counts from the bus.
|
||||||
|
func backlogOf(ctx context.Context, consumers []jetstream.Consumer) (takeInBacklog, error) {
|
||||||
|
var b takeInBacklog
|
||||||
|
for _, c := range consumers {
|
||||||
|
info, err := c.Info(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return takeInBacklog{}, err
|
||||||
|
}
|
||||||
|
b.Pending += info.NumPending
|
||||||
|
b.Taking += uint64(info.NumAckPending)
|
||||||
|
}
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readTakeInBacklog is the take-in's backlog for a reader in any process (`status`): the consumers asked by name.
|
||||||
|
func readTakeInBacklog(ctx context.Context) (takeInBacklog, error) {
|
||||||
|
keys, closeKeys, err := openBuilderKeys()
|
||||||
|
if err != nil {
|
||||||
|
return takeInBacklog{}, err
|
||||||
|
}
|
||||||
|
defer closeKeys()
|
||||||
|
var consumers []jetstream.Consumer
|
||||||
|
for _, bucket := range broker.TakeInBuckets {
|
||||||
|
c, err := keys.JS.Consumer(ctx, "KV_"+bucket, broker.TakeInConsumer)
|
||||||
|
if err != nil {
|
||||||
|
return takeInBacklog{}, fmt.Errorf("the take-in's consumer on %s: %w", bucket, err)
|
||||||
|
}
|
||||||
|
consumers = append(consumers, c)
|
||||||
|
}
|
||||||
|
return backlogOf(ctx, consumers)
|
||||||
|
}
|
||||||
|
|
||||||
|
// kindTakeInStalled is the condition of a take-in that holds work and settles none (ADR 0544 §6); kindTakeInNotConsumed
|
||||||
|
// that of a bucket whose consumer could not be made.
|
||||||
|
const (
|
||||||
|
kindTakeInStalled = "take-in-stalled"
|
||||||
|
kindTakeInNotConsumed = "take-in-not-consumed"
|
||||||
|
)
|
||||||
|
|
||||||
|
func takeInStalledObservation(b takeInBacklog, since time.Time) conditions.Observation {
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeCore, ID: "take-in", Token: kindTakeInStalled,
|
||||||
|
Kind: kindTakeInStalled, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, Source: "done keys",
|
||||||
|
Summary: fmt.Sprintf("the take-in of done keys has %d key(s) waiting and has settled none since %s",
|
||||||
|
b.Total(), since.UTC().Format(time.RFC3339))}
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInNotConsumedObservation is the condition of bucket's consumer not made, with its own id, so it is raised and
|
||||||
|
// cleared apart from a stall and from the other bucket's.
|
||||||
|
func takeInNotConsumedObservation(bucket string, err error) conditions.Observation {
|
||||||
|
summary := "the take-in's consumer on " + bucket + " is made"
|
||||||
|
if err != nil {
|
||||||
|
summary = fmt.Sprintf("the take-in's consumer on %s is not made, so its keys are not taken in until the next "+
|
||||||
|
"start: %v", bucket, err)
|
||||||
|
}
|
||||||
|
return conditions.Observation{Scope: conditions.ScopeCore, ID: "take-in-" + bucket, Token: kindTakeInNotConsumed,
|
||||||
|
Kind: kindTakeInNotConsumed, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||||
|
Source: "done keys", Summary: summary}
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeInMoving counts the keys handed to the take-in loop and the keys it settled.
|
||||||
|
type takeInMoving struct {
|
||||||
|
handed, settled atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// stallWatch judges each look: stalled once there was work to do (keys not delivered yet, or keys handed to the loop
|
||||||
|
// and not settled) and nothing settled for takeInStall. A refused key waiting for its delay is no work of the loop's:
|
||||||
|
// with nothing else, a quiet mesh is not a stall; a loop that hangs on its batch is (the review of ADR 0544). Its first
|
||||||
|
// look always says it changed, so a condition a controller before this one raised is cleared once this one moves.
|
||||||
|
type stallWatch struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
settled uint64
|
||||||
|
movedAt time.Time
|
||||||
|
stalled bool
|
||||||
|
known bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// look takes a reading at now: the backlog, and how many keys were handed and settled; it answers whether the take-in
|
||||||
|
// stalls and whether that changed (always on the first look).
|
||||||
|
func (w *stallWatch) look(b takeInBacklog, handed, settled uint64, now time.Time) (stalled, changed bool) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
work := b.Pending > 0 || handed > settled
|
||||||
|
if !work || settled != w.settled || w.movedAt.IsZero() {
|
||||||
|
w.movedAt = now
|
||||||
|
}
|
||||||
|
w.settled = settled
|
||||||
|
nowStalled := work && now.Sub(w.movedAt) >= takeInStall
|
||||||
|
changed = !w.known || nowStalled != w.stalled
|
||||||
|
w.stalled, w.known = nowStalled, true
|
||||||
|
return nowStalled, changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchTakeInStall reads the backlog every minute and raises take-in-stalled while the take-in holds work and settles
|
||||||
|
// none, clearing it when it moves, and at its first look when it does not stall: a watchdog's timer, as research 072 §2
|
||||||
|
// keeps them until the health part leaves the controller.
|
||||||
|
func watchTakeInStall(ctx context.Context, backlog func(context.Context) (takeInBacklog, error), moving *takeInMoving) {
|
||||||
|
w := &stallWatch{}
|
||||||
|
tick := time.NewTicker(time.Minute)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
b, err := backlog(ctx)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stalled, changed := w.look(b, moving.handed.Load(), moving.settled.Load(), time.Now())
|
||||||
|
keeper := conditionsFrom
|
||||||
|
if !changed || keeper == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
since := w.movedAt
|
||||||
|
w.mu.Unlock()
|
||||||
|
o := takeInStalledObservation(b, since)
|
||||||
|
if stalled {
|
||||||
|
doneKeysSay("done keys: " + o.Summary)
|
||||||
|
_, _ = keeper.Observe(ctx, o)
|
||||||
|
} else {
|
||||||
|
_, _ = keeper.Clear(ctx, o.Key(), "the take-in moves")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1,
|
|
||||||
// issue 654): `module check` and the merge gate read it from the tree beside the manifest, so a change to a
|
|
||||||
// template alone is judged as a change to its module.
|
|
||||||
|
|
||||||
const templatedNotifier = `{"module":"notifier","version":"1",
|
|
||||||
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
|
|
||||||
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"files/notifier.conf"}]}`
|
|
||||||
|
|
||||||
// inlineNotifier is the same module as the mesh holds it: built, its text inline.
|
|
||||||
const inlineNotifier = `{"module":"notifier","version":"1",
|
|
||||||
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}`
|
|
||||||
|
|
||||||
// aTemplatedModule writes a module's directory: its manifest and its files/.
|
|
||||||
func aTemplatedModule(t *testing.T, dir, manifest string, files map[string]string) string {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
path := filepath.Join(dir, "module.json")
|
|
||||||
if err := os.WriteFile(path, []byte(manifest), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for name, text := range files {
|
|
||||||
full := filepath.Join(dir, filepath.FromSlash(name))
|
|
||||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) {
|
|
||||||
good := aTemplatedModule(t, t.TempDir(), templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{good}, &out); err != nil || !strings.Contains(out.String(), "notifier: ok") {
|
|
||||||
t.Fatalf("a module with its text in files/ was refused: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// A change to the template alone, to text the controller refuses inline, is refused the same.
|
|
||||||
bad := aTemplatedModule(t, t.TempDir(), templatedNotifier,
|
|
||||||
map[string]string{"files/notifier.conf": "volume = 3\nstore = ${dir:nowhere}\n"})
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{bad}, &out); err == nil || !strings.Contains(out.String(), "nowhere") {
|
|
||||||
t.Fatalf("a template whose text is refused inline passed:\n%s", out.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// A symlink under files/ that no template names: the build's packing refuses it, so the check does too.
|
|
||||||
linked := t.TempDir()
|
|
||||||
aTemplatedModule(t, linked, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
|
|
||||||
if err := os.Symlink("/etc/hostname", filepath.Join(linked, "files", "elsewhere")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{filepath.Join(linked, "module.json")}, &out); err == nil ||
|
|
||||||
!strings.Contains(out.String(), "files/elsewhere is a symlink") {
|
|
||||||
t.Fatalf("a symlink under files/ passed the check:\n%s", out.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// And a template the tree lacks, by name.
|
|
||||||
missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil)
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{missing}, &out); err == nil || !strings.Contains(out.String(), "files/notifier.conf") {
|
|
||||||
t.Fatalf("a missing template was not refused by name:\n%s", out.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline.
|
|
||||||
|
|
||||||
// What the gate composes for the module in its template form is what the mesh composes for it inline.
|
|
||||||
|
|
||||||
// A module whose files/ is unpacked onto the machine names its own templates directory (issue 731): `module check`
|
|
||||||
// reads its templates from there and says of it exactly what it says of the same module with its text inline.
|
|
||||||
func TestModuleCheckReadsTemplatesFromTheTemplatesDirectoryTheManifestNames(t *testing.T) {
|
|
||||||
const inline = `{"module":"notifier","version":"1",
|
|
||||||
"build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"}]},
|
|
||||||
"resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"},
|
|
||||||
{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}`
|
|
||||||
const templated = `{"module":"notifier","version":"1","templates":"templates",
|
|
||||||
"build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"},
|
|
||||||
{"name":"templates","kind":"archive","from":"templates"}]},
|
|
||||||
"resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"},
|
|
||||||
{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"templates/notifier.conf"}]}`
|
|
||||||
scripts := map[string]string{"files/bin/notify": "#!/bin/sh\n"}
|
|
||||||
var want bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), inline, scripts)}, &want); err != nil {
|
|
||||||
t.Fatalf("the inline module was refused: %v\n%s", err, want.String())
|
|
||||||
}
|
|
||||||
with := map[string]string{"files/bin/notify": "#!/bin/sh\n", "templates/notifier.conf": "volume = 3\n"}
|
|
||||||
var got bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), templated, with)}, &got); err != nil {
|
|
||||||
t.Fatalf("a module with its text in templates/ was refused: %v\n%s", err, got.String())
|
|
||||||
}
|
|
||||||
if got.String() != want.String() {
|
|
||||||
t.Fatalf("module check says otherwise of the templated module:\ninline: %s\ntemplated: %s", want.String(),
|
|
||||||
got.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Its text in files/ instead, where the machine unpacks it, is not where its manifest says its templates are.
|
|
||||||
misplaced := strings.Replace(templated, `"templates/notifier.conf"`, `"files/notifier.conf"`, 1)
|
|
||||||
got.Reset()
|
|
||||||
if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), misplaced,
|
|
||||||
map[string]string{"files/notifier.conf": "volume = 3\n"})}, &got); err == nil ||
|
|
||||||
!strings.Contains(got.String(), "files/notifier.conf") {
|
|
||||||
t.Fatalf("a template outside the templates directory the manifest names passed:\n%s", got.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Issue 876: the gate judges a moved module that names its templates as registration reads it after a real build,
|
|
||||||
// handed its templates; it refused every such module as "marked moved and names no template". This calls the gate's
|
|
||||||
// own shelf (shelfWithChange), which needs no database.
|
|
||||||
@@ -20,11 +20,12 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// unknownFieldObservations is one condition for each module of the catalogue whose stored manifest has
|
// unknownFieldObservations is one condition for each module of the catalogue whose stored manifest has
|
||||||
// a key this controller does not know.
|
// a key this controller does not know and read no further than its name and version (ADR 0578 §6: a manifest read past
|
||||||
|
// the key is registered and the key named, so it raises nothing).
|
||||||
func unknownFieldObservations(known map[string]catalogue.Manifest) []conditions.Observation {
|
func unknownFieldObservations(known map[string]catalogue.Manifest) []conditions.Observation {
|
||||||
names := make([]string, 0, len(known))
|
names := make([]string, 0, len(known))
|
||||||
for name, m := range known {
|
for name, m := range known {
|
||||||
if m.UnknownField() != "" {
|
if m.ReadNoFurther() {
|
||||||
names = append(names, name)
|
names = append(names, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,14 +11,18 @@ import (
|
|||||||
// A module whose stored manifest has a key this controller does not know is a condition, in plain
|
// A module whose stored manifest has a key this controller does not know is a condition, in plain
|
||||||
// words, until it is read again; every other module raises nothing (novox/hq ADR 0262).
|
// words, until it is read again; every other module raises nothing (novox/hq ADR 0262).
|
||||||
func TestAModuleWithAnUnknownFieldIsACondition(t *testing.T) {
|
func TestAModuleWithAnUnknownFieldIsACondition(t *testing.T) {
|
||||||
var later, now catalogue.Manifest
|
var later, now, aside catalogue.Manifest
|
||||||
if err := json.Unmarshal([]byte(`{"module": "dunst", "version": "2", "settings": {}, "a-field-from-later": 1}`), &later); err != nil {
|
if err := json.Unmarshal([]byte(`{"module": "dunst", "version": "2", "tools": "not-a-list", "a-field-from-later": 1}`), &later); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A field set aside with the rest read raises nothing (ADR 0578 §6): the module is registered and the field named.
|
||||||
|
if err := json.Unmarshal([]byte(`{"module": "sway", "version": "2", "settings": {}, "a-field-from-later": 1}`), &aside); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal([]byte(`{"module": "xorg", "version": "1"}`), &now); err != nil {
|
if err := json.Unmarshal([]byte(`{"module": "xorg", "version": "1"}`), &now); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
observed := unknownFieldObservations(map[string]catalogue.Manifest{"dunst": later, "xorg": now})
|
observed := unknownFieldObservations(map[string]catalogue.Manifest{"dunst": later, "xorg": now, "sway": aside})
|
||||||
if len(observed) != 1 || observed[0].ID != "dunst" || observed[0].Kind != kindUnknownField {
|
if len(observed) != 1 || observed[0].ID != "dunst" || observed[0].Kind != kindUnknownField {
|
||||||
t.Fatalf("observed: %+v", observed)
|
t.Fatalf("observed: %+v", observed)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,12 @@ func (v *standInVault) ask(verb string, args map[string]any) (json.RawMessage, e
|
|||||||
// The vault asks the desk's prompt and the provider's tool itself, in its own name (issue 723): one step from
|
// The vault asks the desk's prompt and the provider's tool itself, in its own name (issue 723): one step from
|
||||||
// the controller, which carries no key and no sealed value.
|
// the controller, which carries no key and no sealed value.
|
||||||
switch verb {
|
switch verb {
|
||||||
|
case "issue-account":
|
||||||
|
// The vault is the one issuer once installed (issue 774): the controller forwards, and keeps nothing.
|
||||||
|
if v.refuse != nil {
|
||||||
|
return nil, v.refuse
|
||||||
|
}
|
||||||
|
return json.Marshal(map[string]any{"minted": true, "account": map[string]any{"user": args["user"], "generation": 1}})
|
||||||
case "secret-ask":
|
case "secret-ask":
|
||||||
if v.refuse != nil {
|
if v.refuse != nil {
|
||||||
return nil, v.refuse
|
return nil, v.refuse
|
||||||
|
|||||||
@@ -512,7 +512,9 @@ func (w *watchdogs) lostConsumers(ctx context.Context, heard []link.Advisory) (m
|
|||||||
for _, a := range deleted {
|
for _, a := range deleted {
|
||||||
_, err := w.js.Context().ConsumerInfo(a.Stream, a.Consumer)
|
_, err := w.js.Context().ConsumerInfo(a.Stream, a.Consumer)
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, nats.ErrConsumerNotFound):
|
// A consumer whose stream is gone is gone with it: the retired declarations' stream took every node's
|
||||||
|
// consumer, and its deletion said each as deleted (issue 1945).
|
||||||
|
case errors.Is(err, nats.ErrConsumerNotFound) || errors.Is(err, nats.ErrStreamNotFound):
|
||||||
out[a.Stream+"."+a.Consumer] = want[a.Stream+"."+a.Consumer]
|
out[a.Stream+"."+a.Consumer] = want[a.Stream+"."+a.Consumer]
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return nil, fmt.Errorf("whether %s exists cannot be read: %w", link.ConsumerInWords(a.Stream, a.Consumer), err)
|
return nil, fmt.Errorf("whether %s exists cannot be read: %w", link.ConsumerInWords(a.Stream, a.Consumer), err)
|
||||||
@@ -543,10 +545,9 @@ func later(a, b time.Time) time.Time {
|
|||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
// watchTheMesh opens the condition store and starts the watchdogs, the bus's advisories and the
|
// openServingConditions opens the condition store for the serving controller and makes it conditionsFrom. Nil when the store
|
||||||
// self-check, for the serving controller; the returned function stops them. Nil when the store could
|
// could not be opened, which is said: status then says the conditions cannot be read.
|
||||||
// not be opened, which is said.
|
func openServingConditions(ctx context.Context, open *stores, bus link.OverNATS) *conditions.Keeper {
|
||||||
func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS) func() {
|
|
||||||
keeper, err := keeperOn(ctx, bus.Conn, storeLooks{open.inventory})
|
keeper, err := keeperOn(ctx, bus.Conn, storeLooks{open.inventory})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
|
fmt.Printf("the condition store could NOT be opened, so nothing that goes wrong is kept or said, and "+
|
||||||
@@ -554,6 +555,22 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
conditionsFrom = keeper
|
conditionsFrom = keeper
|
||||||
|
return keeper
|
||||||
|
}
|
||||||
|
|
||||||
|
// closeConditions flushes what the keeper holds and closes it, once nothing that raises a condition runs.
|
||||||
|
func closeConditions(keeper *conditions.Keeper) {
|
||||||
|
flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
keeper.Close(flushing)
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchTheMesh starts the watchdogs, the bus's advisories and the self-check over the condition store's keeper, for the
|
||||||
|
// serving controller; the returned function stops them and waits for them to end. Nil when there is no keeper.
|
||||||
|
func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus link.OverNATS, keeper *conditions.Keeper) func() {
|
||||||
|
if keeper == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
logf := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
|
||||||
stopHearing, err := server.HearAdvisories(logf)
|
stopHearing, err := server.HearAdvisories(logf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -566,26 +583,26 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
|
|||||||
d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
|
d := &doctor{open: open, js: server.JetStream(), keeper: keeper, teller: bus, watchdogs: w, host: host}
|
||||||
w.doctor = d
|
w.doctor = d
|
||||||
doctorFrom = d
|
doctorFrom = d
|
||||||
|
// Its own group as well as serve's: its stop waits for what it started.
|
||||||
watching, stop := context.WithCancel(ctx)
|
watching, stop := context.WithCancel(ctx)
|
||||||
go w.keep(watching)
|
watching, running := withServingGroup(watching)
|
||||||
go d.keep(watching)
|
goServing(watching, func() { w.keep(watching) })
|
||||||
|
goServing(watching, func() { d.keep(watching) })
|
||||||
// And the healers (novox/hq to-be 45 §7): what is open that a registered healer answers, repaired
|
// And the healers (novox/hq to-be 45 §7): what is open that a registered healer answers, repaired
|
||||||
// under the lease and the brake, every act said.
|
// under the lease and the brake, every act said.
|
||||||
healers := newHealing(open, keeper, bus, server.JetStream())
|
healers := newHealing(open, keeper, bus, server.JetStream())
|
||||||
go healers.keep(watching)
|
goServing(watching, func() { healers.keep(watching) })
|
||||||
// And the asker (novox/hq ADR 0259): what needs the operator and names its answers is asked of them,
|
// And the asker (novox/hq ADR 0259): what needs the operator and names its answers is asked of them,
|
||||||
// and the answer chosen is performed on its warrant.
|
// and the answer chosen is performed on its warrant.
|
||||||
startAsking(watching, open, server, bus.Conn, keeper)
|
startAsking(watching, open, server, bus.Conn, keeper)
|
||||||
go forgettingOldHeals(watching, open.inventory)
|
goServing(watching, func() { forgettingOldHeals(watching, open.inventory) })
|
||||||
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
|
||||||
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
|
||||||
broker.ConditionsBucket, conditions.Seat)
|
broker.ConditionsBucket, conditions.Seat)
|
||||||
return func() {
|
return func() {
|
||||||
stop()
|
stop()
|
||||||
|
running.stop()
|
||||||
stopHearing()
|
stopHearing()
|
||||||
flushing, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
keeper.Close(flushing)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,46 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ADR 0326 §3 is judged where a definition is written (novox/hq issue 574): `module check`, and
|
|
||||||
// namesNoInstallation, which `module add`, a build taken in and the merge gate's registration run.
|
|
||||||
func TestWhereADefinitionIsWrittenADeclaringModuleNamesItsFilesKeys(t *testing.T) {
|
|
||||||
const silent = `{"module":"mail-client","version":"1",
|
|
||||||
"settings":{"prefs":{"kind":"preference","default":"// none","why":"its own defaults until a machine names one"}},
|
|
||||||
"resources":[
|
|
||||||
{"id":"user-js","type":"file","path":"/var/lib/mail-client/user.js","mode":"0644","content":"${setting:prefs}\n"},
|
|
||||||
{"id":"accounts","type":"file","path":"/var/lib/mail-client/accounts.json","mode":"0600","merge":"json",
|
|
||||||
"content":"{}\n"%s}]}`
|
|
||||||
dir := t.TempDir()
|
|
||||||
bad := filepath.Join(dir, "bad.json")
|
|
||||||
good := filepath.Join(dir, "good.json")
|
|
||||||
os.WriteFile(bad, []byte(strings.Replace(silent, "%s", "", 1)), 0o600)
|
|
||||||
os.WriteFile(good, []byte(strings.Replace(silent, "%s", `,"takes-settings":["accounts"]`, 1)), 0o600)
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
if err := moduleCheck([]string{bad}, &out); err == nil ||
|
|
||||||
!strings.Contains(out.String(), "mergeable accounts names no takes-settings") {
|
|
||||||
t.Fatalf("module check passed a declaring module's silent mergeable file: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
out.Reset()
|
|
||||||
if err := moduleCheck([]string{good}, &out); err != nil {
|
|
||||||
t.Fatalf("the file naming its keys was refused: %v\n%s", err, out.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
raw, _ := os.ReadFile(bad)
|
|
||||||
m, err := catalogue.ParseManifest(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading it is not where it is judged: %v", err)
|
|
||||||
}
|
|
||||||
if err := namesNoInstallation(m); err == nil || !strings.Contains(err.Error(), "names no takes-settings") {
|
|
||||||
t.Fatalf("registration passed it: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
|
||||||
github.com/jackc/pgx/v5 v5.10.0
|
github.com/jackc/pgx/v5 v5.10.0
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
||||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user