Compare commits
86
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7476aae09 | ||
|
|
be8d8e6cc6 | ||
|
|
0a4c35f731 | ||
|
|
b93b541a13 | ||
|
|
2e681825ab | ||
|
|
fbea777263 | ||
|
|
031e10638c | ||
|
|
4f9227a186 | ||
|
|
fb8ea0e683 | ||
|
|
a6c70803ab | ||
|
|
9bd1c84e0f | ||
|
|
1f3fb52470 | ||
|
|
bb6e472939 | ||
|
|
93f33e2305 | ||
|
|
abdf85aa4f | ||
|
|
91686fdb81 | ||
|
|
d2a868f99f | ||
|
|
a143678fd8 | ||
|
|
9914459830 | ||
|
|
b7159d71a6 | ||
|
|
e0c7f36ca9 | ||
|
|
9120f94aa4 | ||
|
|
1900b40192 | ||
|
|
6cbdb3712e | ||
|
|
2a0c428ef9 | ||
|
|
a5d1f7dde4 | ||
|
|
95dd64c774 | ||
|
|
81fa009bf4 | ||
|
|
05ff90e3c1 | ||
|
|
aaffc19232 | ||
|
|
479ac71050 | ||
|
|
4f6b389a37 | ||
|
|
45f5e27526 | ||
|
|
dd8ba97989 | ||
|
|
ab22efd8b1 | ||
|
|
54aee345fc | ||
|
|
8639b373d2 | ||
|
|
5fc69444f6 | ||
|
|
2fe9d678fc | ||
|
|
58b4f8b6bc | ||
|
|
bf667a602d | ||
|
|
9a8b3870ba | ||
|
|
24c23a1d06 | ||
|
|
a548a61e86 | ||
|
|
bc28606d45 | ||
|
|
7f0eb9c51d | ||
|
|
690aa662a8 | ||
|
|
8bd0b5db15 | ||
|
|
b4fd2cc210 | ||
|
|
1a021052fe | ||
|
|
cf738cdb90 | ||
|
|
7a2af23764 | ||
|
|
daaa1962bd | ||
|
|
df878ba1f7 | ||
|
|
1e01868d92 | ||
|
|
9d04ae7edd | ||
|
|
16df7d0b7a | ||
|
|
cd1d2ff447 | ||
|
|
faf6a23d91 | ||
|
|
2f4e81f3b0 | ||
|
|
4182f8fbfb | ||
|
|
c3a2ffde67 | ||
|
|
6502ba6ad6 | ||
|
|
f6c0520f26 | ||
|
|
c912bcab6e | ||
|
|
63c0b445d7 | ||
|
|
653ff61662 | ||
|
|
a7cd0a7807 | ||
|
|
7271cf151b | ||
|
|
82d0e125b2 | ||
|
|
a4c24a1979 | ||
|
|
cff69893a3 | ||
|
|
047237c911 | ||
|
|
449cda1b00 | ||
|
|
a6e53a4858 | ||
|
|
4993f1b1d7 | ||
|
|
199f6b48b1 | ||
|
|
4636261435 | ||
|
|
4e8fdfdf04 | ||
|
|
d00926e089 | ||
|
|
bb15aab50e | ||
|
|
6a4718b712 | ||
|
|
d5adf9b72b | ||
|
|
7365890298 | ||
|
|
0862f97972 | ||
|
|
45439fd742 |
@@ -1,16 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
)
|
||||
|
||||
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
||||
// record's keys read it; a composition carrying none changes nothing.
|
||||
// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what
|
||||
// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it.
|
||||
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||
res := func(id, content string) json.RawMessage {
|
||||
@@ -26,7 +37,105 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recordKeys(state)(); got.Authority != "CA-1" {
|
||||
if got := recordKeys(state, nil)(); got.Authority != "CA-1" {
|
||||
t.Errorf("the record's keys read the authority %q", got.Authority)
|
||||
}
|
||||
|
||||
// L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the
|
||||
// vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused
|
||||
// and the record names the vault's alone. Each change tells the record's writer.
|
||||
controllerPub, controller, _ := ed25519.GenerateKey(nil)
|
||||
vaultPub, vault, _ := ed25519.GenerateKey(nil)
|
||||
trust := newMeshTrust(controllerPub)
|
||||
told := 0
|
||||
trust.onChange(func() { told++ })
|
||||
keys := recordKeys(state, trust.Trusts)
|
||||
ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub)
|
||||
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) {
|
||||
t.Errorf("before the signers are read the record trusts %v", got)
|
||||
}
|
||||
enrol, _ := delivery.SignContext(delivery.KindEnrolment)
|
||||
msg := []byte("a reply")
|
||||
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil {
|
||||
t.Error("a vault-signed message is accepted before the vault's key is read")
|
||||
}
|
||||
vaultCA := testAuthority(t, vaultPub, vault)
|
||||
keyValue, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now()))
|
||||
fp, _ := delivery.AuthorityFingerprint(vaultCA)
|
||||
authValue, _ := json.Marshal(delivery.SignerAuthority{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()})
|
||||
trust.heardSigner(delivery.SignersKeyKey, keyValue, false, func(string) {})
|
||||
trust.heardSigner(delivery.SignersAuthorityKey, authValue, false, func(string) {})
|
||||
if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault {
|
||||
t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err)
|
||||
}
|
||||
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
|
||||
trust.VaultAuthority() != vaultCA || told != 2 {
|
||||
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
|
||||
}
|
||||
handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"})
|
||||
trust.heardLease(handed, true, nil)
|
||||
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil {
|
||||
t.Error("after the lease is handed away the controller's key is still trusted")
|
||||
}
|
||||
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 {
|
||||
t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told)
|
||||
}
|
||||
trust.heardLease(handed, true, nil)
|
||||
if told != 3 {
|
||||
t.Error("a lease heard again unchanged wrote the record again")
|
||||
}
|
||||
// A signers' key naming another fingerprint is not trusted.
|
||||
bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP})
|
||||
trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {})
|
||||
if !delivery.TrustsKey(keys().Trusts, delivery.SignerVault, vaultFP) {
|
||||
t.Error("a signers' key that does not check replaced the vault's")
|
||||
}
|
||||
|
||||
// L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads
|
||||
// it, and the record is written again; one signed by another authority is refused and nothing is kept.
|
||||
serving, err := identity.GenerateServingKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
servingPub, _ := base64.StdEncoding.DecodeString(serving.Public)
|
||||
signedBy := func(ca string, key ed25519.PrivateKey) string {
|
||||
parent, _ := delivery.ParseCertificate(ca)
|
||||
der, err := x509.CreateCertificate(nil, delivery.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)),
|
||||
parent, ed25519.PublicKey(servingPub), key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||
}
|
||||
var asked []any
|
||||
answer := signedBy(vaultCA, vault)
|
||||
ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) {
|
||||
asked = append(asked, verb, args)
|
||||
return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer})
|
||||
}
|
||||
written := 0
|
||||
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb {
|
||||
t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked)
|
||||
}
|
||||
answer = signedBy(testAuthority(t, controllerPub, controller), controller)
|
||||
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 {
|
||||
t.Errorf("a certificate of another authority was kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// testAuthority is a self-signed authority certificate for key, as the vault makes its own.
|
||||
func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string {
|
||||
t.Helper()
|
||||
tpl := delivery.AuthorityTemplate(pub, time.Now().Add(-time.Hour))
|
||||
der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-host/internal/accounts"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The node seat node-engine's judgements on this machine (novox/hq ADR 0563): what a hand-over and a library's take
|
||||
// would set and what the disk holds now, and whether this machine is root-free.
|
||||
|
||||
// diskOf is what is on disk at a directory, as an act compares it when the operator's yes arrives: its owner and
|
||||
// group, its mode, and which file it is.
|
||||
func diskOf(info os.FileInfo) string {
|
||||
owner := "unknown"
|
||||
if uid, gid, ok := apply.OwnerOf(info); ok {
|
||||
owner = fmt.Sprintf("%d:%d", uid, gid)
|
||||
}
|
||||
which := ""
|
||||
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
which = fmt.Sprintf(" %v:%v", st.Dev, st.Ino)
|
||||
}
|
||||
return fmt.Sprintf("%s %o%s", owner, info.Mode().Perm(), which)
|
||||
}
|
||||
|
||||
// handOverJudgeFrom judges a hand-over through the node seat as the controller's ask is judged (judgeFound): the
|
||||
// declared owner and mode it will set, and the disk as it is.
|
||||
func handOverJudgeFrom(statePath string) func(string) (link.Judged, error) {
|
||||
return func(path string) (link.Judged, error) {
|
||||
found, info, err := judgeFound(statePath, path)
|
||||
if err != nil {
|
||||
return link.Judged{}, err
|
||||
}
|
||||
return link.Judged{Subject: path, Path: path, Sets: orUnsaid(found.Declared), Disk: diskOf(info),
|
||||
Words: "A directory of " + moduleOf(found.ID) + " is used as found: the mesh left its owner and mode."}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func moduleOf(id string) string {
|
||||
if at := strings.LastIndex(id, "."); at > 0 {
|
||||
return id[:at]
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// accessJudgeFrom judges the take act on a library through the node seat (ADR 0328 §7, ADR 0563 §4): declared on this
|
||||
// machine, its home, and its root holding data the mesh has not taken; what the take sets, and the disk as it is.
|
||||
func accessJudgeFrom(statePath string) func(string, string) (link.Judged, error) {
|
||||
return func(act, access string) (link.Judged, error) {
|
||||
d, err := declaredNow(statePath, nil)
|
||||
if errors.Is(err, store.ErrNothingDeclared) {
|
||||
return link.Judged{}, errors.New("this node keeps no declaration, so it is declared no library")
|
||||
}
|
||||
if err != nil {
|
||||
return link.Judged{}, fmt.Errorf("what runs on this node does not read: %w", err)
|
||||
}
|
||||
for _, l := range d.Libraries {
|
||||
if apply.LibraryAccess(l.Library) != access {
|
||||
continue
|
||||
}
|
||||
if !l.Home {
|
||||
return link.Judged{}, fmt.Errorf("this machine is not the home of the library %s, and a library is "+
|
||||
"taken on its home", l.Library)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), lookBound)
|
||||
defer cancel()
|
||||
if v := apply.JudgeLibrary(ctx, l); v.Kind != apply.VerdictWaitKeep {
|
||||
return link.Judged{}, fmt.Errorf("the library %s does not wait for the operator's word to be kept "+
|
||||
"(it is judged %s), so there is nothing to take", l.Library, v.Kind)
|
||||
}
|
||||
info, err := os.Lstat(l.Root)
|
||||
if err != nil {
|
||||
return link.Judged{}, fmt.Errorf("the library %s's root cannot be read: %v", l.Library, err)
|
||||
}
|
||||
return link.Judged{Subject: access, Path: filepath.Clean(l.Root),
|
||||
Sets: "the group " + l.Group + ", group write and setgid on everything under its root",
|
||||
Disk: diskOf(info), Words: "The library " + l.Library + " holds data the mesh has not taken."}, nil
|
||||
}
|
||||
return link.Judged{}, fmt.Errorf("the access %s names no library declared on this machine", access)
|
||||
}
|
||||
}
|
||||
|
||||
// askTakes asks the take of every library the look found waiting for the operator's word, once per library (ADR 0563
|
||||
// §4): asked by the engine itself, never a second time while one is open or after one was asked; access-act asks again
|
||||
// by hand.
|
||||
func askTakes(acts *link.HandActs, waiting []string, say func(string)) {
|
||||
if acts == nil {
|
||||
return
|
||||
}
|
||||
for _, access := range waiting {
|
||||
if acts.AskedFor(link.ActAccess, access) {
|
||||
continue
|
||||
}
|
||||
if _, err := acts.AskAccess(link.EngineSeat+"-"+acts.Node, link.ActTake, access,
|
||||
"its root holds data outside its group", true); err != nil && say != nil {
|
||||
say("the take of " + access + " could not be asked: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lastAccounts is the newest look at every account (the health loop's), which root-free judges from.
|
||||
var lastAccounts atomic.Pointer[accounts.Statement]
|
||||
|
||||
// rootFreshFor is how old the verdict root-free judges from may be (ADR 0563 §6, the controller's verdictFreshFor).
|
||||
const rootFreshFor = 15 * time.Minute
|
||||
|
||||
// judgeRootFree is this machine's root-free (ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most
|
||||
// rootFreshFor old, that every account declared never to become root without a person cannot, at least one such
|
||||
// account declared, and the login shell's execute not served here. Anything else is no pass, and says why.
|
||||
func judgeRootFree(node string, st *accounts.Statement, now time.Time, served bool, servedWhy string) link.RootVerdict {
|
||||
v := link.RootVerdict{Machine: node, Judged: now.UTC()}
|
||||
var not []string
|
||||
confined := ""
|
||||
switch {
|
||||
case st == nil:
|
||||
not = append(not, "this machine's engine has judged no account yet")
|
||||
case now.Sub(st.At) > rootFreshFor:
|
||||
not = append(not, fmt.Sprintf("the engine's newest judgement of its accounts was made at %s, more than %d "+
|
||||
"minutes ago, and a verdict that old is not a verdict on now", st.At.UTC().Format(time.RFC3339),
|
||||
int(rootFreshFor.Minutes())))
|
||||
default:
|
||||
var named []string
|
||||
for _, a := range st.Accounts {
|
||||
if !a.Root {
|
||||
continue
|
||||
}
|
||||
named = append(named, a.Name)
|
||||
if a.State != accounts.Healthy {
|
||||
not = append(not, "the account "+a.Name+" "+orNoneKnownWord(a.Reason))
|
||||
}
|
||||
}
|
||||
if len(named) == 0 {
|
||||
not = append(not, "no account here is declared never to become root, so agents run as the operator's "+
|
||||
"account, which may become root")
|
||||
} else if len(not) == 0 {
|
||||
confined = "the account " + strings.Join(named, ", ") + " cannot become root without a person (judged " +
|
||||
st.At.UTC().Format(time.RFC3339) + ")"
|
||||
}
|
||||
}
|
||||
if served {
|
||||
not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, which "+
|
||||
"can become root ("+servedWhy+")")
|
||||
}
|
||||
if len(not) > 0 {
|
||||
v.Why = strings.Join(not, "; ")
|
||||
return v
|
||||
}
|
||||
v.Free, v.Why = true, confined+"; the login shell's execute is not served here"
|
||||
return v
|
||||
}
|
||||
|
||||
func orNoneKnownWord(s string) string {
|
||||
if s == "" {
|
||||
return "is not judged"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// settingFileSuffix is the resource the login shell's holder renders its setting execute into (ADR 0268: zsh's
|
||||
// tools-conf, `execute=serve` or `execute=withhold`), read beside the bus's discovery.
|
||||
const settingFileSuffix = ".tools-conf"
|
||||
|
||||
// executeServedHere is whether the login shell's execute is served on this machine (ADR 0268), failing closed: served
|
||||
// while the bus's discovery hears it answered here or could not be asked, or while the holder's setting file applied
|
||||
// here says anything but withhold or cannot be read. why says which.
|
||||
func executeServedHere(statePath string, heard func() (bool, error)) (bool, string) {
|
||||
var why []string
|
||||
if heard == nil {
|
||||
why = append(why, "the bus could not be asked whether execute is answered here")
|
||||
} else if h, err := heard(); err != nil {
|
||||
why = append(why, "the bus could not be asked whether execute is answered here: "+err.Error())
|
||||
} else if h {
|
||||
why = append(why, "the bus hears execute answered here")
|
||||
}
|
||||
known, err := store.Load(statePath)
|
||||
if err != nil {
|
||||
why = append(why, "what this machine applied could not be read: "+err.Error())
|
||||
}
|
||||
for _, r := range known.Resources {
|
||||
if r.Type != "file" || !strings.HasSuffix(r.ID, settingFileSuffix) {
|
||||
continue
|
||||
}
|
||||
if value, err := settingIn(r.Target); err != nil {
|
||||
why = append(why, moduleOf(r.ID)+"'s setting execute cannot be read: "+err.Error())
|
||||
} else if value != "withhold" {
|
||||
why = append(why, moduleOf(r.ID)+"'s setting execute here is "+value)
|
||||
}
|
||||
}
|
||||
return len(why) > 0, strings.Join(why, "; ")
|
||||
}
|
||||
|
||||
// settingIn is the value of execute in a setting file, "" when it says none.
|
||||
func settingIn(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
value := ""
|
||||
lines := bufio.NewScanner(f)
|
||||
for lines.Scan() {
|
||||
t := strings.TrimSpace(lines.Text())
|
||||
if key, v, ok := strings.Cut(t, "="); ok && !strings.HasPrefix(t, "#") && strings.TrimSpace(key) == "execute" {
|
||||
value = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
if value == "" {
|
||||
value = "not said"
|
||||
}
|
||||
return value, lines.Err()
|
||||
}
|
||||
|
||||
// keepWaiting is every library a look found waiting for the operator's word to keep it (ADR 0328 §8).
|
||||
func keepWaiting(vs []link.ResourceHealth) []string {
|
||||
var out []string
|
||||
for _, v := range vs {
|
||||
for _, w := range v.AccessWaits {
|
||||
if w.Reason == link.WaitKeep {
|
||||
out = append(out, w.Access)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// engineActs is the node seat this engine holds (ADR 0563): judged and performed by the same code as the controller's
|
||||
// signed asks, its asks to the operator over the session's link, its acts kept beside the state.
|
||||
func engineActs(opts options, node string, queue *link.Session, search *accounts.SetuidCache) *link.HandActs {
|
||||
state := link.BusState{Session: queue, Node: node}
|
||||
h := &link.HandActs{Node: node, State: state,
|
||||
JudgeHandOver: handOverJudgeFrom(opts.state), HandOver: handOverFrom(opts.state),
|
||||
JudgeAccess: accessJudgeFrom(opts.state), AccessAct: accessActFrom(opts.state, opts.timeout),
|
||||
JudgeDirectories: judgeDirectoriesFrom(opts.state), SetuidSearch: queue.SetuidSearch, Searching: search.Running,
|
||||
RootFree: func() link.RootVerdict {
|
||||
served, why := executeServedHere(opts.state, func() (bool, error) {
|
||||
return queue.HeardServed("node-login-shell", "execute")
|
||||
})
|
||||
return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why)
|
||||
},
|
||||
Say: queue.SayEngine}
|
||||
asker := asks.EngineAsker(node)
|
||||
h.Asker = &meshasker.Client{Module: asker, State: state, Publish: queue.PublishAsk,
|
||||
Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }}
|
||||
return h
|
||||
}
|
||||
|
||||
// reconcileEvery is how often the engine catches up its open asks from the router's records (a warrant heard while no
|
||||
// link was open, an ask past its expiry): besides each link's opening.
|
||||
const reconcileEvery = 5 * time.Minute
|
||||
|
||||
func reconcileActs(ctx context.Context, h *link.HandActs) {
|
||||
t := time.NewTicker(reconcileEvery)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
h.Reconcile()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
@@ -82,6 +84,77 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
|
||||
if err != nil || string(got) != string(body) {
|
||||
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
|
||||
}
|
||||
|
||||
// Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named,
|
||||
// and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked.
|
||||
gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)}
|
||||
never := []byte(strings.Repeat("never had here\n", 6000))
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
raw []byte
|
||||
onDisk []byte // nil removes the file
|
||||
refused string // "" applies
|
||||
}{
|
||||
{"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""},
|
||||
{"the file here is gone, its object deleted", raw, nil, ""},
|
||||
{"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"},
|
||||
} {
|
||||
if row.onDisk == nil {
|
||||
_ = os.Remove(target)
|
||||
} else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil)
|
||||
if row.refused == "" {
|
||||
got, _ := os.ReadFile(target)
|
||||
if report.Refused != "" || string(got) != string(body) {
|
||||
t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep",
|
||||
row.name, report.Refused, len(got))
|
||||
}
|
||||
} else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") {
|
||||
t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 924, the live case: what runs came from the kept declaration, whose export names a digest swept long ago,
|
||||
// while the node wrote a newer export since. Composed as it runs, the file names what the node wrote, and applies with
|
||||
// no object; a file someone else changed since is left as named.
|
||||
newer := []byte(strings.Repeat("the export the node wrote since\n", 3000))
|
||||
newerName, _ := declaration.ObjectName(declaration.DigestOf(newer))
|
||||
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{newerName: newer}})
|
||||
if report = applyAndKeep(ctx, opts, referenceDeclaration(target, newer), nil, nil, nil); report.Refused != "" {
|
||||
t.Fatalf("924: the newer export: %s", report.Refused)
|
||||
}
|
||||
stale := json.RawMessage(`{"id":"m.big","type":"file","path":"` + target + `","digest":"` + declaration.DigestOf(body) +
|
||||
`","size":` + strconv.Itoa(len(body)) + `}`)
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
edit []byte // what someone else wrote there since; nil for nobody
|
||||
digest string
|
||||
}{
|
||||
{"the file is what the node wrote", nil, declaration.DigestOf(newer)},
|
||||
{"someone changed the file since", []byte("a hand edit"), declaration.DigestOf(body)},
|
||||
} {
|
||||
if row.edit != nil {
|
||||
if err := os.WriteFile(target, row.edit, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
runs := runsAsWritten([]delivery.Composed{{Node: "n", Module: "m", Resources: []json.RawMessage{stale}}}, opts.state)
|
||||
var got struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
_ = json.Unmarshal(runs[0].Resources[0], &got)
|
||||
if got.Digest != row.digest {
|
||||
t.Errorf("924: %s: what runs names %s, want %s", row.name, got.Digest, row.digest)
|
||||
}
|
||||
if row.edit == nil {
|
||||
raw := []byte(`{"declaration":1,"resources":[` + string(runs[0].Resources[0]) + `]}`)
|
||||
if report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, raw, nil, nil, nil); report.Refused != "" {
|
||||
t.Errorf("924: %s: composed as it runs, refused %q; want applied with no object", row.name, report.Refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
|
||||
|
||||
@@ -30,9 +30,10 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s
|
||||
var fillingNow atomic.Pointer[filling.Engine]
|
||||
|
||||
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
|
||||
// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled).
|
||||
// What it told, or why it could not, is said.
|
||||
func knockFilling(module string, say func(string)) {
|
||||
// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled
|
||||
// (agent.Env.Settled, "<module> settled"), the haves were read at the start, or a module was taken as running
|
||||
// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said.
|
||||
func knockFilling(after string, say func(string)) {
|
||||
e := fillingNow.Load()
|
||||
if e == nil {
|
||||
return
|
||||
@@ -40,9 +41,9 @@ func knockFilling(module string, say func(string)) {
|
||||
told, err := e.Knock(filling.Socket)
|
||||
switch {
|
||||
case err != nil:
|
||||
say(fmt.Sprintf("after %s settled: %v", module, err))
|
||||
say(fmt.Sprintf("after %s: %v", after, err))
|
||||
case told != "":
|
||||
say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told))
|
||||
say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+33
-14
@@ -68,14 +68,23 @@ func foundDirectories(known store.State) []store.Applied {
|
||||
// known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's
|
||||
// path is the resource's target beside the reason, never in it: the controller says a reason on the
|
||||
// operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence.
|
||||
//
|
||||
// From the lease write, heard as the engine hears it (meshTrust), the line has no answerer: the way out is the node
|
||||
// seat's hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5).
|
||||
func handOverLine() string {
|
||||
node, _ := foundNode.Load().(string)
|
||||
if node == "" {
|
||||
node = "<node>"
|
||||
}
|
||||
if t := foundTrust.Load(); t != nil && t.HandedAway() {
|
||||
return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node)
|
||||
}
|
||||
return fmt.Sprintf("`nox node hand-over %s <directory>` on the control-node", node)
|
||||
}
|
||||
|
||||
// foundTrust is what the engine trusts, whose lease decides the way out a directory used as found names.
|
||||
var foundTrust atomic.Pointer[meshTrust]
|
||||
|
||||
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
|
||||
// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line
|
||||
// that hands it over. The path is the target, which the controller keeps as evidence.
|
||||
@@ -129,16 +138,30 @@ func foundNow() []store.Applied {
|
||||
// never follows a path it did not state: a hand-over is root giving a directory to an account at the next
|
||||
// apply, and a path that resolves elsewhere would give that elsewhere.
|
||||
func handOverFound(statePath, path, by string) (string, error) {
|
||||
found, _, err := judgeFound(statePath, path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
|
||||
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
|
||||
}
|
||||
|
||||
// judgeFound is the judgement of a hand-over's path, recording nothing: the directory this machine uses as found at
|
||||
// that path, as the engine states it, and what is on disk there.
|
||||
func judgeFound(statePath, path string) (store.Applied, os.FileInfo, error) {
|
||||
if !filepath.IsAbs(path) {
|
||||
return "", fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
|
||||
return store.Applied{}, nil, fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
|
||||
}
|
||||
if filepath.Clean(path) != path {
|
||||
return "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
|
||||
return store.Applied{}, nil, fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
|
||||
"separator); nothing was handed over", path)
|
||||
}
|
||||
known, err := store.Load(statePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return store.Applied{}, nil, err
|
||||
}
|
||||
var found *store.Applied
|
||||
for _, r := range foundDirectories(known) {
|
||||
@@ -149,31 +172,27 @@ func handOverFound(statePath, path, by string) (string, error) {
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
return "", errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
|
||||
return store.Applied{}, nil, errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
|
||||
"the ones that are); nothing was handed over")
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "", fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
|
||||
return store.Applied{}, nil, fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
|
||||
case info.Mode()&os.ModeSymlink != 0:
|
||||
return "", fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
|
||||
return store.Applied{}, nil, fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
|
||||
case !info.IsDir():
|
||||
return "", fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
|
||||
return store.Applied{}, nil, fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
|
||||
return store.Applied{}, nil, fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
|
||||
}
|
||||
if resolved != path {
|
||||
return "", fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
|
||||
return store.Applied{}, nil, fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
|
||||
"(novox/hq issue 339); nothing was handed over", path, resolved)
|
||||
}
|
||||
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
|
||||
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
|
||||
return *found, info, nil
|
||||
}
|
||||
|
||||
// handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked.
|
||||
|
||||
@@ -40,6 +40,21 @@ func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
|
||||
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
|
||||
t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason)
|
||||
}
|
||||
// ADR 0563 §5: from the lease write the remedy is the node seat's hand-over, answered on the operator's channel, and
|
||||
// no longer the controller's terminal line; the lease is the one the engine hears.
|
||||
handed := newMeshTrust(nil)
|
||||
handed.handedAway = true
|
||||
foundTrust.Store(handed)
|
||||
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||
"`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") || strings.Contains(r.Reason, dir) {
|
||||
t.Fatalf("after the lease write the way out is %q", r.Reason)
|
||||
}
|
||||
foundTrust.Store(newMeshTrust(nil))
|
||||
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||
"`nox node hand-over laptop <directory>`") {
|
||||
t.Fatalf("while the controller holds its lease the way out is %q", r.Reason)
|
||||
}
|
||||
foundTrust.Store((*meshTrust)(nil))
|
||||
foundNode.Store("")
|
||||
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||
"`nox node hand-over <node> <directory>`") {
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestAModuleWithNothingLongRunningIsOkWithinALookOfItsApply(t *testing.T) {
|
||||
applying := fact
|
||||
applying.Phase = string(delivery.PhaseApplying)
|
||||
queue.NoteModule(ctx, applying)
|
||||
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil)
|
||||
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil, nil)
|
||||
// The facts look inside the apply, while the module is still applying.
|
||||
queue.SayFacts(ctx, link.Report{Health: &link.Health{Contract: link.LivenessContract, At: time.Now().UTC()}})
|
||||
if h, _ := bus.have("n1.xorg"); h.Health != delivery.HealthUnknown {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
)
|
||||
|
||||
// heldBy is the containers an open maintenance window holds still, by runtime name (ADR 0189), at now.
|
||||
func heldBy(windows *apply.Windows) func(time.Time) map[string]bool {
|
||||
return func(now time.Time) map[string]bool {
|
||||
held := map[string]bool{}
|
||||
for _, w := range windows.Open(now) {
|
||||
for _, name := range w.Holds {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
return held
|
||||
}
|
||||
}
|
||||
|
||||
// inactiveIn is what the agent asks at its start of a module it believes runs (ADR 0568 §1): whether the machine shows a
|
||||
// long-running resource the composition declares running not active, read from the service manager and the container
|
||||
// runtime and from no record. A resource held by an open maintenance window, or waiting for a library, is not counted;
|
||||
// one that cannot be read is not counted either (unknown is not inactive), and neither is one the manager is restarting.
|
||||
func inactiveIn(rt liveness.Runtime, held func(time.Time) map[string]bool) func(context.Context, delivery.Composed) bool {
|
||||
return func(ctx context.Context, c delivery.Composed) bool {
|
||||
d, err := declarationOf(c)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var holding map[string]bool
|
||||
if held != nil {
|
||||
holding = held(time.Now())
|
||||
}
|
||||
return anyInactive(ctx, rt, liveness.LongRunning(d, withWaiting(nil, apply.WaitingOnLibrary(d))), holding)
|
||||
}
|
||||
}
|
||||
|
||||
// anyInactive reads each group of resources once (the containers, each service manager's units) and says whether any is
|
||||
// not active.
|
||||
func anyInactive(ctx context.Context, rt liveness.Runtime, resources []liveness.Resource, held map[string]bool) bool {
|
||||
type group struct {
|
||||
kind, scope, user string
|
||||
targets []string
|
||||
}
|
||||
var groups []*group
|
||||
at := map[string]*group{}
|
||||
for _, r := range resources {
|
||||
if r.Kind == liveness.KindContainer && held[r.Target] {
|
||||
continue
|
||||
}
|
||||
g := &group{kind: r.Kind, scope: r.Scope, user: r.User}
|
||||
if r.Kind == liveness.KindContainer {
|
||||
g.scope, g.user = "", ""
|
||||
}
|
||||
key := g.kind + "/" + g.scope + "/" + g.user
|
||||
if r.Kind != liveness.KindContainer {
|
||||
key = "units/" + g.scope + "/" + g.user
|
||||
}
|
||||
if at[key] == nil {
|
||||
at[key] = g
|
||||
groups = append(groups, g)
|
||||
}
|
||||
at[key].targets = append(at[key].targets, r.Target)
|
||||
}
|
||||
for _, g := range groups {
|
||||
var got map[string]liveness.Observed
|
||||
var err error
|
||||
if g.kind == liveness.KindContainer {
|
||||
got, err = rt.Containers(ctx, g.targets)
|
||||
} else {
|
||||
got, err = rt.Units(ctx, g.scope, g.user, g.targets)
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, t := range g.targets {
|
||||
if o := got[t]; !o.Found || !(o.Running || o.Restarting) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
)
|
||||
|
||||
// 1957/AC2 and AC4 (unit; ADR 0568 §2, §4): the judge's look acts on a resource that goes down, once per edge. A repeated
|
||||
// statement acts nothing, the edge clears when the resource is healthy again, and a resource held, restarting, starting or
|
||||
// missing (the other path's) is no edge.
|
||||
func testOnlyAResourceNewlyDownIsAnEdge(t *testing.T) {
|
||||
state := func(id, s, reason string, missing bool) liveness.State {
|
||||
return liveness.State{Resource: liveness.Resource{Module: "dnsmasq", ID: id}, State: s, Reason: reason, Missing: missing}
|
||||
}
|
||||
down := state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, false)
|
||||
steps := []struct {
|
||||
name string
|
||||
st []liveness.State
|
||||
want []string
|
||||
}{
|
||||
{"down: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||
{"said again while down: none", []liveness.State{down}, nil},
|
||||
{"healthy again: clears", []liveness.State{state("dnsmasq.unit", liveness.Healthy, "", false)}, nil},
|
||||
{"down again: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||
{"held by a window: none", []liveness.State{state("dnsmasq.unit", liveness.Held, "", false)}, nil},
|
||||
{"restarting: none", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonRestarting, false)}, nil},
|
||||
{"starting: none", []liveness.State{state("dnsmasq.unit", liveness.Starting, "", false)}, nil},
|
||||
{"missing: the other path's", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, true)}, nil},
|
||||
}
|
||||
var last map[string]bool
|
||||
for _, s := range steps {
|
||||
var got []string
|
||||
got, last = newlyInactive(last, liveness.Statement{Resources: s.st})
|
||||
if len(got) != len(s.want) || (len(got) == 1 && got[0] != s.want[0]) {
|
||||
t.Fatalf("%s: acted on %v, want %v", s.name, got, s.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fakeRuntime struct {
|
||||
units map[string]liveness.Observed
|
||||
containers map[string]liveness.Observed
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeRuntime) Containers(_ context.Context, _ []string) (map[string]liveness.Observed, error) {
|
||||
return f.containers, f.err
|
||||
}
|
||||
func (f fakeRuntime) Units(_ context.Context, _, _ string, _ []string) (map[string]liveness.Observed, error) {
|
||||
return f.units, f.err
|
||||
}
|
||||
|
||||
// 1957/AC1 (unit; ADR 0568 §1): the machine's facts say a declared-running resource is not active when its unit is stopped
|
||||
// or unknown to the manager; everything up, a restart in progress, a read that failed and a container held by a
|
||||
// maintenance window are not.
|
||||
func testTheMachineShowsAResourceNotActive(t *testing.T) {
|
||||
svc := []byte(`{"id":"dnsmasq.unit","type":"service","unit":"dnsmasq.service","state":"running","module":"dnsmasq"}`)
|
||||
ctr := []byte(`{"id":"dnsmasq.box","type":"container","name":"box","image":"x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","module":"dnsmasq"}`)
|
||||
up := liveness.Observed{Found: true, Running: true}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
rt fakeRuntime
|
||||
held map[string]bool
|
||||
want bool
|
||||
}{
|
||||
{"all up", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||
{"unit inactive", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||
{"unit unknown to the manager", fakeRuntime{containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||
{"unit restarting", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true, Restarting: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||
{"container not running", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, nil, true},
|
||||
{"container held by a window", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, map[string]bool{"box": true}, false},
|
||||
{"unread", fakeRuntime{err: errors.New("the service manager could not be read")}, nil, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
d, err := declarationOf(delivery.Composed{Node: "n1", Module: "dnsmasq", Resources: []json.RawMessage{svc, ctr}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := anyInactive(context.Background(), c.rt, liveness.LongRunning(d, nil), c.held); got != c.want {
|
||||
t.Fatalf("inactive = %v, want %v", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
@@ -53,4 +55,42 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
||||
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
||||
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
||||
}
|
||||
// 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is
|
||||
// open, nor after it ended; and a library waiting for its disk is not asked.
|
||||
t.Run("the home engine asks the take once", func(t *testing.T) {
|
||||
var asked []asks.Ask
|
||||
state := &link.MemState{}
|
||||
h := &link.HandActs{Node: "n1", State: state,
|
||||
JudgeAccess: func(act, access string) (link.Judged, error) {
|
||||
return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil
|
||||
},
|
||||
AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }}
|
||||
h.Asker = &meshasker.Client{Module: asks.EngineAsker("n1"), State: state,
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
if a, ok := body.(asks.Ask); ok {
|
||||
asked = append(asked, a)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
waiting := keepWaiting(vs)
|
||||
askTakes(h, waiting, nil)
|
||||
askTakes(h, waiting, nil)
|
||||
if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 {
|
||||
t.Fatalf("waiting %v asked %d times", waiting, len(asked))
|
||||
}
|
||||
h.Heard(asks.Warrant{Ask: asked[0].ID, Asker: asks.EngineAsker("n1"), Outcome: asks.OutcomeExpired, At: time.Now()})
|
||||
askTakes(h, waiting, nil)
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("the take was asked again after its ask ended: %d", len(asked))
|
||||
}
|
||||
// Kept in a key of its own (rev-l3 round 1): the acts gone from the record never make it ask again.
|
||||
keys, _ := state.Keys("act.")
|
||||
for _, k := range keys {
|
||||
_ = state.Delete(k)
|
||||
}
|
||||
askTakes(h, waiting, nil)
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("the take was asked again with the acts gone: %d", len(asked))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+76
-16
@@ -516,7 +516,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
|
||||
|
||||
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
|
||||
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
|
||||
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
|
||||
if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
|
||||
return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
|
||||
}
|
||||
|
||||
@@ -1046,15 +1046,7 @@ func runLink(ctx context.Context, opts options) error {
|
||||
say(err.Error())
|
||||
}
|
||||
windows := apply.WindowsIn(filepath.Dir(opts.state))
|
||||
j.HeldNow = func(now time.Time) map[string]bool {
|
||||
held := map[string]bool{}
|
||||
for _, w := range windows.Open(now) {
|
||||
for _, name := range w.Holds {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
return held
|
||||
}
|
||||
j.HeldNow = heldBy(windows)
|
||||
judging = j
|
||||
// And which units its service managers say failed, and whose each is (novox/hq issue 315).
|
||||
unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
|
||||
@@ -1113,10 +1105,17 @@ func runLink(ctx context.Context, opts options) error {
|
||||
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
|
||||
// controller asks.
|
||||
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
||||
// What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's
|
||||
// once read from its signers (novox/hq ADR 0550 §2).
|
||||
trust := newMeshTrust(mine.Membership.Signer)
|
||||
// And the way out a directory used as found names follows the lease it hears (novox/hq ADR 0563 §5).
|
||||
foundTrust.Store(trust)
|
||||
queue := &link.Session{
|
||||
Membership: membership,
|
||||
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
||||
Keys: recordKeys(opts.state),
|
||||
Keys: recordKeys(opts.state, trust.Trusts),
|
||||
// And the controller's key, read per signed ask: none once its lease is handed away (novox/hq ADR 0563 §7).
|
||||
ControllerKey: trust.Controller,
|
||||
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
||||
HaveWritten: proof.haveWritten,
|
||||
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
||||
@@ -1137,6 +1136,10 @@ func runLink(ctx context.Context, opts options) error {
|
||||
Say: say,
|
||||
Timeout: opts.timeout,
|
||||
}
|
||||
// And the node seat node-engine this engine holds (novox/hq ADR 0563): its verbs on every link, its own asks to the
|
||||
// operator under node-engine-<node>, and its acts kept beside its state.
|
||||
queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch)
|
||||
go reconcileActs(ctx, queue.Acts)
|
||||
current := &agentNow{}
|
||||
setRunsHere(runsFromAgent(current))
|
||||
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
|
||||
@@ -1173,11 +1176,18 @@ func runLink(ctx context.Context, opts options) error {
|
||||
if ag := current.get(); ag != nil {
|
||||
ag.Retry(modules...)
|
||||
}
|
||||
}, func(modules ...string) {
|
||||
if ag := current.get(); ag != nil {
|
||||
ag.Reconcile(modules...)
|
||||
}
|
||||
})
|
||||
}
|
||||
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
|
||||
// busy containers are asked for again soon (ADR 0360 §11).
|
||||
go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow)
|
||||
// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
|
||||
trust.onChange(func() { factsOfNow(aside, current.get()) })
|
||||
go followTrust(aside, queue, trust, say)
|
||||
go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow)
|
||||
go busyPasses(aside, func() {
|
||||
if ag := current.get(); ag != nil {
|
||||
ag.Retry(busyModules()...)
|
||||
@@ -1336,8 +1346,8 @@ const (
|
||||
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
|
||||
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
|
||||
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
|
||||
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
|
||||
var missingBefore map[string]bool
|
||||
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply, reconcile func(modules ...string)) {
|
||||
var missingBefore, inactiveBefore map[string]bool
|
||||
ticker := time.NewTicker(liveness.LookEvery)
|
||||
defer ticker.Stop()
|
||||
var lastSaid time.Time
|
||||
@@ -1374,6 +1384,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
if a := accountJudge; a != nil {
|
||||
as, accountsChanged := a.Look(ctx)
|
||||
accountSt = &as
|
||||
lastAccounts.Store(&as)
|
||||
owed = owed || accountsChanged
|
||||
if accountsChanged {
|
||||
for _, v := range as.Accounts {
|
||||
@@ -1388,6 +1399,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
accessSt, accessesChanged := accesses.look(time.Now())
|
||||
keeping.poke(ctx)
|
||||
owed = owed || accessesChanged
|
||||
// And a library whose root holds data outside its group is asked of the operator by this engine, once (novox/hq
|
||||
// ADR 0563 §4).
|
||||
askTakes(queue.Acts, keepWaiting(accessSt), say)
|
||||
if accessesChanged {
|
||||
for _, v := range accessSt {
|
||||
if v.State != link.StateHealthy {
|
||||
@@ -1439,6 +1453,14 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
|
||||
reapply(m...)
|
||||
}
|
||||
// A resource its module declares running that is down but still known to the manager is started by the same
|
||||
// apply, once each time it goes down, never on a clock (ADR 0568 §2): the judge reads, the agent acts.
|
||||
var inactive []string
|
||||
inactive, inactiveBefore = newlyInactive(inactiveBefore, st)
|
||||
if len(inactive) > 0 && reconcile != nil {
|
||||
say(fmt.Sprintf("applying %s again: a resource it declares running is not active", strings.Join(inactive, ", ")))
|
||||
reconcile(inactive...)
|
||||
}
|
||||
}
|
||||
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||
unitSt), accountSt), found), accessSt)) {
|
||||
@@ -1642,6 +1664,15 @@ func announceOr(say link.Announce) func(string) {
|
||||
// run by hand beside this service — the lock beside the state does the same (store.Lock).
|
||||
var applying sync.Mutex
|
||||
|
||||
// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep,
|
||||
// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files.
|
||||
func objectKeep(opts options) *apply.ObjectKeep {
|
||||
if opts.state == "" {
|
||||
return nil
|
||||
}
|
||||
return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects"))
|
||||
}
|
||||
|
||||
// appliedModule is how the apply of one module's declaration came out.
|
||||
type appliedModule struct {
|
||||
// Refused is why nothing was applied, when nothing was.
|
||||
@@ -1681,7 +1712,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De
|
||||
|
||||
// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
|
||||
// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
|
||||
if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
|
||||
if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
|
||||
return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
|
||||
}
|
||||
|
||||
@@ -2134,12 +2165,41 @@ func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[st
|
||||
return missingModules(fresh), now
|
||||
}
|
||||
|
||||
// newlyInactive are the modules of the resources a statement says down (not running after their grace, but still known to
|
||||
// their manager, and not restarting) that the last statement acted on did not, each once, sorted, and the resources down
|
||||
// now: a resource still down is not acted on again, and one that came back and went down again is (ADR 0568 §2).
|
||||
func newlyInactive(last map[string]bool, st liveness.Statement) ([]string, map[string]bool) {
|
||||
now := map[string]bool{}
|
||||
var fresh liveness.Statement
|
||||
for _, r := range st.Resources {
|
||||
if r.State != liveness.Unhealthy || r.Reason != liveness.ReasonDown || r.Missing {
|
||||
continue
|
||||
}
|
||||
now[r.ID] = true
|
||||
if !last[r.ID] {
|
||||
fresh.Resources = append(fresh.Resources, r)
|
||||
}
|
||||
}
|
||||
return missingModules(withMissing(fresh)), now
|
||||
}
|
||||
|
||||
// withMissing is the statement with each of its resources marked missing, so missingModules lists their modules.
|
||||
func withMissing(st liveness.Statement) liveness.Statement {
|
||||
for i := range st.Resources {
|
||||
st.Resources[i].Missing = true
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
|
||||
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
|
||||
// empty, never invented; the certificate is there only after the first declaration that carried it.
|
||||
func recordKeys(state string) func() link.RecordKeys {
|
||||
func recordKeys(state string, trusts func() []string) func() link.RecordKeys {
|
||||
return func() link.RecordKeys {
|
||||
var k link.RecordKeys
|
||||
if trusts != nil {
|
||||
k.Trusts = trusts()
|
||||
}
|
||||
if id, err := identity.Load(identity.Path(state)); err == nil {
|
||||
k.TunnelKey = id.Overlay.Public
|
||||
}
|
||||
|
||||
+216
-52
@@ -5,12 +5,14 @@ import (
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
@@ -20,6 +22,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
@@ -38,6 +41,20 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return w.updates(ctx, in), nil
|
||||
}
|
||||
|
||||
// WatchAwait is Watch for a state that its holder makes when it starts (the placement seat's): a bucket not on the bus
|
||||
// yet is waited for on the bus's word that it was created (link.Session.KVWatchAwait), and waiting is told why.
|
||||
func (w bucketWatch) WatchAwait(ctx context.Context, pattern string, waiting func(error)) (<-chan agent.Update, error) {
|
||||
in, err := w.session.KVWatchAwait(ctx, w.bucket, pattern, waiting)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return w.updates(ctx, in), nil
|
||||
}
|
||||
|
||||
func (w bucketWatch) updates(ctx context.Context, in <-chan link.KVUpdate) <-chan agent.Update {
|
||||
out := make(chan agent.Update, 64)
|
||||
go func() {
|
||||
defer close(out)
|
||||
@@ -49,7 +66,7 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
return out
|
||||
}
|
||||
|
||||
// sessionHaves writes the haves the agent says, through the session.
|
||||
@@ -62,14 +79,32 @@ func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool
|
||||
if !ok || got.Composed == "" {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.Sealed}, true
|
||||
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.SealedAt}, true
|
||||
}
|
||||
got, ok := h.session.ReadHave(ctx, module)
|
||||
if !ok {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.Sealed}, true
|
||||
Sealed: got.SealedAt}, true
|
||||
}
|
||||
|
||||
// ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
|
||||
func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, bool, error) {
|
||||
if module == agent.NodeModule {
|
||||
// The node's own key, its read the bus did not answer told as such (issue 1126, 2026-10-28).
|
||||
got, ok, err := h.session.ReadNodeHaveErr(ctx)
|
||||
if !ok || err != nil || got.Composed == "" {
|
||||
return agent.Have{}, false, err
|
||||
}
|
||||
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.SealedAt}, true, nil
|
||||
}
|
||||
got, ok, err := h.session.ReadHaveErr(ctx, module)
|
||||
if !ok || err != nil {
|
||||
return agent.Have{}, false, err
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.SealedAt}, true, nil
|
||||
}
|
||||
|
||||
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
@@ -310,13 +345,14 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
|
||||
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
|
||||
const operatorKeysFile = "operator-keys"
|
||||
|
||||
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the
|
||||
// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at
|
||||
// the next composition. A file that cannot be read trusts the controller's alone, and says so.
|
||||
func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey {
|
||||
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the
|
||||
// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from
|
||||
// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition.
|
||||
// A file that cannot be read trusts the controller's alone, and says so.
|
||||
func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey {
|
||||
return func() []ed25519.PublicKey {
|
||||
keys := []ed25519.PublicKey{}
|
||||
if len(controller) == ed25519.PublicKeySize {
|
||||
if controller := controllerNow(); len(controller) == ed25519.PublicKeySize {
|
||||
keys = append(keys, controller)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
|
||||
@@ -340,13 +376,16 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu
|
||||
}
|
||||
|
||||
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
||||
// none, ADR 0539 §9) and its filled paths.
|
||||
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
|
||||
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
||||
var once sync.Once
|
||||
var by map[string][]json.RawMessage
|
||||
var filled map[string][]string
|
||||
return func() (map[string][]json.RawMessage, map[string][]string) {
|
||||
once.Do(func() {
|
||||
if keptRetired.Load() {
|
||||
return
|
||||
}
|
||||
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -396,7 +435,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) {
|
||||
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
|
||||
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
|
||||
// account may not read) is said, and tried again with a growing wait, as the link is.
|
||||
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler,
|
||||
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler,
|
||||
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
|
||||
wait := 2 * time.Second
|
||||
for ctx.Err() == nil {
|
||||
@@ -413,6 +452,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
return
|
||||
}
|
||||
var ag *agent.Agent
|
||||
// The kept release-5 declaration is no longer read once every module the seat's placement puts here has a have of its
|
||||
// own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write).
|
||||
if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Swap(true) {
|
||||
say("the kept declaration is no longer read: " + why)
|
||||
}
|
||||
kept := keptOnce(opts, signer)
|
||||
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
|
||||
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
|
||||
@@ -425,7 +469,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
}
|
||||
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
|
||||
// node, and a node whose compositions carry no list writes nothing.
|
||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
|
||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() {
|
||||
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
||||
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
||||
time.AfterFunc(reloadSettles, func() {
|
||||
@@ -450,6 +494,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
|
||||
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
|
||||
Recorded: recordedIn(opts.state, say),
|
||||
Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
|
||||
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
||||
got, err := session.KVHistory(ctx, composedBucket, key)
|
||||
if err != nil {
|
||||
@@ -461,12 +506,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Trusted: trustedKeys(opts, signer),
|
||||
Trusted: trustedKeys(opts, trust.Controller),
|
||||
Backup: backupNode(session),
|
||||
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
|
||||
// again (ace, 2026-10-25 21:17).
|
||||
Settled: func(module string) { knockFilling(module, say) },
|
||||
Say: say,
|
||||
Settled: func(module string) { knockFilling(module+" settled", say) },
|
||||
// And once the haves are read at the start, and whenever a module is taken as running without an apply, so
|
||||
// a refusal from the engine's start does not wait for some module to settle (issue 924).
|
||||
RunsChanged: func(why string) { knockFilling(why, say) },
|
||||
Say: say,
|
||||
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
||||
Backoff: link.Backoff,
|
||||
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
|
||||
@@ -509,49 +557,84 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
run, stopRun := context.WithCancel(linked)
|
||||
// What the node runs and how it opens a sealed value, for the composer to keep the boxes it runs; and the agent
|
||||
// told when the node cannot compose, so each wanted module's have says it.
|
||||
// The agent of this run alone: its composer tells it, never the agent of a run begun since (issue 1126: a pass in
|
||||
// flight at the switch marked the next agent's modules failed).
|
||||
own := &agentNow{}
|
||||
hooks := composerHooks{
|
||||
running: func() []delivery.Composed {
|
||||
if now := current.get(); now != nil {
|
||||
return now.Everything()
|
||||
if now := own.get(); now != nil {
|
||||
return runsAsWritten(now.Everything(), opts.state)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
unseal: sealOpener(opts.state),
|
||||
// And what each module's have says it runs sealed: after a restart the agent has applied nothing yet.
|
||||
held: func(module string) []string {
|
||||
// A have the bus did not answer for is said as such, never as one holding no seals (issue 1126, 2026-10-28).
|
||||
held: func(module string) (delivery.Seals, []string, error) {
|
||||
if module == agent.NodeModule {
|
||||
got, _ := session.ReadNodeHave(linked)
|
||||
return got.Sealed
|
||||
got, _, err := session.ReadNodeHaveErr(linked)
|
||||
return got.SealedAt, got.Sealed, err
|
||||
}
|
||||
got, ok := session.ReadHave(linked, module)
|
||||
got, ok, err := session.ReadHaveErr(linked, module)
|
||||
if !ok {
|
||||
return nil
|
||||
return nil, nil, err
|
||||
}
|
||||
return got.Sealed
|
||||
return got.SealedAt, got.Sealed, nil
|
||||
},
|
||||
refused: func(why string) {
|
||||
if now := current.get(); now != nil {
|
||||
now.NotComposed(linked, why)
|
||||
refused: func(why string, held map[string]string) {
|
||||
if now := own.get(); now != nil && run.Err() == nil {
|
||||
now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126)
|
||||
}
|
||||
},
|
||||
// And the vault's authority beside the controller's, and this node's certificate asked of the vault when a
|
||||
// module here asks for one (ADR 0550 §2).
|
||||
vaultAuthority: trust.VaultAuthority,
|
||||
certify: func(ctx context.Context) error {
|
||||
return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) })
|
||||
},
|
||||
}
|
||||
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
||||
// its own key, `<node>.node`, under candidates.
|
||||
shadowHooks := hooks
|
||||
shadowHooks.refused = nil
|
||||
shadow := selfcompose.NewShadow(composerEnv(session, node, say, shadowHooks),
|
||||
shadowHooks.refused, shadowHooks.certify = nil, nil
|
||||
shadowEnv := composerEnv(session, node, say, shadowHooks)
|
||||
// And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to
|
||||
// the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync).
|
||||
composes := &composesNow{}
|
||||
resync := ownResync(session, node, say, shadowEnv, composes)
|
||||
shadow := selfcompose.NewShadow(shadowEnv,
|
||||
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
session.NoteCandidates(ctx, answers)
|
||||
session.NoteCandidates(ctx, delivery.WithValuesDiffer(answers))
|
||||
go resync.take(ctx, answers)
|
||||
})
|
||||
go shadow.Keep(linked, link.Backoff)
|
||||
if composer := selfComposer(linked, session, node, say, hooks); composer != nil {
|
||||
go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again
|
||||
composer, why := selfComposer(linked, session, node, say, hooks)
|
||||
if linked.Err() != nil {
|
||||
// The link ended before the node knew whose compositions to apply: it chose neither, and waits for the next.
|
||||
stopRun()
|
||||
cancel()
|
||||
continue
|
||||
}
|
||||
if composer != nil {
|
||||
composes.set(delivery.ComposesOwn)
|
||||
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
||||
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
||||
session.NoteComposes(linked, delivery.ComposesOwn, why)
|
||||
// And once the catalogue no longer places it, the agent begins again on the controller's compositions: the
|
||||
// placement key deleted ends this run, the way back (issue 1126, the seed's revert).
|
||||
go endWhenUnplaced(run, session, node, stopRun, say)
|
||||
} else {
|
||||
composes.set(delivery.ComposesController)
|
||||
go resync.again(run)
|
||||
session.NoteComposes(linked, delivery.ComposesController, why)
|
||||
// And once the catalogue places it, the agent begins again composing its own: the placement key appearing ends
|
||||
// this run, so a node does not wait for a restart to follow the seed of its placement (ADR 0549 §4).
|
||||
go endWhenPlaced(run, session, node, stopRun, say)
|
||||
go endWhenPlaced(run, session, node, stopRun, say, func(reason string) {
|
||||
session.NoteComposes(linked, delivery.ComposesController, why+"; "+reason)
|
||||
})
|
||||
}
|
||||
ag = agent.New(env)
|
||||
own.set(ag)
|
||||
current.set(ag)
|
||||
err = ag.Run(run)
|
||||
stopRun()
|
||||
@@ -663,8 +746,19 @@ func busyModules() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given.
|
||||
// keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of
|
||||
// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9).
|
||||
var keptRetired atomic.Bool
|
||||
|
||||
// errKeptRetired is keptDeclaration's answer once the read is retired.
|
||||
var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own")
|
||||
|
||||
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given;
|
||||
// once the read is retired, nothing is read.
|
||||
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||
if keptRetired.Load() {
|
||||
return nil, errKeptRetired
|
||||
}
|
||||
var raw []byte
|
||||
if len(signer) > 0 {
|
||||
var err error
|
||||
@@ -839,27 +933,41 @@ func applyContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
type composerHooks struct {
|
||||
running func() []delivery.Composed
|
||||
unseal func(string) ([]byte, error)
|
||||
refused func(string)
|
||||
held func(module string) []string
|
||||
refused func(why string, held map[string]string)
|
||||
held func(module string) (delivery.Seals, []string, error)
|
||||
// vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify.
|
||||
vaultAuthority func() string
|
||||
certify func(ctx context.Context) error
|
||||
}
|
||||
|
||||
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) *selfcompose.Composer {
|
||||
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
say("this node applies the controller's compositions: " + err.Error())
|
||||
return nil
|
||||
return nil, err.Error()
|
||||
}
|
||||
got, err := session.KVHistory(ctx, delivery.PlacementBucket, key)
|
||||
if err != nil || len(got) == 0 || got[len(got)-1].Deleted {
|
||||
why := "the catalogue does not place it yet"
|
||||
if err != nil {
|
||||
why = "its placement cannot be read: " + err.Error()
|
||||
env := composerEnv(session, node, say, hooks)
|
||||
unreadable := ""
|
||||
placed, err := selfcompose.Placed(ctx, env.Get, node, link.Backoff, func(m string) {
|
||||
if strings.Contains(m, "cannot be read") {
|
||||
unreadable = m
|
||||
}
|
||||
say(m)
|
||||
}, selfcompose.ComposedItsOwn(ctx, env.Get, node))
|
||||
if err != nil {
|
||||
return nil, "its placement could not be read before the link ended: " + err.Error()
|
||||
}
|
||||
if !placed {
|
||||
why := "the seat's placement state holds no key for it"
|
||||
if unreadable != "" {
|
||||
why = "its placement cannot be read, and it never composed its own: " + unreadable
|
||||
}
|
||||
say("this node applies the controller's compositions: " + why)
|
||||
return nil
|
||||
return nil, why
|
||||
}
|
||||
say("this node composes its own modules: the catalogue places it (" + key + ")")
|
||||
return selfcompose.New(composerEnv(session, node, say, hooks))
|
||||
why := "the seat's placement state holds its key (" + key + ")"
|
||||
say("this node composes its own modules: " + why)
|
||||
return selfcompose.New(env), why
|
||||
}
|
||||
|
||||
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
|
||||
@@ -867,6 +975,13 @@ func selfComposer(ctx context.Context, session *link.Session, node string, say f
|
||||
func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env {
|
||||
return selfcompose.Env{Node: node,
|
||||
Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
if bucket == delivery.PlacementBucket || bucket == delivery.PlacementCandidatesBucket {
|
||||
// The seat's states are made when their holder starts: a node-engine that starts first waits for the
|
||||
// bus's word that the bucket was created, and the shadow and the composer say they wait.
|
||||
return bucketWatch{session, bucket}.WatchAwait(ctx, pattern, func(err error) {
|
||||
say("the seat's state " + bucket + " is not on the bus yet (" + err.Error() + "): it is watched when the bus says it was created")
|
||||
})
|
||||
}
|
||||
return bucketWatch{session, bucket}.Watch(ctx, pattern)
|
||||
},
|
||||
Get: func(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||
@@ -883,28 +998,77 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
|
||||
return session.AskVault(ctx, "own", args)
|
||||
},
|
||||
// And the vault's word that a secret changed, on which it composes again (ADR 0547 §2).
|
||||
Hear: session.Hear,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
|
||||
Hear: session.Hear,
|
||||
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
|
||||
Fetch: apply.Fetch,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held,
|
||||
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
|
||||
}
|
||||
|
||||
// endWhenPlaced calls stop once this node's placement key appears in the indexer's placement (a value after the watch
|
||||
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
|
||||
// the node then switches at its next start.
|
||||
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
|
||||
// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch
|
||||
// caught up, or gone already), so the agent begins again and applies the controller's compositions, as a node the
|
||||
// catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be
|
||||
// made is said; the node then returns at its next start.
|
||||
func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, key)
|
||||
if err != nil {
|
||||
say("whether the catalogue places this node is not followed (" + err.Error() + "): it composes its own from its next start")
|
||||
say("whether this node's key in the seat's placement state still exists is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start")
|
||||
return
|
||||
}
|
||||
placed := false
|
||||
for u := range updates {
|
||||
switch {
|
||||
case u.Caught && placed:
|
||||
continue
|
||||
case u.Caught || u.Deleted || len(u.Value) == 0:
|
||||
say("this node's key in the seat's placement state is gone (" + key + "): the node agent begins again, applying the controller's compositions")
|
||||
stop()
|
||||
return
|
||||
default:
|
||||
placed = true
|
||||
}
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state still exists is followed again at the node agent's next start")
|
||||
}
|
||||
}
|
||||
|
||||
// endWhenPlaced calls stop once this node's placement key appears in the seat's placement (a value after the watch
|
||||
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
|
||||
// the node then switches at its next start.
|
||||
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce, note func(string)) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// The seat's placement state is made when its holder starts; the delivery brings the grants part and the holder first,
|
||||
// so it is normally there. A node-engine that starts before it waits for the bus's word that the bucket was created
|
||||
// (no timer), says so once in the journal and in the node's own have, where the watchman reads it, and stays on the
|
||||
// controller's compositions until its key is written. A watch the account may not make is said the same way, and the
|
||||
// node stays until its next start.
|
||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, key, func(err error) {
|
||||
why := "this node's key in the seat's placement state is waited for, the state is not on the bus yet (" + err.Error() + "): it stays on the controller's compositions and is watched when the bus says the state was created"
|
||||
say(why)
|
||||
note(why)
|
||||
})
|
||||
if err != nil {
|
||||
why := "this node's key in the seat's placement state cannot be followed (" + err.Error() + "): it stays on the controller's compositions until its next start"
|
||||
say(why)
|
||||
note(why)
|
||||
return
|
||||
}
|
||||
for u := range updates {
|
||||
if !u.Caught && !u.Deleted && len(u.Value) > 0 {
|
||||
say("the catalogue places this node now (" + key + "): the node agent begins again, composing its own modules")
|
||||
say("this node's key in the seat's placement state is written now (" + key + "): the node agent begins again, composing its own modules")
|
||||
stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state is written is followed again at the node agent's next start")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
|
||||
@@ -47,14 +48,16 @@ type itBed struct {
|
||||
controll ed25519.PrivateKey
|
||||
operator ed25519.PrivateKey
|
||||
said []string
|
||||
// refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met.
|
||||
refusals []string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what
|
||||
// buscompose.PermissionsFor gives a node (KindNode); it answers a connection of each.
|
||||
func aServer(t *testing.T) (admin, node *nats.Conn) {
|
||||
func aServer(t *testing.T, principal buscompose.Principal, refused func(error)) (admin, node *nats.Conn) {
|
||||
t.Helper()
|
||||
perms, err := buscompose.PermissionsFor(buscompose.Principal{Kind: buscompose.KindNode, Node: itNode})
|
||||
perms, err := buscompose.PermissionsFor(principal)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -78,7 +81,8 @@ func aServer(t *testing.T) (admin, node *nats.Conn) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(admin.Close)
|
||||
if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser)); err != nil {
|
||||
if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused(err) })); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(node.Close)
|
||||
@@ -86,6 +90,12 @@ func aServer(t *testing.T) (admin, node *nats.Conn) {
|
||||
}
|
||||
|
||||
func newItBed(t *testing.T) *itBed {
|
||||
t.Helper()
|
||||
return newItBedAs(t, buscompose.Principal{Kind: buscompose.KindNode, Node: itNode})
|
||||
}
|
||||
|
||||
// newItBedAs is newItBed with the node's connection under the grant of principal, a node's (KindNode).
|
||||
func newItBedAs(t *testing.T, principal buscompose.Principal) *itBed {
|
||||
t.Helper()
|
||||
// Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node.
|
||||
bin := t.TempDir()
|
||||
@@ -98,7 +108,11 @@ func newItBed(t *testing.T) *itBed {
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
|
||||
b := &itBed{t: t, dir: t.TempDir()}
|
||||
b.conn, b.nodeConn = aServer(t)
|
||||
b.conn, b.nodeConn = aServer(t, principal, func(err error) {
|
||||
b.mu.Lock()
|
||||
b.refusals = append(b.refusals, err.Error())
|
||||
b.mu.Unlock()
|
||||
})
|
||||
var err error
|
||||
if b.js, err = b.conn.JetStream(); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -145,7 +159,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {},
|
||||
controller := b.controll.Public().(ed25519.PublicKey)
|
||||
nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {},
|
||||
&agentNow{}, func(context.Context, *agent.Agent) {})
|
||||
}()
|
||||
return func() { cancel(); <-done }
|
||||
@@ -160,6 +175,8 @@ func (b *itBed) say(line string) {
|
||||
|
||||
func fpIt(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
|
||||
func fpDigest(c string) string { return "sha256:" + strings.Repeat(c, 64) }
|
||||
|
||||
// compose writes a module's signed composition of one file into the node's bucket and returns its digest.
|
||||
func (b *itBed) compose(module, fp, content string, data bool, signer ed25519.PrivateKey) string {
|
||||
b.t.Helper()
|
||||
@@ -224,6 +241,7 @@ func within(t *testing.T, d time.Duration, what string, ok func() bool) {
|
||||
// applied within one second and its have names the fingerprint and the digest; a composition whose digest no want names
|
||||
// is not applied.
|
||||
func TestNatsAWantAndItsCompositionAreAppliedWithinASecond(t *testing.T) {
|
||||
t.Run("1957: an engine start starts a declared-running unit", testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns)
|
||||
b := newItBed(t)
|
||||
stop := b.start(false)
|
||||
defer stop()
|
||||
@@ -344,8 +362,205 @@ func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) {
|
||||
// with no key in the controller's compositions; a changed config value is composed again on its event and applied as a
|
||||
// revision of what runs. And a catalogue pull request's placement candidate equal to its placement is answered "same"
|
||||
// in the node's own key, and nothing is applied for it (ADR 0559 §4, subtask 10 of the milestone).
|
||||
//
|
||||
// And (issue 1126, ADR 0560) it reads, through its own session under the grant the grants part composes for it
|
||||
// (buscompose.Users, so it carries ServedReads), the build of a placed module that declares one and the one config value
|
||||
// a provider on another node serves from (a mail domain); under a grant without that served key the bus refuses the
|
||||
// read and the node composes nothing.
|
||||
func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
users, err := buscompose.Users(buscompose.Records{Nodes: []string{itNode, "it2"},
|
||||
Assigned: map[string][]buscompose.Declared{"it2": {{Module: "mailer", Served: []string{"domain"}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var principal buscompose.Principal
|
||||
for _, u := range users {
|
||||
if u.Kind == buscompose.KindNode && u.Node == itNode {
|
||||
principal = u
|
||||
}
|
||||
}
|
||||
if len(principal.ServedReads) != 1 {
|
||||
t.Fatalf("the grants part's principal of %s reads %v of the served keys", itNode, principal.ServedReads)
|
||||
}
|
||||
t.Run("without the served key the bus refuses the read", func(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
stop, _ := h8(t, b, false)
|
||||
defer stop()
|
||||
// 15 s, above link.historyWithin (10 s): the bus's refusal of a publish reaches only the connection's error
|
||||
// handler (nats.go fails no request on it), so the refused read ends at its own bound, then the node says it.
|
||||
within(t, 15*time.Second, "the bus did not refuse the served key's read, or the node did not say it", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return strings.Contains(strings.Join(b.refusals, "\n"), "Permissions Violation for Publish to \"$JS.API.CONSUMER.CREATE.KV_"+
|
||||
selfcompose.ResolvedBucket) && strings.Contains(strings.Join(b.refusals, "\n"), ".it2.mailer.domain") &&
|
||||
strings.Contains(strings.Join(b.said, "\n"), "what mailer serves on it2 (domain)")
|
||||
})
|
||||
if b.file("web") != "" {
|
||||
t.Errorf("composed and applied under a grant without the served key: %q", b.file("web"))
|
||||
}
|
||||
})
|
||||
b := newItBedAs(t, principal)
|
||||
stop, st := h8(t, b, true)
|
||||
defer stop()
|
||||
placement, resolved, candidates := st.placement, st.resolved, st.candidates
|
||||
hkey, place, fp := st.hkey, st.place, st.fp
|
||||
within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
|
||||
h8rest(t, b, resolved, candidates, hkey, place, fp)
|
||||
t.Run("one answer per candidate revision", func(t *testing.T) { h8once(t, b, resolved, candidates, place) })
|
||||
h8back(t, b, placement, resolved, candidates, hkey)
|
||||
}
|
||||
|
||||
// h8once is issue 1126, 2026-10-28 (shanks answered probe/1126-seed-7 at revision 84 every 13-17 s, its bus flooded):
|
||||
// a candidate revision is answered once. The node's own answer written, a module's have whose health words alone move
|
||||
// (each look rewrites them) and a config value written again with the value it held answer nothing again; the
|
||||
// candidate's next revision is answered once.
|
||||
func h8once(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, place []byte) {
|
||||
answering := func() int {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range b.said {
|
||||
if strings.HasPrefix(l, "answering placement candidate feat-x ") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
settled := answering()
|
||||
if settled == 0 {
|
||||
t.Fatal("feat-x was never answered")
|
||||
}
|
||||
hkey, _ := delivery.HaveKey(itNode, "web")
|
||||
for _, detail := range []string{"its tool web_health: no link to the bus is open", "its tool web_health: no answer within 10s"} {
|
||||
h := b.have("web")
|
||||
h.Health, h.Detail = delivery.HealthFailed, detail
|
||||
raw, err := delivery.EncodeHave(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.haves.Put(hkey, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
if got := answering(); got != settled {
|
||||
t.Errorf("feat-x answered %d more time(s) at the same revision, with nothing it is composed from changed", got-settled)
|
||||
}
|
||||
same, _ := delivery.DecodePlacement(place)
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "feat-x", Placement: &same})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("feat-x", itNode)
|
||||
if _, err := candidates.Put(ckey, craw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 10*time.Second, "the candidate's next revision was not answered", func() bool { return answering() > settled })
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
if got := answering(); got != settled+1 {
|
||||
t.Errorf("the candidate's next revision was answered %d time(s), want 1", got-settled)
|
||||
}
|
||||
}
|
||||
|
||||
// h8back is the way back (issue 1126, the seed's revert): a candidate deleting the node's placement is compared with
|
||||
// the controller's compositions at its wants (here none: refused, naming web), and the placement key deleted returns the agent to them without a
|
||||
// restart, said in the journal and the node's have, keeping what runs.
|
||||
func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValue, hkey string) {
|
||||
t.Helper()
|
||||
node := func() delivery.HaveNode {
|
||||
e, err := b.haves.Get(hkey)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}
|
||||
}
|
||||
n, _ := delivery.DecodeHaveNode(e.Value())
|
||||
return n
|
||||
}
|
||||
if n := node(); n.Composes != delivery.ComposesOwn {
|
||||
t.Errorf("a node composing its own says it composes %q", n.Composes)
|
||||
}
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "revert-x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("revert-x", itNode)
|
||||
if _, err := candidates.Put(ckey, craw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller holds no composition at web's want (the node composed its own): returning would leave web with
|
||||
// nothing to apply, and the answer says so rather than "same" (review of the shadow, round 1).
|
||||
within(t, 10*time.Second, "the node did not refuse returning to compositions the controller does not hold", func() bool {
|
||||
a := node().Candidates["revert-x"]
|
||||
return a.Proposal == "revert-x" && a.Returns == "" && a.Verdict == delivery.CandidateRefused &&
|
||||
strings.Contains(a.Refused, "web: the controller holds no composition")
|
||||
})
|
||||
ran := b.have("web").Composed
|
||||
pkey, _ := delivery.PlacementKey(itNode)
|
||||
// A pass in flight at the switch (issue 1126, 2026-10-28: the last pass of the own run, failing on its run's end,
|
||||
// marked the next agent's modules failed): a config value changed starts one, and the key goes at once.
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := placement.Delete(pkey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stopWatch := make(chan struct{})
|
||||
marked := make(chan Have1126, 1)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stopWatch:
|
||||
return
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if h := b.have("web"); h.Phase == delivery.PhaseFailed {
|
||||
select {
|
||||
case marked <- Have1126{h.Phase, h.Detail}:
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer close(stopWatch)
|
||||
within(t, 15*time.Second, "the node did not return to the controller's compositions when its placement went", func() bool {
|
||||
b.mu.Lock()
|
||||
said := strings.Join(b.said, "\n")
|
||||
b.mu.Unlock()
|
||||
return strings.Contains(said, "key in the seat's placement state is gone") && node().Composes == delivery.ComposesController
|
||||
})
|
||||
time.Sleep(3 * time.Second) // what a pass in flight would mark lands within it
|
||||
select {
|
||||
case h := <-marked:
|
||||
t.Errorf("a pass in flight at the switch marked web failed: %+v", h)
|
||||
default:
|
||||
}
|
||||
if h := b.have("web"); h.Composed != ran || h.Phase == delivery.PhaseFailed || b.file("web") != "greeting=salut\n" {
|
||||
t.Errorf("returning to the controller's compositions moved what runs: %+v, %q", h, b.file("web"))
|
||||
}
|
||||
}
|
||||
|
||||
// Have1126 is what H8's way back watches of web's have.
|
||||
type Have1126 struct {
|
||||
Phase delivery.Phase
|
||||
Detail string
|
||||
}
|
||||
|
||||
// h8state is what h8 set up that the rest of H8 writes to.
|
||||
type h8state struct {
|
||||
placement, resolved, candidates nats.KeyValue
|
||||
hkey string
|
||||
place []byte
|
||||
fp string
|
||||
}
|
||||
|
||||
// h8 sets H8's bus up and starts the agent, then places the node. With checkBuild, the placed module's build key is
|
||||
// first one that does not decode, which the node must read through its session (and say), then a done one.
|
||||
func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
|
||||
t.Helper()
|
||||
kvOf := func(bucket string) nats.KeyValue {
|
||||
kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5})
|
||||
if err != nil {
|
||||
@@ -355,7 +570,7 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
}
|
||||
placement, modules, manifests := kvOf(delivery.PlacementBucket), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket)
|
||||
resolved := kvOf(selfcompose.ResolvedBucket)
|
||||
kvOf(delivery.BuilderBuildsBucket)
|
||||
builds := kvOf(delivery.BuilderBuildsBucket)
|
||||
candidates := kvOf(delivery.PlacementCandidatesBucket)
|
||||
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
|
||||
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
|
||||
@@ -363,7 +578,8 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
}
|
||||
path := filepath.Join(b.dir, "web.conf")
|
||||
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
|
||||
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}]}`
|
||||
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}],` +
|
||||
`"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
|
||||
digest := delivery.ManifestDigest([]byte(man))
|
||||
mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man})
|
||||
mkey, _ := delivery.ManifestsKey(digest)
|
||||
@@ -373,12 +589,32 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}})
|
||||
pkey, _ := delivery.PlacementKey(itNode)
|
||||
hkey, _ := delivery.HaveNodeKey(itNode)
|
||||
// A provider on it2 serving from its own config value: a mail domain, which it1's composition settles.
|
||||
mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` +
|
||||
`"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}`
|
||||
mdigest := delivery.ManifestDigest([]byte(mailer))
|
||||
mlraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "mailer", JSON: mailer})
|
||||
mlkey, _ := delivery.ManifestsKey(mdigest)
|
||||
mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)})
|
||||
place2, _ := delivery.EncodePlacement(delivery.Placement{Node: "it2", Modules: []delivery.PlacedModule{{Module: "mailer"}}})
|
||||
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}})
|
||||
pkey2, _ := delivery.PlacementKey("it2")
|
||||
hkey2, _ := delivery.HaveNodeKey("it2")
|
||||
buildKey, _ := delivery.BuilderBuildKey("web", fp)
|
||||
done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest,
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}})
|
||||
firstBuild := done
|
||||
if checkBuild {
|
||||
firstBuild = []byte(`{"v":1,"state":"no such state"}`)
|
||||
}
|
||||
for _, w := range []struct {
|
||||
kv nats.KeyValue
|
||||
key string
|
||||
value []byte
|
||||
}{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec},
|
||||
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour"}`)}} {
|
||||
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)},
|
||||
{manifests, mlkey, mlraw}, {modules, "mailer", mlmod}, {b.haves, hkey2, rec2}, {placement, pkey2, place2},
|
||||
{resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}, {builds, buildKey, firstBuild}} {
|
||||
if _, err := w.kv.Put(w.key, w.value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -390,7 +626,6 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
}
|
||||
|
||||
stop := b.start(false)
|
||||
defer stop()
|
||||
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
|
||||
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
@@ -400,7 +635,30 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
if _, err := placement.Put(pkey, place); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
|
||||
if checkBuild {
|
||||
// The build of the placed module is read through the node's session: one that does not decode is said, and nothing
|
||||
// is applied; a done one, then a config event, and the node composes.
|
||||
within(t, 5*time.Second, "the node did not read web's build key through its session", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return strings.Contains(strings.Join(b.said, "\n"), "the builder's "+buildKey)
|
||||
})
|
||||
if b.file("web") != "" {
|
||||
t.Fatal("composed with a build key that does not decode")
|
||||
}
|
||||
if _, err := builds.Put(buildKey, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return stop, h8state{placement: placement, resolved: resolved, candidates: candidates, hkey: hkey, place: place, fp: fp}
|
||||
}
|
||||
|
||||
// h8rest is H8 once the node composes its own: the have, a config revision, and the candidate answered "same".
|
||||
func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey string, place []byte, fp string) {
|
||||
t.Helper()
|
||||
within(t, 2*time.Second, "the have does not name the build", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed != "" && h.Phase == delivery.PhaseRunning
|
||||
@@ -409,7 +667,7 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
t.Errorf("the controller's compositions hold %v; the node composed its own", keys)
|
||||
}
|
||||
first := b.have("web").Composed
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut"}`)); err != nil {
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 3*time.Second, "a changed config value was not composed and applied again", func() bool { return b.file("web") == "greeting=salut\n" })
|
||||
@@ -424,12 +682,11 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
map[string]any{"name": "service-manager", "present": true}}}})
|
||||
second := b.have("web").Composed
|
||||
same, _ := delivery.DecodePlacement(place)
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: "feat/x",
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h1"}, Placement: &same})
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "feat-x", Placement: &same})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("feat/x", itNode)
|
||||
ckey, _ := delivery.PlacementCandidateKey("feat-x", itNode)
|
||||
crev, err := candidates.Put(ckey, craw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -441,8 +698,8 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
return false
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(e.Value())
|
||||
answer = n.Candidates["feat/x"]
|
||||
return err == nil && answer.Head == "h1" && n.Record != nil
|
||||
answer = n.Candidates["feat-x"]
|
||||
return err == nil && answer.Proposal == "feat-x" && n.Record != nil
|
||||
})
|
||||
if answer.Verdict != delivery.CandidateSame || answer.Revision != crev || len(answer.Same) != 1 || answer.Same[0] != "web" {
|
||||
t.Errorf("the answer to a candidate equal to the placement: %+v", answer)
|
||||
@@ -451,3 +708,71 @@ func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web"))
|
||||
}
|
||||
}
|
||||
|
||||
// 1957/AC1 (integration; ADR 0568 §1): a node agent starting on a bus that holds a `running` have for a module at its want,
|
||||
// with a service manager (a systemctl on the PATH) that shows the module's unit inactive, starts the unit once and writes
|
||||
// the have running; the same start with the unit active applies nothing.
|
||||
func testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns(t *testing.T) {
|
||||
for _, active := range []bool{false, true} {
|
||||
b := newItBed(t)
|
||||
bin := t.TempDir()
|
||||
stateFile, calls := filepath.Join(bin, "state"), filepath.Join(bin, "calls")
|
||||
if active {
|
||||
os.WriteFile(stateFile, []byte("active"), 0o644)
|
||||
} else {
|
||||
os.WriteFile(stateFile, []byte("inactive"), 0o644)
|
||||
}
|
||||
script := "#!/bin/sh\necho \"$@\" >> " + calls + "\ncase \"$1\" in\n" +
|
||||
"show) printf 'LoadState=loaded\\nActiveState=%s\\nType=simple\\n' \"$(cat " + stateFile + ")\" ;;\n" +
|
||||
"start) printf active > " + stateFile + " ;;\nis-enabled) echo enabled ;;\nesac\nexit 0\n"
|
||||
if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
|
||||
// The engine's own record of the unit, as a previous run left it, and the have it wrote then.
|
||||
res, _ := json.Marshal(map[string]any{"id": "web.unit", "type": "service", "unit": "web-it.service", "state": "running", "module": "web"})
|
||||
var known store.State
|
||||
known.Record(store.Applied{ID: "web.unit", Type: "service", Target: "web-it.service"})
|
||||
if err := store.Save(b.opts.state, known); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fp := fpIt("a")
|
||||
c := delivery.Composed{Node: itNode, Module: "web", FP: fp, Resources: []json.RawMessage{res}}
|
||||
signed, _ := delivery.SignComposed(c, b.controll)
|
||||
raw, _ := delivery.EncodeComposed(signed)
|
||||
dig, _ := delivery.ComposedDigestOf(signed)
|
||||
key, _ := delivery.ComposedKey(itNode, "web", fp)
|
||||
if _, err := b.composed.Put(key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.want("web", fp, dig, "")
|
||||
hkey, _ := delivery.HaveKey(itNode, "web")
|
||||
have, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.haves.Put(hkey, have); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stop := b.start(false)
|
||||
within(t, 5*time.Second, "the unit was not started and the have not running", func() bool {
|
||||
got, _ := os.ReadFile(stateFile)
|
||||
h := b.have("web")
|
||||
return string(got) == "active" && h.Phase == delivery.PhaseRunning && h.Composed == dig
|
||||
})
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
stop()
|
||||
starts := 0
|
||||
log, _ := os.ReadFile(calls)
|
||||
for _, l := range strings.Split(string(log), "\n") {
|
||||
if strings.HasPrefix(l, "start") {
|
||||
starts++
|
||||
}
|
||||
}
|
||||
if want := map[bool]int{false: 1, true: 0}[active]; starts != want {
|
||||
t.Fatalf("unit active=%v: started %d times, want %d", active, starts, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/ownresync"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
)
|
||||
|
||||
// composesNow is whose compositions this run of the agent applies, once decided ("" before).
|
||||
type composesNow struct {
|
||||
mu sync.Mutex
|
||||
now string
|
||||
}
|
||||
|
||||
func (c *composesNow) set(s string) { c.mu.Lock(); c.now = s; c.mu.Unlock() }
|
||||
func (c *composesNow) get() string { c.mu.Lock(); defer c.mu.Unlock(); return c.now }
|
||||
|
||||
// resyncStep is the step that hands the vault what runs of an own secret a shadow answer says another value of (issue
|
||||
// 1126, cause C4; internal/ownresync): one pass at a time, and the last answers taken again once the agent has decided
|
||||
// whose compositions it applies.
|
||||
type resyncStep struct {
|
||||
r *ownresync.Resync
|
||||
one sync.Mutex
|
||||
mu sync.Mutex
|
||||
last map[string]delivery.CandidateAnswer
|
||||
// tries and pending are the step taken again after a bound when a read or an ask the bus did not answer stopped it
|
||||
// (Outcome.Retry, issue 1126, 2026-10-28), one wait at a time.
|
||||
tries int
|
||||
pending bool
|
||||
}
|
||||
|
||||
func ownResync(session *link.Session, node string, say func(string), env selfcompose.Env, composes *composesNow) *resyncStep {
|
||||
return &resyncStep{r: ownresync.New(ownresync.Env{Node: node, Composes: composes.get, Running: env.Running,
|
||||
Get: env.Get, Unseal: env.Unseal, Seal: identity.Seal, Say: say,
|
||||
Ask: func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
return session.AskVault(ctx, verb, args)
|
||||
}})}
|
||||
}
|
||||
|
||||
// take keeps answers as the last and takes them.
|
||||
func (s *resyncStep) take(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
s.mu.Lock()
|
||||
s.last = answers
|
||||
s.mu.Unlock()
|
||||
s.again(ctx)
|
||||
}
|
||||
|
||||
// again takes the last answers, and once more after a bound (selfcompose.LaterWait) while the bus did not answer for one.
|
||||
func (s *resyncStep) again(ctx context.Context) {
|
||||
s.one.Lock()
|
||||
defer s.one.Unlock()
|
||||
s.mu.Lock()
|
||||
last := s.last
|
||||
s.mu.Unlock()
|
||||
if last == nil {
|
||||
return
|
||||
}
|
||||
retry := false
|
||||
for _, o := range s.r.Take(ctx, last) {
|
||||
retry = retry || o.Retry
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if !retry {
|
||||
s.tries = 0
|
||||
return
|
||||
}
|
||||
if s.pending || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
wait := selfcompose.LaterWait(s.tries)
|
||||
s.tries++
|
||||
s.pending = true
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(wait):
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.pending = false
|
||||
s.mu.Unlock()
|
||||
if ctx.Err() == nil {
|
||||
s.again(ctx)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -36,6 +36,7 @@ func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
|
||||
|
||||
// The modules applied again are those with a missing resource, each once; one down but there is not among them.
|
||||
func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
||||
t.Run("1957: the machine shows a resource not active", testTheMachineShowsAResourceNotActive)
|
||||
st := liveness.Statement{Resources: []liveness.State{
|
||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
|
||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
|
||||
@@ -51,6 +52,7 @@ func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
||||
// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One
|
||||
// that comes back and goes again is newly missing again (the review of issue 1945).
|
||||
func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) {
|
||||
t.Run("1957: a resource newly down is an edge", testOnlyAResourceNewlyDownIsAnEdge)
|
||||
missing := func(ids ...string) liveness.Statement {
|
||||
var st liveness.Statement
|
||||
for _, id := range ids {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// runsAsWritten is what the node runs as the composer reads it (selfcompose.Env.Running), with every file given by digest
|
||||
// named as this node last wrote it, where the two differ (issue 924, novox 2026-10-29 01:16). What runs of a module the
|
||||
// agent has not applied since its start can come from the kept declaration, older than the compositions applied since:
|
||||
// its export (mesh-vault.kept) then named a digest the controller's sweep had removed, while the file on disk was the
|
||||
// newer export the node wrote. A node composing its own carries such a file as it runs (mesh-sdk frombus keptOf), so it
|
||||
// names what is on disk: **only when the node's record says it wrote exactly that content there** (store Applied.Wrote,
|
||||
// Target), never a file someone else changed. A file that cannot be read, or that the record does not vouch for, is
|
||||
// left as the composition names it.
|
||||
func runsAsWritten(running []delivery.Composed, statePath string) []delivery.Composed {
|
||||
if len(running) == 0 || statePath == "" {
|
||||
return running
|
||||
}
|
||||
known, err := store.Load(statePath)
|
||||
if err != nil {
|
||||
return running
|
||||
}
|
||||
out := make([]delivery.Composed, len(running))
|
||||
for i, c := range running {
|
||||
out[i] = c
|
||||
var changed []json.RawMessage
|
||||
for j, raw := range c.Resources {
|
||||
if fixed, ok := asWritten(raw, known); ok {
|
||||
if changed == nil {
|
||||
changed = append([]json.RawMessage(nil), c.Resources...)
|
||||
}
|
||||
changed[j] = fixed
|
||||
}
|
||||
}
|
||||
if changed != nil {
|
||||
out[i].Resources = changed
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// asWritten is the file resource raw named by the digest and size of what this node last wrote at its path, when raw is a
|
||||
// file given by another digest and the file there is what the record says the node wrote.
|
||||
func asWritten(raw json.RawMessage, known store.State) (json.RawMessage, bool) {
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &fields) != nil {
|
||||
return nil, false
|
||||
}
|
||||
var kind, id, path, digest string
|
||||
_ = json.Unmarshal(fields["type"], &kind)
|
||||
_ = json.Unmarshal(fields["id"], &id)
|
||||
_ = json.Unmarshal(fields["path"], &path)
|
||||
_ = json.Unmarshal(fields["digest"], &digest)
|
||||
if kind != string(declaration.TypeFile) || digest == "" || path == "" {
|
||||
return nil, false
|
||||
}
|
||||
rec, ok := known.Find(id)
|
||||
if !ok || rec.Wrote == "" || rec.Target != path {
|
||||
return nil, false
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if hex.EncodeToString(sum[:]) != rec.Wrote {
|
||||
return nil, false // changed since the node wrote it: not the node's to vouch for
|
||||
}
|
||||
if now := declaration.DigestOf(body); now == digest {
|
||||
return nil, false
|
||||
} else {
|
||||
fields["digest"], _ = json.Marshal(now)
|
||||
}
|
||||
fields["size"], _ = json.Marshal(len(body))
|
||||
fixed, err := json.Marshal(fields)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return fixed, true
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the
|
||||
// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's
|
||||
// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the
|
||||
// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key
|
||||
// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its
|
||||
// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus.
|
||||
type meshTrust struct {
|
||||
mu sync.Mutex
|
||||
controller ed25519.PublicKey
|
||||
handedAway bool
|
||||
vault ed25519.PublicKey
|
||||
authority string
|
||||
// changed is told when what it trusts changed, so the record is written again.
|
||||
changed func()
|
||||
}
|
||||
|
||||
// newMeshTrust trusts the controller's key alone until the vault's is read.
|
||||
func newMeshTrust(controller ed25519.PublicKey) *meshTrust {
|
||||
t := &meshTrust{}
|
||||
if len(controller) == ed25519.PublicKeySize {
|
||||
t.controller = controller
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// onChange names who is told when what it trusts changed.
|
||||
func (t *meshTrust) onChange(f func()) {
|
||||
t.mu.Lock()
|
||||
t.changed = f
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away.
|
||||
func (t *meshTrust) Controller() ed25519.PublicKey {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.handedAway {
|
||||
return nil
|
||||
}
|
||||
return t.controller
|
||||
}
|
||||
|
||||
// HandedAway says the controller's lease was handed away, as heard on the lease key.
|
||||
func (t *meshTrust) HandedAway() bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.handedAway
|
||||
}
|
||||
|
||||
// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the
|
||||
// vault's once read.
|
||||
func (t *meshTrust) Signers() []delivery.Signer {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
var out []delivery.Signer
|
||||
if t.controller != nil && !t.handedAway {
|
||||
out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller})
|
||||
}
|
||||
if t.vault != nil {
|
||||
out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Verify is the signer that signed msg as kind, or why none it trusts did.
|
||||
func (t *meshTrust) Verify(kind string, msg, sig []byte) (delivery.Signer, error) {
|
||||
context, err := delivery.SignContext(kind)
|
||||
if err != nil {
|
||||
return delivery.Signer{}, err
|
||||
}
|
||||
return delivery.VerifyBy(t.Signers(), context, msg, sig)
|
||||
}
|
||||
|
||||
// VaultAuthority is the vault's authority certificate as read from `signers`, or "".
|
||||
func (t *meshTrust) VaultAuthority() string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.authority
|
||||
}
|
||||
|
||||
// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint.
|
||||
func (t *meshTrust) Trusts() []string {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
authorities := map[string]string{}
|
||||
if a := t.VaultAuthority(); a != "" {
|
||||
authorities[delivery.SignerVault] = a
|
||||
}
|
||||
return delivery.Trusts(t.Signers(), authorities)
|
||||
}
|
||||
|
||||
// change applies f under the lock and tells who follows when Trusts differs after it.
|
||||
func (t *meshTrust) change(f func()) {
|
||||
before := strings.Join(t.Trusts(), "\n")
|
||||
t.mu.Lock()
|
||||
f()
|
||||
changed := t.changed
|
||||
t.mu.Unlock()
|
||||
if strings.Join(t.Trusts(), "\n") != before && changed != nil {
|
||||
changed()
|
||||
}
|
||||
}
|
||||
|
||||
// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not
|
||||
// check; a deleted key is no longer trusted.
|
||||
func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) {
|
||||
switch key {
|
||||
case delivery.SignersKeyKey:
|
||||
var pub ed25519.PublicKey
|
||||
if !deleted && len(value) > 0 {
|
||||
k, err := delivery.DecodeSignerKey(value)
|
||||
if err == nil {
|
||||
pub, err = k.PublicKey()
|
||||
}
|
||||
if err != nil {
|
||||
say("the vault's signing key in " + delivery.SignersRead + " is not trusted: " + err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
t.change(func() { t.vault = pub })
|
||||
case delivery.SignersAuthorityKey:
|
||||
pem := ""
|
||||
if !deleted && len(value) > 0 {
|
||||
a, err := delivery.DecodeSignerAuthority(value)
|
||||
if err != nil {
|
||||
say("the vault's authority in " + delivery.SignersRead + " is not trusted: " + err.Error())
|
||||
return
|
||||
}
|
||||
pem = a.Certificate
|
||||
}
|
||||
t.change(func() { t.authority = pem })
|
||||
}
|
||||
}
|
||||
|
||||
// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the
|
||||
// controller's key is trusted no more. A lease that cannot be read changes nothing.
|
||||
func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) {
|
||||
if readErr != nil {
|
||||
return
|
||||
}
|
||||
handed, _ := buscompose.LeaseHandedAway(raw, found, nil)
|
||||
t.change(func() { t.handedAway = handed })
|
||||
}
|
||||
|
||||
// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may
|
||||
// not serve `sign` yet, and until it does this node is not granted the reads.
|
||||
var trustFollowAgain = 2 * time.Minute
|
||||
|
||||
// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as
|
||||
// long as ctx lasts. A watch that cannot be made is said once per reason and tried again.
|
||||
func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) {
|
||||
said := ""
|
||||
sayOnce := func(line string) {
|
||||
if line != said {
|
||||
said = line
|
||||
if line != "" {
|
||||
say(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
linked, cancel, err := session.WhenLinked(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
watching, stop := context.WithCancel(linked)
|
||||
signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">")
|
||||
lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
|
||||
if serr != nil || lerr != nil {
|
||||
stop()
|
||||
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+
|
||||
"the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain))
|
||||
signers, lease = nil, nil
|
||||
} else {
|
||||
sayOnce("")
|
||||
}
|
||||
for signers != nil || lease != nil {
|
||||
select {
|
||||
case u, ok := <-signers:
|
||||
if !ok {
|
||||
signers = nil
|
||||
continue
|
||||
}
|
||||
if !u.Caught {
|
||||
trust.heardSigner(u.Key, u.Value, u.Deleted, say)
|
||||
}
|
||||
case u, ok := <-lease:
|
||||
if !ok {
|
||||
lease = nil
|
||||
continue
|
||||
}
|
||||
if !u.Caught {
|
||||
trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
stop()
|
||||
select {
|
||||
case <-linked.Done():
|
||||
case <-time.After(trustFollowAgain):
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// placedAllHave says whether every module the seat mesh-placement places on node has a have of its own, read from the
|
||||
// bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose placement or haves
|
||||
// cannot be read, keeps reading it.
|
||||
func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
h, err := session.KVHistory(ctx, delivery.PlacementBucket, key)
|
||||
if err != nil || len(h) == 0 || h[len(h)-1].Deleted {
|
||||
return false, "its placement is not read"
|
||||
}
|
||||
p, err := delivery.DecodePlacement(h[len(h)-1].Value)
|
||||
if err != nil {
|
||||
return false, err.Error()
|
||||
}
|
||||
return allHave(p, func(module string) (bool, error) {
|
||||
_, ok, err := session.ReadHaveErr(ctx, module)
|
||||
return ok, err
|
||||
})
|
||||
}
|
||||
|
||||
// allHave says whether has finds a have for every module p places, and why not.
|
||||
func allHave(p delivery.Placement, has func(module string) (bool, error)) (bool, string) {
|
||||
if len(p.Modules) == 0 {
|
||||
return false, "the seat mesh-placement places no module here"
|
||||
}
|
||||
for _, m := range p.Modules {
|
||||
ok, err := has(m.Module)
|
||||
if err != nil {
|
||||
return false, "the have of " + m.Module + " is not read: " + err.Error()
|
||||
}
|
||||
if !ok {
|
||||
return false, m.Module + " has no have yet"
|
||||
}
|
||||
}
|
||||
return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules))
|
||||
}
|
||||
|
||||
// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks
|
||||
// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the
|
||||
// vault's this node trusts, keeps it where the record reads it, and writes the record again (written).
|
||||
func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust,
|
||||
ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error {
|
||||
serving, err := identity.LoadServingKey(identity.ServingKeyPath(state))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := base64.StdEncoding.DecodeString(serving.Public)
|
||||
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||
return fmt.Errorf("this node's serving key is not an ed25519 key")
|
||||
}
|
||||
raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var a delivery.SignAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return fmt.Errorf("the vault's answer is not a signing: %w", err)
|
||||
}
|
||||
trusted := trust.VaultAuthority()
|
||||
if trusted == "" {
|
||||
return fmt.Errorf("this node has read no authority of the vault's to check the certificate against")
|
||||
}
|
||||
if err := delivery.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node),
|
||||
ed25519.PublicKey(key), time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
path := identity.CertificatePath(state)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
if written != nil {
|
||||
written()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -9,6 +12,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/accounts"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/units"
|
||||
)
|
||||
|
||||
@@ -103,6 +107,61 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
||||
if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" {
|
||||
t.Fatalf("root said: %v", got)
|
||||
}
|
||||
// 774 (ADR 0563 §6): the node seat's root-free passes only on the engine's own positive verdict at most 15 minutes old
|
||||
// with execute not served here; a stale, missing or negative verdict, or execute served, is no pass and says why.
|
||||
t.Run("this machine's root-free", func(t *testing.T) {
|
||||
agent := accounts.Verdict{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true},
|
||||
State: accounts.Healthy, Since: at}
|
||||
free := &accounts.Statement{At: at, Accounts: []accounts.Verdict{agent}}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
st *accounts.Statement
|
||||
after time.Duration
|
||||
served bool
|
||||
free bool
|
||||
why string
|
||||
}{
|
||||
{name: "a positive verdict", st: free, after: 14 * time.Minute, free: true, why: "the account agent cannot become root"},
|
||||
{name: "a stale verdict", st: free, after: 16 * time.Minute, why: "more than 15 minutes ago"},
|
||||
{name: "no verdict", st: nil, why: "judged no account yet"},
|
||||
{name: "a way to root", st: as, why: "in the group docker, which grants root"},
|
||||
{name: "no agent account", st: &accounts.Statement{At: at}, why: "agents run as the operator's account"},
|
||||
{name: "execute served", st: free, served: true, why: "the login shell runs any command"},
|
||||
} {
|
||||
v := judgeRootFree("ace", row.st, at.Add(row.after), row.served, "the bus hears execute answered here")
|
||||
if v.Free != row.free || !strings.Contains(v.Why, row.why) || v.Machine != "ace" {
|
||||
t.Errorf("%s: %+v", row.name, v)
|
||||
}
|
||||
}
|
||||
// execute is served while the holder's setting file says anything but withhold, or the bus hears it.
|
||||
dir := t.TempDir()
|
||||
statePath, conf := filepath.Join(dir, "state.json"), filepath.Join(dir, "tools.conf")
|
||||
if err := store.Save(statePath, store.State{Resources: []store.Applied{{ID: "zsh.tools-conf", Type: "file", Target: conf}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
quiet := func() (bool, error) { return false, nil }
|
||||
for _, row := range []struct {
|
||||
name, setting string
|
||||
heard func() (bool, error)
|
||||
served bool
|
||||
}{
|
||||
{name: "withheld and not heard", setting: "# the mesh's\nexecute=withhold\n", heard: quiet},
|
||||
{name: "served", setting: "execute=serve\n", heard: quiet, served: true},
|
||||
{name: "withheld and heard", setting: "execute=withhold\n", heard: func() (bool, error) { return true, nil }, served: true},
|
||||
{name: "the bus not asked", setting: "execute=withhold\n", heard: func() (bool, error) { return false, errors.New("no link") }, served: true},
|
||||
{name: "no setting file", heard: quiet, served: true},
|
||||
} {
|
||||
_ = os.Remove(conf)
|
||||
if row.setting != "" {
|
||||
if err := os.WriteFile(conf, []byte(row.setting), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if served, why := executeServedHere(statePath, row.heard); served != row.served {
|
||||
t.Errorf("%s: served %v (%s)", row.name, served, why)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
func aFileOf(module, id string) json.RawMessage {
|
||||
@@ -59,6 +61,41 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
|
||||
if err != nil || d.Provisioner != "from-agent" {
|
||||
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
|
||||
}
|
||||
|
||||
// L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the
|
||||
// file is there and readable and the controller's key is dropped; until then it reads it.
|
||||
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||
if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` +
|
||||
`"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 {
|
||||
t.Fatalf("before the read is retired: %+v, %v", d, err)
|
||||
}
|
||||
placed := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
has map[string]bool
|
||||
want bool
|
||||
}{
|
||||
{"every placed module has a have", map[string]bool{"zsh": true, "lemurs": true}, true},
|
||||
{"one has none yet", map[string]bool{"zsh": true}, false},
|
||||
} {
|
||||
if got, why := allHave(placed, func(m string) (bool, error) { return row.has[m], nil }); got != row.want {
|
||||
t.Errorf("%s: %v (%s)", row.name, got, why)
|
||||
}
|
||||
}
|
||||
if got, _ := allHave(delivery.Placement{Node: "n1"}, nil); got {
|
||||
t.Error("a node with nothing placed retires the read")
|
||||
}
|
||||
keptRetired.Store(true)
|
||||
defer keptRetired.Store(false)
|
||||
if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) {
|
||||
t.Errorf("after the read is retired: %v", err)
|
||||
}
|
||||
if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil {
|
||||
t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029003747-575cb82fba69
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366 h1:B7oQ82ar2McqZ9OrbJfyvG+50Ol6fBUoK3eYE2uTPvo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029003747-575cb82fba69 h1:0uFXqo13BukGqe1fltCwk9O/nAuDWx5paHUsmdfgar4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029003747-575cb82fba69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -240,6 +240,17 @@ func (c *SetuidCache) SearchAgain() {
|
||||
c.forgetProgress()
|
||||
}
|
||||
|
||||
// Running says whether a search walks now (novox/hq ADR 0563 §2: the node seat starts no second while one does).
|
||||
// Nil c runs none.
|
||||
func (c *SetuidCache) Running() bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.running
|
||||
}
|
||||
|
||||
// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
|
||||
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
|
||||
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
|
||||
|
||||
+416
-59
@@ -24,6 +24,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -57,9 +58,10 @@ type Have struct {
|
||||
// waits for a person's yes (ADR 0547 §3).
|
||||
Judgement string
|
||||
Waits string
|
||||
// Sealed is, on a node composing its own, each value the composition it names carries sealed to the node's key
|
||||
// (delivery.Have.Sealed): kept in the have so the composer reuses them after a restart (ADR 0547 §3).
|
||||
Sealed []string
|
||||
// Sealed is, on a node composing its own, each value the composition it names carries sealed to the node's key, by
|
||||
// slot (delivery.Have.SealedAt): kept in the have so the composer reuses them after a restart (ADR 0547 §3), each at
|
||||
// its own slot alone (issue 1126, C1).
|
||||
Sealed delivery.Seals
|
||||
}
|
||||
|
||||
// Haves is the node's have bucket as the agent uses it.
|
||||
@@ -107,7 +109,9 @@ type Env struct {
|
||||
Kept func() map[string][]json.RawMessage
|
||||
// KeptFilled is the paths the kept declaration names as filled by the provisioner, by the module whose file is at
|
||||
// each (the node's own under NodeModule). A module that still runs from the kept declaration keeps its filled files
|
||||
// filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none.
|
||||
// filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. A module
|
||||
// whose have names a composition the bucket keeps runs from that composition, never from the kept declaration
|
||||
// (issue 924): the kept declaration is older than every composition applied since the switch.
|
||||
KeptFilled func() map[string][]string
|
||||
// Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A
|
||||
// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
|
||||
@@ -116,6 +120,11 @@ type Env struct {
|
||||
// Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed
|
||||
// since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no.
|
||||
Recorded func(ctx context.Context, c delivery.Composed) bool
|
||||
// Inactive says a resource the composition declares running is not active on the machine now (a unit stopped, a
|
||||
// container not running), read from the machine and not from any record (ADR 0568 §1). A have or a kept declaration
|
||||
// that says the module runs is believed at the agent's start only when this says no; nil says no, and an unreadable
|
||||
// machine says no (unknown is not inactive).
|
||||
Inactive func(ctx context.Context, c delivery.Composed) bool
|
||||
// Revisions are a composition key's revisions the bucket keeps, oldest first; nil reads none. The composition a want
|
||||
// names is taken from them when a newer one replaced it under the same key (ADR 0539 clause 1).
|
||||
Revisions func(ctx context.Context, key string) ([]Update, error)
|
||||
@@ -124,6 +133,12 @@ type Env struct {
|
||||
// them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host
|
||||
// internal/filling Knock; ace, 2026-10-25 21:17). Nil tells nothing.
|
||||
Settled func(module string)
|
||||
// RunsChanged is told, outside the agent's lock, that what runs (Everything) changed other than by an apply or a
|
||||
// removal: once at the start, when the haves and the compositions they name are read, and each time a module is
|
||||
// taken as running from its have or the kept declaration. Like Settled, the filling then tells the provisioner's
|
||||
// client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09
|
||||
// UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing.
|
||||
RunsChanged func(why string)
|
||||
// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
|
||||
// transient failure only on RetryTransient or a change of either key.
|
||||
Backoff func(try int) time.Duration
|
||||
@@ -136,6 +151,11 @@ type Env struct {
|
||||
// move of a module of policy record that recreates a container or moves its data waits for a person's yes in its
|
||||
// want (ADR 0547 §3). Nil judges no move.
|
||||
Policy func(ctx context.Context, module string) string
|
||||
// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), and whether its manifest
|
||||
// could be read: what a composition's Data is, for what runs of a module from the kept declaration, which carries no
|
||||
// Data of its own. A composition says Data, so without it a data module from the kept declaration is judged to move
|
||||
// its data against its first composition after a restart (mongodb and mssql on novox, issue 924). Nil reads none.
|
||||
HoldsData func(ctx context.Context, module string) (data, known bool)
|
||||
}
|
||||
|
||||
// transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has
|
||||
@@ -153,6 +173,13 @@ type Agent struct {
|
||||
// applied is the composition each module was last applied from: a revision a want named from the key's history is
|
||||
// not the key's latest value, and what it holds is still held out of the others' applies.
|
||||
applied map[string]compNow
|
||||
// had is, for each module whose have names a composition the bucket keeps (its latest value or a revision), that
|
||||
// composition, read once at the start (loadHad): what runs of a module the agent has not taken yet, ahead of the kept
|
||||
// declaration (issue 924).
|
||||
had map[string]compNow
|
||||
// keptData is, for each module of the kept declaration whose manifest could be read, whether it declares data
|
||||
// (Env.HoldsData): the Data of what runs of it from the kept declaration.
|
||||
keptData map[string]bool
|
||||
// attempt is the last apply of a module that failed or is held, with the revisions it was made from.
|
||||
attempt map[string]attempt
|
||||
checked map[string]bool
|
||||
@@ -161,7 +188,12 @@ type Agent struct {
|
||||
said map[string]string
|
||||
// notComposed is the modules whose have says the node could not compose them: why, and the have before (NotComposed).
|
||||
notComposed map[string]notComposedMark
|
||||
retry chan []string
|
||||
// heldAlone is the modules the composer holds alone (NotComposedEach): what runs of each in the node's own
|
||||
// composition is held out of the node's own apply until a composition of its own is handed (heldOut).
|
||||
heldAlone map[string]bool
|
||||
retry chan []string
|
||||
// reconcile carries the judge's edges: a resource found not active (ADR 0568 §2).
|
||||
reconcile chan []string
|
||||
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
|
||||
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
|
||||
tries map[string]int
|
||||
@@ -206,9 +238,9 @@ func New(env Env) *Agent {
|
||||
if env.Say == nil {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{},
|
||||
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
|
||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
|
||||
retry: make(chan []string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||
retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
|
||||
}
|
||||
|
||||
@@ -224,7 +256,7 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("the agent cannot watch its compositions: %w", err)
|
||||
}
|
||||
var wantsCaught, compsCaught bool
|
||||
var wantsCaught, compsCaught, hadRead bool
|
||||
dirty := map[string]bool{}
|
||||
for {
|
||||
select {
|
||||
@@ -240,6 +272,17 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
dirty[m] = true
|
||||
}
|
||||
a.mu.Unlock()
|
||||
case modules := <-a.reconcile:
|
||||
a.mu.Lock()
|
||||
for _, m := range modules {
|
||||
if w := a.wants[m]; !w.Present || w.Want.Removed || a.attempt[m].Held {
|
||||
continue
|
||||
}
|
||||
delete(a.running, m)
|
||||
delete(a.attempt, m)
|
||||
dirty[m] = true
|
||||
}
|
||||
a.mu.Unlock()
|
||||
case ask := <-a.again:
|
||||
// The wait of a transiently failed apply ran out: tried again from the keys it has, if nothing tried it since.
|
||||
a.mu.Lock()
|
||||
@@ -280,7 +323,17 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
}
|
||||
// Nothing is applied until what was there at the start is all in: a module's composition may arrive after its
|
||||
// want, and the node's own composition comes before any module's.
|
||||
if !(wantsCaught && compsCaught) || len(dirty) == 0 {
|
||||
if !(wantsCaught && compsCaught) {
|
||||
continue
|
||||
}
|
||||
if !hadRead {
|
||||
// Before the first settle: what runs of each module with a have, from the composition it names, so the
|
||||
// filling judges by it at once and not by the kept declaration until some module settles (issue 924).
|
||||
hadRead = true
|
||||
a.loadHad(ctx)
|
||||
a.runsChanged("the haves were read at the start")
|
||||
}
|
||||
if len(dirty) == 0 {
|
||||
continue
|
||||
}
|
||||
a.settle(ctx, dirty)
|
||||
@@ -298,6 +351,16 @@ func (a *Agent) Retry(modules ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Reconcile asks for the modules to be applied again because a resource each declares running was found not active, once
|
||||
// per edge (the judge's look, ADR 0568 §2). It is Retry, but a module whose apply is held (a failed backup) is left
|
||||
// alone, as is one the node wants no more (ADR 0568 §4). It never blocks.
|
||||
func (a *Agent) Reconcile(modules ...string) {
|
||||
select {
|
||||
case a.reconcile <- modules:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it
|
||||
// when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never
|
||||
// blocks.
|
||||
@@ -314,29 +377,97 @@ type notComposedMark struct {
|
||||
prior Have
|
||||
}
|
||||
|
||||
// NotComposedDetail opens the detail of a have NotComposed marks: the module keeps running what its build and digest
|
||||
// name, and only the node's composition of it failed.
|
||||
const NotComposedDetail = "the node cannot compose it, and keeps running what it ran: "
|
||||
|
||||
// composerMark says h is a mark NotComposed wrote: failed, its detail opening with NotComposedDetail.
|
||||
func composerMark(h Have) bool {
|
||||
return h.Phase == string(delivery.PhaseFailed) && strings.HasPrefix(h.Detail, NotComposedDetail)
|
||||
}
|
||||
|
||||
// HavesReadErr is what a Haves may answer besides Read: a have absent (false, nil) told from one that could not be read
|
||||
// (false, the error).
|
||||
type HavesReadErr interface {
|
||||
ReadErr(ctx context.Context, module string) (Have, bool, error)
|
||||
}
|
||||
|
||||
// readHave reads a module's have: found, absent, or unread. A read that fails is tried again after a short wait while
|
||||
// the bus does not answer (a reload pauses it for a few seconds, issue 1126), and is unread when it still fails. A
|
||||
// Haves that cannot tell an absent key from a failed read says absent.
|
||||
func (a *Agent) readHave(ctx context.Context, module string) (h Have, found, unread bool) {
|
||||
re, tells := a.env.Haves.(HavesReadErr)
|
||||
if !tells {
|
||||
h, found = a.env.Haves.Read(ctx, module)
|
||||
return h, found, false
|
||||
}
|
||||
for try := 0; ; try++ {
|
||||
h, found, err := re.ReadErr(ctx, module)
|
||||
if err == nil {
|
||||
return h, found, false
|
||||
}
|
||||
if try == 2 || ctx.Err() != nil {
|
||||
return Have{}, false, true
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Have{}, false, true
|
||||
case <-time.After(time.Duration(1<<try) * readAgain):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readAgain is the first wait before a have is read again.
|
||||
var readAgain = time.Second
|
||||
|
||||
// NotComposed says the node composing its own could not compose its modules (ADR 0547 §2: a stand-in refused, an
|
||||
// input that cannot be read): each module with a want gets its have failed, naming why, while it keeps running what it
|
||||
// ran, so the failure is seen where every have is read and not only in the journal. The have keeps what runs: the
|
||||
// fingerprint, the digest and the sealed values the agent runs, or, before it ran anything since its start, those its
|
||||
// last have said (a refusal at the first compose after a start). Called with "" once the node composes again: each
|
||||
// module so marked has its have back as it was, unless an apply wrote it since.
|
||||
func (a *Agent) NotComposed(ctx context.Context, why string) {
|
||||
func (a *Agent) NotComposed(ctx context.Context, why string) { a.NotComposedEach(ctx, why, nil) }
|
||||
|
||||
// NotComposedEach is NotComposed for the node refused whole (why), or, with why empty, for the modules the node holds
|
||||
// alone (held, each with its own why: refused alone, the rest composed and applied, issue 1126): only those are marked,
|
||||
// each naming its own why, and every other module marked before has its have back. Both empty, every mark is cleared.
|
||||
func (a *Agent) NotComposedEach(ctx context.Context, why string, held map[string]string) {
|
||||
a.mu.Lock()
|
||||
var modules []string
|
||||
if why != "" {
|
||||
for module, w := range a.wants {
|
||||
if w.Present && !w.Want.Removed && module != NodeModule && a.notComposed[module].why != why {
|
||||
modules = append(modules, module)
|
||||
}
|
||||
if why == "" {
|
||||
a.heldAlone = make(map[string]bool, len(held))
|
||||
for m := range held {
|
||||
a.heldAlone[m] = true
|
||||
}
|
||||
} else {
|
||||
for module := range a.notComposed {
|
||||
}
|
||||
whyOf := map[string]string{} // the mark each module is to carry; a marked module not in it is cleared
|
||||
for module, w := range a.wants {
|
||||
if !w.Present || w.Want.Removed || module == NodeModule {
|
||||
continue
|
||||
}
|
||||
if why != "" {
|
||||
whyOf[module] = why
|
||||
} else if h, ok := held[module]; ok && h != "" {
|
||||
whyOf[module] = h
|
||||
}
|
||||
}
|
||||
var modules []string
|
||||
for module, w := range whyOf {
|
||||
if a.notComposed[module].why != w {
|
||||
modules = append(modules, module)
|
||||
}
|
||||
}
|
||||
for module := range a.notComposed {
|
||||
if _, still := whyOf[module]; !still && why == "" { // the node refused whole marks; it clears none
|
||||
modules = append(modules, module)
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
why := whyOf[module]
|
||||
if ctx.Err() != nil {
|
||||
return // the run ended: what it would mark or clear is the next run's to say (issue 1126)
|
||||
}
|
||||
a.mu.Lock()
|
||||
mark, marked := a.notComposed[module]
|
||||
r, ran := a.running[module]
|
||||
@@ -364,16 +495,39 @@ func (a *Agent) NotComposed(ctx context.Context, why string) {
|
||||
if mark.prior.Phase == "" {
|
||||
mark.prior.Phase = string(delivery.PhaseRunning)
|
||||
}
|
||||
} else if h, ok := a.env.Haves.Read(ctx, module); ok {
|
||||
} else if h, found, unread := a.readHave(ctx, module); found {
|
||||
mark.prior = h // what its last have said runs: this agent has run nothing of it since its start
|
||||
} else if unread {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
// What its have names cannot be read (the bus paused, issue 1126): a mark written now would drop the build
|
||||
// and digest it names. Nothing is written; the next refusal marks it once the have reads.
|
||||
a.mu.Lock()
|
||||
a.sayOnce("unmarked:"+module, fmt.Sprintf("%s's have cannot be read, so it is not marked: the node cannot compose it: %s", module, why))
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A have that already says another cause (an apply that failed, a backup held, an apply under way) keeps it: a
|
||||
// composer's mark over it would hide that cause, and baseline would take the mark for what runs (review of the
|
||||
// switch, round 1).
|
||||
if p := mark.prior.Phase; !marked && p != "" && p != string(delivery.PhaseRunning) && !composerMark(mark.prior) {
|
||||
a.mu.Lock()
|
||||
a.sayOnce("unmarked-cause:"+module, fmt.Sprintf("%s is not marked: its have says %s (%s), and the node cannot compose it: %s",
|
||||
module, p, mark.prior.Detail, why))
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
mark.why = why
|
||||
a.mu.Lock()
|
||||
a.notComposed[module] = mark
|
||||
a.mu.Unlock()
|
||||
a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed,
|
||||
Phase: string(delivery.PhaseFailed), Detail: "the node cannot compose it, and keeps running what it ran: " + why})
|
||||
Phase: string(delivery.PhaseFailed), Detail: NotComposedDetail + why})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,15 +556,15 @@ func (a *Agent) writeHave(ctx context.Context, h Have) {
|
||||
a.env.Haves.Write(ctx, h)
|
||||
}
|
||||
|
||||
func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) []string {
|
||||
func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) delivery.Seals {
|
||||
a.mu.Lock()
|
||||
if c, ok := a.applied[module]; ok && c.Err == nil && c.Dig == composed {
|
||||
a.mu.Unlock()
|
||||
return SealedIn(c.C)
|
||||
return delivery.SealsIn(c.C)
|
||||
}
|
||||
if c, ok := a.compositionFor(module, fp); ok && c.Err == nil && c.Dig == composed {
|
||||
a.mu.Unlock()
|
||||
return SealedIn(c.C)
|
||||
return delivery.SealsIn(c.C)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if h, ok := a.env.Haves.Read(ctx, module); ok && h.Composed == composed {
|
||||
@@ -419,32 +573,6 @@ func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) []str
|
||||
return nil
|
||||
}
|
||||
|
||||
// SealedIn is each value a composition carries sealed to its node's key: a file's `sealed` and a vault box
|
||||
// (catalogue.VaultSealedField), as their JSON text, in the order of its resources.
|
||||
func SealedIn(c delivery.Composed) []string {
|
||||
var out []string
|
||||
for _, raw := range c.Resources {
|
||||
var r map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &r) != nil {
|
||||
continue
|
||||
}
|
||||
for _, field := range []string{"sealed", catalogue.VaultSealedField} {
|
||||
f, ok := r[field]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var text string
|
||||
if json.Unmarshal(f, &text) != nil {
|
||||
text = string(f) // a vault box is carried as its JSON
|
||||
}
|
||||
if text != "" {
|
||||
out = append(out, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Running is the compositions the node runs now: each module's that the agent applied or found already running, and
|
||||
// the node's own. What the node judges its health by and says of itself is built from them.
|
||||
func (a *Agent) Running() []delivery.Composed {
|
||||
@@ -628,7 +756,12 @@ func (a *Agent) revisionNamed(ctx context.Context, module, fp, digest string) (c
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dig, err := delivery.VerifyComposed(c, trusted)
|
||||
var dig string
|
||||
if a.env.SelfComposed {
|
||||
dig, err = delivery.ComposedDigestOf(c) // the node's own: unsigned (ADR 0547 §2)
|
||||
} else {
|
||||
dig, err = delivery.VerifyComposed(c, trusted)
|
||||
}
|
||||
if err != nil || dig != digest {
|
||||
continue
|
||||
}
|
||||
@@ -657,22 +790,33 @@ func (a *Agent) others(except string) []delivery.Composed {
|
||||
out = append(out, c.C)
|
||||
} else if c, ok := a.applied[m]; ok && c.Dig == h.Composed {
|
||||
out = append(out, c.C)
|
||||
} else if c, ok := a.had[m]; ok && c.Dig == h.Composed {
|
||||
out = append(out, c.C) // a revision its have named, replaced since under the same key
|
||||
}
|
||||
}
|
||||
hadNames := make([]string, 0, len(a.had))
|
||||
for m := range a.had {
|
||||
if _, held := a.running[m]; !held && m != except {
|
||||
hadNames = append(hadNames, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(hadNames)
|
||||
for _, m := range hadNames {
|
||||
out = append(out, a.had[m].C)
|
||||
}
|
||||
if a.env.Kept != nil {
|
||||
kept := a.env.Kept()
|
||||
names := make([]string, 0, len(kept))
|
||||
for m := range kept {
|
||||
if _, held := a.running[m]; !held && m != except {
|
||||
_, held := a.running[m]
|
||||
_, had := a.had[m]
|
||||
if !held && !had && m != except {
|
||||
names = append(names, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, m := range names {
|
||||
c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]}
|
||||
if a.env.KeptFilled != nil {
|
||||
c.Filled = a.env.KeptFilled()[m]
|
||||
}
|
||||
c, _ := a.keptComposed(m, kept[m])
|
||||
if m == NodeModule {
|
||||
c.Module = ""
|
||||
}
|
||||
@@ -682,6 +826,85 @@ func (a *Agent) others(except string) []delivery.Composed {
|
||||
return out
|
||||
}
|
||||
|
||||
// loadHad reads, once at the start, the have of every module with a want or in the kept declaration and, where it names
|
||||
// a composition, that composition: the bucket's value at its key when its digest is the have's, else the revision the
|
||||
// bucket keeps with that digest. A have naming none, or one the bucket no longer keeps, leaves the module to the kept
|
||||
// declaration, said once (issue 924).
|
||||
func (a *Agent) loadHad(ctx context.Context) {
|
||||
a.mu.Lock()
|
||||
seen := map[string]bool{}
|
||||
for m, w := range a.wants {
|
||||
if m != NodeModule && w.Present && !w.Want.Removed {
|
||||
seen[m] = true
|
||||
}
|
||||
}
|
||||
if a.env.Kept != nil {
|
||||
for m := range a.env.Kept() {
|
||||
if m != NodeModule {
|
||||
seen[m] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
names := make([]string, 0, len(seen))
|
||||
for m := range seen {
|
||||
names = append(names, m)
|
||||
}
|
||||
sort.Strings(names)
|
||||
a.readKeptData(ctx, names...)
|
||||
for _, m := range names {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
h, ok := a.env.Haves.Read(ctx, m)
|
||||
if !ok || h.Composed == "" || !haveRuns(h) {
|
||||
continue
|
||||
}
|
||||
a.mu.Lock()
|
||||
c, found := a.compositionFor(m, h.FP)
|
||||
a.mu.Unlock()
|
||||
if !found || c.Err != nil || c.Dig != h.Composed {
|
||||
c, found = a.revisionNamed(ctx, m, h.FP, h.Composed)
|
||||
}
|
||||
a.mu.Lock()
|
||||
if found {
|
||||
a.had[m] = c
|
||||
} else if _, kept := a.keptHas(m); kept {
|
||||
a.sayOnce("had:"+m, fmt.Sprintf("%s's have names the composition %s, which the bucket no longer keeps: it runs "+
|
||||
"from the kept declaration until it is applied or taken", m, short(h.Composed)))
|
||||
}
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// haveRuns says a have names what runs, in the phases baseline takes as running: running (or empty), held for a failed
|
||||
// backup, or a composer's mark over what still runs. A have backing up, applying, checking, failed or waiting names a
|
||||
// composition not applied, or not yet (issue 924, review of 031e106).
|
||||
func haveRuns(h Have) bool {
|
||||
switch delivery.Phase(h.Phase) {
|
||||
case "", delivery.PhaseRunning, delivery.PhaseHeld:
|
||||
return true
|
||||
}
|
||||
return composerMark(h)
|
||||
}
|
||||
|
||||
// keptHas says whether the kept declaration has resources for the module. Called holding a.mu or not: Kept takes no lock
|
||||
// of the agent's.
|
||||
func (a *Agent) keptHas(module string) ([]json.RawMessage, bool) {
|
||||
if a.env.Kept == nil {
|
||||
return nil, false
|
||||
}
|
||||
rs, ok := a.env.Kept()[module]
|
||||
return rs, ok
|
||||
}
|
||||
|
||||
// runsChanged tells RunsChanged, outside the agent's lock.
|
||||
func (a *Agent) runsChanged(why string) {
|
||||
if a.env.RunsChanged != nil {
|
||||
a.env.RunsChanged(why)
|
||||
}
|
||||
}
|
||||
|
||||
// unreadableModules are the keys of the module compositions in the node's bucket that cannot be read or are refused, of a
|
||||
// module the assigner has not removed: for each module only the composition its want names, else its newest, since a
|
||||
// stale one nobody applies (written before a rule the SDK now holds) must not hold the node's own back. Called holding
|
||||
@@ -762,6 +985,50 @@ func (a *Agent) heldOut(except string) []delivery.Composed {
|
||||
sort.Strings(names)
|
||||
for _, m := range names {
|
||||
out = append(out, waiting[m].C)
|
||||
held[m] = true
|
||||
}
|
||||
// A module the composer holds alone keeps what runs of it in the node's own composition, until a composition of its
|
||||
// own is handed: held out as one of its own, so the node's own apply never removes it (issue 1126, review of the
|
||||
// apply, round 1: de-spiegel leaving the fold with its names unset; issue 1945).
|
||||
if len(a.heldAlone) == 0 {
|
||||
return out
|
||||
}
|
||||
var own *delivery.Composed
|
||||
for _, c := range a.others("\x00") {
|
||||
if c.Module == "" {
|
||||
own = &c
|
||||
break
|
||||
}
|
||||
}
|
||||
alone := make([]string, 0, len(a.heldAlone))
|
||||
for m := range a.heldAlone {
|
||||
if !held[m] && m != except && own != nil {
|
||||
alone = append(alone, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(alone)
|
||||
for _, m := range alone {
|
||||
var rs []json.RawMessage
|
||||
for _, raw := range own.Resources {
|
||||
var head struct {
|
||||
ID string `json:"id"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
if json.Unmarshal(raw, &head) != nil {
|
||||
continue
|
||||
}
|
||||
owner := head.Module
|
||||
if owner == "" {
|
||||
owner, _ = declaration.ModuleGuess(head.ID)
|
||||
}
|
||||
if owner == m {
|
||||
rs = append(rs, raw)
|
||||
}
|
||||
}
|
||||
if len(rs) > 0 {
|
||||
// Held out, never judged: its Data from its manifest where known, as every other composition says it.
|
||||
out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs, Data: a.keptData[m]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -897,14 +1164,38 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
||||
a.mu.Lock()
|
||||
a.prior[module] = true
|
||||
a.mu.Unlock()
|
||||
if h.Phase == string(delivery.PhaseHeld) {
|
||||
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
|
||||
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4).
|
||||
a.mu.Lock()
|
||||
a.attempt[module] = attempt{FP: w.Want.FP, Composed: w.Want.Composed, WantRev: w.Rev, CompRev: comp.Rev, Held: true,
|
||||
AnsweredRev: w.Rev}
|
||||
if h.Composed != "" {
|
||||
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
a.runsChanged(module + " was taken as held")
|
||||
return true
|
||||
}
|
||||
if h.Composed != "" {
|
||||
// This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A
|
||||
// failed apply or a different composition is applied again.
|
||||
// A have a composer marked failed (NotComposed) names what still runs: taken as running, so a mark this
|
||||
// agent did not make clears once what it names is what the want names (issue 1126, the stale marks after
|
||||
// the seed's revert).
|
||||
marked := composerMark(h)
|
||||
if marked {
|
||||
h.Detail = ""
|
||||
}
|
||||
if h.FP == w.Want.FP && h.Composed == w.Want.Composed &&
|
||||
(h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) {
|
||||
(h.Phase == "" || h.Phase == string(delivery.PhaseRunning) || marked) {
|
||||
if a.inactiveAtStart(ctx, module, comp) {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
||||
a.mu.Unlock()
|
||||
a.runsChanged(module + " was taken from its have")
|
||||
// Said again, so the session holds the module and its key carries the health this engine judges,
|
||||
// not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25).
|
||||
a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
|
||||
@@ -927,13 +1218,31 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
||||
if !ok || !SameResources(kept, comp.C.Resources) {
|
||||
return false
|
||||
}
|
||||
if a.inactiveAtStart(ctx, module, comp) {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning}
|
||||
a.mu.Unlock()
|
||||
a.runsChanged(module + " was taken from the kept declaration")
|
||||
a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
|
||||
return true
|
||||
}
|
||||
|
||||
// inactiveAtStart says the machine shows a resource the module declares running not active, so what a have or the kept
|
||||
// declaration says runs is not believed and the module is applied once, as for a changed want (ADR 0568 §1).
|
||||
func (a *Agent) inactiveAtStart(ctx context.Context, module string, comp compNow) bool {
|
||||
if a.env.Inactive == nil || !a.env.Inactive(ctx, comp.C) {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.prior[module] = true
|
||||
a.sayOnce("inactive:"+module+":"+comp.Dig, fmt.Sprintf("%s is applied again: a resource it declares running is not "+
|
||||
"active on this machine", module))
|
||||
a.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
|
||||
func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNow, others []delivery.Composed) {
|
||||
fp, dig := w.Want.FP, w.Want.Composed
|
||||
a.mu.Lock()
|
||||
@@ -1106,6 +1415,7 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
|
||||
if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
|
||||
return false
|
||||
}
|
||||
a.readKeptData(ctx, module) // the manifest may read now though it did not at the start
|
||||
a.mu.Lock()
|
||||
r, ran := a.running[module]
|
||||
if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved {
|
||||
@@ -1151,14 +1461,61 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
|
||||
|
||||
// runningComposition is what the module runs now: the composition last applied, else its resources in the declaration
|
||||
// the node kept from before the switch. Called with a.mu held.
|
||||
//
|
||||
// The composition its have named at the start comes before the kept declaration (issue 924). From the kept declaration,
|
||||
// what runs carries Data as its manifest declares it; a module whose manifest could not be read cannot be judged against
|
||||
// it (false), and that is said, rather than taking it to hold no data and judging a false move of its data.
|
||||
func (a *Agent) runningComposition(module string) (delivery.Composed, bool) {
|
||||
if c, ok := a.applied[module]; ok && c.Err == nil {
|
||||
return c.C, true
|
||||
}
|
||||
if a.env.Kept != nil {
|
||||
if rs, ok := a.env.Kept()[module]; ok {
|
||||
return delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}, true
|
||||
if c, ok := a.had[module]; ok {
|
||||
return c.C, true
|
||||
}
|
||||
if rs, ok := a.keptHas(module); ok {
|
||||
c, known := a.keptComposed(module, rs)
|
||||
if !known {
|
||||
a.sayOnce("kept-data:"+module, fmt.Sprintf("%s runs from the kept declaration, and whether it holds data cannot be "+
|
||||
"read from its manifest: a move of it cannot be judged", module))
|
||||
}
|
||||
return c, known
|
||||
}
|
||||
return delivery.Composed{}, false
|
||||
}
|
||||
|
||||
// keptComposed is what runs of a module from the kept declaration: its resources there, the paths it names as filled,
|
||||
// and Data as the module's manifest declares it; false when the manifest could not be read, so Data is not known. Called
|
||||
// with a.mu held.
|
||||
func (a *Agent) keptComposed(module string, rs []json.RawMessage) (delivery.Composed, bool) {
|
||||
c := delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}
|
||||
if a.env.KeptFilled != nil {
|
||||
c.Filled = a.env.KeptFilled()[module]
|
||||
}
|
||||
data, known := a.keptData[module]
|
||||
c.Data = data
|
||||
return c, known || module == NodeModule
|
||||
}
|
||||
|
||||
// readKeptData reads, outside the agent's lock, whether each module of the kept declaration not known yet declares data
|
||||
// (Env.HoldsData), and keeps the answers it could read.
|
||||
func (a *Agent) readKeptData(ctx context.Context, modules ...string) {
|
||||
if a.env.HoldsData == nil {
|
||||
return
|
||||
}
|
||||
for _, m := range modules {
|
||||
if m == NodeModule {
|
||||
continue
|
||||
}
|
||||
a.mu.Lock()
|
||||
_, done := a.keptData[m]
|
||||
a.mu.Unlock()
|
||||
if _, kept := a.keptHas(m); done || !kept {
|
||||
continue
|
||||
}
|
||||
if data, known := a.env.HoldsData(ctx, m); known {
|
||||
a.mu.Lock()
|
||||
a.keptData[m] = data
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+358
-21
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -72,6 +74,19 @@ type haves struct {
|
||||
stored map[string]Have
|
||||
log []Have
|
||||
removed []string
|
||||
// unreadable is each module whose have the bus cannot read now (ReadErr).
|
||||
unreadable map[string]bool
|
||||
}
|
||||
|
||||
func (h *haves) ReadErr(ctx context.Context, module string) (Have, bool, error) {
|
||||
h.mu.Lock()
|
||||
bad := h.unreadable[module]
|
||||
h.mu.Unlock()
|
||||
if bad {
|
||||
return Have{}, false, fmt.Errorf("nats: no responders available for request")
|
||||
}
|
||||
x, ok := h.Read(ctx, module)
|
||||
return x, ok, nil
|
||||
}
|
||||
|
||||
func (h *haves) Read(_ context.Context, module string) (Have, bool) {
|
||||
@@ -327,7 +342,44 @@ func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
|
||||
|
||||
// What the node-engine itself last applied is what runs after a restart, not the kept declaration.
|
||||
func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
|
||||
t.Run("1957: an engine start reads the machine's facts", testAnEngineStartReconcilesAModuleFromTheMachinesFacts)
|
||||
t.Run("says the have again", testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines)
|
||||
// An apply that failed is not what runs, though its have names the want's build and digest: the agent starting on
|
||||
// it applies again (review of the switch, round 1).
|
||||
t.Run("applies again after an apply failure", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
|
||||
wk, wraw := wantFor(t, "avahi", fp, dig, "")
|
||||
b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
|
||||
Detail: "apply: unit failed to start"}}
|
||||
b.comps.before(ck, craw)
|
||||
b.wants.before(wk, wraw)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "a failed apply was not applied again", func() bool { return len(b.appliedList()) == 1 })
|
||||
})
|
||||
// Issue 1126, the stale marks after the seed's revert: a have a composer marked failed names what still runs; the
|
||||
// agent starting on it applies nothing and says it running, so a mark it did not make clears.
|
||||
t.Run("clears a composer's mark", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
|
||||
wk, wraw := wantFor(t, "avahi", fp, dig, "")
|
||||
b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
|
||||
Detail: NotComposedDetail + "frombus: the credentials of ace: context canceled"}}
|
||||
b.comps.before(ck, craw)
|
||||
b.wants.before(wk, wraw)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "the mark was not cleared", func() bool {
|
||||
h := b.haves.last("avahi")
|
||||
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp && h.Composed == dig && h.Detail == ""
|
||||
})
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a module whose mark names what the want names was applied: %v", got)
|
||||
}
|
||||
})
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
@@ -380,6 +432,7 @@ func testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing
|
||||
// A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
|
||||
// of either key.
|
||||
func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
|
||||
t.Run("1957: the judge's edge reconciles, and a failed re-apply is failed", testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone)
|
||||
// A real failure is not retried by the clock, however short the backoff: only by a change of either key.
|
||||
t.Run("a real failure", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
@@ -787,6 +840,37 @@ func TestTheNodesOwnApplyHoldsOutAModuleThatMovedToItsOwnComposition(t *testing.
|
||||
if !strings.Contains(strings.Join(names, ","), "systemd-resolved") {
|
||||
t.Fatalf("the others of the node's own apply were %v: systemd-resolved's resources would be removed", names)
|
||||
}
|
||||
|
||||
// Issue 1126, review of the apply, round 1: a module the composer holds alone (de-spiegel leaving the fold with its
|
||||
// names unset) has no composition of its own in the bucket: what runs of it in the node's own is held out of the
|
||||
// node's own apply until one is handed.
|
||||
b2 := newBed(t)
|
||||
spiegel := file("de-spiegel", "server-env", "A=1")
|
||||
var bare map[string]any
|
||||
_ = json.Unmarshal(spiegel, &bare)
|
||||
delete(bare, "module")
|
||||
spiegel, _ = json.Marshal(bare)
|
||||
b2.env.Kept = func() map[string][]json.RawMessage { return map[string][]json.RawMessage{NodeModule: {spiegel}} }
|
||||
nk2, nraw2, ndig2 := composition(t, NodeModule, "", false)
|
||||
nwraw2, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig2})
|
||||
b2.comps.before(nk2, nraw2)
|
||||
b2.wants.before(nwk, nwraw2)
|
||||
ctx2, cancel2 := context.WithCancel(context.Background())
|
||||
b2.ag = New(b2.env)
|
||||
b2.ag.NotComposedEach(ctx2, "", map[string]string{"de-spiegel": "de-spiegel says ${config:praktijk-name}, and nothing sets it"})
|
||||
done2 := make(chan struct{})
|
||||
go func() { defer close(done2); _ = b2.ag.Run(ctx2) }()
|
||||
defer func() { cancel2(); <-done2 }()
|
||||
eventually(t, "the node's own not applied beside a held module", func() bool { return len(b2.appliedList()) == 1 })
|
||||
b2.mu.Lock()
|
||||
defer b2.mu.Unlock()
|
||||
heldOut := false
|
||||
for _, c := range b2.others[0] {
|
||||
heldOut = heldOut || (c.Module == "de-spiegel" && len(c.Resources) == 1)
|
||||
}
|
||||
if !heldOut {
|
||||
t.Errorf("the others of the node's own apply were %+v: held de-spiegel's resources would be removed", b2.others[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A module the assigner removed holds nothing back, its compositions still in the bucket or not: its resources are its
|
||||
@@ -942,18 +1026,102 @@ func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC18 (unit): a module that still runs from the kept declaration carries the kept declaration's filled paths of
|
||||
// its files, so the filling writes them ("not named as filled" for claude-code and screen-lock, 2026-10-25).
|
||||
func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.kept = map[string][]json.RawMessage{"claude-code": {file("claude-code", "settings", "")}}
|
||||
b.env.KeptFilled = func() map[string][]string {
|
||||
return map[string][]string{"claude-code": {"/tmp/r6/claude-code/settings"}}
|
||||
// 1945/AC18 and issue 924 (unit): what runs of a module the agent has not taken yet carries the filled paths the
|
||||
// filling judges by. From the kept declaration while the module has no have naming a composition ("not named as filled"
|
||||
// for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept
|
||||
// declaration being older, read at the start before any module settles and told to the filling then, so a file the
|
||||
// newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29).
|
||||
// Only a have in a phase baseline takes as running names what runs; and what runs carries Data, from the composition
|
||||
// or, from the kept declaration, from the module's manifest, so a data module is not judged to move its data against
|
||||
// its first composition after a restart (mongodb and mssql on novox).
|
||||
func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) {
|
||||
const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound"
|
||||
filledFile := func(id string) json.RawMessage {
|
||||
return json.RawMessage(`{"id":"power.` + id + `","type":"file","module":"power","path":"/tmp/r6/power/` + id +
|
||||
`","mode":"0644","owner-at-runtime":"owned"}`)
|
||||
}
|
||||
a := New(b.env)
|
||||
got := a.Everything()
|
||||
if len(got) != 1 || got[0].Module != "claude-code" || strings.Join(got[0].Filled, ",") != "/tmp/r6/claude-code/settings" {
|
||||
t.Fatalf("what runs from the kept declaration: %+v", got)
|
||||
fp := fpOf("a")
|
||||
ck, _ := delivery.ComposedKey(node, "power", fp)
|
||||
applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, Data: true,
|
||||
Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}}
|
||||
_, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey)
|
||||
newer := applied
|
||||
newer.Resources = append([]json.RawMessage{}, applied.Resources...)
|
||||
newer.Resources = append(newer.Resources, file("power", "later", "1"))
|
||||
_, newerRaw, _ := signedBy(t, newer, ck, controllerKey)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
// have is the module's have; bucket the value its composition key holds now, history the revisions kept.
|
||||
have *Have
|
||||
bucket []byte
|
||||
history []Update
|
||||
filled, told string
|
||||
// data is what runs' Data; holdsData the manifest's answer (nil: none read).
|
||||
data bool
|
||||
holdsData func(context.Context, string) (bool, bool)
|
||||
}{
|
||||
{name: "no have: the kept declaration's", filled: old},
|
||||
{name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
||||
bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start", data: true},
|
||||
{name: "a have applying names nothing that runs yet", have: &Have{Module: "power", FP: fp, Composed: appliedDig,
|
||||
Phase: string(delivery.PhaseApplying)}, bucket: appliedRaw, filled: old, told: "the haves were read at the start"},
|
||||
{name: "a data module from the kept declaration says Data as its manifest does", filled: old,
|
||||
told: "the haves were read at the start", data: true,
|
||||
holdsData: func(context.Context, string) (bool, bool) { return true, true }},
|
||||
{name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
||||
bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}},
|
||||
filled: old + "," + added, told: "the haves were read at the start", data: true},
|
||||
{name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
||||
bucket: newerRaw, filled: old, told: "the haves were read at the start"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.kept = map[string][]json.RawMessage{"power": {filledFile("settings")}}
|
||||
b.env.KeptFilled = func() map[string][]string { return map[string][]string{"power": {old}} }
|
||||
if tc.have != nil {
|
||||
b.haves.stored = map[string]Have{"power": *tc.have}
|
||||
}
|
||||
if tc.bucket != nil {
|
||||
b.comps.before(ck, tc.bucket)
|
||||
}
|
||||
b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil }
|
||||
b.env.HoldsData = tc.holdsData
|
||||
var told []string
|
||||
var mu sync.Mutex
|
||||
b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() }
|
||||
filledNow := func() string {
|
||||
for _, c := range b.ag.Everything() {
|
||||
if c.Module == "power" {
|
||||
if c.Data != tc.data {
|
||||
t.Errorf("what runs says Data %v, want %v", c.Data, tc.data)
|
||||
}
|
||||
return strings.Join(c.Filled, ",")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if tc.told == "" { // no want, no have: nothing to start, and the kept declaration is what runs
|
||||
a := New(b.env)
|
||||
b.ag = a
|
||||
if got := filledNow(); got != tc.filled {
|
||||
t.Fatalf("filled %q, want %q", got, tc.filled)
|
||||
}
|
||||
return
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "the filling was not told what runs at the start", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(told) > 0 && told[0] == tc.told
|
||||
})
|
||||
if got := filledNow(); got != tc.filled {
|
||||
t.Fatalf("filled %q, want %q", got, tc.filled)
|
||||
}
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("applied %v: nothing has a want", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1014,39 +1182,89 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
||||
b.wants.put(nwk, nwraw)
|
||||
eventually(t, "the node's own key does not carry its sealed values", func() bool {
|
||||
h := b.haves.last(NodeModule)
|
||||
return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed[0] == "core"
|
||||
return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed["core-box#sealed"] == "core"
|
||||
})
|
||||
// The node cannot compose (a stand-in refused): each wanted module's have says so, failed, naming why, and still
|
||||
// names what runs; composed again, the have is what it runs. web was applied by this agent; idle was not since the
|
||||
// agent's start (the switch and every delivery of the node-engine restart it), so what runs is what its have said.
|
||||
idleFP, idleDig := fpOf("c"), "sha256:"+strings.Repeat("9", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "idle", FP: idleFP, Composed: idleDig, Phase: string(delivery.PhaseRunning),
|
||||
Sealed: []string{`{"box":"idle"}`}})
|
||||
Sealed: delivery.Seals{"idle.key#sealed": `{"box":"idle"}`}})
|
||||
want("idle", idleFP, "")
|
||||
eventually(t, "idle's want was not taken", func() bool {
|
||||
b.ag.mu.Lock()
|
||||
defer b.ag.mu.Unlock()
|
||||
return b.ag.wants["idle"].Present
|
||||
})
|
||||
if h := b.haves.last("web"); len(h.Sealed) != 1 || h.Sealed[0] != "1" {
|
||||
if h := b.haves.last("web"); len(h.Sealed) != 1 || h.Sealed["web.pw#sealed"] != "1" {
|
||||
t.Errorf("web's running have does not carry its sealed value: %+v", h)
|
||||
}
|
||||
b.ag.NotComposed(context.Background(), "a stand-in for web's password")
|
||||
for _, row := range []struct {
|
||||
module, fp, composed, sealed string
|
||||
}{{"web", fp, dig, "1"}, {"idle", idleFP, idleDig, `{"box":"idle"}`}} {
|
||||
module, fp, composed, slot, sealed string
|
||||
}{{"web", fp, dig, "web.pw#sealed", "1"}, {"idle", idleFP, idleDig, "idle.key#sealed", `{"box":"idle"}`}} {
|
||||
h := b.haves.last(row.module)
|
||||
if h.Phase != string(delivery.PhaseFailed) || h.FP != row.fp || h.Composed != row.composed ||
|
||||
len(h.Sealed) != 1 || h.Sealed[0] != row.sealed || !strings.Contains(h.Detail, "web's password") {
|
||||
len(h.Sealed) != 1 || h.Sealed[row.slot] != row.sealed || !strings.Contains(h.Detail, "web's password") {
|
||||
t.Errorf("%s's have while the node cannot compose: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
// A wanted module whose have cannot be read (the bus paused, issue 1126) is not marked: a mark would drop the build
|
||||
// and digest its have names.
|
||||
readAgain = time.Millisecond
|
||||
b.haves.mu.Lock()
|
||||
b.haves.unreadable = map[string]bool{"unread": true}
|
||||
b.haves.mu.Unlock()
|
||||
want("unread", fpOf("d"), "")
|
||||
want("fresh", fpOf("e"), "") // never ran, its have absent: marked, naming nothing it runs
|
||||
// A have that already says another cause keeps it (review of the switch, round 1): an apply that failed.
|
||||
brokenFP, brokenDig := fpOf("f"), "sha256:"+strings.Repeat("8", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "broken", FP: brokenFP, Composed: brokenDig, Phase: string(delivery.PhaseFailed),
|
||||
Detail: "apply: unit failed to start"})
|
||||
want("broken", brokenFP, "")
|
||||
eventually(t, "unread's want was not taken", func() bool {
|
||||
b.ag.mu.Lock()
|
||||
defer b.ag.mu.Unlock()
|
||||
return b.ag.wants["unread"].Present
|
||||
})
|
||||
b.ag.NotComposed(context.Background(), "a stand-in for web's password, again")
|
||||
if h := b.haves.last("unread"); h.Module != "" {
|
||||
t.Errorf("a module whose have could not be read was marked: %+v", h)
|
||||
}
|
||||
if h := b.haves.last("fresh"); h.Phase != string(delivery.PhaseFailed) || h.FP != "" {
|
||||
t.Errorf("a module that never ran, its have absent, is not marked: %+v", h)
|
||||
}
|
||||
if h := b.haves.last("broken"); h.Detail != "apply: unit failed to start" || h.Composed != brokenDig {
|
||||
t.Errorf("an apply failure was overwritten by a composer's mark: %+v", h)
|
||||
}
|
||||
// A run that ended marks nothing: what it would say is the next run's (review of the switch, round 1).
|
||||
ended, end := context.WithCancel(context.Background())
|
||||
end()
|
||||
before := len(b.haves.log)
|
||||
b.ag.NotComposed(ended, "a refusal from a run that ended")
|
||||
if len(b.haves.log) != before {
|
||||
t.Errorf("a run that ended wrote %d haves", len(b.haves.log)-before)
|
||||
}
|
||||
b.ag.NotComposed(context.Background(), "")
|
||||
for _, row := range []struct{ module, composed string }{{"web", dig}, {"idle", idleDig}} {
|
||||
if h := b.haves.last(row.module); h.Phase != string(delivery.PhaseRunning) || h.Composed != row.composed || len(h.Sealed) != 1 {
|
||||
t.Errorf("%s's have once the node composes again: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
// Issue 1126, the applying path: a module the node holds alone (its own refusal: de-spiegel's unset route names on
|
||||
// novox) is marked alone, naming its own words; web, which composes, keeps running, unmarked. Composed again, idle's
|
||||
// have is what it runs.
|
||||
b.ag.NotComposedEach(context.Background(), "", map[string]string{"idle": "idle says ${config:name}, and nothing sets it"})
|
||||
if h := b.haves.last("idle"); h.Phase != string(delivery.PhaseFailed) || h.Composed != idleDig || !strings.Contains(h.Detail, "${config:name}") {
|
||||
t.Errorf("idle's have while it alone is held: %+v", h)
|
||||
}
|
||||
if h := b.haves.last("web"); h.Phase != string(delivery.PhaseRunning) || h.Composed != dig {
|
||||
t.Errorf("web, which composes, is marked beside idle: %+v", h)
|
||||
}
|
||||
b.ag.NotComposedEach(context.Background(), "", nil)
|
||||
if h := b.haves.last("idle"); h.Phase != string(delivery.PhaseRunning) || h.Composed != idleDig {
|
||||
t.Errorf("idle's have once it composes again: %+v", h)
|
||||
}
|
||||
// A revision at the same build (a config value changed) is applied.
|
||||
_, craw2, dig2 := unsigned("web", fp, file("web", "x", "2"))
|
||||
b.comps.put(ck, craw2)
|
||||
@@ -1069,7 +1287,7 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
||||
// what runs (fingerprint, digest, sealed values) rather than naming nothing (review of L2, round 3).
|
||||
recFP, recDig := fpOf("d"), "sha256:"+strings.Repeat("8", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "rec", FP: recFP, Composed: recDig, Phase: string(delivery.PhaseRunning),
|
||||
Sealed: []string{`{"box":"rec"}`}})
|
||||
Sealed: delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}})
|
||||
rk, rraw, rdig := unsigned("rec", fpB, container("rec@sha256:"+strings.Repeat("3", 64)))
|
||||
b.comps.put(dk2, draw2)
|
||||
b.comps.put(rk, rraw)
|
||||
@@ -1078,15 +1296,15 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
||||
applied := len(b.appliedList())
|
||||
for _, row := range []struct {
|
||||
module, fp, composed, waits string
|
||||
sealed []string
|
||||
}{{"db", fp, "", ddig2, nil}, {"rec", recFP, recDig, rdig, []string{`{"box":"rec"}`}}} {
|
||||
sealed delivery.Seals
|
||||
}{{"db", fp, "", ddig2, nil}, {"rec", recFP, recDig, rdig, delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}}} {
|
||||
eventually(t, row.module+"'s move did not wait for a yes", func() bool {
|
||||
h := b.haves.last(row.module)
|
||||
return h.Phase == string(delivery.PhaseWaitingYes) && h.Waits == row.waits
|
||||
})
|
||||
h := b.haves.last(row.module)
|
||||
if h.FP != row.fp || (row.composed != "" && h.Composed != row.composed) || h.Composed == "" ||
|
||||
(row.sealed != nil && !slices.Equal(h.Sealed, row.sealed)) {
|
||||
(row.sealed != nil && !maps.Equal(h.Sealed, row.sealed)) {
|
||||
t.Errorf("%s's have waiting for a yes does not keep what runs: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
@@ -1101,3 +1319,122 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
||||
r.Phase == string(delivery.PhaseRunning) && r.FP == fpB && r.Composed == rdig
|
||||
})
|
||||
}
|
||||
|
||||
// 1957/AC1 and AC4 (unit; ADR 0568 §1, §4): an engine start takes a module from what the machine shows, not only from the
|
||||
// have or the kept declaration it kept. Every resource up applies nothing; one declared running and not active applies the
|
||||
// module once, from a have or from the kept declaration; a module with no want is not looked at.
|
||||
func testAnEngineStartReconcilesAModuleFromTheMachinesFacts(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
have bool // the engine's own have says running at the want
|
||||
held bool // else its have says held, from a failed backup
|
||||
kept bool // else the kept declaration says it runs
|
||||
inactive bool
|
||||
unwanted bool
|
||||
applied int
|
||||
asked bool // the machine was looked at
|
||||
}{
|
||||
{"all up: nothing is applied", true, false, false, false, false, 0, true},
|
||||
{"a unit not active: applied once", true, false, false, true, false, 1, true},
|
||||
{"kept declaration, a unit not active: applied once", false, false, true, true, false, 1, true},
|
||||
{"kept declaration, all up: nothing is applied", false, false, true, false, false, 0, true},
|
||||
{"a held module applies nothing at start", false, true, false, true, false, 0, false},
|
||||
{"no want: nothing is looked at or applied", true, false, false, true, true, 0, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
rs := []json.RawMessage{file("web", "x", "1")}
|
||||
ck, craw, dig := composition(t, "web", fp, false, rs...)
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.comps.before(ck, craw)
|
||||
if !c.unwanted {
|
||||
b.wants.before(wk, wraw)
|
||||
}
|
||||
if c.have {
|
||||
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
|
||||
}
|
||||
if c.held {
|
||||
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64),
|
||||
Phase: string(delivery.PhaseHeld), Detail: "restic: the repository is locked"}}
|
||||
}
|
||||
if c.kept {
|
||||
b.kept = map[string][]json.RawMessage{"web": rs}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
asked := false
|
||||
b.env.Inactive = func(context.Context, delivery.Composed) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
asked = true
|
||||
return c.inactive
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
if !c.unwanted && !c.held {
|
||||
eventually(t, "the module was not settled", func() bool { return b.haves.last("web").Composed == dig })
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != c.applied {
|
||||
t.Fatalf("applied %v, want %d", got, c.applied)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if asked != c.asked {
|
||||
t.Fatalf("the machine was asked: %v, want %v", asked, c.asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1957/AC2, AC3 and AC4 (unit; ADR 0568 §2, §3, §4): the judge's edge, Reconcile, applies a running module again; one held
|
||||
// for a failed backup and one with no want are left alone; and an apply that fails writes the have `failed` with the
|
||||
// apply's own words.
|
||||
func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ckW, crawW, digW := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
wkW, wrawW := wantFor(t, "web", fp, digW, "")
|
||||
ckD, crawD, digD := composition(t, "db", fp, true, file("db", "x", "1"))
|
||||
wkD, wrawD := wantFor(t, "db", fp, digD, "")
|
||||
b.comps.before(ckW, crawW)
|
||||
b.wants.before(wkW, wrawW)
|
||||
b.comps.before(ckD, crawD)
|
||||
b.wants.before(wkD, wrawD)
|
||||
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: digW, Phase: "running"},
|
||||
"db": {Module: "db", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64), Phase: "running"}}
|
||||
b.backup = func(string) Backup { return Backup{Words: "restic: the repository is locked"} }
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "web was not settled", func() bool { return b.haves.last("web").Composed == digW })
|
||||
eventually(t, "db was not held", func() bool { return b.haves.last("db").Phase == string(delivery.PhaseHeld) })
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("applied at the start: %v", got)
|
||||
}
|
||||
|
||||
b.ag.Reconcile("db", "ghost")
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a held module and an unwanted one were applied: %v", got)
|
||||
}
|
||||
if h := b.haves.last("db"); h.Phase != string(delivery.PhaseHeld) {
|
||||
t.Fatalf("the held module's have changed: %+v", h)
|
||||
}
|
||||
|
||||
b.ag.Reconcile("web")
|
||||
eventually(t, "web was not applied again", func() bool { return len(b.appliedList()) == 1 })
|
||||
eventually(t, "web is not running again", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
||||
|
||||
b.mu.Lock()
|
||||
b.failWith = "systemctl start web.service: exit status 1"
|
||||
b.mu.Unlock()
|
||||
b.ag.Reconcile("web")
|
||||
eventually(t, "the failed re-apply did not write the have failed with its words", func() bool {
|
||||
h := b.haves.last("web")
|
||||
return h.Phase == string(delivery.PhaseFailed) && h.Detail == "systemctl start web.service: exit status 1"
|
||||
})
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 2 {
|
||||
t.Fatalf("a failed apply was tried again with no edge: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
+72
-2
@@ -1223,6 +1223,19 @@ func applyScoped(
|
||||
}
|
||||
}
|
||||
|
||||
// A file written or removed in a unit directory is read by the service manager only after a reload, whether or not
|
||||
// a service reflects it: a drop-in on reboot.target is reflected by no service, and without the reload the next
|
||||
// reboot still runs the target as it was loaded (issue 1126). Once per apply.
|
||||
if at := unitFileMoved(report.Outcomes); at != "" {
|
||||
if u, ok := sys.(unitReloader); ok {
|
||||
if err := u.ReloadUnits(ctx, run); err != nil {
|
||||
failures = append(failures, &Error{Resource: at, Err: fmt.Errorf("reloading the service manager's units: %w", err)})
|
||||
} else {
|
||||
log(fmt.Sprintf(" reloaded the service manager's units: %s changed", at))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder.
|
||||
if origin == store.OriginDeclared && scope.isNode() {
|
||||
// And an identity, then a library group, no longer declared goes, once what ran as it went above (ADR 0328
|
||||
@@ -1997,11 +2010,68 @@ type serviceReloader interface {
|
||||
ReloadService(ctx context.Context, run system.Runner, unit string) error
|
||||
}
|
||||
|
||||
// UnitDirs are the system service manager's unit directories: a file the apply writes or removes in one of them, or in a
|
||||
// drop-in directory below one, is read by the manager only after a reload (issue 1126). Replaceable in tests.
|
||||
var UnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/usr/local/lib/systemd/system"}
|
||||
|
||||
// unitFileMoved is the first file this apply wrote or removed in a unit directory or a drop-in directory below one,
|
||||
// or "" when it moved none.
|
||||
func unitFileMoved(outcomes []Outcome) string {
|
||||
for _, o := range outcomes {
|
||||
// Only what changed the file on disk, as MovesRoot reads it: skipped, held, kept, forgotten and waiting come
|
||||
// back on every apply and change nothing.
|
||||
if declaration.Type(o.Type) != declaration.TypeFile {
|
||||
continue
|
||||
}
|
||||
switch o.Action {
|
||||
case "created", "updated", "corrected", "removed":
|
||||
default:
|
||||
continue
|
||||
}
|
||||
at := filepath.Clean(o.Target)
|
||||
for _, dir := range UnitDirs {
|
||||
if rel, err := filepath.Rel(dir, at); err == nil && rel != "." && !strings.HasPrefix(rel, "..") {
|
||||
return at
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unitReloader is a service manager that caches unit files and must be told to read them again.
|
||||
type unitReloader interface {
|
||||
ReloadUnits(ctx context.Context, run system.Runner) error
|
||||
}
|
||||
|
||||
// restReader is a service manager that can say whether a one-shot has nothing running now
|
||||
// (novox/hq issue 1126).
|
||||
type restReader interface {
|
||||
ServiceAtRest(ctx context.Context, run system.Runner, unit string) (bool, error)
|
||||
}
|
||||
|
||||
// declaredState is a unit's state as a declaration of want reads it. **A one-shot at rest is
|
||||
// stopped to a declaration that it be stopped** (novox/hq issue 1126): a one-shot that does not
|
||||
// linger reads "running" once it ran and worked — that is what a declaration that it run asks — but
|
||||
// it has no process between its runs, so it is also as stopped as it can be, and stopping it changes
|
||||
// nothing. Read only as "running", a boot-time cleanup declared stopped and enabled failed every
|
||||
// apply after the boot that ran it: "was asked to be stopped and is running".
|
||||
func declaredState(ctx context.Context, sys system.System, run Runner, unit, want string) (string, error) {
|
||||
state, err := sys.ServiceState(ctx, run, unit)
|
||||
if err != nil || want != "stopped" || state != "running" {
|
||||
return state, err
|
||||
}
|
||||
if r, ok := sys.(restReader); ok {
|
||||
rest, err := r.ServiceAtRest(ctx, run, unit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rest {
|
||||
return "stopped", nil
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// serviceSettle is how long the host waits before looking at a unit a second time. A test sets it
|
||||
// to nothing; on a machine it is the window in which a daemon that refuses its configuration dies.
|
||||
var serviceSettle = 2 * time.Second
|
||||
@@ -2154,7 +2224,7 @@ func applyServiceIn(ctx context.Context, sys system.System, r *declaration.Servi
|
||||
// own stopping from anybody else's (novox/hq ADR 0269).
|
||||
out.declared = &store.FoundUnit{Unit: r.Unit, State: r.State, Boot: r.Boot}
|
||||
|
||||
before, err := sys.ServiceState(ctx, run, r.Unit)
|
||||
before, err := declaredState(ctx, sys, run, r.Unit, r.State)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -2165,7 +2235,7 @@ func applyServiceIn(ctx context.Context, sys system.System, r *declaration.Servi
|
||||
|
||||
// Read back. A service manager accepting a command says the transaction was accepted,
|
||||
// not that the unit is running — one that starts and immediately dies satisfies it.
|
||||
after, err := sys.ServiceState(ctx, run, r.Unit)
|
||||
after, err := declaredState(ctx, sys, run, r.Unit, r.State)
|
||||
if err == nil && r.State == "running" {
|
||||
after, err = stayedRunning(ctx, sys, run, r.Unit)
|
||||
}
|
||||
|
||||
@@ -350,6 +350,42 @@ func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A boot-time one-shot a package ships, declared stopped and enabled at boot, is satisfied while
|
||||
// nothing of it runs, whatever its last run did (novox/hq issue 1126). The kernel module's
|
||||
// linux-modules-cleanup.service failed on ace with "was asked to be stopped and is running": an
|
||||
// inactive one-shot with ExecMainStatus=0 reads "running", and stopping it changes nothing.
|
||||
func TestABootOneShotDeclaredStoppedIsSatisfiedAtRest(t *testing.T) {
|
||||
for _, c := range []struct{ name, status string }{
|
||||
{"exited 0: never ran, ran at boot, or unloaded", "0"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var asked []string
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
args = oldShape(name, args)
|
||||
switch args[0] {
|
||||
case "show":
|
||||
return "LoadState=loaded\nActiveState=inactive\nType=oneshot\nRemainAfterExit=no\n" +
|
||||
"ExecMainStatus=" + c.status + "\n", nil
|
||||
case "is-enabled":
|
||||
return "enabled\n", nil
|
||||
}
|
||||
asked = append(asked, strings.Join(args, " "))
|
||||
return "", nil
|
||||
}
|
||||
d := parse(t, `{"declaration":1,"resources":[
|
||||
{"id":"cleanup","type":"service","unit":"linux-modules-cleanup.service","state":"stopped","boot":"enabled"}
|
||||
]}`)
|
||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a one-shot at rest declared stopped failed: %v", err)
|
||||
}
|
||||
if len(asked) > 0 {
|
||||
t.Errorf("the service manager was asked %q for a unit already as declared", asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestADroppedServiceIsGivenBackTheStateItWasFoundIn(t *testing.T) {
|
||||
// novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed, and
|
||||
// leaves what was the machine's. A service resource never installs a unit — so undeclaring it
|
||||
@@ -1225,6 +1261,52 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
|
||||
if reloaded < 0 || reloaded > stop {
|
||||
t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands)
|
||||
}
|
||||
|
||||
// A drop-in in a unit directory that no service reflects — the power module's bound on reboot.target
|
||||
// (issue 1126) — is read by the service manager only after a reload: one per apply, and no restart.
|
||||
// Unchanged, it reloads nothing.
|
||||
units := filepath.Join(dir, "system")
|
||||
saved := UnitDirs
|
||||
UnitDirs = []string{units}
|
||||
t.Cleanup(func() { UnitDirs = saved })
|
||||
dropIn := filepath.Join(units, "reboot.target.d", "mesh-power.conf")
|
||||
withDropIn := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
|
||||
{"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"},
|
||||
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]},
|
||||
{"id":"units","type":"directory","path":%q,"mode":"0755"},
|
||||
{"id":"drop-ins","type":"directory","path":%q,"mode":"0755"},
|
||||
{"id":"bound","type":"file","path":%q,"content":"[Unit]\nJobTimeoutSec=10min\n","mode":"0644"},
|
||||
{"id":"bound-too","type":"file","path":%q,"content":"[Unit]\nJobTimeoutSec=10min\n","mode":"0644"}
|
||||
]}`, path, units, filepath.Dir(dropIn), dropIn, filepath.Join(filepath.Dir(dropIn), "other.conf")))
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
reloads int
|
||||
}{{"a drop-in written", 1}, {"the same drop-in again", 0}} {
|
||||
commands = nil
|
||||
if _, state, err = Apply(context.Background(), archHost(t), withDropIn, state,
|
||||
store.OriginCarried, run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloads := 0
|
||||
for _, c := range commands {
|
||||
if strings.Contains(c, "daemon-reload") {
|
||||
reloads++
|
||||
}
|
||||
if strings.Contains(c, "stop thing.service") {
|
||||
t.Errorf("%s: a drop-in no service reflects restarted one: %v", row.name, commands)
|
||||
}
|
||||
}
|
||||
if reloads != row.reloads {
|
||||
t.Errorf("%s: %d reloads, want %d; commands were %v", row.name, reloads, row.reloads, commands)
|
||||
}
|
||||
}
|
||||
// A drop-in whose module is skipped behind its step, or held as found, comes back so on every apply: it moved
|
||||
// nothing, and reloads nothing.
|
||||
for _, action := range []string{"skipped", "held"} {
|
||||
if at := unitFileMoved([]Outcome{{ID: "bound", Type: "file", Target: dropIn, Action: action}}); at != "" {
|
||||
t.Errorf("a drop-in %s reloads the service manager (%s)", action, at)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -290,3 +290,7 @@ func unpack(body []byte, into string) (int, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch is an archive at source as the host fetches every archive it unpacks, at most maxArchive bytes: what the node's
|
||||
// own composer reads a moved module's templates archive with (issue 1126). Its digest is the caller's to check.
|
||||
func Fetch(ctx context.Context, source string) ([]byte, error) { return fetch(ctx, source) }
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one
|
||||
// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for
|
||||
// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the
|
||||
// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account
|
||||
// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back.
|
||||
//
|
||||
// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and
|
||||
// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy.
|
||||
type ObjectKeep struct {
|
||||
Dir string
|
||||
// Now is the clock; nil is time.Now.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// KeptObjectsFor is how long an object of the keep is held after it was last used.
|
||||
const KeptObjectsFor = 14 * 24 * time.Hour
|
||||
|
||||
// KeepObjectsIn is the keep at dir; an empty dir keeps nothing.
|
||||
func KeepObjectsIn(dir string) *ObjectKeep {
|
||||
if dir == "" {
|
||||
return nil
|
||||
}
|
||||
return &ObjectKeep{Dir: dir}
|
||||
}
|
||||
|
||||
func (k *ObjectKeep) now() time.Time {
|
||||
if k.Now != nil {
|
||||
return k.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// get is the kept content of the object name when it is what f names, and marks it used.
|
||||
func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) {
|
||||
if k == nil {
|
||||
return nil, false
|
||||
}
|
||||
path := filepath.Join(k.Dir, name)
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil {
|
||||
return nil, false
|
||||
}
|
||||
now := k.now()
|
||||
_ = os.Chtimes(path, now, now)
|
||||
return body, true
|
||||
}
|
||||
|
||||
// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long.
|
||||
func (k *ObjectKeep) put(name string, body []byte) {
|
||||
if k == nil {
|
||||
return
|
||||
}
|
||||
path := filepath.Join(k.Dir, name)
|
||||
now := k.now()
|
||||
if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) {
|
||||
_ = os.Chtimes(path, now, now)
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(k.Dir, 0o700); err != nil {
|
||||
return
|
||||
}
|
||||
tmp, err := os.CreateTemp(k.Dir, ".put-*")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_, werr := tmp.Write(body)
|
||||
cerr := tmp.Close()
|
||||
if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil {
|
||||
_ = os.Remove(tmp.Name())
|
||||
return
|
||||
}
|
||||
_ = os.Chtimes(path, now, now)
|
||||
k.prune(now)
|
||||
}
|
||||
|
||||
// prune lets go of every object unused for KeptObjectsFor.
|
||||
func (k *ObjectKeep) prune(now time.Time) {
|
||||
entries, err := os.ReadDir(k.Dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
continue
|
||||
}
|
||||
if now.Sub(info.ModTime()) >= KeptObjectsFor {
|
||||
_ = os.Remove(filepath.Join(k.Dir, e.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own
|
||||
// keep did not hold the object either.
|
||||
func (k *ObjectKeep) never() string {
|
||||
if k == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir)
|
||||
}
|
||||
@@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er
|
||||
// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
|
||||
// once however many files name its digest.
|
||||
func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
|
||||
return ResolveFileReferencesKept(ctx, d, fetch, nil)
|
||||
}
|
||||
|
||||
// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had
|
||||
// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node
|
||||
// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the
|
||||
// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while
|
||||
// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its
|
||||
// own names objects the controller no longer knows of). A nil keep keeps nothing.
|
||||
func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error {
|
||||
fetched := map[string][]byte{}
|
||||
for i, r := range d.Resources {
|
||||
f, ok := r.(*declaration.File)
|
||||
if !ok || !f.IsReference() {
|
||||
continue
|
||||
}
|
||||
body, err := referencedContent(ctx, f, fetch, fetched)
|
||||
body, err := referencedContent(ctx, f, fetch, fetched, keep)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc
|
||||
}
|
||||
|
||||
func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
|
||||
fetched map[string][]byte) ([]byte, error) {
|
||||
fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) {
|
||||
if body, ok := fetched[f.Digest]; ok {
|
||||
return body, nil
|
||||
}
|
||||
if body, ok := alreadyThere(f); ok {
|
||||
fetched[f.Digest] = body
|
||||
return body, nil
|
||||
}
|
||||
name, err := declaration.ObjectName(f.Digest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
|
||||
}
|
||||
if body, ok := alreadyThere(f); ok {
|
||||
fetched[f.Digest] = body
|
||||
keep.put(name, body)
|
||||
return body, nil
|
||||
}
|
||||
if body, ok := keep.get(name, f); ok {
|
||||
fetched[f.Digest] = body
|
||||
return body, nil
|
||||
}
|
||||
if fetch == nil {
|
||||
return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
|
||||
"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
|
||||
}
|
||||
body, err := fetch(ctx, name, f.Size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w",
|
||||
f.ID, f.Path, name, err)
|
||||
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w",
|
||||
f.ID, f.Path, name, keep.never(), err)
|
||||
}
|
||||
if err := f.Verify(body); err != nil {
|
||||
return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
|
||||
}
|
||||
fetched[f.Digest] = body
|
||||
keep.put(name, body)
|
||||
return body, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,10 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a
|
||||
@@ -68,6 +72,112 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
|
||||
if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" {
|
||||
t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given)
|
||||
}
|
||||
// 774 (ADR 0563 §3): a hand-over asked through the node seat is the engine's own ask, performed only on the
|
||||
// operator's warrant to it: never on the call, not on a warrant for another digest, once on a warrant heard twice,
|
||||
// and not when the disk changed since it was asked.
|
||||
t.Run("the node seat's hand-over acts only on the operator's warrant", func(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
warrant func(asked asks.Ask, w asks.Warrant) asks.Warrant
|
||||
diskAfter string
|
||||
twice bool
|
||||
done int
|
||||
state string
|
||||
}{
|
||||
{name: "no warrant", done: 0, state: HandAsked},
|
||||
{name: "the operator's yes", warrant: yes, done: 1, state: HandDone},
|
||||
{name: "the yes heard twice", warrant: yes, twice: true, done: 1, state: HandDone},
|
||||
{name: "a warrant for another digest", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
w = yes(a, w)
|
||||
w.AskDigest = "sha256:another"
|
||||
return w
|
||||
}, done: 0, state: HandAsked},
|
||||
{name: "a no", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
w = yes(a, w)
|
||||
w.Option, w.Label, w.Level = "no", "No, leave it", asks.Acknowledge
|
||||
return w
|
||||
}, done: 0, state: HandRefused},
|
||||
{name: "the disk changed", warrant: yes, diskAfter: "0:0 700 9:2", done: 0, state: HandRefused},
|
||||
{name: "expired", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
return asks.Warrant{Ask: a.ID, Asker: w.Asker, Outcome: asks.OutcomeExpired, At: w.At}
|
||||
}, done: 0, state: HandRefused},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
h, asked, done := anEngine(t)
|
||||
disk := "1000:1000 755 9:1"
|
||||
h.JudgeHandOver = func(path string) (Judged, error) {
|
||||
return Judged{Subject: path, Path: path, Sets: "owner jo, group jo, mode 0750", Disk: disk,
|
||||
Words: "A directory used as found."}, nil
|
||||
}
|
||||
if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "shanks.node-tools",
|
||||
Args: map[string]any{"path": "/srv/x", "why": "a test"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 1 || *done != 0 {
|
||||
t.Fatalf("the call asked %d and performed %d", len(*asked), *done)
|
||||
}
|
||||
// The act is one key, and the open ask one key made by a create (ADR 0563 §3).
|
||||
if acts, _ := h.State.Keys("act."); len(acts) != 1 || acts[0] != "act."+(*asked)[0].ID {
|
||||
t.Fatalf("the act is kept as %v", acts)
|
||||
}
|
||||
if open, _ := h.State.Keys("open."); len(open) != 1 || !strings.HasPrefix(open[0], "open.hand-over.") {
|
||||
t.Fatalf("the open ask is kept as %v", open)
|
||||
}
|
||||
if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "x", Args: map[string]any{"path": "/srv/x"}}); err == nil {
|
||||
t.Fatal("a second ask of the same path was asked while one is open")
|
||||
}
|
||||
if row.diskAfter != "" {
|
||||
disk = row.diskAfter
|
||||
}
|
||||
if row.warrant != nil {
|
||||
w := row.warrant((*asked)[0], asks.Warrant{Asker: asks.EngineAsker("n1"), At: time.Now()})
|
||||
h.Heard(w)
|
||||
if row.twice {
|
||||
h.Heard(w)
|
||||
}
|
||||
}
|
||||
got, _ := h.Acts(0)
|
||||
acts := got.(map[string]any)["acts"].([]HandAct)
|
||||
if *done != row.done || len(acts) != 1 || acts[0].State != row.state {
|
||||
t.Fatalf("performed %d (want %d), acts %+v (want %s)", *done, row.done, acts, row.state)
|
||||
}
|
||||
if open, _ := h.State.Keys("open."); (len(open) == 1) != (row.state == HandAsked) {
|
||||
t.Fatalf("an ask %s leaves its open key as %v", row.state, open)
|
||||
}
|
||||
if row.state == HandDone && (acts[0].By != "operator" || acts[0].Via == "" || acts[0].Ask == "") {
|
||||
t.Fatalf("the act does not record the warrant: %+v", acts[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// yes is the operator's yes to an ask, as the router writes it.
|
||||
func yes(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||
w.Ask, w.Outcome, w.Option, w.Label, w.Level = a.ID, asks.OutcomeChosen, "yes", "Yes", asks.Approve
|
||||
w.By = &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"}
|
||||
w.AskDigest, w.Proofs = a.Digest(), []string{"P1"}
|
||||
return w
|
||||
}
|
||||
|
||||
// anEngine is a node seat with its own state in a temporary folder, a router that keeps what it was asked, and a
|
||||
// hand-over that counts what it performed.
|
||||
func anEngine(t *testing.T) (*HandActs, *[]asks.Ask, *int) {
|
||||
t.Helper()
|
||||
var asked []asks.Ask
|
||||
done := 0
|
||||
state := &MemState{}
|
||||
h := &HandActs{Node: "n1", State: state,
|
||||
HandOver: func(path, by string) (string, error) { done++; return path + " handed over by " + by, nil }}
|
||||
h.Asker = &asker.Client{Module: asks.EngineAsker("n1"), State: state,
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
if a, ok := body.(asks.Ask); ok && subject == asks.AskSubject(asks.EngineAsker("n1")) {
|
||||
asked = append(asked, a)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Record: func(string) (*stdio.StateEntry, error) { return nil, nil }}
|
||||
return h, &asked, &done
|
||||
}
|
||||
|
||||
// The controller names an access by its declared resource id exactly, which may hold a `/`: it reaches the act,
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
)
|
||||
|
||||
// The node seat node-engine, served on the engine's own link (novox/hq ADR 0563 §1).
|
||||
//
|
||||
// Each verb is answered on this node's subject alone, `mesh.seat.node-engine.tool.<verb>.<node>`, and on the same
|
||||
// address naming its caller, `mesh.seat.node-engine.call.<verb>.<node>.<caller>`, whose last token the bus lets each user
|
||||
// publish with its own name alone (novox/hq issue 365): the caller is that token, and a call on the subject naming none is
|
||||
// from an unproven caller, recorded so. The body is the call's arguments; the answer is {result, error, node}, as every
|
||||
// tool's (mesh-tools node-tools). One service, `node-engine`, answers discovery with each verb's seat, tool and node.
|
||||
|
||||
// EngineSeat is the seat.
|
||||
const EngineSeat = catalogue.EngineSeat
|
||||
|
||||
// EngineToolSubject is where a verb of this node's seat is asked naming no caller.
|
||||
func EngineToolSubject(verb, node string) string {
|
||||
return "mesh.seat." + EngineSeat + ".tool." + verb + "." + node
|
||||
}
|
||||
|
||||
// EngineCallPattern is where a verb of this node's seat is asked naming its caller, any caller.
|
||||
func EngineCallPattern(verb, node string) string {
|
||||
return "mesh.seat." + EngineSeat + ".call." + verb + "." + node + ".*"
|
||||
}
|
||||
|
||||
// EngineEventSubject is where the seat says one of its events about this node.
|
||||
func EngineEventSubject(event, node string) string {
|
||||
return "mesh.seat." + EngineSeat + ".event." + event + "." + node
|
||||
}
|
||||
|
||||
// EngineCall is one call of the seat: its verb, its caller as the bus proved it, and its arguments.
|
||||
type EngineCall struct {
|
||||
Verb string
|
||||
Caller string
|
||||
Args map[string]any
|
||||
}
|
||||
|
||||
// EngineFunc answers one call: its result, or why it is refused.
|
||||
type EngineFunc func(EngineCall) (any, error)
|
||||
|
||||
// CallerOfSubject is the caller a call's subject names for this node's verb: the last token of a caller-named subject,
|
||||
// each `~` a dot, or stdio.Unproven on the subject that names none or on any other.
|
||||
func CallerOfSubject(subject, verb, node string) string {
|
||||
token, ok := strings.CutPrefix(subject, "mesh.seat."+EngineSeat+".call."+verb+"."+node+".")
|
||||
if !ok || token == "" || strings.ContainsAny(token, ".*> ") {
|
||||
return stdio.Unproven
|
||||
}
|
||||
return strings.ReplaceAll(token, "~", ".")
|
||||
}
|
||||
|
||||
// MaxEngineCall bounds a call's arguments: a path, an access and a few words.
|
||||
const MaxEngineCall = 16 << 10
|
||||
|
||||
// AnswerEngineCall is the envelope one call is answered with: the verb's result, or its refusal in words.
|
||||
func AnswerEngineCall(handle EngineFunc, verb, node, subject string, body []byte) []byte {
|
||||
envelope := map[string]any{"node": node}
|
||||
say := func() []byte {
|
||||
raw, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
raw, _ = json.Marshal(map[string]any{"node": node, "error": "the answer could not be written: " + err.Error()})
|
||||
}
|
||||
return raw
|
||||
}
|
||||
args := map[string]any{}
|
||||
switch {
|
||||
case handle == nil:
|
||||
envelope["error"] = "this engine serves no verb of " + EngineSeat
|
||||
return say()
|
||||
case len(body) > MaxEngineCall:
|
||||
envelope["error"] = fmt.Sprintf("the call is %d bytes, more than a call of %s takes", len(body), EngineSeat)
|
||||
return say()
|
||||
case len(strings.TrimSpace(string(body))) > 0:
|
||||
if err := json.Unmarshal(body, &args); err != nil {
|
||||
envelope["error"] = "the call's arguments are not an object: " + err.Error()
|
||||
return say()
|
||||
}
|
||||
}
|
||||
result, err := handle(EngineCall{Verb: verb, Caller: CallerOfSubject(subject, verb, node), Args: args})
|
||||
if err != nil {
|
||||
envelope["error"] = err.Error()
|
||||
} else {
|
||||
envelope["result"] = result
|
||||
}
|
||||
return say()
|
||||
}
|
||||
|
||||
// EngineSeats is a link that serves this node's seat. Optional, as Asked is.
|
||||
type EngineSeats interface {
|
||||
ServeEngineSeat(EngineFunc)
|
||||
}
|
||||
|
||||
// ServeEngineSeat answers every verb of the seat on this node's subjects, as the service node-engine, which says each
|
||||
// verb's seat, tool and node to discovery (novox/hq ADR 0197). A bus whose user list is older than the grant refuses the
|
||||
// subscriptions, said by the client; a caller then hears nobody answer.
|
||||
func (l *natsLink) ServeEngineSeat(handle EngineFunc) {
|
||||
if handle == nil || l.conn == nil {
|
||||
return
|
||||
}
|
||||
svc, err := micro.AddService(l.conn, micro.Config{Name: EngineSeat, Version: "1.0.0",
|
||||
Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, v := range catalogue.EngineVerbs() {
|
||||
verb := v.Name
|
||||
answer := micro.HandlerFunc(func(r micro.Request) {
|
||||
_ = r.Respond(AnswerEngineCall(handle, verb, l.node, r.Subject(), r.Data()))
|
||||
})
|
||||
meta := map[string]string{"seat": EngineSeat, "tool": verb, "node": l.node}
|
||||
_ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)),
|
||||
micro.WithEndpointMetadata(meta))
|
||||
_ = svc.AddEndpoint(verb+"-called", answer, micro.WithEndpointSubject(EngineCallPattern(verb, l.node)),
|
||||
micro.WithEndpointMetadata(meta))
|
||||
}
|
||||
l.engine = svc
|
||||
}
|
||||
|
||||
// SayEngine says one of the seat's events about this node, on the events stream.
|
||||
func (l *natsLink) SayEngine(event string, body any) error {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = l.js.Publish(EngineEventSubject(event, l.node), raw, nats.Context(ctx))
|
||||
return err
|
||||
}
|
||||
|
||||
// The engine asks the operator under its own asker name (asks.EngineAsker) on the router's seat (ADR 0259 §3, as ADR
|
||||
// 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and
|
||||
// its asks' state read from the router's records under its own name.
|
||||
|
||||
// RouterRecords is the router's bucket of asks, which each asker reads under its own name: the state `asks` of the
|
||||
// operator-channel's holder, the messenger (mesh-catalog modules/messenger).
|
||||
const RouterRecords = "messenger_asks"
|
||||
|
||||
// OperatorAsks is a link that asks the operator as this node's engine. Optional.
|
||||
type OperatorAsks interface {
|
||||
PublishAsk(subject string, body any, id string) error
|
||||
RouterRecord(key string) (*stdio.StateEntry, error)
|
||||
HearWarrants(asker string, heard func(asks.Warrant)) error
|
||||
}
|
||||
|
||||
// PublishAsk publishes an ask or a cancel on the router's seat, once per id within the stream's duplicate window.
|
||||
func (l *natsLink) PublishAsk(subject string, body any, id string) error {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = l.js.Publish(subject, raw, nats.MsgId(id), nats.Context(ctx))
|
||||
return err
|
||||
}
|
||||
|
||||
// RouterRecord reads one key of the router's records, `<asker>.<id>`; nil when it holds none.
|
||||
func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+RouterRecords+".$KV."+RouterRecords+"."+key, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status := msg.Header.Get("Status"); status == "404" {
|
||||
return nil, nil
|
||||
} else if status != "" {
|
||||
return nil, fmt.Errorf("the router's record %s could not be read: %s %s", key, status, msg.Header.Get("Description"))
|
||||
}
|
||||
return &stdio.StateEntry{Key: key, Value: msg.Data}, nil
|
||||
}
|
||||
|
||||
// HearWarrants hears the warrants for asker on its decided subject, each handed to heard as it arrives.
|
||||
func (l *natsLink) HearWarrants(asker string, heard func(asks.Warrant)) error {
|
||||
sub, err := l.conn.Subscribe(asks.DecidedSubject(asker), func(msg *nats.Msg) {
|
||||
var w asks.Warrant
|
||||
if json.Unmarshal(msg.Data, &w) == nil {
|
||||
heard(w)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.warrants = sub
|
||||
return nil
|
||||
}
|
||||
|
||||
// errNoAsking is a link that cannot ask the operator.
|
||||
var errNoAsking = errors.New("the link open now cannot ask the operator")
|
||||
|
||||
// discoveryQuiet is how long the engine waits after the last answer to the bus's discovery, and discoveryMost the
|
||||
// longest it waits in all.
|
||||
const (
|
||||
discoveryQuiet = 300 * time.Millisecond
|
||||
discoveryMost = 2 * time.Second
|
||||
)
|
||||
|
||||
// HeardServed asks the bus's discovery (novox/hq ADR 0197) whether a seat's verb is answered on this machine: an
|
||||
// endpoint naming the seat, the verb as its tool and this node.
|
||||
func (l *natsLink) HeardServed(seat, verb string) (bool, error) {
|
||||
inbox := l.conn.NewRespInbox()
|
||||
sub, err := l.conn.SubscribeSync(inbox)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if err := l.conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||
return false, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||
}
|
||||
heard := false
|
||||
deadline := time.Now().Add(discoveryMost)
|
||||
for time.Now().Before(deadline) {
|
||||
msg, err := sub.NextMsg(discoveryQuiet)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
var info micro.Info
|
||||
if json.Unmarshal(msg.Data, &info) != nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
node := e.Metadata["node"]
|
||||
if node == "" {
|
||||
node = info.ID
|
||||
}
|
||||
if e.Metadata["seat"] == seat && e.Metadata["tool"] == verb && node == l.node {
|
||||
heard = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return heard, nil
|
||||
}
|
||||
|
||||
// HeardServed is the link open now asking the bus's discovery whether a seat's verb is answered on this machine.
|
||||
func (q *Session) HeardServed(seat, verb string) (bool, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
asker, ok := bus.(interface {
|
||||
HeardServed(string, string) (bool, error)
|
||||
})
|
||||
if bus == nil || !ok {
|
||||
return false, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
|
||||
}
|
||||
return asker.HeardServed(seat, verb)
|
||||
}
|
||||
|
||||
// ---- the engine's acts in the bus's own storage (ADR 0563 §2, §3) -------------------------------------------------
|
||||
|
||||
// actsStore is this node's keys of ActsBucket over the link open now: bound, never created (the grants part creates
|
||||
// it, as the node-engine's declared state).
|
||||
func (l *natsLink) actsStore() (nats.KeyValue, error) {
|
||||
kv, err := l.js.KeyValue(ActsBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+
|
||||
"manifest; asked again once it is): %w", ActsBucket, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
|
||||
// BusState is the engine's ActStore: its own node's keys of ActsBucket, `<node>.<key>`, over whichever link the session
|
||||
// has open; with none open every read and write is refused, and nothing is kept anywhere else.
|
||||
type BusState struct {
|
||||
Session *Session
|
||||
Node string
|
||||
}
|
||||
|
||||
func (b BusState) kv() (nats.KeyValue, error) {
|
||||
b.Session.init()
|
||||
b.Session.mu.Lock()
|
||||
bus := b.Session.bus
|
||||
b.Session.mu.Unlock()
|
||||
l, ok := bus.(interface{ actsStore() (nats.KeyValue, error) })
|
||||
if bus == nil || !ok {
|
||||
return nil, fmt.Errorf("%w: this machine has no link to the bus open now, where its acts are kept", ErrNotAsked)
|
||||
}
|
||||
return l.actsStore()
|
||||
}
|
||||
|
||||
func (b BusState) key(k string) string { return b.Node + "." + k }
|
||||
|
||||
// changed is a write that lost its compare-and-set, as stdio.ErrChanged.
|
||||
func changed(err error) error {
|
||||
var api *nats.APIError
|
||||
if errors.Is(err, nats.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == nats.JSErrCodeStreamWrongLastSequence) {
|
||||
return stdio.ErrChanged
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (b BusState) Get(key string) (*stdio.StateEntry, error) {
|
||||
kv, err := b.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, err := kv.Get(b.key(key))
|
||||
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil
|
||||
}
|
||||
|
||||
func (b BusState) Create(key string, value any) (uint64, error) {
|
||||
kv, err := b.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := kv.Create(b.key(key), raw)
|
||||
return rev, changed(err)
|
||||
}
|
||||
|
||||
func (b BusState) Update(key string, value any, revision uint64) (uint64, error) {
|
||||
kv, err := b.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := kv.Update(b.key(key), raw, revision)
|
||||
return rev, changed(err)
|
||||
}
|
||||
|
||||
func (b BusState) Delete(key string) error {
|
||||
kv, err := b.kv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return kv.Delete(b.key(key))
|
||||
}
|
||||
|
||||
// Keys are this node's keys under prefix, without the node: a watch of `<node>.>` alone, the one filter the node's
|
||||
// grant names, its values left out.
|
||||
func (b BusState) Keys(prefix string) ([]string, error) {
|
||||
kv, err := b.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w, err := kv.Watch(b.Node+".>", nats.MetaOnly(), nats.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
var out []string
|
||||
for {
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
if e == nil {
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
if k := strings.TrimPrefix(e.Key(), b.Node+"."); strings.HasPrefix(k, prefix) {
|
||||
out = append(out, k)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
return nil, errors.New("the node-engine's acts could not be listed within 10s")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,769 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// The node seat node-engine's acts (novox/hq ADR 0563).
|
||||
//
|
||||
// A hand-over and an access act are the engine's own asks, performed only on the warrant (§3): a call performs nothing.
|
||||
// The engine judges every value as it does for the controller's signed ask, then refuses in words or asks the operator
|
||||
// under its own asker name (asks.EngineAsker) at the level approve, binding the digest of the verb, the node, every
|
||||
// argument and the owner, group and mode it will set. A warrant is claimed once by compare-and-set in the engine's own
|
||||
// state (asker.Client.Take), held to the ask it answers, and to the act bound and the disk as judged, judged again now;
|
||||
// only then is the act performed and recorded with the warrant. A decline, an expiry or a changed disk says refused and
|
||||
// changes nothing. One ask is open per path or access.
|
||||
//
|
||||
// The read-only verbs need no warrant (§2): judge-directories changes nothing; setuid-search is bounded here, one at a
|
||||
// time and at most one started in an hour. Every act, with who asked as the runtime proved it, is kept in the engine's own
|
||||
// declared state acts (ActsBucket, created by the grants part), under its own node's keys alone, read by the verb acts.
|
||||
|
||||
// The acts of the seat, as its verbs name them.
|
||||
const (
|
||||
ActHandOver = "hand-over"
|
||||
ActAccess = "access-act"
|
||||
ActJudgeDirectories = "judge-directories"
|
||||
ActSetuidSearch = "setuid-search"
|
||||
)
|
||||
|
||||
// The states of an act.
|
||||
const (
|
||||
HandAsked = "asked"
|
||||
HandDone = "done"
|
||||
HandRefused = "refused"
|
||||
HandAnswered = "answered"
|
||||
)
|
||||
|
||||
// SearchEvery is how often a setuid search may be started on a node (ADR 0563 §2): one in any hour.
|
||||
const SearchEvery = time.Hour
|
||||
|
||||
// askMargin is how long past an ask's expiry the router is given to say it ended, before the engine closes it itself.
|
||||
const askMargin = 5 * time.Minute
|
||||
|
||||
// ActsBucket is the node-engine's declared state acts (its manifest's state, ADR 0563 §2, §3), the bucket the grants
|
||||
// part creates (mesh-sdk buscompose EngineActsBucket): keys `<node>.act.<id>`, `<node>.open.<act>.<digest>` and
|
||||
// `<node>.ask.<id>`, each written by this node's engine alone.
|
||||
const ActsBucket = "mesh-host_acts"
|
||||
|
||||
// Judged is what the engine judged of one act before it asks: what it is about, what it will set (the owner, group and
|
||||
// mode, in words), and what the disk holds now (Disk), compared again when the warrant arrives.
|
||||
type Judged struct {
|
||||
// Subject is the path of a hand-over, or the access of an access act.
|
||||
Subject string
|
||||
// Path is the directory the act changes, as the engine states it.
|
||||
Path string
|
||||
// Sets is the owner, group and mode the act will set, in words; bound in the ask's digest.
|
||||
Sets string
|
||||
// Disk is what the engine read on disk when it judged: owner, group, mode and the file it is.
|
||||
Disk string
|
||||
// Words is what the operator is shown of it, without a path (ADR 0234 §6): the path is in the whole words alone.
|
||||
Words string
|
||||
}
|
||||
|
||||
// HandAct is one act asked through the seat, as acts reads it.
|
||||
type HandAct struct {
|
||||
ID string `json:"id"`
|
||||
Act string `json:"act"`
|
||||
Subject string `json:"subject,omitempty"`
|
||||
Caller string `json:"caller"`
|
||||
Why string `json:"why,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
State string `json:"state"`
|
||||
Said string `json:"said,omitempty"`
|
||||
Ended time.Time `json:"ended,omitempty"`
|
||||
// Ask is the operator's ask of a hand-over or an access act; Sets and Disk what was judged when it was asked.
|
||||
Ask string `json:"ask,omitempty"`
|
||||
Sets string `json:"sets,omitempty"`
|
||||
Disk string `json:"disk,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
// Expires is when the ask ends unanswered.
|
||||
Expires time.Time `json:"expires,omitempty"`
|
||||
// By, Via, Proofs are the operator's warrant, recorded with the act (ADR 0563 §3).
|
||||
By string `json:"by,omitempty"`
|
||||
Via string `json:"via,omitempty"`
|
||||
Proofs []string `json:"proofs,omitempty"`
|
||||
// Auto says the engine asked it itself, on finding a library's root holding data outside its group (§4).
|
||||
Auto bool `json:"auto,omitempty"`
|
||||
}
|
||||
|
||||
// RootVerdict is the engine's answer to root-free (ADR 0563 §6), in the controller's words.
|
||||
type RootVerdict struct {
|
||||
Machine string `json:"machine"`
|
||||
Free bool `json:"free"`
|
||||
Why string `json:"why"`
|
||||
Judged time.Time `json:"judged"`
|
||||
}
|
||||
|
||||
// HandActs is the engine's side of the seat. Every func is the engine's own judgement or act; nil refuses.
|
||||
type HandActs struct {
|
||||
Node string
|
||||
// State is where the acts and the asks are kept: this node's keys of ActsBucket (BusState), each key one act.
|
||||
State ActStore
|
||||
// Asker asks the operator as asks.EngineAsker(Node), with State as its store.
|
||||
Asker *asker.Client
|
||||
// JudgeHandOver judges a hand-over's path, refusing in words; HandOver records it on the warrant.
|
||||
JudgeHandOver func(path string) (Judged, error)
|
||||
HandOver HandOverFunc
|
||||
// JudgeAccess judges an access act on an access, refusing in words; AccessAct runs it on the warrant.
|
||||
JudgeAccess func(act, access string) (Judged, error)
|
||||
AccessAct AccessActFunc
|
||||
// JudgeDirectories, SetuidSearch and Searching are the read-only verbs' and whether a search runs now.
|
||||
JudgeDirectories JudgeDirectoriesFunc
|
||||
SetuidSearch SetuidSearchFunc
|
||||
Searching func() bool
|
||||
// RootFree is this machine's root-free verdict now.
|
||||
RootFree func() RootVerdict
|
||||
// Say says one of the seat's events, asked, done or refused; nil says nothing.
|
||||
Say func(event string, body any)
|
||||
Now func() time.Time
|
||||
// Nonce keeps two asks of one second apart; nil is random.
|
||||
Nonce func() string
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (h *HandActs) now() time.Time {
|
||||
if h.Now != nil {
|
||||
return h.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (h *HandActs) nonce() string {
|
||||
if h.Nonce != nil {
|
||||
return h.Nonce()
|
||||
}
|
||||
var b [2]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
func (h *HandActs) say(event string, a HandAct) {
|
||||
if h.Say != nil {
|
||||
h.Say(event, map[string]any{"act": a.Act, "subject": a.Subject, "caller": a.Caller, "ask": a.Ask,
|
||||
"state": a.State, "said": a.Said, "node": h.Node})
|
||||
}
|
||||
}
|
||||
|
||||
// Call is the seat's one entry: a verb, the caller as the runtime proved it, and its arguments.
|
||||
func (h *HandActs) Call(c EngineCall) (any, error) {
|
||||
arg := func(k string) string { s, _ := c.Args[k].(string); return strings.TrimSpace(s) }
|
||||
switch c.Verb {
|
||||
case ActHandOver:
|
||||
return h.AskHandOver(c.Caller, arg("path"), arg("why"), false)
|
||||
case ActAccess:
|
||||
return h.AskAccess(c.Caller, arg("act"), arg("access"), arg("why"), false)
|
||||
case ActJudgeDirectories:
|
||||
return h.Judge(c.Caller)
|
||||
case ActSetuidSearch:
|
||||
return h.Search(c.Caller, arg("why"))
|
||||
case "root-free":
|
||||
if h.RootFree == nil {
|
||||
return nil, errors.New("this engine judges no machine root-free")
|
||||
}
|
||||
return map[string]any{"machines": []RootVerdict{h.RootFree()}}, nil
|
||||
case "acts":
|
||||
if id := arg("id"); id != "" {
|
||||
return h.Act(id)
|
||||
}
|
||||
limit, _ := strconv.Atoi(arg("limit"))
|
||||
if f, ok := c.Args["limit"].(float64); ok {
|
||||
limit = int(f)
|
||||
}
|
||||
return h.Acts(limit)
|
||||
}
|
||||
return nil, fmt.Errorf("the node seat %s serves no verb %q", EngineSeat, c.Verb)
|
||||
}
|
||||
|
||||
// ---- the record --------------------------------------------------------------------------------------------------
|
||||
|
||||
// ActStore is the engine's own state, this node's keys alone (BusState prefixes them with the node): each act one key,
|
||||
// written by compare-and-set on its revision, an open ask one key made by a create.
|
||||
type ActStore interface {
|
||||
Get(key string) (*stdio.StateEntry, error)
|
||||
Create(key string, value any) (uint64, error)
|
||||
Update(key string, value any, revision uint64) (uint64, error)
|
||||
Delete(key string) error
|
||||
// Keys are the keys under prefix.
|
||||
Keys(prefix string) ([]string, error)
|
||||
}
|
||||
|
||||
func actKey(id string) string { return "act." + id }
|
||||
|
||||
// openKey is the one key an open ask of act on subject holds: made by a create, so a second ask of it is refused.
|
||||
func openKey(act, subject string) string {
|
||||
sum := sha256.Sum256([]byte(subject))
|
||||
return "open." + act + "." + hex.EncodeToString(sum[:12])
|
||||
}
|
||||
|
||||
// read is every act kept, oldest first.
|
||||
func (h *HandActs) read() ([]HandAct, error) {
|
||||
keys, err := h.State.Keys("act.")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the engine's acts cannot be listed: %w", err)
|
||||
}
|
||||
var out []HandAct
|
||||
for _, k := range keys {
|
||||
a, _, err := h.readOne(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a != nil {
|
||||
out = append(out, *a)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *HandActs) readOne(key string) (*HandAct, uint64, error) {
|
||||
e, err := h.State.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var a HandAct
|
||||
if err := json.Unmarshal(e.Value, &a); err != nil {
|
||||
return nil, 0, fmt.Errorf("the engine's act %s cannot be read: %w", key, err)
|
||||
}
|
||||
return &a, e.Revision, nil
|
||||
}
|
||||
|
||||
// judgedKey keeps the node's judgements and refused searches, the last keptJudged of them: any caller may ask them, so
|
||||
// they roll in one key and never add one (ADR 0563 §2: a wrong caller gains nothing, and fills no bucket).
|
||||
const (
|
||||
judgedKey = "judged"
|
||||
keptJudged = 50
|
||||
)
|
||||
|
||||
// roll adds a to the rolling key by compare-and-set, the oldest falling out.
|
||||
func (h *HandActs) roll(a HandAct) error {
|
||||
for range 5 {
|
||||
e, err := h.State.Get(judgedKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var kept []HandAct
|
||||
if e != nil {
|
||||
if err := json.Unmarshal(e.Value, &kept); err != nil {
|
||||
kept = nil // an unreadable roll is started again, never a reason to refuse
|
||||
}
|
||||
}
|
||||
kept = append(kept, a)
|
||||
if len(kept) > keptJudged {
|
||||
kept = kept[len(kept)-keptJudged:]
|
||||
}
|
||||
if e == nil {
|
||||
_, err = h.State.Create(judgedKey, kept)
|
||||
} else {
|
||||
_, err = h.State.Update(judgedKey, kept, e.Revision)
|
||||
}
|
||||
if !errors.Is(err, stdio.ErrChanged) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return errors.New("the engine's judgements kept changing under its write")
|
||||
}
|
||||
|
||||
func (h *HandActs) rolled() ([]HandAct, error) {
|
||||
e, err := h.State.Get(judgedKey)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
var kept []HandAct
|
||||
if err := json.Unmarshal(e.Value, &kept); err != nil {
|
||||
return nil, fmt.Errorf("the engine's judgements cannot be read: %w", err)
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
func (h *HandActs) record(a HandAct) error {
|
||||
_, err := h.State.Create(actKey(a.ID), a)
|
||||
return err
|
||||
}
|
||||
|
||||
// end marks the open act of an ask ended, once, by compare-and-set on its revision, and frees its open key: false when
|
||||
// it was not open.
|
||||
func (h *HandActs) end(ask string, set func(*HandAct)) (HandAct, bool, error) {
|
||||
for range 5 {
|
||||
a, rev, err := h.readOne(actKey(ask))
|
||||
if err != nil || a == nil || a.State != HandAsked {
|
||||
return HandAct{}, false, err
|
||||
}
|
||||
set(a)
|
||||
a.Ended = h.now().UTC()
|
||||
if _, err := h.State.Update(actKey(ask), *a, rev); errors.Is(err, stdio.ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return HandAct{}, false, err
|
||||
}
|
||||
_ = h.State.Delete(openKey(a.Act, a.Subject))
|
||||
return *a, true, nil
|
||||
}
|
||||
return HandAct{}, false, errors.New("the engine's act " + ask + " kept changing under its write")
|
||||
}
|
||||
|
||||
// Act is the verb acts with an id: that one act.
|
||||
func (h *HandActs) Act(id string) (any, error) {
|
||||
a, _, err := h.readOne(actKey(id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a == nil {
|
||||
return nil, fmt.Errorf("this engine keeps no act %q", id)
|
||||
}
|
||||
return map[string]any{"node": h.Node, "acts": []HandAct{*a}}, nil
|
||||
}
|
||||
|
||||
// Acts is the verb acts: every act kept, newest first, at most limit when given.
|
||||
func (h *HandActs) Acts(limit int) (any, error) {
|
||||
acts, err := h.read()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
judged, err := h.rolled()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
acts = append(acts, judged...)
|
||||
sort.SliceStable(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) })
|
||||
out := make([]HandAct, 0, len(acts))
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
out = append(out, acts[i])
|
||||
}
|
||||
if limit > 0 && len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return map[string]any{"node": h.Node, "acts": out}, nil
|
||||
}
|
||||
|
||||
// ---- the read-only verbs (§2) -------------------------------------------------------------------------------------
|
||||
|
||||
// Judge is judge-directories: the engine's judgement, read-only, recorded with its caller.
|
||||
func (h *HandActs) Judge(caller string) (any, error) {
|
||||
if h.JudgeDirectories == nil {
|
||||
return nil, errors.New("this engine judges no directory")
|
||||
}
|
||||
said, err := h.JudgeDirectories(caller)
|
||||
a := HandAct{ID: h.id(ActJudgeDirectories), Act: ActJudgeDirectories, Caller: caller, At: h.now().UTC(), State: HandAnswered}
|
||||
if err != nil {
|
||||
a.State, a.Said = HandRefused, err.Error()
|
||||
}
|
||||
if rerr := h.roll(a); rerr != nil && err == nil {
|
||||
return nil, rerr
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"node": h.Node, "said": said}, nil
|
||||
}
|
||||
|
||||
// Search is setuid-search: refused while one runs and when one was started within the hour, naming when the next may
|
||||
// start; otherwise the last search is thrown away and a full one starts, recorded with its caller.
|
||||
func (h *HandActs) Search(caller, why string) (any, error) {
|
||||
if h.SetuidSearch == nil {
|
||||
return nil, errors.New("this engine keeps no search for setuid programs")
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.now().UTC()
|
||||
if h.Searching != nil && h.Searching() {
|
||||
return nil, errors.New("a search for setuid programs runs on this machine now; no second is started while it does")
|
||||
}
|
||||
// The last start is a key of its own, claimed by compare-and-set before the search starts: no number of other acts,
|
||||
// nor the record of acts, moves it (ADR 0563 §2).
|
||||
if err := h.claimSearch(now, caller); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
said, err := h.SetuidSearch(caller)
|
||||
a := HandAct{ID: h.id(ActSetuidSearch), Act: ActSetuidSearch, Caller: caller, Why: why, At: now, State: HandDone, Said: said}
|
||||
if err != nil {
|
||||
a.State, a.Said = HandRefused, err.Error()
|
||||
}
|
||||
// A search started is bounded to one an hour and kept as an act of its own; a refused one, which any caller may
|
||||
// cause, rolls with the judgements.
|
||||
record := h.record
|
||||
if err != nil {
|
||||
record = h.roll
|
||||
}
|
||||
if rerr := record(a); rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"node": h.Node, "said": said}, nil
|
||||
}
|
||||
|
||||
// searchKey keeps the last search's start on this node, and libKey each library's own take ask: keys of their own, which
|
||||
// no other act's key moves, kept for the bucket's age (ninety days, the node-engine's manifest).
|
||||
const searchKey = "search.last"
|
||||
|
||||
func libKey(access string) string {
|
||||
sum := sha256.Sum256([]byte(access))
|
||||
return "lib." + hex.EncodeToString(sum[:12]) + ".asked"
|
||||
}
|
||||
|
||||
// started is the last search's start as kept, and who asked it.
|
||||
type started struct {
|
||||
At time.Time `json:"at"`
|
||||
By string `json:"by"`
|
||||
}
|
||||
|
||||
// claimSearch refuses a search within SearchEvery of the last start, naming when the next may start, and otherwise
|
||||
// claims this one by compare-and-set: a second claim at the same moment loses.
|
||||
func (h *HandActs) claimSearch(now time.Time, caller string) error {
|
||||
e, err := h.State.Get(searchKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var rev uint64
|
||||
if e != nil {
|
||||
var last started
|
||||
if err := json.Unmarshal(e.Value, &last); err != nil {
|
||||
return fmt.Errorf("the last search's start cannot be read, so none is started: %w", err)
|
||||
}
|
||||
if now.Sub(last.At) < SearchEvery {
|
||||
return fmt.Errorf("a search was started at %s by %s, and one is started in any hour: the next may start "+
|
||||
"at %s", last.At.Format(time.RFC3339), last.By, last.At.Add(SearchEvery).Format(time.RFC3339))
|
||||
}
|
||||
rev = e.Revision
|
||||
}
|
||||
claim := started{At: now, By: caller}
|
||||
if rev == 0 {
|
||||
_, err = h.State.Create(searchKey, claim)
|
||||
} else {
|
||||
_, err = h.State.Update(searchKey, claim, rev)
|
||||
}
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return errors.New("another search was started at this moment; no second is started")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// ---- the asks (§3, §4) --------------------------------------------------------------------------------------------
|
||||
|
||||
func (h *HandActs) id(act string) string {
|
||||
return act + "-" + h.now().UTC().Format("20060102t150405") + "-" + h.nonce()
|
||||
}
|
||||
|
||||
// bound is the act an ask's yes binds: the verb, the node, every argument and what it will set (ADR 0563 §3).
|
||||
func (h *HandActs) bound(a HandAct) asks.Act {
|
||||
act := asks.Act{"verb": a.Act, "node": h.Node, "arg.subject": a.Subject, "arg.why": a.Why, "sets": a.Sets}
|
||||
if a.Path != "" {
|
||||
act["path"] = a.Path
|
||||
}
|
||||
return act
|
||||
}
|
||||
|
||||
// AskHandOver is hand-over: judged, refused in words or asked of the operator. auto is the engine asking by itself.
|
||||
func (h *HandActs) AskHandOver(caller, path, why string, auto bool) (any, error) {
|
||||
if h.JudgeHandOver == nil || h.HandOver == nil {
|
||||
return nil, errors.New("this engine takes no hand-over")
|
||||
}
|
||||
j, err := h.JudgeHandOver(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.ask(ActHandOver, caller, why, auto, j)
|
||||
}
|
||||
|
||||
// AskAccess is access-act: the act take on an access, judged on the library's home, refused or asked of the operator.
|
||||
func (h *HandActs) AskAccess(caller, act, access, why string, auto bool) (any, error) {
|
||||
if h.JudgeAccess == nil || h.AccessAct == nil {
|
||||
return nil, errors.New("this engine runs no act on an accessed folder")
|
||||
}
|
||||
if act != ActTake {
|
||||
return nil, fmt.Errorf("the act asked is %q, and the one act on an access is %q", act, ActTake)
|
||||
}
|
||||
j, err := h.JudgeAccess(act, access)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.ask(ActAccess, caller, why, auto, j)
|
||||
}
|
||||
|
||||
// AskedFor says whether an act on subject was ever asked by the engine itself: a library's take is asked by itself once
|
||||
// (ADR 0563 §4), and again only by hand.
|
||||
func (h *HandActs) AskedFor(act, subject string) bool {
|
||||
if act == ActAccess {
|
||||
e, err := h.State.Get(libKey(subject))
|
||||
return err != nil || e != nil // unread is never a reason to ask again
|
||||
}
|
||||
acts, err := h.read()
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
for _, a := range acts {
|
||||
if a.Act == act && a.Subject == subject && (a.Auto || a.State == HandAsked) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error) {
|
||||
if h.Asker == nil {
|
||||
return nil, errors.New("this engine reaches no operator to ask: nothing was asked and nothing changed")
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.now().UTC()
|
||||
a := HandAct{ID: h.id(act), Act: act, Subject: j.Subject, Path: j.Path, Caller: caller, Why: why, At: now,
|
||||
State: HandAsked, Sets: j.Sets, Disk: j.Disk, Expires: now.Add(asks.ApproveLasts - time.Minute), Auto: auto}
|
||||
a.Ask = a.ID
|
||||
// One ask open per path or access: its open key made by a create, which a second ask cannot make.
|
||||
if _, err := h.State.Create(openKey(act, j.Subject), map[string]string{"ask": a.Ask}); errors.Is(err, stdio.ErrChanged) {
|
||||
held := "another"
|
||||
if e, _ := h.State.Get(openKey(act, j.Subject)); e != nil {
|
||||
var o struct {
|
||||
Ask string `json:"ask"`
|
||||
}
|
||||
if json.Unmarshal(e.Value, &o) == nil && o.Ask != "" {
|
||||
held = o.Ask
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("%s of %s is asked already: the ask %s waits for the operator's answer", act, j.Subject, held)
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := h.record(a); err != nil {
|
||||
_ = h.State.Delete(openKey(act, j.Subject))
|
||||
return nil, err
|
||||
}
|
||||
bind, err := asks.ActDigest(h.bound(a))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
headline, does, stays := "Hand a directory on "+h.Node+" to the mesh?", "The mesh gives it "+j.Sets+".",
|
||||
"The directory stays as it was found."
|
||||
if act == ActAccess {
|
||||
headline, does, stays = "Keep the library "+strings.TrimPrefix(j.Subject, "library:")+" on "+h.Node+"?",
|
||||
"The library's group, group write and setgid are set on everything under its root.",
|
||||
"The library waits and nothing under its root changes."
|
||||
}
|
||||
if len([]rune(headline)) > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
asked := asks.Ask{ID: a.Ask, Headline: headline, Who: asks.Operator,
|
||||
Explanation: j.Words + " Your yes sets " + j.Sets + "; nothing changes until you answer.",
|
||||
Whole: j.Words + " (" + j.Path + ") Your yes sets " + j.Sets + "; nothing changes until you answer.",
|
||||
Details: "Asked by " + caller + " on " + h.Node + orWhy(why),
|
||||
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
|
||||
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
|
||||
Expires: a.Expires, OnExpiry: stays, About: EngineSeat + "-" + h.Node + "-" + act}
|
||||
if err := h.Asker.Ask(asked); err != nil {
|
||||
_, _, _ = h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the operator could not be asked: "+err.Error() })
|
||||
return nil, fmt.Errorf("the operator could not be asked, and nothing changed: %w", err)
|
||||
}
|
||||
// Once asked, a library's own take ask is kept's own take ask: once, in a key of its own (ADR 0563 §4).
|
||||
if auto && act == ActAccess {
|
||||
_, _ = h.State.Create(libKey(j.Subject), map[string]any{"ask": a.Ask, "at": now})
|
||||
}
|
||||
h.say(HandAsked, a)
|
||||
return map[string]any{"asked": a.Ask, "node": h.Node, "says": "the operator is asked on their channel; only " +
|
||||
"their yes changes anything. acts shows where it stands."}, nil
|
||||
}
|
||||
|
||||
func orWhy(why string) string {
|
||||
if why == "" {
|
||||
return ""
|
||||
}
|
||||
return ": " + why
|
||||
}
|
||||
|
||||
// Warranted is the act on the operator's answer, handed once by asker.Client.Take after it held the warrant to the
|
||||
// ask: a no is recorded refused; a yes is held to the act the option bound and the disk as judged, judged again now,
|
||||
// and only then performed and recorded done with the warrant.
|
||||
func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error {
|
||||
a, _, err := h.readOne(actKey(w.Ask))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a == nil || a.State != HandAsked {
|
||||
return fmt.Errorf("the warrant answers the ask %s, which is not open here: nothing is done", w.Ask)
|
||||
}
|
||||
refuse := func(why string) error {
|
||||
ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) })
|
||||
if err == nil && ok {
|
||||
h.say(HandRefused, ended)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if o.ID != "yes" {
|
||||
return refuse("the operator said no")
|
||||
}
|
||||
var j Judged
|
||||
switch a.Act {
|
||||
case ActHandOver:
|
||||
j, err = h.JudgeHandOver(a.Subject)
|
||||
case ActAccess:
|
||||
j, err = h.JudgeAccess(ActTake, a.Subject)
|
||||
default:
|
||||
err = fmt.Errorf("the act %s is not performed on a warrant", a.Act)
|
||||
}
|
||||
if err != nil {
|
||||
return refuse("judged again on the operator's yes, it is refused: " + err.Error())
|
||||
}
|
||||
now := *a
|
||||
now.Sets, now.Path = j.Sets, j.Path
|
||||
if err := o.Performs(h.bound(now)); err != nil {
|
||||
return refuse("what the yes bound is not what would be done now: " + err.Error())
|
||||
}
|
||||
if j.Disk != a.Disk {
|
||||
return refuse("the disk changed since the operator was asked (" + a.Disk + ", now " + j.Disk + "): ask again")
|
||||
}
|
||||
// Recorded with the way it came (ADR 0563 §5): the operator's yes to this engine's ask, not a terminal's line.
|
||||
by := operatorOf(w) + ", to " + EngineSeat + "'s ask " + w.Ask
|
||||
var said string
|
||||
switch a.Act {
|
||||
case ActHandOver:
|
||||
said, err = h.HandOver(a.Subject, by)
|
||||
case ActAccess:
|
||||
said, err = h.AccessAct(AccessAct{Act: ActTake, Access: a.Subject, Path: j.Path, Warrant: w.Ask, By: by})
|
||||
}
|
||||
if err != nil {
|
||||
return refuse(err.Error())
|
||||
}
|
||||
ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandDone, said; warrantOn(x, w) })
|
||||
if err == nil && ok {
|
||||
h.say(HandDone, ended)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func warrantOn(a *HandAct, w asks.Warrant) {
|
||||
a.Proofs = w.Proofs
|
||||
if w.By != nil {
|
||||
a.By, a.Via = w.By.Who, w.By.Kind
|
||||
if w.By.Verified != "" {
|
||||
a.Via += " (" + w.By.Verified + ")"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func operatorOf(w asks.Warrant) string {
|
||||
if w.By == nil {
|
||||
return "the operator"
|
||||
}
|
||||
return "the " + w.By.Who + " via " + w.By.Kind
|
||||
}
|
||||
|
||||
// Heard is a warrant heard on the engine's decided subject: taken once (asker.Client.Take), and an ask that ended with
|
||||
// no choice recorded refused.
|
||||
func (h *HandActs) Heard(w asks.Warrant) {
|
||||
if h.Asker == nil {
|
||||
return
|
||||
}
|
||||
// A warrant the ask kept here does not hold to (another digest, another option, too late) ends nothing: it is no
|
||||
// answer to the ask, which stays open for the operator's.
|
||||
if handled, _ := h.Asker.Take(w, h.Warranted); handled == asker.Recorded {
|
||||
why := "the ask ended " + string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
why += ": " + w.Words
|
||||
}
|
||||
if ended, ok, _ := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) }); ok {
|
||||
h.say(HandRefused, ended)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reconcile catches up what the engine missed while it was away: a warrant the router holds for an open ask is taken,
|
||||
// and an ask past its expiry and a margin is closed refused, expired.
|
||||
func (h *HandActs) Reconcile() {
|
||||
if h.Asker == nil {
|
||||
return
|
||||
}
|
||||
acts, err := h.read()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, a := range acts {
|
||||
if a.State != HandAsked || a.Ask == "" {
|
||||
continue
|
||||
}
|
||||
if _, w, err := h.Asker.Where(a.Ask); err == nil && w != nil {
|
||||
h.Heard(*w)
|
||||
continue
|
||||
}
|
||||
if !a.Expires.IsZero() && h.now().After(a.Expires.Add(askMargin)) {
|
||||
if ended, ok, _ := h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the ask expired unanswered" }); ok {
|
||||
h.say(HandRefused, ended)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MemState is an ActStore in memory: an engine's acts while no bus is reached in a test.
|
||||
type MemState struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]stdio.StateEntry
|
||||
top uint64
|
||||
}
|
||||
|
||||
func (m *MemState) Get(key string) (*stdio.StateEntry, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if e, ok := m.keys[key]; ok {
|
||||
return &e, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *MemState) Create(key string, value any) (uint64, error) { return m.put(key, value, 0, true) }
|
||||
|
||||
func (m *MemState) Update(key string, value any, revision uint64) (uint64, error) {
|
||||
return m.put(key, value, revision, false)
|
||||
}
|
||||
|
||||
func (m *MemState) put(key string, value any, revision uint64, create bool) (uint64, error) {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
was, kept := m.keys[key]
|
||||
if (create && kept) || (!create && (!kept || was.Revision != revision)) {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
if m.keys == nil {
|
||||
m.keys = map[string]stdio.StateEntry{}
|
||||
}
|
||||
m.top++
|
||||
m.keys[key] = stdio.StateEntry{Key: key, Value: raw, Revision: m.top}
|
||||
return m.top, nil
|
||||
}
|
||||
|
||||
func (m *MemState) Delete(key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.keys, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MemState) Keys(prefix string) ([]string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out []string
|
||||
for k := range m.keys {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
@@ -76,9 +76,13 @@ const MaxHandOverBy = 200
|
||||
// HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken.
|
||||
type HandOverCheck struct {
|
||||
Signer ed25519.PublicKey
|
||||
Node string
|
||||
Now func() time.Time
|
||||
Nonces *Nonces
|
||||
// SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer:
|
||||
// nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7,
|
||||
// ADR 0550 §2).
|
||||
SignerNow func() ed25519.PublicKey
|
||||
Node string
|
||||
Now func() time.Time
|
||||
Nonces *Nonces
|
||||
}
|
||||
|
||||
// Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory —
|
||||
@@ -157,8 +161,14 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod
|
||||
if err := json.Unmarshal(body, &signed); err != nil {
|
||||
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
|
||||
}
|
||||
if len(check.Signer) != ed25519.PublicKeySize ||
|
||||
!ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) {
|
||||
signer := check.Signer
|
||||
if check.SignerNow != nil {
|
||||
if signer = check.SignerNow(); signer == nil {
|
||||
return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken"
|
||||
}
|
||||
}
|
||||
if len(signer) != ed25519.PublicKeySize ||
|
||||
!ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) {
|
||||
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
|
||||
}
|
||||
var ask HandOverAsk
|
||||
|
||||
@@ -9,6 +9,9 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// The hand-over's ask and answer hold these field names; the controller's side holds the same list
|
||||
@@ -145,6 +148,21 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing.
|
||||
if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" {
|
||||
t.Fatalf("the handler was given %v", given)
|
||||
}
|
||||
// ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key
|
||||
// is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over.
|
||||
trusted := m.Signer
|
||||
following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node,
|
||||
Now: func() time.Time { return now }, Nonces: &Nonces{}}
|
||||
before := len(given)
|
||||
trusted = nil
|
||||
late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) }))
|
||||
if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before {
|
||||
t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a)
|
||||
}
|
||||
trusted = m.Signer
|
||||
if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 {
|
||||
t.Fatalf("while the controller holds the lease its ask was answered %+v", a)
|
||||
}
|
||||
// A nonce is forgotten once its ask has expired, so the set does not grow for ever.
|
||||
now = now.Add(HandOverGood + time.Second)
|
||||
check.Nonces.take("ff", now.Add(time.Second), now)
|
||||
@@ -174,17 +192,23 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) {
|
||||
// handingLink is a quiet link that hears hand-overs, keeping the handler it was given.
|
||||
type handingLink struct {
|
||||
*quietLink
|
||||
given chan HandOverFunc
|
||||
given chan HandOverFunc
|
||||
signer chan func() ed25519.PublicKey
|
||||
}
|
||||
|
||||
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
|
||||
|
||||
// FollowSigner keeps what the link is told to verify the controller's asks with.
|
||||
func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f }
|
||||
|
||||
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
|
||||
// judgement, and nothing else.
|
||||
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
||||
m, _ := aMember(t)
|
||||
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)}
|
||||
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)}
|
||||
q := &Session{Membership: m, Timeout: time.Second}
|
||||
// The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7).
|
||||
q.ControllerKey = func() ed25519.PublicKey { return nil }
|
||||
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
served := make(chan error, 1)
|
||||
@@ -197,8 +221,40 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the link was never given the queue's hand-over")
|
||||
}
|
||||
select {
|
||||
case f := <-l.signer:
|
||||
if f() != nil {
|
||||
t.Fatal("the link was given another key than the queue's")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the link was never given the key the engine trusts now")
|
||||
}
|
||||
stop()
|
||||
<-served
|
||||
// 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on
|
||||
// the subject that names none; a verb the seat does not serve is refused, and the answer is a tool's envelope.
|
||||
t.Run("a call of the node seat reaches its verb with its proven caller", func(t *testing.T) {
|
||||
h := &HandActs{Node: "n1", State: &MemState{},
|
||||
RootFree: func() RootVerdict { return RootVerdict{Machine: "n1", Free: true, Why: "judged"} },
|
||||
JudgeDirectories: func(by string) (string, error) { return "judged for " + by, nil }}
|
||||
for _, row := range []struct{ verb, subject, want string }{
|
||||
{"judge-directories", "mesh.seat.node-engine.call.judge-directories.n1.shanks~node-tools", `"said":"judged for shanks.node-tools"`},
|
||||
{"judge-directories", EngineToolSubject("judge-directories", "n1"), `"said":"judged for unproven"`},
|
||||
{"judge-directories", "mesh.seat.node-engine.call.judge-directories.n2.shanks~node-tools", `"said":"judged for unproven"`},
|
||||
{"root-free", "mesh.seat.node-engine.call.root-free.n1.ace~messenger", `"free":true`},
|
||||
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
|
||||
} {
|
||||
if got := string(AnswerEngineCall(h.Call, row.verb, "n1", row.subject, []byte(`{}`))); !strings.Contains(got, row.want) ||
|
||||
!strings.Contains(got, `"node":"n1"`) {
|
||||
t.Errorf("%s on %s: %s", row.verb, row.subject, got)
|
||||
}
|
||||
}
|
||||
acts, _ := h.Acts(0)
|
||||
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 3 || got[0].Caller != "unproven" ||
|
||||
got[2].Caller != "shanks.node-tools" || got[2].Act != ActJudgeDirectories {
|
||||
t.Errorf("the judgements are recorded as %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// **Against a real server**: a signed request on the node's own subject is answered; one on another node's is
|
||||
@@ -237,4 +293,63 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) {
|
||||
if len(handled) != 1 {
|
||||
t.Fatalf("handled %d asks; want only the one with a reply", len(handled))
|
||||
}
|
||||
// 774 (ADR 0563 §2, §3): the engine's acts are its node's keys of its declared state on the bus, each written by
|
||||
// compare-and-set: a second create and a stale revision lose, and another node's keys are not this node's.
|
||||
t.Run("the acts are kept in the bus's own storage under this node's keys", func(t *testing.T) {
|
||||
_ = js.DeleteKeyValue(ActsBucket)
|
||||
if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: ActsBucket}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := &Session{Membership: m, Timeout: time.Second}
|
||||
q.bus = l
|
||||
mine, theirs := BusState{Session: q, Node: m.Node}, BusState{Session: q, Node: "another"}
|
||||
rev, err := mine.Create("act.a-1", map[string]string{"state": "asked"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := mine.Create("act.a-1", map[string]string{}); !errors.Is(err, stdio.ErrChanged) {
|
||||
t.Errorf("a second create: %v", err)
|
||||
}
|
||||
if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev+7); !errors.Is(err, stdio.ErrChanged) {
|
||||
t.Errorf("an update at a stale revision: %v", err)
|
||||
}
|
||||
if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev); err != nil {
|
||||
t.Errorf("an update at its revision: %v", err)
|
||||
}
|
||||
_, _ = theirs.Create("act.b-1", map[string]string{})
|
||||
if keys, err := mine.Keys("act."); err != nil || len(keys) != 1 || keys[0] != "act.a-1" {
|
||||
t.Errorf("this node's acts are %v, %v", keys, err)
|
||||
}
|
||||
if e, _ := js.KeyValue(ActsBucket); e != nil {
|
||||
if got, err := e.Get(m.Node + ".act.a-1"); err != nil || !strings.Contains(string(got.Value()), "done") {
|
||||
t.Errorf("the bucket holds %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
if err := mine.Delete("act.a-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, err := mine.Get("act.a-1"); err != nil || e != nil {
|
||||
t.Errorf("a deleted act reads %v, %v", e, err)
|
||||
}
|
||||
})
|
||||
// 774 (ADR 0563 §1): the node seat is served on this node's subjects alone, naming its caller, and said to discovery.
|
||||
t.Run("the node seat is served on this node and discovered", func(t *testing.T) {
|
||||
l.ServeEngineSeat(func(c EngineCall) (any, error) { return c.Verb + " for " + c.Caller, nil })
|
||||
reply, err := conn.Request("mesh.seat.node-engine.call.acts."+m.Node+".shanks~node-tools", []byte(`{}`), 5*time.Second)
|
||||
if err != nil || !strings.Contains(string(reply.Data), `"result":"acts for shanks.node-tools"`) {
|
||||
t.Fatalf("the call answered %v, %v", reply, err)
|
||||
}
|
||||
if _, err := conn.Request(EngineToolSubject("acts", "another"), []byte(`{}`), 300*time.Millisecond); err == nil {
|
||||
t.Fatal("another node's verb was answered here")
|
||||
}
|
||||
for seat, want := range map[string]bool{EngineSeat: true, "node-login-shell": false} {
|
||||
verb := "acts"
|
||||
if !want {
|
||||
verb = "execute"
|
||||
}
|
||||
if heard, err := l.HeardServed(seat, verb); err != nil || heard != want {
|
||||
t.Errorf("discovery hears %s.%s here: %v, %v", seat, verb, heard, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+66
-19
@@ -49,9 +49,12 @@ type haveState struct {
|
||||
nodeComposed string
|
||||
// nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3,
|
||||
// issue 1126): kept in its key so a restart reuses them and applies nothing again.
|
||||
nodeSealed []string
|
||||
// nodeCandidates is the node's answer to each placement candidate group (novox/hq ADR 0559 §4), written into its key.
|
||||
nodeSealed delivery.Seals
|
||||
// nodeCandidates is the node's answer to each placement candidate proposal (novox/hq ADR 0559 §4), written into its key.
|
||||
nodeCandidates map[string]delivery.CandidateAnswer
|
||||
// nodeComposes and nodeComposesWhy are whose compositions the node applies and why (HaveNode.Composes), written into
|
||||
// its key.
|
||||
nodeComposes, nodeComposesWhy string
|
||||
// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
|
||||
// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
|
||||
// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
|
||||
@@ -380,7 +383,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
case f.Failed:
|
||||
h.Phase = delivery.PhaseFailed
|
||||
}
|
||||
h.Sealed = f.Sealed
|
||||
h.SealedAt = f.Sealed
|
||||
h.Health = s.healthOf(module, f)
|
||||
if h.Detail == "" {
|
||||
h.Detail = s.wordsOf(module, h.Health)
|
||||
@@ -421,8 +424,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
var raw []byte
|
||||
facts := *s.node
|
||||
facts.Composed = s.nodeComposed
|
||||
facts.Sealed = s.nodeSealed
|
||||
facts.SealedAt = s.nodeSealed
|
||||
facts.Candidates = s.nodeCandidates
|
||||
facts.Composes, facts.ComposesWhy = s.nodeComposes, s.nodeComposesWhy
|
||||
if raw, err = delivery.EncodeHaveNode(facts); err == nil {
|
||||
want := facts
|
||||
if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
|
||||
@@ -712,7 +716,7 @@ func (q *Session) NoteRemoved(ctx context.Context, module, build string) {
|
||||
|
||||
// NoteNode takes the digest of the node's own composition it runs and the values it carries sealed to the node's key
|
||||
// (nil for a composition the node did not make itself), and writes the node's key where it differs.
|
||||
func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string) {
|
||||
func (q *Session) NoteNode(ctx context.Context, composed string, sealed delivery.Seals) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
@@ -729,8 +733,8 @@ func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// NoteCandidates takes the node's answers to the placement candidates, by group (its shadow composer's, novox/hq ADR
|
||||
// 0559 §4), and writes the node's key where it differs; a group left out is no longer answered.
|
||||
// NoteCandidates takes the node's answers to the placement candidates, by proposal (its shadow composer's, novox/hq ADR
|
||||
// 0559 §4), and writes the node's key where it differs; a proposal left out is no longer answered.
|
||||
func (q *Session) NoteCandidates(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
@@ -752,68 +756,111 @@ func (q *Session) NoteCandidates(ctx context.Context, answers map[string]deliver
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn or
|
||||
// delivery.ComposesController) and why: written at each start of its agent, so a node returning to the controller's
|
||||
// compositions says so where every have is read (issue 1126).
|
||||
func (q *Session) NoteComposes(ctx context.Context, composes, why string) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
changed := s.nodeComposes != composes || s.nodeComposesWhy != why
|
||||
s.nodeComposes, s.nodeComposesWhy = composes, why
|
||||
known := s.node != nil
|
||||
s.mu.Unlock()
|
||||
if !known || !changed {
|
||||
return // said with the node's first facts
|
||||
}
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition
|
||||
// it last ran is and carries sealed, read after a restart (ADR 0547 §3).
|
||||
func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) {
|
||||
n, ok, _ := q.ReadNodeHaveErr(ctx)
|
||||
return n, ok
|
||||
}
|
||||
|
||||
// ReadNodeHaveErr is ReadNodeHave telling the key absent (false, nil) from one that could not be read (false, the
|
||||
// error): a bus that did not answer is never kept as a node that ran nothing (issue 1126, 2026-10-28).
|
||||
func (q *Session) ReadNodeHaveErr(ctx context.Context) (delivery.HaveNode, bool, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
hb, ok := bus.(HaveBus)
|
||||
if !ok || bus == nil {
|
||||
return delivery.HaveNode{}, false
|
||||
return delivery.HaveNode{}, false, ErrNotLinkedHaves
|
||||
}
|
||||
key, err := delivery.HaveNodeKey(q.Membership.Node)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
return delivery.HaveNode{}, false, err
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, q.timeout())
|
||||
defer cancel()
|
||||
bucket, err := hb.HaveBind(read)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
return delivery.HaveNode{}, false, err
|
||||
}
|
||||
value, _, err := bucket.Get(read, key)
|
||||
if errors.Is(err, ErrHaveAbsent) {
|
||||
return delivery.HaveNode{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
return delivery.HaveNode{}, false, err
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
return delivery.HaveNode{}, false, nil // a key no engine can read names nothing it runs
|
||||
}
|
||||
return n, true
|
||||
return n, true, nil
|
||||
}
|
||||
|
||||
// ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot
|
||||
// be read answers as none: the agent then compares with the declaration it kept, and applies only what differs.
|
||||
func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) {
|
||||
h, ok, _ := q.ReadHaveErr(ctx, module)
|
||||
return h, ok
|
||||
}
|
||||
|
||||
// ErrNotLinkedHaves is a have read with no link to the bus open that can bind the have bucket.
|
||||
var ErrNotLinkedHaves = errors.New("no link to the bus that binds the have bucket is open")
|
||||
|
||||
// ReadHaveErr is ReadHave telling a key absent (false, nil) from one that could not be read (false, the error): a bus
|
||||
// that does not answer for a moment is not a module that never ran (issue 1126).
|
||||
func (q *Session) ReadHaveErr(ctx context.Context, module string) (delivery.Have, bool, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
hb, ok := bus.(HaveBus)
|
||||
if !ok || bus == nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.Have{}, false, ErrNotLinkedHaves
|
||||
}
|
||||
key, err := delivery.HaveKey(q.Membership.Node, module)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.Have{}, false, err
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, q.timeout())
|
||||
defer cancel()
|
||||
bucket, err := hb.HaveBind(read)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.Have{}, false, err
|
||||
}
|
||||
value, _, err := bucket.Get(read, key)
|
||||
if errors.Is(err, ErrHaveAbsent) {
|
||||
return delivery.Have{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.Have{}, false, err
|
||||
}
|
||||
h, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.Have{}, false, nil // a have no engine can read names nothing it runs
|
||||
}
|
||||
return h, true
|
||||
return h, true, nil
|
||||
}
|
||||
|
||||
// nothingJudged is the health of a module a look judged no resource of: ok when it was looked at and the module applied
|
||||
|
||||
@@ -274,7 +274,8 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||
// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
|
||||
// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
|
||||
// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
|
||||
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"}
|
||||
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert",
|
||||
Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2)
|
||||
q.Keys = func() RecordKeys { return keys }
|
||||
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
|
||||
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
|
||||
@@ -286,7 +287,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||
t.Fatalf("the node key holds no record: %+v, %v", got, err)
|
||||
}
|
||||
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
|
||||
Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
||||
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
||||
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
|
||||
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
|
||||
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
|
||||
@@ -582,17 +583,17 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
|
||||
if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseRemoved {
|
||||
t.Fatalf("removed: %+v", got)
|
||||
}
|
||||
q.NoteNode(context.Background(), dig, []string{"core"})
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.Sealed) != 1 || n.Sealed[0] != "core" {
|
||||
t.Fatalf("the node key, with its own composition's sealed values: %+v %v", n, err)
|
||||
q.NoteNode(context.Background(), dig, delivery.Seals{"core-box#sealed": "core"})
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.SealedAt) != 1 || n.SealedAt["core-box#sealed"] != "core" {
|
||||
t.Fatalf("the node key, with its own composition's sealed values by slot: %+v %v", n, err)
|
||||
}
|
||||
if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.Sealed) != 1 {
|
||||
if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.SealedAt) != 1 {
|
||||
t.Fatalf("the node key read back: %+v %v", n, ok)
|
||||
}
|
||||
// The node's answers to placement candidates ride in the same key (ADR 0559 §4), and go when no group is answered.
|
||||
answers := map[string]delivery.CandidateAnswer{"feat/x": {Head: "h", Verdict: delivery.CandidateSame, Same: []string{"web"}}}
|
||||
answers := map[string]delivery.CandidateAnswer{"p-1": {Proposal: "p-1", Revision: 1, Verdict: delivery.CandidateSame, Same: []string{"web"}}}
|
||||
q.NoteCandidates(context.Background(), answers)
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Candidates["feat/x"].Head != "h" || n.Composed != dig {
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Candidates["p-1"].Proposal != "p-1" || n.Composed != dig {
|
||||
t.Fatalf("the node key with its answers: %+v %v", n, err)
|
||||
}
|
||||
q.NoteCandidates(context.Background(), nil)
|
||||
|
||||
@@ -6,9 +6,11 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
)
|
||||
|
||||
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
|
||||
@@ -35,11 +37,14 @@ func natsURL(address string) string {
|
||||
|
||||
// natsLink is this node's connection as a JetStream subscription.
|
||||
type natsLink struct {
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
asking *nats.Subscription
|
||||
asked chan struct{}
|
||||
handing *nats.Subscription
|
||||
// mu guards signerNow (FollowSigner).
|
||||
mu sync.Mutex
|
||||
signerNow func() ed25519.PublicKey
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
asking *nats.Subscription
|
||||
asked chan struct{}
|
||||
handing *nats.Subscription
|
||||
// searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361).
|
||||
searching *nats.Subscription
|
||||
// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
|
||||
@@ -48,7 +53,10 @@ type natsLink struct {
|
||||
acting *nats.Subscription
|
||||
// moving hears the controller asking to move a value sealed to this machine into the vault (issue 676).
|
||||
moving *nats.Subscription
|
||||
node string
|
||||
// engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563).
|
||||
engine micro.Service
|
||||
warrants *nats.Subscription
|
||||
node string
|
||||
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
|
||||
signer ed25519.PublicKey
|
||||
lost chan error
|
||||
@@ -141,6 +149,21 @@ func (l *natsLink) hearAsks() {
|
||||
}
|
||||
}
|
||||
|
||||
// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the
|
||||
// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2).
|
||||
func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) {
|
||||
l.mu.Lock()
|
||||
l.signerNow = f
|
||||
l.mu.Unlock()
|
||||
}
|
||||
|
||||
// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust.
|
||||
func (l *natsLink) check() HandOverCheck {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node}
|
||||
}
|
||||
|
||||
// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
|
||||
// (novox/hq issue 356): a request on this machine's own subject, answered once on the reply it carried. The
|
||||
// ask is verified with the mesh's key before anything is read out of it (AnswerHandOver), because a responder
|
||||
@@ -151,7 +174,7 @@ func (l *natsLink) HearHandOvers(handle HandOverFunc) {
|
||||
if handle == nil {
|
||||
return
|
||||
}
|
||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
||||
check := l.check()
|
||||
handing, err := l.conn.Subscribe(AskHandOverSubject(l.node), func(msg *nats.Msg) {
|
||||
if body, answer := HandOverReply(check, handle, msg.Reply, msg.Data); answer {
|
||||
_ = msg.Respond(body)
|
||||
@@ -170,7 +193,7 @@ func (l *natsLink) HearSetuidSearches(handle SetuidSearchFunc) {
|
||||
if handle == nil {
|
||||
return
|
||||
}
|
||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
||||
check := l.check()
|
||||
searching, err := l.conn.Subscribe(AskSetuidSearchSubject(l.node), func(msg *nats.Msg) {
|
||||
if body, answer := SetuidSearchReply(check, handle, msg.Reply, msg.Data); answer {
|
||||
_ = msg.Respond(body)
|
||||
@@ -189,7 +212,7 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) {
|
||||
if handle == nil {
|
||||
return
|
||||
}
|
||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
||||
check := l.check()
|
||||
judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) {
|
||||
if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer {
|
||||
_ = msg.Respond(body)
|
||||
@@ -207,7 +230,7 @@ func (l *natsLink) HearAccessActs(handle AccessActFunc) {
|
||||
if handle == nil {
|
||||
return
|
||||
}
|
||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
||||
check := l.check()
|
||||
acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) {
|
||||
if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer {
|
||||
_ = msg.Respond(body)
|
||||
@@ -225,7 +248,7 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) {
|
||||
if handle == nil {
|
||||
return
|
||||
}
|
||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
||||
check := l.check()
|
||||
moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) {
|
||||
if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer {
|
||||
_ = msg.Respond(body)
|
||||
@@ -258,6 +281,12 @@ func (l *natsLink) Close() {
|
||||
if l.judging != nil {
|
||||
_ = l.judging.Unsubscribe()
|
||||
}
|
||||
if l.engine != nil {
|
||||
_ = l.engine.Stop()
|
||||
}
|
||||
if l.warrants != nil {
|
||||
_ = l.warrants.Unsubscribe()
|
||||
}
|
||||
if l.conn != nil {
|
||||
l.conn.Close()
|
||||
}
|
||||
@@ -301,10 +330,12 @@ func (l *natsLink) Ask(ctx context.Context, subject string, body []byte) ([]byte
|
||||
// What the queue asks of a live link beside Bus, each by a type assertion that fails quietly: said here
|
||||
// so a link that drops one does not build.
|
||||
var (
|
||||
_ Bus = (*natsLink)(nil)
|
||||
_ HealthBus = (*natsLink)(nil)
|
||||
_ HaveBus = (*natsLink)(nil)
|
||||
_ ToolBus = (*natsLink)(nil)
|
||||
_ Asker = (*natsLink)(nil)
|
||||
_ Asked = (*natsLink)(nil)
|
||||
_ Bus = (*natsLink)(nil)
|
||||
_ HealthBus = (*natsLink)(nil)
|
||||
_ HaveBus = (*natsLink)(nil)
|
||||
_ ToolBus = (*natsLink)(nil)
|
||||
_ Asker = (*natsLink)(nil)
|
||||
_ Asked = (*natsLink)(nil)
|
||||
_ EngineSeats = (*natsLink)(nil)
|
||||
_ OperatorAsks = (*natsLink)(nil)
|
||||
)
|
||||
|
||||
@@ -4,6 +4,7 @@ package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,7 +15,10 @@ import (
|
||||
// The revisions a bucket keeps of a key, read over a real bus: oldest first, a delete and a purge said as deleted, and
|
||||
// none for a key that never was (issue 1945: the node agent takes the composition its want names from them).
|
||||
func TestNatsAKeysRevisionsAreReadBack(t *testing.T) {
|
||||
srv, err := natsserver.NewServer(&natsserver.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
// A second account reads as a node's does: refused the one bucket "denied-it" (the seat's state before its grant arrives).
|
||||
srv, err := natsserver.NewServer(&natsserver.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Users: []*natsserver.User{{Username: "admin", Password: "a"}, {Username: "node", Password: "n", Permissions: &natsserver.Permissions{
|
||||
Publish: &natsserver.SubjectPermission{Allow: []string{">"}, Deny: []string{"$JS.API.*.*.KV_denied-it", "$JS.API.*.*.*.KV_denied-it", "$JS.API.*.*.*.KV_denied-it.>", "$KV.denied-it.>"}}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -23,7 +27,7 @@ func TestNatsAKeysRevisionsAreReadBack(t *testing.T) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
conn, err := nats.Connect(srv.ClientURL())
|
||||
conn, err := nats.Connect(srv.ClientURL(), nats.UserInfo("admin", "a"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -65,4 +69,58 @@ func TestNatsAKeysRevisionsAreReadBack(t *testing.T) {
|
||||
if got, err := bus.KVHistory(ctx, "composed-it", "it.never.x"); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a key that never was: %+v (%v)", got, err)
|
||||
}
|
||||
|
||||
// A bucket that is not there, and one the account may not read, are errors at once, never a wait: the node agent
|
||||
// follows its placement key by one watch at its start (endWhenPlaced) and, refused, stays on the controller's compositions and says why.
|
||||
if _, err := bus.KVWatch(ctx, "never-it", ">"); err == nil {
|
||||
t.Error("a watch of a bucket that is not there gave no error")
|
||||
}
|
||||
if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "denied-it", History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodeConn, err := nats.Connect(srv.ClientURL(), nats.UserInfo("node", "n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nodeConn.Close)
|
||||
started := time.Now()
|
||||
wctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := (OverNATS{Conn: nodeConn}).KVWatch(wctx, "denied-it", ">"); err == nil || wctx.Err() != nil {
|
||||
t.Errorf("a watch the account may not make: error %v after %s, context %v", err, time.Since(started), wctx.Err())
|
||||
} else {
|
||||
t.Logf("a refused watch is an error after %s: %v", time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
|
||||
// A bucket made after the watch was asked is then watched, on the bus's word that it was created and with no timer;
|
||||
// a bucket the account may not read is no waiting but the error (endWhenPlaced, the shadow's candidates).
|
||||
q := &Session{Membership: Membership{Node: "n1"}}
|
||||
q.Attach(ctx, OverNATS{Conn: conn, JS: js})
|
||||
var waited atomic.Int32
|
||||
actx, acancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer acancel()
|
||||
ch, err := q.KVWatchAwait(actx, "later-it", ">", func(error) { waited.Add(1) })
|
||||
if err != nil || waited.Load() != 1 {
|
||||
t.Fatalf("a bucket not there yet: watch %v, waiting told %d time(s)", err, waited.Load())
|
||||
}
|
||||
select {
|
||||
case u := <-ch:
|
||||
t.Fatalf("an update before the bucket exists: %+v", u)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
later, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "later-it", History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := later.PutString("placement.n1", "v"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for seen := false; !seen; {
|
||||
select {
|
||||
case u := <-ch:
|
||||
seen = u.Key == "placement.n1" && string(u.Value) == "v"
|
||||
case <-actx.Done():
|
||||
t.Fatal("the bucket made after the watch was never watched")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
@@ -746,6 +747,6 @@ type ModuleFact struct {
|
||||
// Judgement and Waits are, in phase waiting-yes, the judgement of the move and the composition waiting (ADR 0547 §3).
|
||||
Judgement string
|
||||
Waits string
|
||||
// Sealed is the values the composition it names carries sealed to this node's key (delivery.Have.Sealed).
|
||||
Sealed []string
|
||||
// Sealed is the values the composition it names carries sealed to this node's key, by slot (delivery.Have.SealedAt).
|
||||
Sealed delivery.Seals
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ type RecordKeys struct {
|
||||
Certificate string
|
||||
// Authority is the mesh's own certificate as this machine last applied it (public).
|
||||
Authority string
|
||||
// Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2).
|
||||
Trusts []string
|
||||
}
|
||||
|
||||
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
|
||||
@@ -45,7 +47,7 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
|
||||
return nil
|
||||
}
|
||||
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
|
||||
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority}
|
||||
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts}
|
||||
for _, c := range p.Capabilities {
|
||||
if c.Present && c.Name != "" {
|
||||
rec.Capabilities = append(rec.Capabilities, c.Name)
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
meshasks "git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// Membership is what a node needs to reach its mesh again, held by the caller.
|
||||
@@ -185,6 +187,12 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
||||
}
|
||||
// And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue
|
||||
// 356), answered on the link the question came over.
|
||||
// Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7).
|
||||
if f, ok := link.(interface {
|
||||
FollowSigner(func() ed25519.PublicKey)
|
||||
}); ok && queue.ControllerKey != nil {
|
||||
f.FollowSigner(queue.ControllerKey)
|
||||
}
|
||||
if h, ok := link.(HandOvers); ok {
|
||||
h.HearHandOvers(queue.HandOver)
|
||||
}
|
||||
@@ -200,6 +208,19 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
||||
if s, ok := link.(AccessActs); ok {
|
||||
s.HearAccessActs(queue.AccessAct)
|
||||
}
|
||||
// And its node seat, node-engine, with the operator's answers to its own asks (novox/hq ADR 0563): served on this
|
||||
// link, and what was answered while no link was open caught up from the router's records.
|
||||
if queue.Acts != nil {
|
||||
if s, ok := link.(EngineSeats); ok {
|
||||
s.ServeEngineSeat(queue.Acts.Call)
|
||||
}
|
||||
if a, ok := link.(OperatorAsks); ok {
|
||||
if err := a.HearWarrants(meshasks.EngineAsker(m.Node), queue.Acts.Heard); err != nil {
|
||||
say("cannot hear the operator's answers to this engine's asks: " + err.Error())
|
||||
}
|
||||
go queue.Acts.Reconcile()
|
||||
}
|
||||
}
|
||||
// And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676).
|
||||
if s, ok := link.(MoveSecrets); ok {
|
||||
s.HearMoveSecrets(queue.MoveSecret)
|
||||
|
||||
@@ -2,12 +2,14 @@ package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
@@ -47,6 +49,12 @@ type Session struct {
|
||||
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
|
||||
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
|
||||
MoveSecret MoveSecretFunc
|
||||
// ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq
|
||||
// ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's.
|
||||
ControllerKey func() ed25519.PublicKey
|
||||
// Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the
|
||||
// operator asked and heard over it. Nil serves no seat.
|
||||
Acts *HandActs
|
||||
// Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes
|
||||
// no record.
|
||||
Keys func() RecordKeys
|
||||
@@ -337,3 +345,49 @@ func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, with
|
||||
}
|
||||
return env.Result, nil
|
||||
}
|
||||
|
||||
// PublishAsk publishes an ask or a cancel to the operator's router over the link open now (novox/hq ADR 0563 §3).
|
||||
func (q *Session) PublishAsk(subject string, body any, id string) error {
|
||||
asking, err := q.operatorAsks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return asking.PublishAsk(subject, body, id)
|
||||
}
|
||||
|
||||
// RouterRecord reads one key of the router's records over the link open now.
|
||||
func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) {
|
||||
asking, err := q.operatorAsks()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return asking.RouterRecord(key)
|
||||
}
|
||||
|
||||
// SayEngine says one of the node seat's events over the link open now; with none open it is not said.
|
||||
func (q *Session) SayEngine(event string, body any) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
if sayer, ok := bus.(interface{ SayEngine(string, any) error }); ok {
|
||||
if err := sayer.SayEngine(event, body); err != nil {
|
||||
q.Say("could not say the node seat's " + event + ": " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (q *Session) operatorAsks() (OperatorAsks, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
if bus == nil {
|
||||
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
|
||||
}
|
||||
asking, ok := bus.(OperatorAsks)
|
||||
if !ok {
|
||||
return nil, errNoAsking
|
||||
}
|
||||
return asking, nil
|
||||
}
|
||||
|
||||
@@ -57,4 +57,82 @@ func TestASetuidSearchAskIsBelievedOnlySignedForThisNodeAndOnce(t *testing.T) {
|
||||
if a := AnswerSetuidSearch(check, nil, body); a.Refused == "" {
|
||||
t.Fatalf("no handler: %+v", a)
|
||||
}
|
||||
// 774 (ADR 0563 §2): the node seat's setuid-search starts none while one runs, at most one in any hour, naming when the
|
||||
// next may start, and records who asked.
|
||||
t.Run("the node seat's search is bounded", func(t *testing.T) {
|
||||
at, running, started := now, false, 0
|
||||
h := &HandActs{Node: "n1", State: &MemState{}, Now: func() time.Time { return at },
|
||||
Searching: func() bool { return running },
|
||||
SetuidSearch: func(by string) (string, error) { started++; return "a new search starts", nil }}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
after time.Duration
|
||||
running bool
|
||||
refused string
|
||||
started int
|
||||
}{
|
||||
{name: "one runs", running: true, refused: "runs on this machine now", started: 0},
|
||||
{name: "the first", started: 1},
|
||||
{name: "within the hour", after: 59 * time.Minute, refused: "the next may start at 2026-10-10T03:00:00Z", started: 1},
|
||||
{name: "after the hour", after: time.Hour + time.Second, started: 2},
|
||||
} {
|
||||
at, running = now.Add(row.after), row.running
|
||||
_, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"})
|
||||
if (row.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.refused)) || started != row.started {
|
||||
t.Errorf("%s: %v, %d started", row.name, err, started)
|
||||
}
|
||||
}
|
||||
acts, _ := h.Acts(1)
|
||||
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 1 || got[0].Caller != "ace.node-tools" || got[0].State != HandDone {
|
||||
t.Errorf("the last search is recorded as %+v", got)
|
||||
}
|
||||
// The bound is kept in a key of its own (rev-l3 round 1): two hundred judgements before a search, or the acts
|
||||
// gone from the record, never let a second search start within the hour.
|
||||
h.JudgeDirectories = func(string) (string, error) { return "judged", nil }
|
||||
for range 200 {
|
||||
if _, err := h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "ace.node-tools"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
at = now.Add(time.Hour + 2*time.Minute)
|
||||
if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") {
|
||||
t.Errorf("a search after two hundred judgements: %v", err)
|
||||
}
|
||||
keys, _ := h.State.Keys("act.")
|
||||
for _, k := range keys {
|
||||
_ = h.State.Delete(k)
|
||||
}
|
||||
if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") || started != 2 {
|
||||
t.Errorf("a search with the acts gone: %v, %d started", err, started)
|
||||
}
|
||||
// rev-l3 round 2 (ADR 0563 §2, a wrong caller gains nothing): judgements and refused searches, which any caller
|
||||
// may ask, keep one rolling key of the node's, so ten thousand of them write a bounded number of bytes and add no
|
||||
// key to the bucket every node shares.
|
||||
keysNow := func() (int, int) {
|
||||
all, _ := h.State.Keys("")
|
||||
size := 0
|
||||
for _, k := range all {
|
||||
if e, _ := h.State.Get(k); e != nil {
|
||||
size += len(e.Value)
|
||||
}
|
||||
}
|
||||
return len(all), size
|
||||
}
|
||||
_, _ = h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "unproven"})
|
||||
keysBefore, _ := keysNow()
|
||||
for i := range 10000 {
|
||||
verb := ActJudgeDirectories
|
||||
if i%10 == 0 {
|
||||
verb = ActSetuidSearch
|
||||
}
|
||||
_, _ = h.Call(EngineCall{Verb: verb, Caller: "unproven"})
|
||||
}
|
||||
if n, size := keysNow(); n != keysBefore || size > 64<<10 {
|
||||
t.Errorf("ten thousand judgements and refused searches left %d keys (was %d) of %d bytes", n, keysBefore, size)
|
||||
}
|
||||
listed, _ := h.Acts(0)
|
||||
if got := listed.(map[string]any)["acts"].([]HandAct); len(got) == 0 || got[0].Caller != "unproven" {
|
||||
t.Errorf("acts does not show the latest judgement first: %d", len(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -239,3 +239,60 @@ func (q *Session) Hear(subject string) (<-chan []byte, func(), error) {
|
||||
}
|
||||
return h.Hear(subject)
|
||||
}
|
||||
|
||||
// CreatedAdvisory is the subject the bus says a bucket's stream was created on (the server's own advisory; the same
|
||||
// subject mesh-sdk's process.AwaitBucket binds on, and the grants part gives a principal for each bucket it may read).
|
||||
func CreatedAdvisory(bucket string) string {
|
||||
return "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_" + bucket
|
||||
}
|
||||
|
||||
// KVWatchAwait is KVWatch for a bucket that may not be on the bus yet (the placement seat's states, made when their
|
||||
// holder starts): the channel is returned at once, and while the bucket is missing it yields nothing and waiting is told
|
||||
// why; when the bus says the bucket's stream was created the watch is made and its updates follow. No timer: the
|
||||
// advisory is heard before the first try, so a creation between the try and the wait is not missed. Any other error of
|
||||
// the first try is the caller's, as for KVWatch: a bucket the account may not read is not waited for.
|
||||
func (q *Session) KVWatchAwait(ctx context.Context, bucket, pattern string, waiting func(error)) (<-chan KVUpdate, error) {
|
||||
created, stop, err := q.Hear(CreatedAdvisory(bucket))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
in, err := q.KVWatch(ctx, bucket, pattern)
|
||||
if err == nil {
|
||||
stop()
|
||||
return in, nil
|
||||
}
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
stop()
|
||||
return nil, err
|
||||
}
|
||||
waiting(err)
|
||||
out := make(chan KVUpdate, 64)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer stop()
|
||||
for in == nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case _, ok := <-created:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w, err := q.KVWatch(ctx, bucket, pattern)
|
||||
if err != nil {
|
||||
waiting(err)
|
||||
continue
|
||||
}
|
||||
in = w
|
||||
}
|
||||
}
|
||||
for u := range in {
|
||||
select {
|
||||
case out <- u:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
// Package ownresync hands the vault the value of this machine's own secret that runs, where the vault holds another
|
||||
// (novox/hq issue 1126, cause C4).
|
||||
//
|
||||
// The vault's `move-in` never replaces a value it holds, so its first import of an own secret stays fixed; where the
|
||||
// controller later made that secret anew (a rotation at start, a re-issue), the new value runs and the vault keeps the
|
||||
// old one. The node's shadow composer sees it: composing from the vault's value, it opens both seals with this
|
||||
// machine's key and its answer's shown body says delivery.MaskedAnotherValue. After each shadow answer this step asks the
|
||||
// vault's `move-in` with replace_running: true and what runs, for each own secret so masked, once per running value, and
|
||||
// says what it asked and what the vault answered. It is move-in's, the verb this node may ask already, so no seat and no
|
||||
// bus grant changes for it.
|
||||
//
|
||||
// **Only while this node applies the controller's compositions**, and only for a module whose composition that runs is
|
||||
// the one its want names: then the controller's value is what runs, and what runs is the truth the vault lacks. Once the
|
||||
// node composes its own, the vault's value is the source and nothing is handed. **The switch** is the vault's
|
||||
// (`own_take_switch`, a person's act): while it is off the vault refuses every ask, and this step says so once per value.
|
||||
// The vault also refuses any value it changed in the last 15 minutes, so a re-issue the node has not applied yet is
|
||||
// never put back.
|
||||
package ownresync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
)
|
||||
|
||||
// Verb is the vault's verb this step asks, with ReplaceRunning true in both steps.
|
||||
const Verb = "move-in"
|
||||
|
||||
// ReplaceRunning is move-in's argument that makes it take the running value over the one the vault holds.
|
||||
const ReplaceRunning = "replace_running"
|
||||
|
||||
// Env is what the step reads and asks.
|
||||
type Env struct {
|
||||
Node string
|
||||
// Composes is whose compositions the node applies now: delivery.ComposesController, delivery.ComposesOwn, or ""
|
||||
// while its agent has not decided. The step hands nothing but for ComposesController.
|
||||
Composes func() string
|
||||
// Running is what the node runs now (agent.Everything).
|
||||
Running func() []delivery.Composed
|
||||
// Get reads one key's latest value; nil and no error when there is none.
|
||||
Get func(ctx context.Context, bucket, key string) ([]byte, error)
|
||||
// Unseal opens a value sealed to this machine; Seal seals a value to a raw X25519 key in base64 (identity.Seal).
|
||||
Unseal func(sealed string) ([]byte, error)
|
||||
Seal func(to string, value []byte) (string, error)
|
||||
// Ask asks one of the vault's verbs in this engine's own name (link.Session.AskVault).
|
||||
Ask func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error)
|
||||
Say func(string)
|
||||
}
|
||||
|
||||
// Outcome is what the step did for one own secret: asked or not, and what was said.
|
||||
type Outcome struct {
|
||||
Module string
|
||||
Name string
|
||||
Asked bool
|
||||
Said string
|
||||
// Retry says a read or an ask the bus did not answer stopped it: no answer, kept as none, and the step is to be
|
||||
// taken again after a bound (issue 1126, 2026-10-28: "want.shanks.brother could not be read ... no responders").
|
||||
Retry bool
|
||||
}
|
||||
|
||||
// Resync is the step of one node agent's run.
|
||||
type Resync struct {
|
||||
env Env
|
||||
|
||||
mu sync.Mutex
|
||||
// done is, by module/name, the running value's fingerprint the vault was asked with or why it was not: each is
|
||||
// asked, and each reason said, once.
|
||||
done map[string]string
|
||||
}
|
||||
|
||||
// New is the step for one node.
|
||||
func New(env Env) *Resync {
|
||||
if env.Say == nil {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Resync{env: env, done: map[string]string{}}
|
||||
}
|
||||
|
||||
// Take reads the node's shadow answers and hands the vault what runs of each own secret they say another value of.
|
||||
func (r *Resync) Take(ctx context.Context, answers map[string]delivery.CandidateAnswer) []Outcome {
|
||||
seen := map[delivery.OwnValueDiffers]bool{}
|
||||
var drift []delivery.OwnValueDiffers
|
||||
proposals := make([]string, 0, len(answers))
|
||||
for g := range answers {
|
||||
proposals = append(proposals, g)
|
||||
}
|
||||
sort.Strings(proposals)
|
||||
for _, g := range proposals {
|
||||
for _, o := range delivery.OwnValuesDiffer(answers[g]) {
|
||||
if !seen[o] {
|
||||
seen[o] = true
|
||||
drift = append(drift, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(drift) == 0 {
|
||||
return nil
|
||||
}
|
||||
composes := ""
|
||||
if r.env.Composes != nil {
|
||||
composes = r.env.Composes()
|
||||
}
|
||||
var out []Outcome
|
||||
for _, o := range drift {
|
||||
res := r.one(ctx, composes, o)
|
||||
if res.Said != "" || res.Retry {
|
||||
out = append(out, res)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// one is the step for one own secret: Said is empty when it was done before for the same value and reason.
|
||||
func (r *Resync) one(ctx context.Context, composes string, o delivery.OwnValueDiffers) Outcome {
|
||||
res := Outcome{Module: o.Module, Name: o.Name}
|
||||
key := o.Module + "/" + o.Name
|
||||
if composes == "" {
|
||||
return res // the agent has not decided whose compositions it applies: taken again once it has
|
||||
}
|
||||
if composes != delivery.ComposesController {
|
||||
return r.said(key, "not:composes", res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
||||
"nothing is handed: this node composes its own (%q), so the vault's value is the source", key, composes))
|
||||
}
|
||||
sealed, why, failed := r.running(ctx, o)
|
||||
if failed != nil {
|
||||
// The bus did not answer: no reason to keep, and taken again (Retry); said once per reason.
|
||||
res.Retry = true
|
||||
return r.said(key, "retry:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
||||
"nothing is handed yet: %s; asked again", key, why))
|
||||
}
|
||||
if why != "" {
|
||||
return r.said(key, "not:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
||||
"nothing is handed: %s", key, why))
|
||||
}
|
||||
sum := sha256.Sum256([]byte(sealed))
|
||||
fp := hex.EncodeToString(sum[:])
|
||||
r.mu.Lock()
|
||||
asked := r.done[key] == fp
|
||||
r.mu.Unlock()
|
||||
if asked {
|
||||
return res
|
||||
}
|
||||
value, err := r.env.Unseal(sealed)
|
||||
if err != nil {
|
||||
return r.said(key, "not:unseal:"+fp, res, fmt.Sprintf("%s: what runs does not open with this machine's key, so "+
|
||||
"nothing is handed: %v", key, err))
|
||||
}
|
||||
defer wipe(value)
|
||||
said, err := r.hand(ctx, o, value)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return res // the run ended: asked again in the next
|
||||
}
|
||||
if selfcompose.BusFailed(err) {
|
||||
// The vault did not answer on the bus: no word of the vault's, so not kept as asked (Retry).
|
||||
res.Retry = true
|
||||
return r.said(key, "retry:ask", res, fmt.Sprintf("%s: the vault's %s with replace_running did not answer, "+
|
||||
"and is asked again: %v", key, Verb, err))
|
||||
}
|
||||
said = fmt.Sprintf("%s: the vault's %s with replace_running refused: %v", key, Verb, err)
|
||||
}
|
||||
// Once per running value, whatever the vault answered: a refusal is the vault's word, said once, and a new value
|
||||
// that runs asks again.
|
||||
r.mu.Lock()
|
||||
r.done[key] = fp
|
||||
r.mu.Unlock()
|
||||
res.Asked, res.Said = true, said
|
||||
r.env.Say(said)
|
||||
return res
|
||||
}
|
||||
|
||||
// said says why once per reason, and keeps the reason as done.
|
||||
func (r *Resync) said(key, reason string, res Outcome, why string) Outcome {
|
||||
r.mu.Lock()
|
||||
again := r.done[key] == reason
|
||||
r.done[key] = reason
|
||||
r.mu.Unlock()
|
||||
if again {
|
||||
return res
|
||||
}
|
||||
res.Said = why
|
||||
r.env.Say(why)
|
||||
return res
|
||||
}
|
||||
|
||||
// running is the sealed value of o in the composition that runs, when that composition is the one the module's want
|
||||
// names; or why it is not handed, with the error of a read the bus did not answer, which is no answer.
|
||||
func (r *Resync) running(ctx context.Context, o delivery.OwnValueDiffers) (sealed, why string, failed error) {
|
||||
var comp *delivery.Composed
|
||||
if r.env.Running != nil {
|
||||
for _, c := range r.env.Running() {
|
||||
if c.Module == o.Module {
|
||||
c := c
|
||||
comp = &c
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if comp == nil {
|
||||
return "", o.Module + " runs no composition here", nil
|
||||
}
|
||||
wantKey, err := delivery.WantKey(r.env.Node, o.Module)
|
||||
if err != nil {
|
||||
return "", err.Error(), nil
|
||||
}
|
||||
raw, err := r.env.Get(ctx, delivery.AssignerWantsBucket, wantKey)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its want %s could not be read (%v)", wantKey, err), err
|
||||
}
|
||||
if raw == nil {
|
||||
return "", fmt.Sprintf("it has no want %s", wantKey), nil
|
||||
}
|
||||
want, err := delivery.DecodeWant(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its want %s is not one: %v", wantKey, err), nil
|
||||
}
|
||||
if want.Self || want.Removed || want.Composed == "" {
|
||||
return "", fmt.Sprintf("its want %s names no composition of the controller's", wantKey), nil
|
||||
}
|
||||
// What runs is the composition the want names, as the module's have says: applied and running.
|
||||
haveKey, err := delivery.HaveKey(r.env.Node, o.Module)
|
||||
if err != nil {
|
||||
return "", err.Error(), nil
|
||||
}
|
||||
raw, err = r.env.Get(ctx, delivery.HaveBucket, haveKey)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its have %s could not be read (%v)", haveKey, err), err
|
||||
}
|
||||
if raw == nil {
|
||||
return "", fmt.Sprintf("it has no have %s", haveKey), nil
|
||||
}
|
||||
have, err := delivery.DecodeHave(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its have %s is not one: %v", haveKey, err), nil
|
||||
}
|
||||
if have.Composed != want.Composed || have.Phase != delivery.PhaseRunning {
|
||||
return "", fmt.Sprintf("what runs (%s, %s) is not yet the composition its want names (%s)", have.Composed,
|
||||
have.Phase, want.Composed), nil
|
||||
}
|
||||
// On a node the resource's id carries its module's name (`only-office.needs-jwt`); delivery.OwnNeedName reads it.
|
||||
id := o.Module + "." + catalogue.NeedID(o.Name)
|
||||
for _, res := range comp.Resources {
|
||||
var body struct {
|
||||
ID string `json:"id"`
|
||||
Sealed any `json:"sealed"`
|
||||
}
|
||||
if json.Unmarshal(res, &body) != nil {
|
||||
continue
|
||||
}
|
||||
if name, own := delivery.OwnNeedName(o.Module, body.ID); !own || name != o.Name {
|
||||
continue
|
||||
}
|
||||
if s, ok := body.Sealed.(string); ok && s != "" {
|
||||
return s, "", nil
|
||||
}
|
||||
return "", fmt.Sprintf("its resource %s carries no value sealed to this machine", id), nil
|
||||
}
|
||||
return "", fmt.Sprintf("what runs holds no resource %s", id), nil
|
||||
}
|
||||
|
||||
// hand asks the vault's two steps with value, and checks what the vault holds after by its fingerprint.
|
||||
func (r *Resync) hand(ctx context.Context, o delivery.OwnValueDiffers, value []byte) (string, error) {
|
||||
raw, err := r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "module": o.Module, "name": o.Name,
|
||||
"why": "this node's shadow answer says the vault holds another value than runs here"})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var opened struct {
|
||||
Call string `json:"call"`
|
||||
SealTo string `json:"seal_to"`
|
||||
Secret string `json:"secret"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
|
||||
return "", fmt.Errorf("the vault answered no call and no key to seal to")
|
||||
}
|
||||
sealed, err := r.env.Seal(opened.SealTo, value)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
|
||||
}
|
||||
raw, err = r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "call": opened.Call, "sealed": sealed})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var kept struct {
|
||||
Secret string `json:"secret"`
|
||||
From int `json:"from_generation"`
|
||||
To int `json:"to_generation"`
|
||||
Unchanged bool `json:"unchanged"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &kept); err != nil || kept.Secret == "" {
|
||||
return "", fmt.Errorf("the vault answered %s with no secret", opened.Secret)
|
||||
}
|
||||
sum := sha256.Sum256(value)
|
||||
if kept.Fingerprint != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
return "", fmt.Errorf("the vault says it holds %s at generation %d, but not the value handed in", kept.Secret, kept.To)
|
||||
}
|
||||
if kept.Unchanged {
|
||||
return fmt.Sprintf("handed the vault what runs of %s: it held that value already (generation %d)", kept.Secret, kept.To), nil
|
||||
}
|
||||
return fmt.Sprintf("handed the vault what runs of %s: generation %d replaced by %d, the value that runs", kept.Secret,
|
||||
kept.From, kept.To), nil
|
||||
}
|
||||
|
||||
func wipe(b []byte) {
|
||||
for i := range b {
|
||||
b[i] = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package ownresync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Issue 1126, cause C4: after a shadow answer saying the vault holds another value of an own secret than runs, the node
|
||||
// hands the vault what runs, in the verb's two steps, once per running value; and only while it applies the
|
||||
// controller's compositions and runs the one its want names.
|
||||
func TestTheNodeHandsTheVaultWhatRunsOfADriftedOwnSecret(t *testing.T) {
|
||||
const running = "the-value-that-runs"
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
fp := "src2:" + strings.Repeat("b", 64)
|
||||
sum := sha256.Sum256([]byte(running))
|
||||
answers := map[string]delivery.CandidateAnswer{"g": {Proposal: "p-1", Revision: 1, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: "only-office", Why: "the resource differs", Bodies: []delivery.ResourceBodies{
|
||||
{ID: "only-office.needs-jwt", Composed: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`, Running: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`},
|
||||
{ID: "only-office.needs-salt", Composed: `{"sealed":"<masked: another seal of the same value>"}`}}}}}}
|
||||
for name, row := range map[string]struct {
|
||||
composes string
|
||||
runs string // the have's composition digest
|
||||
refuse string // the vault's refusal
|
||||
asks int // the vault's verb asked, both passes
|
||||
says string
|
||||
// Issue 1126, 2026-10-28: a read or an ask the bus did not answer is no answer: taken again (Retry), never kept.
|
||||
wantRead string // the want's read: "" answers it, "fails" the bus does not answer, "absent" no key
|
||||
retry bool
|
||||
}{
|
||||
"applies the controller's, runs its want": {delivery.ComposesController, digest, "", 2, "generation 1 replaced by 2", "", false},
|
||||
"composes its own": {delivery.ComposesOwn, digest, "", 0, "composes its own", "", false},
|
||||
"not decided yet": {"", digest, "", 0, "", "", false},
|
||||
"runs not yet what its want names": {delivery.ComposesController, "sha256:" + strings.Repeat("c", 64), "", 0, "not yet the composition", "", false},
|
||||
"the vault refuses": {delivery.ComposesController, digest, "switched off", 1, "switched off", "", false},
|
||||
"the vault does not answer": {delivery.ComposesController, digest, "nats: no responders available for request", 2, "asked again", "", true},
|
||||
"the want's read does not answer": {delivery.ComposesController, digest, "", 0, "asked again", "fails", true},
|
||||
"it has no want": {delivery.ComposesController, digest, "", 0, "it has no want", "absent", false},
|
||||
} {
|
||||
var asked []map[string]any
|
||||
var said []string
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest})
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: row.runs, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
r := New(Env{Node: "novox", Composes: func() string { return row.composes },
|
||||
Running: func() []delivery.Composed {
|
||||
return []delivery.Composed{{Node: "novox", Module: "only-office", FP: fp, Resources: []json.RawMessage{
|
||||
// As novox runs it (its shadow answer, 2026-10-28): the id carries the module's name.
|
||||
json.RawMessage(`{"id":"only-office.needs-jwt","module":"only-office","type":"file",` +
|
||||
`"path":"/var/lib/only-office/jwt.secret","sealed":"box-of-the-running-value"}`)}}}
|
||||
},
|
||||
Get: func(_ context.Context, bucket, key string) ([]byte, error) {
|
||||
switch {
|
||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "fails":
|
||||
return nil, errors.New("the bucket mesh-delivery-assigner_wants: nats: no responders available for request")
|
||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "absent":
|
||||
return nil, nil
|
||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office":
|
||||
return want, nil
|
||||
case bucket == delivery.HaveBucket && key == "novox.only-office":
|
||||
return have, nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
Unseal: func(sealed string) ([]byte, error) {
|
||||
if sealed != "box-of-the-running-value" {
|
||||
return nil, errors.New("not this machine's")
|
||||
}
|
||||
return []byte(running), nil
|
||||
},
|
||||
Seal: func(to string, value []byte) (string, error) { return to + "(" + string(value) + ")", nil },
|
||||
Ask: func(_ context.Context, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
asked = append(asked, args)
|
||||
if verb != "move-in" || args["replace_running"] != true {
|
||||
t.Errorf("%s: asked %s with %v, not move-in with replace_running", name, verb, args)
|
||||
}
|
||||
if row.refuse != "" {
|
||||
return nil, errors.New(row.refuse)
|
||||
}
|
||||
if args["call"] == nil {
|
||||
if args["module"] != "only-office" || args["name"] != "jwt" || args["node"] != nil {
|
||||
t.Errorf("%s: the first step names %v", name, args)
|
||||
}
|
||||
return json.RawMessage(`{"call":"c1","seal_to":"key","secret":"own/novox/only-office/jwt"}`), nil
|
||||
}
|
||||
if args["sealed"] != "key("+running+")" {
|
||||
t.Errorf("%s: handed %v, not what runs sealed to the call's key", name, args["sealed"])
|
||||
}
|
||||
return json.RawMessage(`{"secret":"own/novox/only-office/jwt","from_generation":1,"to_generation":2,` +
|
||||
`"fingerprint":"sha256:` + hex.EncodeToString(sum[:]) + `"}`), nil
|
||||
},
|
||||
Say: func(s string) { said = append(said, s) }})
|
||||
first := r.Take(context.Background(), answers)
|
||||
again := r.Take(context.Background(), answers)
|
||||
if len(asked) != row.asks {
|
||||
t.Errorf("%s: the vault asked %d times, want %d: %v", name, len(asked), row.asks, asked)
|
||||
}
|
||||
if retry := len(first) == 1 && first[0].Retry; retry != row.retry {
|
||||
t.Errorf("%s: taken again %v, want %v: %+v", name, retry, row.retry, first)
|
||||
}
|
||||
if row.retry && (len(again) != 1 || !again[0].Retry || again[0].Said != "") || !row.retry && len(again) != 0 {
|
||||
t.Errorf("%s: the same answer did or said something again: %+v", name, again)
|
||||
}
|
||||
all := strings.Join(said, "\n")
|
||||
if row.says == "" && (len(first) != 0 || all != "") || row.says != "" && (len(first) != 1 || !strings.Contains(all, row.says)) {
|
||||
t.Errorf("%s: did %+v and said %q, want %q", name, first, all, row.says)
|
||||
}
|
||||
if strings.Contains(all, running) {
|
||||
t.Errorf("%s: the value is in what the node said", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
+558
-134
@@ -11,18 +11,28 @@
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
@@ -46,18 +56,35 @@ type Env struct {
|
||||
// Hear subscribes a subject: here the vault's word that a secret changed (VaultSecretChanged), on which the node
|
||||
// composes again, so a rotated secret or credential reaches it at once (ADR 0547 §2). Nil hears nothing.
|
||||
Hear func(subject string) (<-chan []byte, func(), error)
|
||||
// Refused is told why the node cannot compose its modules, and "" once it composes again, so each wanted module's
|
||||
// have says it (agent.NotComposed): a refusal is loud where haves are read, not one journal line. Nil tells nobody.
|
||||
Refused func(why string)
|
||||
// Refused is told why the node cannot compose its modules (why, the node refused whole), or, of a node that composes,
|
||||
// each module it holds and why (held: refused alone, the rest composed and applied, frombus.Compose), and both empty
|
||||
// once it composes every module again, so each module's have says it (agent.NotComposedEach): a refusal is loud where
|
||||
// haves are read, not one journal line, and one module's never marks the others (issue 1126). Nil tells nobody.
|
||||
Refused func(why string, held map[string]string)
|
||||
// Running is what the node runs now (agent.Everything), and Unseal opens a value sealed to this machine: a secret
|
||||
// the vault seals again for each ask is kept as the box the node already runs when that box opens to the same
|
||||
// value, so neither the switch to composing its own nor a restart changes a composition (ADR 0547 §3). Either nil
|
||||
// keeps the vault's box.
|
||||
// the vault seals again for each ask is kept as the box the node already runs at the same slot (the same field of
|
||||
// the same resource of the same module, delivery.KeepSeals) when that box opens to the same value, so neither the
|
||||
// switch to composing its own nor a restart changes a composition (ADR 0547 §3); a box held at another slot is
|
||||
// never taken, though it opens to the same value (issue 1126, C1: a provider's grant and its consumer's secret).
|
||||
// Either nil keeps the vault's box.
|
||||
Running func() []delivery.Composed
|
||||
Unseal func(sealed string) ([]byte, error)
|
||||
// Held is the sealed values a module's have says it runs (delivery.Have.Sealed), kept by the agent on the bus: what
|
||||
// runs after a restart, before the agent applied anything, and after the switch once the kept declaration is old.
|
||||
Held func(module string) []string
|
||||
// Held is the sealed values a module's have says it runs, kept by the agent on the bus: what runs after a restart,
|
||||
// before the agent applied anything, and after the switch once the kept declaration is old. By slot
|
||||
// (delivery.Have.SealedAt); legacy is the list a have written before slots holds (delivery.Have.Sealed), reused
|
||||
// only within its own module.
|
||||
// An error is a have the bus did not answer for, never one absent: the pass is then tried again (BusFailed).
|
||||
Held func(module string) (slots delivery.Seals, legacy []string, err error)
|
||||
// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
|
||||
// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
|
||||
Fetch func(ctx context.Context, url string) ([]byte, error)
|
||||
// VaultAuthority is the vault's authority as this node read it from the vault's `signers` (ADR 0550 §2), composed
|
||||
// beside the controller's as the mesh's authority (frombus.SignersReader); "" or nil composes the controller's alone.
|
||||
VaultAuthority func() string
|
||||
// Certify asks the vault's `sign` for this node's own certificate, keeps it and writes the node's record again (ADR
|
||||
// 0550 §2): asked only when a module placed here asks for a certificate and the record holds none from an authority
|
||||
// the node trusts. Nil asks nothing, and such a module stands in, as before.
|
||||
Certify func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value.
|
||||
@@ -77,14 +104,43 @@ type Composer struct {
|
||||
modules map[string]delivery.Module
|
||||
values map[string]map[string]any // by module, this node's
|
||||
manifests map[string][]byte // by digest, read once
|
||||
archives map[string][]byte // templates archives by digest, read once
|
||||
handed map[string]string // composed key to the digest handed
|
||||
// sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again
|
||||
// is other bytes, so the one first had is kept while the generation stands, and the composition stays the same.
|
||||
sealedOwn map[string]ownSealed
|
||||
// certified is the serving key a certificate was last asked for successfully, so it is asked once per key.
|
||||
certified string
|
||||
said string
|
||||
refusal string
|
||||
caught bool
|
||||
rev uint64
|
||||
// held is what the composer's last pass held: each module refused alone, and the node's own ("node") when it could
|
||||
// compose none (heldNow).
|
||||
held []string
|
||||
// doing is what the composer waits on now (waiting), for a running answer to say (issue 1126, T2); its own lock, so
|
||||
// it is read while c.mu is held.
|
||||
doingMu sync.Mutex
|
||||
doing string
|
||||
caught bool
|
||||
rev uint64
|
||||
// missed counts the reads the bus did not answer (BusFailed), for a pass or an answer to tell whether it met one:
|
||||
// a read that failed on the bus is tried again after a bound, never kept as an answer (issue 1126, 2026-10-28).
|
||||
missed atomic.Uint64
|
||||
// owed is set when a pass refused for a bus that did not answer: the composer composes again after a bound.
|
||||
owed atomic.Bool
|
||||
}
|
||||
|
||||
// note counts err when it is the bus not answering, and returns it.
|
||||
func (c *Composer) note(err error) error {
|
||||
if BusFailed(err) {
|
||||
c.missed.Add(1)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// get is env.Get, its failures noted.
|
||||
func (c *Composer) get(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||
raw, err := c.env.Get(ctx, bucket, key)
|
||||
return raw, c.note(err)
|
||||
}
|
||||
|
||||
// New is a composer for one node.
|
||||
@@ -94,7 +150,7 @@ func New(env Env) *Composer {
|
||||
}
|
||||
return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{},
|
||||
records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{},
|
||||
manifests: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}}
|
||||
manifests: map[string][]byte{}, archives: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}}
|
||||
}
|
||||
|
||||
// ownSealed is one own secret's sealed value at a generation and the key it is sealed to.
|
||||
@@ -187,11 +243,18 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
||||
caught := make([]bool, len(ins))
|
||||
ready := false
|
||||
dirty := false
|
||||
// A pass refused for a bus that did not answer is made again after a bound (laterAgain), not kept until another
|
||||
// input moves (issue 1126, 2026-10-28).
|
||||
var again <-chan time.Time
|
||||
tries := 0
|
||||
for {
|
||||
// Drain whatever came, then compose once for all of it.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-again:
|
||||
again = nil
|
||||
dirty = true
|
||||
case t := <-merged:
|
||||
if !t.ok {
|
||||
c.env.Say(fmt.Sprintf("the watch of %s ended; the node composes no more until its agent starts again", ins[t.in].bucket))
|
||||
@@ -230,6 +293,14 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
||||
return
|
||||
}
|
||||
}
|
||||
if c.owed.Load() {
|
||||
wait := LaterWait(tries)
|
||||
tries++
|
||||
c.env.Say(fmt.Sprintf("the node composes again in %s: a read on the bus did not answer", wait))
|
||||
again = time.After(wait)
|
||||
} else {
|
||||
tries, again = 0, nil
|
||||
}
|
||||
if !c.caughtSent() {
|
||||
select {
|
||||
case out <- agent.Update{Caught: true}:
|
||||
@@ -320,7 +391,7 @@ func (c *Composer) takeRecordChanged(u agent.Update) bool {
|
||||
}
|
||||
|
||||
// takeModuleHave takes one of this node's modules' haves, and says whether it changed which it runs inside its own
|
||||
// composition: one whose have names no fingerprint of the new form and is not removed (nats on novox: a build-...
|
||||
// composition: one whose have names no fingerprint of the new form, is not removed, and runs something (nats on novox: a build-...
|
||||
// id; mesh-wireguard: none). Read from the haves, which outlast a restart, never from what the agent has applied
|
||||
// this run: a fresh agent composes before it applies anything (review of the fold, round 1).
|
||||
func (c *Composer) takeModuleHave(u agent.Update) bool {
|
||||
@@ -331,7 +402,11 @@ func (c *Composer) takeModuleHave(u agent.Update) bool {
|
||||
in := false
|
||||
if !u.Deleted {
|
||||
if h, err := delivery.DecodeHave(u.Value); err == nil {
|
||||
in = h.Phase != delivery.PhaseRemoved && !strings.HasPrefix(h.FP, delivery.SourcePrefix2)
|
||||
// A have that names nothing it runs, failed (the node could not compose a module it never ran: NotComposed), is
|
||||
// no module run inside the node's own composition: taken as one, it was composed from its manifest past its
|
||||
// build (issue 1126, review round 1 of the builds).
|
||||
runsNothing := h.Phase == delivery.PhaseFailed && h.FP == "" && h.Composed == ""
|
||||
in = h.Phase != delivery.PhaseRemoved && !runsNothing && !strings.HasPrefix(h.FP, delivery.SourcePrefix2)
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
@@ -385,11 +460,10 @@ func (c *Composer) takeValue(u agent.Update) {
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var r struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
var v any
|
||||
if u.Deleted || json.Unmarshal(u.Value, &r) != nil || len(r.Value) == 0 || json.Unmarshal(r.Value, &v) != nil {
|
||||
// Only a value its source lays: a default is the composition's already, as the controller reads them
|
||||
// (catalogue.LaidValue, issue 1126, C2: Element's links blanked by a laid default).
|
||||
v, laid := catalogue.LaidValue(u.Value)
|
||||
if u.Deleted || !laid {
|
||||
delete(c.values[module], key)
|
||||
return
|
||||
}
|
||||
@@ -418,29 +492,86 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||
_, nodeWanted := c.wants[agent.NodeModule]
|
||||
// What it runs inside its own composition and no want names at a build of the new form stays there, as the
|
||||
// controller's composed writer folds it (issue 1126); compose folds none a want names.
|
||||
inOwn := c.inOwnOf(c.placements[c.env.Node])
|
||||
own, placed := c.placements[c.env.Node]
|
||||
inOwn := c.inOwnOf(own)
|
||||
r := reader{c: c}
|
||||
c.mu.Unlock()
|
||||
sort.Strings(behind)
|
||||
for _, module := range behind {
|
||||
digest, err := c.builtManifest(ctx, module, fps[module])
|
||||
if err != nil {
|
||||
why := fmt.Sprintf("%s is wanted at %s, whose manifest cannot be read, so the node does not compose: %v", module, fps[module], err)
|
||||
c.refused(why)
|
||||
c.sayOnce(why)
|
||||
return nil
|
||||
}
|
||||
kept[module] = digest
|
||||
}
|
||||
|
||||
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, Kept: kept, InOwn: inOwn}, fps)
|
||||
if err != nil {
|
||||
c.refused(fmt.Sprintf("%v", err))
|
||||
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
|
||||
if !placed {
|
||||
// The catalogue no longer places it: the agent returns to the controller's compositions (endWhenUnplaced), and
|
||||
// this is no refusal of what it runs (issue 1126, the seed's revert).
|
||||
c.sayOnce("this node's key in the seat's placement state is gone: it composes nothing of its own, and keeps running what it runs")
|
||||
c.owed.Store(false) // nothing of its own to compose, so no pass is owed and no timer armed again
|
||||
return nil
|
||||
}
|
||||
c.refused("")
|
||||
c.sayOnce("")
|
||||
// Composed from the build its want names (frombus.Scope.KeptAt, issue 1126).
|
||||
for _, module := range behind {
|
||||
kept[module] = fps[module]
|
||||
}
|
||||
|
||||
// What it runs: a folded module of an old-form build is carried in its own composition as it runs (issue 1126).
|
||||
var running []delivery.Composed
|
||||
if c.env.Running != nil {
|
||||
running = c.env.Running()
|
||||
}
|
||||
// Its own certificate, asked of the vault when a module here asks for one and the record holds none it trusts.
|
||||
c.certifyIfWanted(ctx, r, own)
|
||||
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
|
||||
missed := c.missed.Load()
|
||||
compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
||||
err = c.missedIn(missed, err)
|
||||
// A bus that does not answer for a moment (a reload of its accounts pauses it a few seconds) is no refusal: the pass is
|
||||
// tried again after a growing wait before anything is marked (issue 1126).
|
||||
for try := 0; err != nil && Transient(err) && ctx.Err() == nil && try < len(composeAgain); try++ {
|
||||
c.env.Say(fmt.Sprintf("the node's composition met a bus that did not answer, and is tried again in %s: %v", composeAgain[try], err))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(composeAgain[try]):
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
missed = c.missed.Load()
|
||||
compositions, _, held, err = frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
||||
err = c.missedIn(missed, err)
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
// The run ended (the agent begins again, on its own compositions or the controller's): what failed with it is
|
||||
// no refusal, and nothing is marked (issue 1126: a pass in flight at the switch marked the next agent's modules).
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
|
||||
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
|
||||
c.refused(fmt.Sprintf("%v", err), nil)
|
||||
// A bus that did not answer past every try is no answer: composed again after a bound (issue 1126, 2026-10-28).
|
||||
c.owed.Store(BusFailed(err))
|
||||
return nil
|
||||
}
|
||||
// The boxes it runs, slot by slot: read before c.mu is taken, since the agent's lock is taken to read what runs.
|
||||
missed = c.missed.Load()
|
||||
for key, comp := range compositions {
|
||||
compositions[key] = c.keepRunning(comp, running)
|
||||
}
|
||||
if c.missed.Load() > missed {
|
||||
// A have of what runs the bus did not answer for: its boxes would be the vault's, sealed again, and the
|
||||
// composition another; nothing is handed, and the pass is made again after a bound (issue 1126, 2026-10-28).
|
||||
c.sayOnce("the node composes nothing new for now, and keeps running what it runs: the have of what it runs did not answer on the bus")
|
||||
c.owed.Store(true)
|
||||
return nil
|
||||
}
|
||||
c.owed.Store(false)
|
||||
// A module refused alone is held: it keeps what it runs, its have says why, and every other module is applied (issue
|
||||
// 1126: one module's refusal froze the whole node).
|
||||
c.refused("", held)
|
||||
if len(held) == 0 {
|
||||
c.sayOnce("")
|
||||
} else {
|
||||
var lines []string
|
||||
for _, m := range heldModules(held) {
|
||||
lines = append(lines, m+": "+held[m])
|
||||
}
|
||||
c.sayOnce("the node composes its modules but holds " + strings.Join(heldModules(held), ", ") +
|
||||
", which keep running what they run: " + strings.Join(lines, "; "))
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var out []agent.Update
|
||||
@@ -474,6 +605,11 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||
out = append(out, agent.Update{Key: key, Value: raw, Revision: c.rev})
|
||||
}
|
||||
for key := range c.handed {
|
||||
if _, module, _, _, err := delivery.ParseComposedKey(key); err == nil && module != "" {
|
||||
if _, isHeld := held[module]; isHeld {
|
||||
continue // held: what was handed of it stays, so the node keeps running it
|
||||
}
|
||||
}
|
||||
if !now[key] {
|
||||
c.rev++
|
||||
delete(c.handed, key)
|
||||
@@ -483,56 +619,107 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||
return out
|
||||
}
|
||||
|
||||
// refused tells the agent why the node cannot compose, or "" that it composes again, once per change.
|
||||
func (c *Composer) refused(why string) {
|
||||
// heldModules is the modules of held, sorted.
|
||||
func heldModules(held map[string]string) []string {
|
||||
out := make([]string, 0, len(held))
|
||||
for m := range held {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// refused tells the agent why the node cannot compose (why), or which modules it holds (held), or both empty that it
|
||||
// composes every module again, once per change.
|
||||
func (c *Composer) refused(why string, held map[string]string) {
|
||||
said := why
|
||||
for _, m := range heldModules(held) {
|
||||
said += "\x00" + m + "\x00" + held[m]
|
||||
}
|
||||
c.mu.Lock()
|
||||
same := c.refusal == why
|
||||
c.refusal = why
|
||||
same := c.refusal == said
|
||||
c.refusal = said
|
||||
c.held = heldModules(held)
|
||||
if why != "" {
|
||||
c.held = append(c.held, delivery.HaveNodeWord)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if !same && c.env.Refused != nil {
|
||||
c.env.Refused(why)
|
||||
c.env.Refused(why, held)
|
||||
}
|
||||
}
|
||||
|
||||
// runningBox is the box the node already runs that opens to the same value as box, or box: what the agent runs, and
|
||||
// what each wanted module's have says it runs sealed (Held). Read before c.mu is taken, since the agent's lock is taken
|
||||
// to read what runs.
|
||||
func (c *Composer) runningBox(box string) string {
|
||||
if c.env.Unseal == nil || (c.env.Running == nil && c.env.Held == nil) {
|
||||
return box
|
||||
// heldNow is what the composer's last pass held, sorted.
|
||||
func (c *Composer) heldNow() []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return slices.Clone(c.held)
|
||||
}
|
||||
|
||||
// keepRunning is comp with each sealed value the node already runs at the same slot kept, where it opens to the same
|
||||
// value (delivery.KeepSeals, issue 1126, C1): of what runs (running, the agent's), else of what its module's have says
|
||||
// it runs (Held). A have written before slots names no slot: its boxes are reused within their own module alone, at a
|
||||
// slot no box is kept for, and only one opening to the slot's value.
|
||||
func (c *Composer) keepRunning(comp delivery.Composed, running []delivery.Composed) delivery.Composed {
|
||||
if c.env.Unseal == nil {
|
||||
return comp
|
||||
}
|
||||
value, err := c.env.Unseal(box)
|
||||
if err != nil {
|
||||
return box
|
||||
}
|
||||
var held []string
|
||||
if c.env.Running != nil {
|
||||
for _, comp := range c.env.Running() {
|
||||
held = append(held, agent.SealedIn(comp)...)
|
||||
}
|
||||
module := comp.Module
|
||||
if module == "" {
|
||||
module = agent.NodeModule
|
||||
}
|
||||
kept := delivery.Seals{}
|
||||
var legacy []string
|
||||
if c.env.Held != nil {
|
||||
c.mu.Lock()
|
||||
// The node's own key among them (agent.NodeModule): its core boxes, as its last run said them.
|
||||
modules := make([]string, 0, len(c.wants))
|
||||
for m := range c.wants {
|
||||
modules = append(modules, m)
|
||||
slots, old, err := c.env.Held(module)
|
||||
c.note(err)
|
||||
for slot, box := range slots {
|
||||
kept[slot] = box
|
||||
}
|
||||
c.mu.Unlock()
|
||||
sort.Strings(modules)
|
||||
for _, m := range modules {
|
||||
held = append(held, c.env.Held(m)...)
|
||||
legacy = old
|
||||
}
|
||||
for _, r := range running {
|
||||
if r.Module == comp.Module {
|
||||
for slot, box := range delivery.SealsIn(r) {
|
||||
kept[slot] = box
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, h := range held {
|
||||
if h == box {
|
||||
for slot, box := range delivery.SealsIn(comp) {
|
||||
if _, ok := kept[slot]; ok {
|
||||
continue
|
||||
}
|
||||
if v, err := c.env.Unseal(h); err == nil && bytes.Equal(v, value) {
|
||||
return h
|
||||
for _, h := range legacy {
|
||||
if same, known := c.sameValue(box, h); known && same {
|
||||
kept[slot] = h
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return box
|
||||
return delivery.KeepSeals(comp, kept, c.sameValue)
|
||||
}
|
||||
|
||||
// waiting notes what the composer waits on until the returned func is called, which restores what it waited on before.
|
||||
func (c *Composer) waiting(what string) func() {
|
||||
c.doingMu.Lock()
|
||||
before := c.doing
|
||||
c.doing = what
|
||||
c.doingMu.Unlock()
|
||||
return func() {
|
||||
c.doingMu.Lock()
|
||||
c.doing = before
|
||||
c.doingMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// waitingOn is what the composer waits on now, in words.
|
||||
func (c *Composer) waitingOn() string {
|
||||
c.doingMu.Lock()
|
||||
defer c.doingMu.Unlock()
|
||||
if c.doing == "" {
|
||||
return "nothing it reads (composing)"
|
||||
}
|
||||
return c.doing
|
||||
}
|
||||
|
||||
func (c *Composer) sayOnce(line string) {
|
||||
@@ -548,30 +735,6 @@ func (c *Composer) sayOnce(line string) {
|
||||
}
|
||||
}
|
||||
|
||||
// builtManifest is the digest of the manifest a module's build at a fingerprint was made from (the builder's build key):
|
||||
// what a node composes a module from when its want names another build than the trunk's.
|
||||
func (c *Composer) builtManifest(ctx context.Context, module, fp string) (string, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw, err := c.env.Get(ctx, delivery.BuilderBuildsBucket, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if raw == nil {
|
||||
return "", fmt.Errorf("the builder holds no build %s", key)
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if b.Manifest == "" {
|
||||
return "", fmt.Errorf("the build %s names no manifest", key)
|
||||
}
|
||||
return b.Manifest, nil
|
||||
}
|
||||
|
||||
// Policy is a module's upgrade policy as its manifest at the indexer's trunk derives it (catalogue.DerivedUpgrade), for
|
||||
// the agent's judgement of a move (ADR 0547 §3); empty when the manifest is not known or cannot be read. The manifest is
|
||||
// read within ctx, the agent's (review of L2, round 1, finding 5), so a stopping agent stops the read.
|
||||
@@ -594,9 +757,93 @@ func (c *Composer) Policy(ctx context.Context, module string) string {
|
||||
return policy
|
||||
}
|
||||
|
||||
// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), as a composition's Data says
|
||||
// it, and whether the manifest could be read: the agent's Data for what runs of a module from the kept declaration
|
||||
// (agent.Env.HoldsData, issue 924).
|
||||
func (c *Composer) HoldsData(ctx context.Context, module string) (data, known bool) {
|
||||
c.mu.Lock()
|
||||
m, ok := c.modules[module]
|
||||
c.mu.Unlock()
|
||||
if !ok || m.Manifest == "" {
|
||||
return false, false
|
||||
}
|
||||
raw, err := reader{c: c}.Manifest(ctx, m.Manifest)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
parsed, err := catalogue.ParseManifestForInputs(raw)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
return len(parsed.DataItems()) > 0, true
|
||||
}
|
||||
|
||||
// reader is the composer's view of the bus as frombus reads it.
|
||||
type reader struct{ c *Composer }
|
||||
|
||||
// VaultAuthority is the vault's authority the node follows (frombus.SignersReader).
|
||||
func (r reader) VaultAuthority(context.Context) string {
|
||||
if r.c.env.VaultAuthority == nil {
|
||||
return ""
|
||||
}
|
||||
return r.c.env.VaultAuthority()
|
||||
}
|
||||
|
||||
var _ frombus.SignersReader = reader{}
|
||||
|
||||
// certificateWanted says whether a module the placement p puts on this node asks for a certificate for the machine's
|
||||
// internal name (its manifest's `certificate`), read from the indexer's manifests; a manifest that cannot be read is
|
||||
// taken as asking none, and the composition says what it misses.
|
||||
func (c *Composer) certificateWanted(ctx context.Context, r reader, p delivery.Placement) bool {
|
||||
for _, m := range p.Modules {
|
||||
c.mu.Lock()
|
||||
digest := c.modules[m.Module].Manifest
|
||||
c.mu.Unlock()
|
||||
if digest == "" {
|
||||
continue
|
||||
}
|
||||
raw, err := r.Manifest(ctx, digest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if parsed, err := catalogue.ParseManifestForInputs(raw); err == nil && parsed.Certificate != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// certifyIfWanted asks Certify once per serving key when a module placed here asks for a certificate and this node's
|
||||
// record holds none for its internal name and serving key from an authority it trusts (the one it holds, or the vault's).
|
||||
func (c *Composer) certifyIfWanted(ctx context.Context, r reader, p delivery.Placement) {
|
||||
if c.env.Certify == nil || !c.certificateWanted(ctx, r, p) {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
rec, ok := c.records[c.env.Node]
|
||||
asked := c.certified
|
||||
c.mu.Unlock()
|
||||
if !ok || rec.ServingKey == "" || asked == rec.ServingKey {
|
||||
return
|
||||
}
|
||||
key, err := base64.StdEncoding.DecodeString(rec.ServingKey)
|
||||
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||
return
|
||||
}
|
||||
trusted := []string{rec.Authority, r.VaultAuthority(ctx)}
|
||||
if rec.Certificate != "" && delivery.CheckNodeCertificate(rec.Certificate, trusted, overlay.InternalName(c.env.Node),
|
||||
ed25519.PublicKey(key), time.Now()) == nil {
|
||||
return
|
||||
}
|
||||
if err := c.env.Certify(ctx); err != nil {
|
||||
c.sayOnce(fmt.Sprintf("a module here asks for a certificate, and the vault's sign gave none: %v", err))
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.certified = rec.ServingKey
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
@@ -638,7 +885,8 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := r.c.env.Get(ctx, delivery.ManifestsBucket, key)
|
||||
defer r.c.waiting("the manifest " + digest + " (" + delivery.ManifestsBucket + ")")()
|
||||
value, err := r.c.get(ctx, delivery.ManifestsBucket, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -655,6 +903,77 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) {
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
func (r reader) Build(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.c.waiting("the build of " + module + " at " + fp + " (" + delivery.BuilderBuildsBucket + ")")()
|
||||
raw, err := r.c.get(ctx, delivery.BuilderBuildsBucket, key)
|
||||
if err != nil || raw == nil {
|
||||
return nil, err
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the builder's %s: %w", key, err)
|
||||
}
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
// archiveWithin bounds one read of a templates archive in the artifact store.
|
||||
const archiveWithin = 30 * time.Second
|
||||
|
||||
// Archive reads a templates archive once per digest (an archive is the whole of its digest), checked against it.
|
||||
func (r reader) Archive(ctx context.Context, url, digest string) ([]byte, error) {
|
||||
r.c.mu.Lock()
|
||||
body, ok := r.c.archives[digest]
|
||||
r.c.mu.Unlock()
|
||||
if ok {
|
||||
return body, nil
|
||||
}
|
||||
if r.c.env.Fetch == nil {
|
||||
return nil, fmt.Errorf("this node reads no archive (%s)", url)
|
||||
}
|
||||
// Bounded: a store that does not answer must not hold the composer or the node's answer (issue 1126).
|
||||
defer r.c.waiting("the archive " + url)()
|
||||
fetching, cancel := context.WithTimeout(ctx, archiveWithin)
|
||||
defer cancel()
|
||||
body, err := r.c.env.Fetch(fetching, url)
|
||||
if r.c.note(err) != nil {
|
||||
return nil, err
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return nil, fmt.Errorf("%s was recorded as %s and what arrived is %s", url, digest, got)
|
||||
}
|
||||
r.c.mu.Lock()
|
||||
r.c.archives[digest] = body
|
||||
r.c.mu.Unlock()
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// Served reads, of another node's module, the keys named of mesh-config's resolved values, one key each: the node is
|
||||
// granted those keys alone (buscompose Principal.ServedReads, issue 1126).
|
||||
func (r reader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
|
||||
defer r.c.waiting("what " + module + " serves on " + node + " (" + ResolvedBucket + ")")()
|
||||
out := map[string]any{}
|
||||
for _, k := range keys {
|
||||
raw, err := r.c.get(ctx, ResolvedBucket, node+"."+module+"."+k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, laid := catalogue.LaidValue(raw)
|
||||
if raw == nil || !laid {
|
||||
continue
|
||||
}
|
||||
out[k] = value
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r reader) Values(_ context.Context, node, module string) (map[string]any, error) {
|
||||
if node != r.c.env.Node {
|
||||
return nil, nil // a node reads its own slice alone (ADR 0547 §2)
|
||||
@@ -675,7 +994,10 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
|
||||
if node != r.c.env.Node || r.c.env.Own == nil {
|
||||
return nil, nil
|
||||
}
|
||||
done := r.c.waiting("the vault's own secrets")
|
||||
raw, err := r.c.env.Own(ctx, map[string]any{})
|
||||
r.c.note(err)
|
||||
done()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the vault's own: %w", err)
|
||||
}
|
||||
@@ -696,25 +1018,16 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
|
||||
if a.Node != node {
|
||||
return nil, fmt.Errorf("the vault answered %q's own secrets to %s", a.Node, node)
|
||||
}
|
||||
ids := make([]string, len(a.Own))
|
||||
fresh := make([]string, len(a.Own))
|
||||
for i, o := range a.Own {
|
||||
ids[i] = o.Module + "/" + o.Name
|
||||
if !r.c.keeps(ids[i], o.Generation, a.Key) {
|
||||
fresh[i] = r.c.runningBox(string(o.Sealed))
|
||||
}
|
||||
}
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
out := make([]compose.OwnSecretRow, 0, len(a.Own))
|
||||
for i, o := range a.Own {
|
||||
id := ids[i]
|
||||
for _, o := range a.Own {
|
||||
// The box first had is kept while its generation stands, so sealing it again moves no composition; the box the
|
||||
// node runs at the same slot is kept after the composition (keepRunning).
|
||||
id := o.Module + "/" + o.Name
|
||||
kept, ok := r.c.sealedOwn[id]
|
||||
if !ok || kept.generation != o.Generation || kept.key != a.Key {
|
||||
kept = ownSealed{generation: o.Generation, key: a.Key, sealed: fresh[i]}
|
||||
if kept.sealed == "" {
|
||||
kept.sealed = string(o.Sealed)
|
||||
}
|
||||
kept = ownSealed{generation: o.Generation, key: a.Key, sealed: string(o.Sealed)}
|
||||
r.c.sealedOwn[id] = kept
|
||||
}
|
||||
out = append(out, compose.OwnSecretRow{Node: node, Module: o.Module, Name: o.Name, Origin: o.Origin,
|
||||
@@ -738,7 +1051,10 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
|
||||
for _, k := range asProvider {
|
||||
ask = append(ask, asked{Role: "provider", CredentialKey: k})
|
||||
}
|
||||
done := r.c.waiting("the vault's credentials")
|
||||
raw, err := r.c.env.Own(ctx, map[string]any{"credentials": ask})
|
||||
r.c.note(err)
|
||||
done()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the vault's own: %w", err)
|
||||
}
|
||||
@@ -759,28 +1075,17 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
|
||||
if a.Node != node {
|
||||
return nil, nil, fmt.Errorf("the vault answered %q's credentials to %s", a.Node, node)
|
||||
}
|
||||
ids := make([]string, len(a.Credentials))
|
||||
fresh := make([]string, len(a.Credentials))
|
||||
for i, c := range a.Credentials {
|
||||
ids[i] = c.Role + "|" + c.Name + "|" + c.Provision + "|" + c.Consumer + "|" + c.ConsumerModule + "|" + c.Local
|
||||
if !r.c.keeps(ids[i], c.Generation, a.Key) {
|
||||
fresh[i] = r.c.runningBox(string(c.Sealed))
|
||||
}
|
||||
}
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
var creds []compose.Credential
|
||||
var issued []compose.Issued
|
||||
for i, c := range a.Credentials {
|
||||
// The value first had is kept while its generation stands, so sealing it again moves no composition; and the
|
||||
// first had is the box the node runs when it opens to the same value (runningBox).
|
||||
id := ids[i]
|
||||
for _, c := range a.Credentials {
|
||||
// The value first had is kept while its generation stands, so sealing it again moves no composition; the box the
|
||||
// node runs at the same slot is kept after the composition (keepRunning).
|
||||
id := c.Role + "|" + c.Name + "|" + c.Provision + "|" + c.Consumer + "|" + c.ConsumerModule + "|" + c.Local
|
||||
kept, ok := r.c.sealedOwn[id]
|
||||
if !ok || kept.generation != c.Generation || kept.key != a.Key {
|
||||
kept = ownSealed{generation: c.Generation, key: a.Key, sealed: fresh[i]}
|
||||
if kept.sealed == "" {
|
||||
kept.sealed = string(c.Sealed)
|
||||
}
|
||||
kept = ownSealed{generation: c.Generation, key: a.Key, sealed: string(c.Sealed)}
|
||||
r.c.sealedOwn[id] = kept
|
||||
}
|
||||
k := c.CredentialKey
|
||||
@@ -796,10 +1101,129 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
|
||||
return creds, issued, nil
|
||||
}
|
||||
|
||||
// keeps says the composer holds id's sealed value at this generation and key already.
|
||||
func (c *Composer) keeps(id string, generation int, key string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
kept, ok := c.sealedOwn[id]
|
||||
return ok && kept.generation == generation && kept.key == key
|
||||
// Placed reads whether the catalogue places node now, for the node agent choosing whose compositions it applies at
|
||||
// each start: true when its placement key holds a value, false only when a read succeeds and finds no key or a delete
|
||||
// marker. A read that fails chooses neither (review of the shadow, round 1: a timeout or a bus not ready at a restart
|
||||
// flipped a self-composing node to the controller's compositions for a whole run): it is tried again after wait(try),
|
||||
// each distinct failure said once, until ctx ends, which is then its error.
|
||||
//
|
||||
// switched says the node composed from its own placement key before (its have says it composes its own,
|
||||
// delivery.ComposesOwn, and that is kept on the bus, so it survives a restart). Only a node that never switched takes a
|
||||
// read that fails (a timeout, or a state its account may not read yet, as the seat's before its grant arrives) as "not
|
||||
// placed": it stays on the controller's compositions, says why once, and switches on a positive read of its key (the
|
||||
// watch endWhenPlaced keeps). A node that switched never goes back on a failed read, however long it lasts.
|
||||
func Placed(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
|
||||
wait func(try int) time.Duration, say func(string), switched bool) (bool, error) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
said := ""
|
||||
for try := 1; ; try++ {
|
||||
raw, err := get(ctx, delivery.PlacementBucket, key)
|
||||
if errors.Is(err, nats.ErrBucketNotFound) {
|
||||
return false, nil // the seat has written no placement state yet: nothing places it, read and known
|
||||
}
|
||||
if err == nil {
|
||||
return len(raw) > 0, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return false, ctx.Err()
|
||||
}
|
||||
if !switched {
|
||||
say(fmt.Sprintf("this node's key in the seat's placement state cannot be read (%s): it has never composed its own, so it applies the controller's compositions and switches when its key is read: %v", key, err))
|
||||
return false, nil
|
||||
}
|
||||
if why := err.Error(); why != said {
|
||||
say(fmt.Sprintf("this node's key in the seat's placement state cannot be read (%s): the node agent waits, choosing neither its own compositions nor the controller's: %v", key, err))
|
||||
said = why
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
case <-time.After(wait(try)):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ComposedItsOwn says whether the node's own have, kept on the bus, says it composed from its placement key at its last
|
||||
// start. A have that cannot be read says yes: the safe side, a node waits and does not fall back.
|
||||
func ComposedItsOwn(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string) bool {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
raw, err := get(ctx, delivery.HaveBucket, key)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return false
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(raw)
|
||||
return err != nil || n.Composes == delivery.ComposesOwn
|
||||
}
|
||||
|
||||
// composeAgain are the waits before a pass that met a bus not answering is tried again.
|
||||
var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
|
||||
|
||||
// laterAgain are the waits, after a pass or an answer refused for a bus that did not answer past composeAgain, before
|
||||
// it is made again; the last repeats while the bus does not answer (issue 1126, 2026-10-28: a failed read was kept as
|
||||
// the answer until another input moved).
|
||||
var laterAgain = []time.Duration{5 * time.Second, 15 * time.Second, 30 * time.Second, time.Minute, 2 * time.Minute, 5 * time.Minute}
|
||||
|
||||
// LaterWait is the wait before the try'th making again (from 0), of a pass, an answer or an ask the bus did not answer.
|
||||
func LaterWait(try int) time.Duration { return laterAgain[min(try, len(laterAgain)-1)] }
|
||||
|
||||
// ErrBusMissed is a pass that met a read the bus did not answer, though the composition was made around it (a have of
|
||||
// sealed values that could not be read): never handed as made.
|
||||
var ErrBusMissed = errors.New("a read on the bus did not answer during the pass")
|
||||
|
||||
// missedIn is err, or ErrBusMissed when no error came but a read since missed failed on the bus.
|
||||
func (c *Composer) missedIn(missed uint64, err error) error {
|
||||
if err == nil && c.missed.Load() > missed {
|
||||
return ErrBusMissed
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// BusFailed says err is a read the bus did not answer, for any reason but the run's own end: Transient, a read that
|
||||
// timed out, or no link to the bus. Such a read is tried again after a bound, never kept as an answer; a key absent, a
|
||||
// value refused or a verdict is an answer (issue 1126, 2026-10-28). Matched by value and by words, as Transient.
|
||||
func BusFailed(err error) bool {
|
||||
if err == nil || errors.Is(err, context.Canceled) {
|
||||
return false
|
||||
}
|
||||
if Transient(err) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) ||
|
||||
errors.Is(err, nats.ErrConnectionClosed) || errors.Is(err, nats.ErrDisconnected) || errors.Is(err, nats.ErrNoServers) {
|
||||
return true
|
||||
}
|
||||
words := err.Error()
|
||||
for _, w := range []string{"timeout", "deadline exceeded", "no link to the bus", "did not answer", "connection closed"} {
|
||||
if strings.Contains(words, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Transient says err is the bus not answering for a moment, as a reload of its accounts makes it for a few seconds:
|
||||
// JetStream not enabled, or no responders; each fails at once, so a pass tried again costs only its wait. A read that
|
||||
// timed out is not one: it already waited longer than such a pause, and is most often a read the bus refuses. Matched
|
||||
// by value and by words, since the vault's and the reads' errors arrive wrapped in words (issue 1126).
|
||||
func Transient(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, nats.ErrJetStreamNotEnabled) || errors.Is(err, nats.ErrJetStreamNotEnabledForAccount) || errors.Is(err, nats.ErrNoResponders) ||
|
||||
errors.Is(err, ErrBusMissed) {
|
||||
return true
|
||||
}
|
||||
words := err.Error()
|
||||
for _, w := range []string{"jetstream not enabled", "no responders"} {
|
||||
if strings.Contains(words, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -2,10 +2,17 @@ package selfcompose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +20,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
@@ -236,8 +244,156 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
t.Fatalf("handed %v, want %s and %s", got, lemursKey, nodeKey)
|
||||
}
|
||||
|
||||
// Issue 1126, the switch made safe: a bus not answering for a moment (a reload of its accounts) is tried again and
|
||||
// refuses nothing; a pass whose run ended refuses nothing either.
|
||||
setFor(t, &composeAgain, []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond})
|
||||
fresh := func(get func(context.Context, string, string) ([]byte, error)) (*Composer, *[]string) {
|
||||
var refusals []string
|
||||
e := b.env()
|
||||
e.Get = get
|
||||
e.Refused = func(why string, held map[string]string) {
|
||||
for _, m := range heldModules(held) {
|
||||
why += "held " + m + ": " + held[m]
|
||||
}
|
||||
refusals = append(refusals, why)
|
||||
}
|
||||
c2 := New(e)
|
||||
c.mu.Lock()
|
||||
for k, v := range c.wants {
|
||||
c2.wants[k] = v
|
||||
}
|
||||
for k, v := range c.placements {
|
||||
c2.placements[k] = v
|
||||
}
|
||||
for k, v := range c.records {
|
||||
c2.records[k] = v
|
||||
}
|
||||
for k, v := range c.modules {
|
||||
c2.modules[k] = v
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return c2, &refusals
|
||||
}
|
||||
for name, row := range map[string]struct {
|
||||
failures int // reads failing before the bus answers
|
||||
cancel bool
|
||||
composes bool
|
||||
}{
|
||||
"the bus paused for one read": {failures: 1, composes: true},
|
||||
"the bus paused past every try": {failures: 100},
|
||||
"the run ended mid-pass": {failures: 100, cancel: true},
|
||||
} {
|
||||
failed := 0
|
||||
pctx, pcancel := context.WithCancel(ctx)
|
||||
c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) {
|
||||
if failed < row.failures {
|
||||
failed++
|
||||
if row.cancel {
|
||||
pcancel()
|
||||
return nil, fmt.Errorf("the vault's own could not be asked: %w", context.Canceled)
|
||||
}
|
||||
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, nats.ErrJetStreamNotEnabled)
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.kv[bucket+"/"+key], nil
|
||||
})
|
||||
handed := c2.compose(pctx)
|
||||
pcancel()
|
||||
switch {
|
||||
case row.composes && (len(handed) == 0 || len(*refusals) != 0):
|
||||
t.Errorf("%s: handed %d, refusals %q; want composed and nothing refused", name, len(handed), *refusals)
|
||||
case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != 1+len(composeAgain)):
|
||||
t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, 1+len(composeAgain))
|
||||
case row.cancel && len(*refusals) != 0:
|
||||
t.Errorf("%s: a pass whose run ended refused %q", name, *refusals)
|
||||
}
|
||||
}
|
||||
if !Transient(fmt.Errorf("the manifest of avahi: the revisions of x in mesh-delivery-indexer_manifests: nats: jetstream not enabled")) ||
|
||||
!Transient(fmt.Errorf("x: %w", nats.ErrNoResponders)) || Transient(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) ||
|
||||
Transient(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) {
|
||||
t.Error("Transient misjudges a bus that does not answer, or a refusal")
|
||||
}
|
||||
// Issue 1126, the applying path (review of the seed): one module's own refusal holds that module alone. On novox,
|
||||
// de-spiegel's unset route names (its manifest at the indexer's commit e25e1beb, captured) refused the whole node and
|
||||
// marked all its modules not composed. Now lemurs is handed; de-spiegel is held, and proxy with it, which receives its
|
||||
// routes: what was handed of de-spiegel stays handed, so the node keeps running it, and the agent is told those two
|
||||
// alone, de-spiegel in its config form's words.
|
||||
spiegel, err := os.ReadFile("testdata/de-spiegel-e25e1beb.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
proxy := `{"module":"proxy","version":"1","capabilities":["systemd"],"provides":[{"name":"route","scope":"mesh"},{"name":"smtp","scope":"mesh"}],` +
|
||||
`"resources":[{"id":"service","type":"service","unit":"proxy.service","state":"running","boot":"enabled"}]}`
|
||||
c3, told := fresh(env.Get)
|
||||
for m, raw := range map[string]string{"de-spiegel": string(spiegel), "proxy": proxy} {
|
||||
dig := delivery.ManifestDigest([]byte(raw))
|
||||
enc, _ := delivery.EncodeManifest(delivery.Manifest{Module: m, JSON: raw})
|
||||
k, _ := delivery.ManifestsKey(dig)
|
||||
b.mu.Lock()
|
||||
b.kv[delivery.ManifestsBucket+"/"+k] = enc
|
||||
b.mu.Unlock()
|
||||
mfp := delivery.SourcePrefix2 + strings.Repeat(map[string]string{"de-spiegel": "d", "proxy": "e"}[m], 64)
|
||||
c3.modules[m] = delivery.Module{FP2: mfp, Manifest: dig, Repository: "novox/" + m, Commit: "c"}
|
||||
c3.wants[m] = delivery.Want{FP: mfp, Self: true}
|
||||
}
|
||||
p3 := c3.placements["n1"]
|
||||
p3.Modules = append(append([]delivery.PlacedModule{}, p3.Modules...), delivery.PlacedModule{Module: "proxy"}, delivery.PlacedModule{Module: "de-spiegel"})
|
||||
c3.placements["n1"] = p3
|
||||
r3 := c3.records["n1"]
|
||||
r3.Capabilities = []string{"systemd", "container-runtime"}
|
||||
c3.records["n1"] = r3
|
||||
spiegelKey, _ := delivery.ComposedKey("n1", "de-spiegel", c3.wants["de-spiegel"].FP)
|
||||
proxyKey, _ := delivery.ComposedKey("n1", "proxy", c3.wants["proxy"].FP)
|
||||
c3.handed[spiegelKey] = "sha256:" + strings.Repeat("5", 64) // what the node runs of it, handed before
|
||||
handed3 := map[string]agent.Update{}
|
||||
for _, u := range c3.compose(ctx) {
|
||||
handed3[u.Key] = u
|
||||
}
|
||||
lemursKey3, _ := delivery.ComposedKey("n1", "lemurs", c3.wants["lemurs"].FP)
|
||||
if u, ok := handed3[lemursKey3]; !ok || u.Deleted {
|
||||
t.Errorf("lemurs, which composes, is not handed beside de-spiegel's refusal: %v (told %q)", handed3, *told)
|
||||
}
|
||||
if u, ok := handed3[proxyKey]; ok {
|
||||
t.Errorf("proxy is handed without held de-spiegel's routes: %+v", u)
|
||||
}
|
||||
if u, ok := handed3[spiegelKey]; ok {
|
||||
t.Errorf("held de-spiegel is handed %+v, want what was handed of it kept", u)
|
||||
}
|
||||
if len(*told) != 1 || !strings.HasPrefix((*told)[0], "held de-spiegel: ") || !strings.Contains((*told)[0], "${config:praktijk-name}") ||
|
||||
!strings.Contains((*told)[0], "held proxy: ") || strings.Contains((*told)[0], "held lemurs") {
|
||||
t.Errorf("the agent was told %q, want de-spiegel held in its config form's words, and proxy", *told)
|
||||
}
|
||||
|
||||
// A read the bus did not answer, timed out or not, is tried again later; the run's own end and a refusal are not.
|
||||
if !BusFailed(fmt.Errorf("the revisions of it2.mailer.domain: %w", context.DeadlineExceeded)) || !BusFailed(fmt.Errorf("x: %w", nats.ErrNoResponders)) ||
|
||||
!BusFailed(fmt.Errorf("the have of web: %w", nats.ErrTimeout)) || BusFailed(fmt.Errorf("x: %w", context.Canceled)) ||
|
||||
BusFailed(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) || BusFailed(nil) {
|
||||
t.Error("BusFailed misjudges a read the bus did not answer, the run's end, or a refusal")
|
||||
}
|
||||
|
||||
// Issue 1126, C2: a config value is laid as the controller lays it (catalogue.LaidValue), from the mesh's, the node's
|
||||
// or the module's own and never from a default or an unset one, both of this node's and of what another node serves:
|
||||
// a default laid blanked Element's links on ace.
|
||||
for _, row := range []struct {
|
||||
source string
|
||||
laid bool
|
||||
}{{"mesh", true}, {"node", true}, {"module", true}, {"default", false}, {"unset", false}, {"", false}} {
|
||||
probe := New(env)
|
||||
value := []byte(`{"value":["x"],"source":"` + row.source + `"}`)
|
||||
probe.takeValue(agent.Update{Key: "n1.lemurs.links", Value: value})
|
||||
_, laid := probe.values["lemurs"]["links"]
|
||||
b.mu.Lock()
|
||||
b.kv[ResolvedBucket+"/n2.lemurs.links"] = value
|
||||
b.mu.Unlock()
|
||||
served, err := reader{c: probe}.Served(ctx, "n2", "lemurs", []string{"links"})
|
||||
if laid != row.laid || err != nil || (served["links"] != nil) != row.laid {
|
||||
t.Errorf("a value from %q: laid %v, served %v (%v), want laid %v", row.source, laid, served, err, row.laid)
|
||||
}
|
||||
}
|
||||
|
||||
// A config value of another module, and a value changing nothing composed, hand nothing.
|
||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1}`))
|
||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1,"source":"node"}`))
|
||||
quiet(t, out)
|
||||
|
||||
// A want removed takes its composition back.
|
||||
@@ -297,7 +453,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
t.Fatalf("keeper's composition does not carry the sealed value: %s", handed)
|
||||
}
|
||||
before := handed
|
||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":2}`))
|
||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":2,"source":"node"}`))
|
||||
quiet(t, out)
|
||||
b.mu.Lock()
|
||||
generation++
|
||||
@@ -313,32 +469,52 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
// node runs comes from keeper's have (Held); the vault seals the value again, other bytes, and the box the have holds
|
||||
// is kept, since it opens to the same value: the composition does not move. The box from before the rotation opens
|
||||
// to the old value, so the vault's is taken.
|
||||
boxOf := func(raw string) string {
|
||||
// Issue 1126, C1: a box is kept only at the slot that holds it (the same field of the same resource of the same
|
||||
// module). A box another module holds (a provider's grant of the value its consumer holds as a secret, on one node)
|
||||
// or another resource of keeper holds is never taken, though it opens to the same value; a have written before slots
|
||||
// (a list) is reused within its own module alone.
|
||||
boxOf := func(raw string) (string, string) {
|
||||
c, err := delivery.DecodeComposed([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sealed := agent.SealedIn(c)
|
||||
sealed := delivery.SealsIn(c)
|
||||
if len(sealed) != 1 {
|
||||
t.Fatalf("keeper's composition carries %v", sealed)
|
||||
}
|
||||
return sealed[0]
|
||||
for slot, box := range sealed {
|
||||
return slot, box
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
// And the node's own key (review of L2, round 3, finding 1): a core box its own composition carries is held there, and
|
||||
// kept after a restart as a module's is, so the node's own composition does not move either.
|
||||
slot, rotatedBox := boxOf(string(rotated.Value))
|
||||
_, beforeBox := boxOf(before)
|
||||
for _, row := range []struct {
|
||||
name, held, in string
|
||||
name, in, held string
|
||||
slots delivery.Seals
|
||||
kept bool
|
||||
}{{"the have holds the rotated value", boxOf(string(rotated.Value)), "keeper", true},
|
||||
{"the have holds the value before the rotation", boxOf(before), "keeper", false},
|
||||
{"the node's own key holds the rotated value", boxOf(string(rotated.Value)), agent.NodeModule, true}} {
|
||||
fails bool // the have's read the bus did not answer: no composition, made again later (2026-10-28)
|
||||
}{
|
||||
{"the have holds the rotated value at its slot", "keeper", rotatedBox, delivery.Seals{slot: rotatedBox}, true, false},
|
||||
{"the have holds the value before the rotation", "keeper", beforeBox, delivery.Seals{slot: beforeBox}, false, false},
|
||||
{"a have written before slots holds the rotated value", "keeper", rotatedBox, nil, true, false},
|
||||
{"another module holds the same value", "lemurs", rotatedBox, delivery.Seals{"lemurs.grant-x#sealed": rotatedBox}, false, false},
|
||||
{"another resource of keeper holds the same value", "keeper", rotatedBox, delivery.Seals{"keeper.other#sealed": rotatedBox}, false, false},
|
||||
{"the bus does not answer for the have", "keeper", rotatedBox, nil, false, true},
|
||||
} {
|
||||
restarted := env
|
||||
restarted.Running = func() []delivery.Composed { return nil }
|
||||
restarted.Held = func(module string) []string {
|
||||
if module == row.in {
|
||||
return []string{row.held}
|
||||
restarted.Held = func(module string) (delivery.Seals, []string, error) {
|
||||
if module != row.in {
|
||||
return nil, nil, nil
|
||||
}
|
||||
return nil
|
||||
if row.fails {
|
||||
return nil, nil, fmt.Errorf("the have of %s: %w", module, nats.ErrTimeout)
|
||||
}
|
||||
if row.slots == nil {
|
||||
return nil, []string{row.held}, nil
|
||||
}
|
||||
return row.slots, nil, nil
|
||||
}
|
||||
restarted.Unseal = func(box string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
@@ -349,13 +525,102 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
return nil, fmt.Errorf("not sealed to n1: %s", box)
|
||||
}
|
||||
again := New(restarted)
|
||||
again.wants = map[string]delivery.Want{"keeper": {FP: fp, Self: true}, agent.NodeModule: {}}
|
||||
rows, err := reader{c: again}.OwnSecrets(ctx, "n1")
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("%s: own secrets %+v, %v", row.name, rows, err)
|
||||
c.mu.Lock()
|
||||
for k, v := range c.wants {
|
||||
again.wants[k] = v
|
||||
}
|
||||
if kept := rows[0].Sealed == row.held; kept != row.kept {
|
||||
t.Errorf("%s: composed with %s (held %s), kept %v, want %v", row.name, rows[0].Sealed, row.held, kept, row.kept)
|
||||
for k, v := range c.placements {
|
||||
again.placements[k] = v
|
||||
}
|
||||
for k, v := range c.records {
|
||||
again.records[k] = v
|
||||
}
|
||||
for k, v := range c.modules {
|
||||
again.modules[k] = v
|
||||
}
|
||||
c.mu.Unlock()
|
||||
var composed string
|
||||
for _, u := range again.compose(ctx) {
|
||||
if u.Key == keeperKey {
|
||||
composed = string(u.Value)
|
||||
}
|
||||
}
|
||||
if row.fails {
|
||||
if composed != "" || !again.owed.Load() {
|
||||
t.Errorf("%s: composed %v and owes a pass again %v; want nothing handed and a pass owed", row.name, composed != "", again.owed.Load())
|
||||
}
|
||||
// Then the catalogue no longer places the node: it composes nothing of its own, and owes no pass either, so
|
||||
// no timer is armed again (rev-seed-same-3, round 1).
|
||||
again.mu.Lock()
|
||||
clear(again.placements)
|
||||
again.mu.Unlock()
|
||||
if again.compose(ctx); again.owed.Load() {
|
||||
t.Errorf("%s: an unplaced node still owes a pass", row.name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if composed == "" {
|
||||
t.Fatalf("%s: keeper was not composed", row.name)
|
||||
}
|
||||
if _, box := boxOf(composed); (box == row.held) != row.kept {
|
||||
t.Errorf("%s: composed with %s (held %s), kept %v", row.name, box, row.held, row.kept)
|
||||
}
|
||||
}
|
||||
// And the node's own key (review of L2, round 3, finding 1): a core box its own composition carries is held there, and
|
||||
// kept after a restart as a module's is, so the node's own composition does not move either.
|
||||
ownComp := delivery.Composed{Node: "n1", Resources: []json.RawMessage{json.RawMessage(`{"id":"core-box","sealed":"{\"ask\":999}"}`)}}
|
||||
b.mu.Lock()
|
||||
sealedAt[`{"ask":999}`] = generation + 1
|
||||
b.mu.Unlock()
|
||||
coreEnv := env
|
||||
coreEnv.Unseal = func(box string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if g, ok := sealedAt[box]; ok {
|
||||
return []byte(fmt.Sprintf("password-%d", g)), nil
|
||||
}
|
||||
return nil, fmt.Errorf("not sealed to n1: %s", box)
|
||||
}
|
||||
coreEnv.Held = func(module string) (delivery.Seals, []string, error) {
|
||||
if module == agent.NodeModule {
|
||||
return delivery.Seals{"core-box#sealed": rotatedBox}, nil, nil
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
if got := delivery.SealsIn(New(coreEnv).keepRunning(ownComp, nil))["core-box#sealed"]; got != rotatedBox {
|
||||
t.Errorf("the node's own core box after a restart is %s, want the one its key holds %s", got, rotatedBox)
|
||||
}
|
||||
// Issue 1126, C1, the seed-7 probe: each entry of a file's secrets (catalogue's intoFile, the shape captured from it
|
||||
// with the module a composition names) is a slot of its own: the box that runs at that entry is kept where it opens
|
||||
// to the same value; one at another entry is not, though it opens to the same.
|
||||
b.mu.Lock()
|
||||
sealedAt["box-again"] = sealedAt[rotatedBox]
|
||||
sealedAt["box-other"] = sealedAt[rotatedBox] + 100
|
||||
b.mu.Unlock()
|
||||
ini := func(db, key string) delivery.Composed {
|
||||
return delivery.Composed{Node: "n1", Module: "grafana", Resources: []json.RawMessage{json.RawMessage(
|
||||
`{"content":"[database]\npassword = ${secret:postgres-database}\n[security]\nsecret_key = ${secret:secret-key}\n",` +
|
||||
`"id":"grafana.ini","mode":"0600","module":"grafana","path":"/etc/grafana/grafana.ini","secrets":{"postgres-database":` +
|
||||
strconv.Quote(db) + `,"secret-key":` + strconv.Quote(key) + `},"type":"file"}`)}}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
held delivery.Seals
|
||||
want string
|
||||
}{
|
||||
{"the box that runs at its entry", delivery.Seals{"grafana.ini#secrets.postgres-database": rotatedBox}, rotatedBox},
|
||||
{"a box at another entry", delivery.Seals{"grafana.ini#secrets.secret-key": rotatedBox}, "box-again"},
|
||||
} {
|
||||
secretsEnv := coreEnv
|
||||
secretsEnv.Held = func(module string) (delivery.Seals, []string, error) {
|
||||
if module == "grafana" {
|
||||
return row.held, nil, nil
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
got := delivery.SealsIn(New(secretsEnv).keepRunning(ini("box-again", "box-other"), nil))
|
||||
if got["grafana.ini#secrets.postgres-database"] != row.want || got["grafana.ini#secrets.secret-key"] != "box-other" {
|
||||
t.Errorf("%s: the secrets kept are %v", row.name, got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -382,7 +647,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
return nil, fmt.Errorf("not sealed to n1: %s", box)
|
||||
}
|
||||
var refusals []string
|
||||
pairEnv.Refused = func(why string) { refusals = append(refusals, why) }
|
||||
pairEnv.Refused = func(why string, held map[string]string) { refusals = append(refusals, why) }
|
||||
pc := New(pairEnv)
|
||||
consumer := []compose.CredentialKey{{Provision: "pg", Consumer: "n1", ConsumerModule: "web", Provider: "n2"}}
|
||||
provider := []compose.CredentialKey{{Provision: "s3", Consumer: "n3", ConsumerModule: "files", Provider: "n1"}}
|
||||
@@ -390,14 +655,19 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
if err != nil || len(creds) != 1 || len(issued) != 1 {
|
||||
t.Fatalf("credentials %+v issued %+v, %v", creds, issued, err)
|
||||
}
|
||||
if creds[0].State != compose.CredentialHeld || creds[0].Provider != "n2" || creds[0].Sealed != `{"run":1}` ||
|
||||
if creds[0].State != compose.CredentialHeld || creds[0].Provider != "n2" || creds[0].Sealed != `{"c":1}` ||
|
||||
issued[0].Consumer != "n3" || issued[0].ForProvider != `{"p":1}` {
|
||||
t.Errorf("credentials %+v issued %+v", creds, issued)
|
||||
}
|
||||
// What runs at the credential's slot is kept, after the composition (keepRunning, issue 1126, C1).
|
||||
webComp := delivery.Composed{Node: "n1", Module: "web", Resources: []json.RawMessage{json.RawMessage(`{"id":"web.secret-pg","sealed":"{\"c\":1}"}`)}}
|
||||
if got := delivery.SealsIn(pc.keepRunning(webComp, pairEnv.Running()))["web.secret-pg#sealed"]; got != `{"run":1}` {
|
||||
t.Errorf("web's credential composed as %s, want the box that runs", got)
|
||||
}
|
||||
// A node that cannot compose tells the agent once, and once again when it composes (the agent's haves say it).
|
||||
pc.refused("a stand-in for web's password")
|
||||
pc.refused("a stand-in for web's password")
|
||||
pc.refused("")
|
||||
pc.refused("a stand-in for web's password", nil)
|
||||
pc.refused("a stand-in for web's password", nil)
|
||||
pc.refused("", nil)
|
||||
if len(refusals) != 2 || refusals[0] == "" || refusals[1] != "" {
|
||||
t.Errorf("the agent was told %q", refusals)
|
||||
}
|
||||
@@ -420,11 +690,15 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}}
|
||||
senv := readOnly.env()
|
||||
senv.Watch = sb.watch
|
||||
var ownFails atomic.Bool // the vault's own on a bus that does not answer (2026-10-28, ace's answer to seed-7)
|
||||
senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) {
|
||||
if ownFails.Load() {
|
||||
return nil, fmt.Errorf("the vault's own could not be asked: %w", nats.ErrNoResponders)
|
||||
}
|
||||
return json.RawMessage(`{"node":"n1","key":"sk","own":[]}`), nil // nothing issued for keeper: it is not placed here
|
||||
}
|
||||
// What n1 runs: lemurs and its own composition, as its composer composes them from the trunk.
|
||||
running, _, err := frombus.Compose(ctx, fixedReader{placements: map[string]delivery.Placement{"n1": trunk},
|
||||
running, _, _, err := frombus.Compose(ctx, fixedReader{placements: map[string]delivery.Placement{"n1": trunk},
|
||||
records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}},
|
||||
modules: mods, manifests: map[string][]byte{digest: []byte(lemurs), kdigest: []byte(keeper)}}, frombus.Mesh{}, frombus.Scope{Node: "n1"}, map[string]string{"lemurs": fp})
|
||||
if err != nil {
|
||||
@@ -437,6 +711,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The controller's compositions at n1's wants, in n1's composition bucket: what a head deleting its placement
|
||||
// returns it to (the seed's revert); here the ones it runs.
|
||||
composedBucket, _ := delivery.ComposedBucketFor("n1")
|
||||
for key, c := range running {
|
||||
raw, err := delivery.EncodeComposed(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readOnly.mu.Lock()
|
||||
readOnly.kv[composedBucket+"/"+key] = raw
|
||||
readOnly.mu.Unlock()
|
||||
}
|
||||
answered := make(chan map[string]delivery.CandidateAnswer, 8)
|
||||
sh := NewShadow(senv, func(_ context.Context, a map[string]delivery.CandidateAnswer) { answered <- a })
|
||||
sctx, scancel := context.WithCancel(ctx)
|
||||
@@ -447,7 +733,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
go sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond })
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
sb.missing(delivery.PlacementCandidatesBucket, false)
|
||||
sb.made(6)
|
||||
sb.made(7)
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
|
||||
tp, _ := delivery.EncodePlacement(trunk)
|
||||
sb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
||||
@@ -456,14 +742,28 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
raw, _ := delivery.EncodeModule(m)
|
||||
sb.put(delivery.ModulesBucket, name, raw, 1)
|
||||
}
|
||||
wantLemurs, _ := delivery.WantKey("n1", "lemurs")
|
||||
var lemursDigest string
|
||||
for _, c := range running {
|
||||
if c.Module == "lemurs" {
|
||||
lemursDigest, _ = delivery.ComposedDigestOf(c)
|
||||
}
|
||||
}
|
||||
wantRaw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: lemursDigest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sb.put(delivery.AssignerWantsBucket, wantLemurs, wantRaw, 1)
|
||||
wantNode, _ := delivery.WantNodeKey("n1")
|
||||
nodeRaw, _ := delivery.EncodeWant(delivery.Want{})
|
||||
sb.put(delivery.AssignerWantsBucket, wantNode, nodeRaw, 1)
|
||||
candidate := func(group string, rev uint64, modules ...string) {
|
||||
p := trunk
|
||||
p.Modules = nil
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group,
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h-" + group}, Placement: &p})
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: group, Placement: &p})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -474,6 +774,23 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
candidate("adds", 4, "lemurs", "keeper")
|
||||
candidate("drops", 5)
|
||||
candidate("ghost", 6, "lemurs", "ghost")
|
||||
// A head deleting n1's placement file (the seed's revert, mesh-catalog #526): n1 returns to the controller's
|
||||
// compositions, answered as such, not as every module removed.
|
||||
if raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "deletes"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
k, _ := delivery.PlacementCandidateKey("deletes", "n1")
|
||||
sb.put(delivery.PlacementCandidatesBucket, k, raw, 7)
|
||||
}
|
||||
// A proposal that changes another node's key and reaches n1's composition (ADR 0567 §3): n1's candidate holds no
|
||||
// placement of its own change (Reached, nil), and n1 answers against its current placement, never as one whose key
|
||||
// the proposal takes away.
|
||||
if raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "reach", Reached: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
k, _ := delivery.PlacementCandidateKey("reach", "n1")
|
||||
sb.put(delivery.PlacementCandidatesBucket, k, raw, 8)
|
||||
}
|
||||
sb.caught()
|
||||
var answers map[string]delivery.CandidateAnswer
|
||||
waitFor := func(why string, ok func(map[string]delivery.CandidateAnswer) bool) {
|
||||
@@ -487,7 +804,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
waitFor("every group answered", func(a map[string]delivery.CandidateAnswer) bool { return len(a) == 4 })
|
||||
waitFor("every group answered", func(a map[string]delivery.CandidateAnswer) bool { return len(a) == 6 })
|
||||
// Every watch and get the shadow makes is one the node's grant allows (issue 1126, 2026-10-27 09:21).
|
||||
sb.mu.Lock()
|
||||
for _, w := range sb.watches {
|
||||
@@ -505,7 +822,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
readOnly.mu.Unlock()
|
||||
for group, want := range map[string]func(delivery.CandidateAnswer) bool{
|
||||
"same": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateSame && a.Head == "h-same" && a.Revision == 3 && len(a.Same) == 2
|
||||
return a.Verdict == delivery.CandidateSame && a.Proposal == "same" && a.Revision == 3 && len(a.Same) == 2
|
||||
},
|
||||
"adds": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateDiffers && len(a.Added) == 1 && a.Added[0].Module == "keeper" &&
|
||||
@@ -517,6 +834,14 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
"ghost": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateRefused && a.Refused != ""
|
||||
},
|
||||
"reach": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateSame && a.Returns == "" && a.Revision == 8 && len(a.Removed) == 0 &&
|
||||
fmt.Sprint(a.Same) == "[lemurs node]"
|
||||
},
|
||||
"deletes": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateSame && a.Returns == delivery.ReturnsToController && a.Revision == 7 &&
|
||||
len(a.Removed) == 0 && fmt.Sprint(a.Same) == "[lemurs node]"
|
||||
},
|
||||
} {
|
||||
if !want(answers[group]) {
|
||||
t.Errorf("%s: %+v", group, answers[group])
|
||||
@@ -529,9 +854,100 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
t.Errorf("answers handed again for nothing changed: %+v", a)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
// Issue 1126, 2026-10-28 (shanks answered probe/1126-seed-7 at revision 84 every 13-17 s): a candidate revision is
|
||||
// answered once. A module's have whose health or words alone move (each look rewrites them), and a placement, a
|
||||
// module or a config value written again with the value it held, answer nothing again; a new revision answers its
|
||||
// group alone, and what runs moving answers every group once.
|
||||
answering := func() int {
|
||||
readOnly.mu.Lock()
|
||||
defer readOnly.mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range readOnly.said {
|
||||
if strings.HasPrefix(l, "answering placement candidate ") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
drain := func() {
|
||||
for {
|
||||
select {
|
||||
case answers = <-answered:
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
lemursHave := func(detail string, rev uint64) {
|
||||
raw, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: lemursDigest, Phase: delivery.PhaseRunning,
|
||||
Health: delivery.HealthFailed, Detail: detail})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hk, _ := delivery.HaveKey("n1", "lemurs")
|
||||
sb.put(delivery.HaveBucket, hk, raw, rev)
|
||||
}
|
||||
lemursHave("its tool lemurs_health: no link to the bus is open", 10)
|
||||
drain()
|
||||
settled := answering()
|
||||
lemursHave("its tool lemurs_health: no answer within 10s", 11)
|
||||
lemursHave("its tool lemurs_health: no answer within 20s", 12)
|
||||
sb.put(delivery.PlacementBucket, "placement.n1", tp, 2)
|
||||
lraw, _ := delivery.EncodeModule(mods["lemurs"])
|
||||
sb.put(delivery.ModulesBucket, "lemurs", lraw, 2)
|
||||
drain()
|
||||
if got := answering(); got != settled {
|
||||
t.Errorf("answered %d more time(s) with no candidate revision and no input of the node changed", got-settled)
|
||||
}
|
||||
candidate("same", 8, "lemurs")
|
||||
waitFor("the new revision answered", func(a map[string]delivery.CandidateAnswer) bool { return a["same"].Revision == 8 })
|
||||
drain()
|
||||
if got := answering(); got != settled+1 {
|
||||
t.Errorf("a new revision of one group answered %d group(s), want 1", got-settled)
|
||||
}
|
||||
// Issue 1126, 2026-10-28 (seed-9's probe): each answer says which input woke it, so a loop is told from inputs moving.
|
||||
lastSaid := func(prefix string) string {
|
||||
readOnly.mu.Lock()
|
||||
defer readOnly.mu.Unlock()
|
||||
for i := len(readOnly.said) - 1; i >= 0; i-- {
|
||||
if strings.HasPrefix(readOnly.said[i], prefix) {
|
||||
return readOnly.said[i]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if got := lastSaid("answering placement candidate same "); got != "answering placement candidate same (revision 8; woken by the candidate)" {
|
||||
t.Errorf("a new revision's answer said %q", got)
|
||||
}
|
||||
settled = answering()
|
||||
hk, _ := delivery.HaveKey("n1", "lemurs")
|
||||
moved, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: "sha256:" + strings.Repeat("e", 64), Phase: delivery.PhaseRunning})
|
||||
sb.put(delivery.HaveBucket, hk, moved, 13)
|
||||
drain()
|
||||
if got := answering(); got != settled+len(answers) {
|
||||
t.Errorf("what runs moved, and %d answer(s) were composed again for %d group(s)", got-settled, len(answers))
|
||||
}
|
||||
if got := lastSaid("answering placement candidate drops "); got != "answering placement candidate drops (revision 5; woken by haves)" {
|
||||
t.Errorf("an answer woken by what runs moving said %q", got)
|
||||
}
|
||||
// A read the bus did not answer is no answer to keep: the group is answered again after a bound, though nothing moved.
|
||||
setFor(t, &laterAgain, []time.Duration{20 * time.Millisecond})
|
||||
ownFails.Store(true)
|
||||
candidate("adds", 9, "lemurs", "keeper")
|
||||
waitFor("the answer met the vault not answering", func(a map[string]delivery.CandidateAnswer) bool {
|
||||
return a["adds"].Revision == 9 && strings.Contains(a["adds"].Refused, "no responders")
|
||||
})
|
||||
ownFails.Store(false)
|
||||
waitFor("answered again once the bus answers, with nothing else moved", func(a map[string]delivery.CandidateAnswer) bool {
|
||||
return a["adds"].Revision == 9 && a["adds"].Verdict == delivery.CandidateDiffers
|
||||
})
|
||||
drain()
|
||||
if got := lastSaid("answering placement candidate adds "); got != "answering placement candidate adds (revision 9; woken by a retry after a read the bus did not answer)" {
|
||||
t.Errorf("an answer composed again after the bus did not answer said %q", got)
|
||||
}
|
||||
k, _ := delivery.PlacementCandidateKey("same", "n1")
|
||||
sb.del(delivery.PlacementCandidatesBucket, k)
|
||||
waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 3 })
|
||||
waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 5 })
|
||||
|
||||
// Review of L2b, round 2: a watch ends while the link stays; a group's key is deleted before the shadow watches
|
||||
// again. The new run reads afresh, so that group is no longer answered.
|
||||
@@ -543,9 +959,209 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
sb.missing(delivery.PlacementCandidatesBucket, false)
|
||||
waitFor("a group deleted while the shadow was down is still answered", func(a map[string]delivery.CandidateAnswer) bool {
|
||||
_, ok := a["drops"]
|
||||
return !ok && len(a) == 2
|
||||
return !ok && len(a) == 4
|
||||
})
|
||||
|
||||
// Whose compositions a node applies at a start (review of the shadow, round 1): only a read that succeeds chooses; a
|
||||
// failing read is tried again, said once, and chooses neither, even when the link ends.
|
||||
for name, row := range map[string]struct {
|
||||
answers []error // per try; nil then the value
|
||||
value []byte
|
||||
end bool
|
||||
placed bool
|
||||
tries int
|
||||
quiet bool // read and known: nothing to say
|
||||
// switched: the node's have says it composed its own before (it never falls back on a failed read).
|
||||
switched bool
|
||||
}{
|
||||
"switched: placed after two failures": {answers: []error{fmt.Errorf("nats: timeout"), fmt.Errorf("nats: timeout")}, value: []byte(`{}`), placed: true, tries: 3, switched: true},
|
||||
"never switched: a timeout is not placed, said once": {answers: []error{fmt.Errorf("nats: timeout")}, placed: false, tries: 1},
|
||||
"no key": {placed: false, tries: 1},
|
||||
"no placement state yet": {answers: []error{fmt.Errorf("the bucket %s: %w", delivery.PlacementBucket, nats.ErrBucketNotFound)}, placed: false, tries: 1, quiet: true},
|
||||
"switched: failing until the link ends": {answers: []error{fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready"),
|
||||
fmt.Errorf("jetstream not ready")}, end: true, tries: 3, switched: true},
|
||||
} {
|
||||
lctx, lcancel := context.WithCancel(ctx)
|
||||
tries := 0
|
||||
var said []string
|
||||
get := func(context.Context, string, string) ([]byte, error) {
|
||||
tries++
|
||||
if tries <= len(row.answers) {
|
||||
if row.end && tries == len(row.answers) {
|
||||
lcancel()
|
||||
}
|
||||
return nil, row.answers[tries-1]
|
||||
}
|
||||
return row.value, nil
|
||||
}
|
||||
placed, err := Placed(lctx, get, "n1", func(int) time.Duration { return time.Millisecond }, func(s string) { said = append(said, s) }, row.switched)
|
||||
lcancel()
|
||||
if placed != row.placed || tries != row.tries || (err != nil) != row.end || (len(row.answers) > 0 && !row.quiet) != (len(said) == 1) {
|
||||
t.Errorf("%s: placed %v, err %v after %d reads, said %q", name, placed, err, tries, said)
|
||||
}
|
||||
}
|
||||
|
||||
// Whether the node composed its own before is read from its own have: own says yes, the controller's or no have says
|
||||
// no, and a have that cannot be read says yes (the safe side: a node waits and does not fall back).
|
||||
for name, row := range map[string]struct {
|
||||
raw []byte
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
"own": {raw: []byte(`{"v":1,"composes":"own"}`), want: true},
|
||||
"the controller's": {raw: []byte(`{"v":1,"composes":"controller"}`), want: false},
|
||||
"no have yet": {want: false},
|
||||
"a have that cannot be read": {err: fmt.Errorf("nats: timeout"), want: true},
|
||||
"a have that is no have": {raw: []byte(`{`), want: true},
|
||||
} {
|
||||
get := func(context.Context, string, string) ([]byte, error) { return row.raw, row.err }
|
||||
if got := ComposedItsOwn(ctx, get, "n1"); got != row.want {
|
||||
t.Errorf("composed its own, %s: %v, want %v", name, got, row.want)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126, the shadow shows its work: one input not read after the wait is said, naming it, in the journal and in
|
||||
// a refused answer to each group known; once read, the group is answered for real.
|
||||
setFor(t, ¬ReadyAfter, 30*time.Millisecond)
|
||||
nb := newPatternBus()
|
||||
nenv := readOnly.env()
|
||||
nenv.Watch = nb.watch
|
||||
nenv.Running = senv.Running
|
||||
var nsaid []string
|
||||
var nmu sync.Mutex
|
||||
nenv.Say = func(l string) { nmu.Lock(); nsaid = append(nsaid, l); nmu.Unlock() }
|
||||
nanswered := make(chan map[string]delivery.CandidateAnswer, 8)
|
||||
nsh := NewShadow(nenv, func(_ context.Context, a map[string]delivery.CandidateAnswer) { nanswered <- a })
|
||||
nctx, ncancel := context.WithCancel(ctx)
|
||||
defer ncancel()
|
||||
go func() { _ = nsh.Run(nctx) }()
|
||||
nb.made(7)
|
||||
nb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
||||
nb.put(delivery.HaveBucket, "n1.node", rec, 1)
|
||||
for name, m := range mods {
|
||||
raw, _ := delivery.EncodeModule(m)
|
||||
nb.put(delivery.ModulesBucket, name, raw, 1)
|
||||
}
|
||||
sameRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "same", Placement: &trunk})
|
||||
sameKey, _ := delivery.PlacementCandidateKey("same", "n1")
|
||||
nb.put(delivery.PlacementCandidatesBucket, sameKey, sameRaw, 3)
|
||||
// Issue 1126, T3: an older group (a closed pull request's candidate, rev 2) is answered after the newest.
|
||||
oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "older", Placement: &trunk})
|
||||
oldKey, _ := delivery.PlacementCandidateKey("older", "n1")
|
||||
nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2)
|
||||
for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} {
|
||||
nb.send(bucket, agent.Update{Caught: true})
|
||||
}
|
||||
select {
|
||||
case a := <-nanswered:
|
||||
if g := a["same"]; g.Verdict != delivery.CandidateRefused || !strings.Contains(g.Refused, delivery.AssignerWantsBucket) || g.Proposal != "same" {
|
||||
t.Errorf("a shadow waiting for its wants answered %+v", g)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("a shadow not ready after its wait answered nothing")
|
||||
}
|
||||
nmu.Lock()
|
||||
if !strings.Contains(strings.Join(nsaid, "\n"), "this node answers no placement candidate yet: the node has not read "+delivery.AssignerWantsBucket) {
|
||||
t.Errorf("a shadow not ready said %q", nsaid)
|
||||
}
|
||||
nmu.Unlock()
|
||||
setFor(t, &slowAfter, time.Nanosecond) // every answer is slow: said, and noted in words that do not move with the time
|
||||
nb.send(delivery.AssignerWantsBucket, agent.Update{Caught: true})
|
||||
select {
|
||||
case a := <-nanswered:
|
||||
if g := a["same"]; g.Verdict != delivery.CandidateSame || !strings.Contains(g.Note, "this answer took over") {
|
||||
t.Errorf("the shadow, once ready, answered %+v", g)
|
||||
}
|
||||
// The newest first, handed at once: the older group still holds the answer it had.
|
||||
if g := a["older"]; g.Verdict != delivery.CandidateRefused {
|
||||
t.Errorf("the older group was answered before the newest was handed: %+v", g)
|
||||
}
|
||||
if g := (<-nanswered)["older"]; g.Verdict != delivery.CandidateSame {
|
||||
t.Errorf("the older group, answered next: %+v", g)
|
||||
}
|
||||
// Issue 1126, T2: an answer says when it starts and when it ends (and, every minute between, what it waits on).
|
||||
nmu.Lock()
|
||||
said := strings.Join(nsaid, "\n")
|
||||
for _, line := range []string{"the answer to same took", "answering placement candidate same (revision 3; woken by its first answer since the shadow began)",
|
||||
"answered placement candidate same in", "answering placement candidate older (revision 2;"} {
|
||||
if !strings.Contains(said, line) {
|
||||
t.Errorf("%q was not said: %q", line, nsaid)
|
||||
}
|
||||
}
|
||||
if strings.Index(said, "answering placement candidate same") > strings.Index(said, "answering placement candidate older") {
|
||||
t.Errorf("the older group was answered first: %q", nsaid)
|
||||
}
|
||||
nmu.Unlock()
|
||||
done := nsh.c.waiting("the vault's own secrets")
|
||||
if got := nsh.c.waitingOn(); got != "the vault's own secrets" {
|
||||
t.Errorf("a running answer would say it waits on %q", got)
|
||||
}
|
||||
done()
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the shadow, once ready, answered nothing")
|
||||
}
|
||||
|
||||
// Issue 1126, probe #532 on shanks: what runs is the composition the have names, never the copy of the declaration
|
||||
// kept before the switch (no fp, no digest): a module composed at the digest its have says runs is the same.
|
||||
{
|
||||
probe := delivery.CandidateAnswer{Proposal: "pro", Revision: 68, Verdict: delivery.CandidateDiffers, Same: []string{"adwaita"},
|
||||
Differs: []delivery.ModuleDiffers{
|
||||
{Module: "xterm", Composed: "sha256:3c0a31df64283c0678b1cdbeff8ac030609cab65cc7e47faaf714da743d9340e", Why: "the fingerprint differs"},
|
||||
{Module: "zsh", Composed: "sha256:6093f31056d39eaf604410b7289ec6b5fe602b19c8547283562fcfc078c4844f", Resource: "zsh.bundle-tools", Why: "the resource differs"},
|
||||
{Module: "brother", Composed: "sha256:0702b13991288da72d230460624e949d3c1d7d833eac2ec2134073c82e5a61cf",
|
||||
Running: "sha256:c11c14af2738e3fcdb04da4004bbe84728bdefaf8c986d76601ffbe502435871", Resource: "brother.needs-broker-device", Why: "the resource differs"}}}
|
||||
got := asHavesSay(probe, map[string]string{
|
||||
"xterm": "sha256:3c0a31df64283c0678b1cdbeff8ac030609cab65cc7e47faaf714da743d9340e",
|
||||
"zsh": "sha256:6093f31056d39eaf604410b7289ec6b5fe602b19c8547283562fcfc078c4844f",
|
||||
"brother": "sha256:c11c14af2738e3fcdb04da4004bbe84728bdefaf8c986d76601ffbe502435871"})
|
||||
if fmt.Sprint(got.Same) != "[adwaita xterm zsh]" || len(got.Differs) != 1 || got.Differs[0].Module != "brother" || got.Verdict != delivery.CandidateDiffers {
|
||||
t.Errorf("shanks' probe answered %+v", got)
|
||||
}
|
||||
if one := asHavesSay(delivery.CandidateAnswer{Proposal: "pro", Revision: 1, Verdict: delivery.CandidateDiffers, Differs: probe.Differs[:1]},
|
||||
map[string]string{"xterm": probe.Differs[0].Composed}); one.Verdict != delivery.CandidateSame {
|
||||
t.Errorf("a module the same as its have says is answered %+v", one)
|
||||
}
|
||||
}
|
||||
|
||||
// The way back, row by row (review of the shadow, round 1): the controller's compositions at n1's wants compared
|
||||
// with what runs; one differing is answered so; one the controller holds none of refuses, without returning.
|
||||
var runsList []delivery.Composed
|
||||
for _, c := range running {
|
||||
runsList = append(runsList, c)
|
||||
}
|
||||
lemursKey, _ = delivery.ComposedKey("n1", "lemurs", fp)
|
||||
changed := running[lemursKey]
|
||||
changed.Resources = []json.RawMessage{json.RawMessage(`{"id":"lemurs.service","type":"service","module":"lemurs","unit":"lemurs.service","state":"stopped","boot":"disabled"}`)}
|
||||
changedRaw, _ := delivery.EncodeComposed(changed)
|
||||
for name, row := range map[string]struct {
|
||||
lemurs []byte
|
||||
check func(delivery.CandidateAnswer) bool
|
||||
}{
|
||||
"the controller's differs": {changedRaw, func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateDiffers && a.Returns == delivery.ReturnsToController &&
|
||||
len(a.Differs) == 1 && a.Differs[0].Module == "lemurs" && fmt.Sprint(a.Same) == "[node]"
|
||||
}},
|
||||
"the controller holds none": {nil, func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateRefused && a.Returns == "" && strings.Contains(a.Refused, "lemurs: the controller holds no composition")
|
||||
}},
|
||||
} {
|
||||
get := func(_ context.Context, bucket, key string) ([]byte, error) {
|
||||
if key == lemursKey {
|
||||
return row.lemurs, nil
|
||||
}
|
||||
readOnly.mu.Lock()
|
||||
defer readOnly.mu.Unlock()
|
||||
return readOnly.kv[bucket+"/"+key], nil
|
||||
}
|
||||
a := returning(ctx, get, "n1", map[string]delivery.Want{"lemurs": {FP: fp}, agent.NodeModule: {}}, runsList, "pro", 1)
|
||||
if !row.check(a) {
|
||||
t.Errorf("%s: %+v", name, a)
|
||||
}
|
||||
if _, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"pro": a}}); err != nil {
|
||||
t.Errorf("%s: the answer is not one a have holds: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is
|
||||
// composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the
|
||||
// trunk's.
|
||||
@@ -556,6 +1172,80 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
if fmt.Sprint(gotFps) != fmt.Sprint(map[string]string{"lemurs": fp, "keeper": other}) {
|
||||
t.Errorf("shadow builds %v; want lemurs at what runs, keeper at the trunk's, nats at none (folded)", gotFps)
|
||||
}
|
||||
|
||||
// Issue 1126, the seed's refusals: what a node composes from its builds and reads of a provider elsewhere. A build
|
||||
// is the builder's key; a templates archive is fetched once per digest and refused when it is not that digest; another
|
||||
// node's served key is read alone, as the node's grant allows it (buscompose Principal.ServedReads).
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
|
||||
buildKey, _ := delivery.BuilderBuildKey("lemurs", fp)
|
||||
b.mu.Lock()
|
||||
b.kv[delivery.BuilderBuildsBucket+"/"+buildKey] = built
|
||||
b.kv[ResolvedBucket+"/n2.mailer.domain"] = []byte(`{"value":"example.org","source":"module"}`)
|
||||
b.reads = nil
|
||||
b.mu.Unlock()
|
||||
fetched := 0
|
||||
archive := []byte("an archive")
|
||||
sum := sha256.Sum256(archive)
|
||||
env.Fetch = func(context.Context, string) ([]byte, error) { fetched++; return archive, nil }
|
||||
r := reader{c: New(env)}
|
||||
if got, err := r.Build(ctx, "lemurs", fp); err != nil || got == nil || got.Manifest != digest {
|
||||
t.Errorf("the build of lemurs: %+v, %v", got, err)
|
||||
}
|
||||
if got, err := r.Build(ctx, "lemurs", other); err != nil || got != nil {
|
||||
t.Errorf("no build at another fingerprint: %+v, %v", got, err)
|
||||
}
|
||||
for range 2 {
|
||||
if got, err := r.Archive(ctx, "http://store/v2/x/files/blobs/d", "sha256:"+hex.EncodeToString(sum[:])); err != nil || string(got) != "an archive" {
|
||||
t.Errorf("the archive: %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
if _, err := r.Archive(ctx, "http://store/v2/x/files/blobs/e", "sha256:"+strings.Repeat("0", 64)); err == nil || fetched != 2 {
|
||||
t.Errorf("fetched %d times (want once per digest), and an archive of another digest: %v", fetched, err)
|
||||
}
|
||||
if got, err := r.Served(ctx, "n2", "mailer", []string{"domain"}); err != nil || fmt.Sprint(got) != "map[domain:example.org]" {
|
||||
t.Errorf("what n2's mailer serves: %v, %v", got, err)
|
||||
}
|
||||
served, err := buscompose.PermissionsFor(buscompose.Principal{Kind: buscompose.KindNode, Node: "n1", ServedReads: []string{"n2.mailer.domain"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.mu.Lock()
|
||||
for _, read := range b.reads {
|
||||
ok := grantAllows(t, "n1", read[0], read[1])
|
||||
if read[0] == ResolvedBucket {
|
||||
ok = slices.Contains(served.Publish, "$JS.API.CONSUMER.CREATE.KV_"+ResolvedBucket+".*.$KV."+ResolvedBucket+"."+read[1])
|
||||
}
|
||||
if !ok {
|
||||
t.Errorf("the composer reads %s of %s, which the node's grant does not allow", read[1], read[0])
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
|
||||
// L3 s7 (ADR 0550 §2): the node asks the vault for its own certificate only when a module placed here asks for one
|
||||
// and its record holds none from an authority it trusts; once per serving key.
|
||||
certBus := newFakeBus()
|
||||
certAsks := 0
|
||||
certEnv := certBus.env()
|
||||
certEnv.Certify = func(context.Context) error { certAsks++; return nil }
|
||||
certEnv.VaultAuthority = func() string { return "" }
|
||||
cc := New(certEnv)
|
||||
withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}`
|
||||
cdigest := delivery.ManifestDigest([]byte(withCert))
|
||||
cc.manifests[digest] = []byte(lemurs)
|
||||
cc.manifests[cdigest] = []byte(withCert)
|
||||
cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}}
|
||||
cc.records["n1"] = delivery.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))}
|
||||
onlyLemurs := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}}}
|
||||
cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs)
|
||||
if certAsks != 0 {
|
||||
t.Errorf("no module asks for a certificate, and the vault was asked %d times", certAsks)
|
||||
}
|
||||
withWeb := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "web"}}}
|
||||
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
|
||||
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
|
||||
if certAsks != 1 {
|
||||
t.Errorf("a module asks for a certificate: the vault was asked %d times, want once", certAsks)
|
||||
}
|
||||
}
|
||||
|
||||
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches
|
||||
@@ -719,3 +1409,68 @@ func (r fixedReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow
|
||||
func (r fixedReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
func (r fixedReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (r fixedReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("no archive %s", url)
|
||||
}
|
||||
func (r fixedReader) Served(context.Context, string, string, []string) (map[string]any, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// TestAHaveSaysWhetherItsModuleRunsInsideTheNodesOwn feeds takeModuleHave a module's have and reads whether the node
|
||||
// counts the module as run inside its own composition (issue 1126, review round 2): a failed have that names nothing
|
||||
// it runs (NotComposed before the module ever ran) is not, so the next composition does not take the module from its
|
||||
// manifest past its build.
|
||||
func TestAHaveSaysWhetherItsModuleRunsInsideTheNodesOwn(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
have delivery.Have
|
||||
in bool
|
||||
// heldToday says the shadow counts the module held today (issue 1126: a seed holding it too holds it as today).
|
||||
heldToday bool
|
||||
}{
|
||||
{"failed, naming nothing it runs", delivery.Have{Phase: delivery.PhaseFailed, Health: delivery.HealthFailed,
|
||||
Detail: "the node cannot compose it, and keeps running what it ran: no build"}, false, true},
|
||||
{"failed at a build fingerprint", delivery.Have{FP: "build-0123abcd", Composed: "sha256:" + strings.Repeat("c", 64), Phase: delivery.PhaseFailed,
|
||||
Health: delivery.HealthFailed, Detail: "the node cannot compose it, and keeps running what it ran: no build"}, true, true},
|
||||
// The live have of novox's de-spiegel (2026-10-28): a build of the old form, named by `build`, and no `fp`.
|
||||
{"running a build of the old form", delivery.Have{Build: "build-1790368817085312877", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, true, true},
|
||||
{"running a build of the new form", delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("c", 64),
|
||||
Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, false, false},
|
||||
{"running with no fingerprint (mesh-wireguard)", delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, true, false},
|
||||
{"held today", delivery.Have{FP: "build-0123abcd", Composed: "sha256:" + strings.Repeat("c", 64), Phase: delivery.PhaseHeld,
|
||||
Health: delivery.HealthWaits, Detail: "held: a required config value is unset"}, true, true},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
c := New(Env{Node: "n1"})
|
||||
key, err := delivery.HaveKey("n1", "web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeHave(row.have)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.takeModuleHave(agent.Update{Key: key, Value: value})
|
||||
if c.inOwn["web"] != row.in {
|
||||
t.Errorf("web counted inside the node's own composition: %v, want %v", c.inOwn["web"], row.in)
|
||||
}
|
||||
s := &Shadow{c: c, ran: map[string]string{}}
|
||||
s.takeRan(agent.Update{Key: key, Value: value})
|
||||
if _, got := s.heldToday()["web"]; got != row.heldToday {
|
||||
t.Errorf("web held today: %v, want %v", got, row.heldToday)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// setFor sets a package global (a wait the composer or the shadow reads) for one test, and puts back what it held when
|
||||
// the test ends, so no later test runs on another test's waits.
|
||||
func setFor[T any](t *testing.T, p *T, v T) {
|
||||
t.Helper()
|
||||
old := *p
|
||||
*p = v
|
||||
t.Cleanup(func() { *p = old })
|
||||
}
|
||||
|
||||
+538
-75
@@ -1,9 +1,12 @@
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -13,8 +16,8 @@ import (
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
// Shadow is the node's answer to placement candidates (novox/hq ADR 0559 §4, issue 1126): for every group with a key in
|
||||
// the indexer's state `placement-candidates`, it composes this node in shadow from the trunk's placement with the group's
|
||||
// Shadow is the node's answer to placement candidates (novox/hq ADR 0559 §4, issue 1126): for every proposal with a key in
|
||||
// the placement seat's state `candidates`, it composes this node in shadow from the trunk's placement with the proposal's
|
||||
// keys laid over it and its own inputs (the sealed values and boxes it holds among them), compares the result with what
|
||||
// the node runs, and hands the answers to be written into the node's own have key, `<node>.node`, under `candidates`.
|
||||
// It applies nothing and hands no composition: what runs is still composed from `placement` alone, by the Composer.
|
||||
@@ -22,15 +25,72 @@ import (
|
||||
// **Event-driven, like the Composer.** It watches every candidate key, the trunk's placement, every node's record, the
|
||||
// indexer's modules, this node's config values, and this node's module haves (what it runs moved); a burst of changes
|
||||
// answers once. It runs whether or not the node is placed yet: a node that applies the controller's compositions answers
|
||||
// against what it runs from them. An answer is handed only when it changes, and a group's answer is dropped when its
|
||||
// against what it runs from them. An answer is handed only when it changes, and a proposal's answer is dropped when its
|
||||
// keys go.
|
||||
type Shadow struct {
|
||||
c *Composer
|
||||
answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)
|
||||
|
||||
mu sync.Mutex
|
||||
candidates map[string]map[string]candidateKey // by group, by node
|
||||
candidates map[string]map[string]candidateKey // by proposal, by node
|
||||
answers map[string]delivery.CandidateAnswer
|
||||
// wants are the node's wants by module (agent.NodeModule for its own): what a proposal deleting its placement returns
|
||||
// it to, the controller's compositions at them.
|
||||
wants map[string]delivery.Want
|
||||
// ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy
|
||||
// from the declaration kept before the switch (issue 1126, probe #532 on shanks).
|
||||
ran map[string]string
|
||||
// heldNow is, by module, how its have says the node keeps it today as it runs: "held", "failed", or "runs a build of
|
||||
// the old form" (a `build` and no `fp`, which no registered build of the trunk names); "" when it is composed as any
|
||||
// other. A candidate that cannot compose it, or holds it only behind one of these, keeps it as today
|
||||
// (frombus.Scope.HeldToday, issue 1126).
|
||||
heldNow map[string]string
|
||||
// answered is, by proposal, the candidate revisions and the input changes (moved) each answer was composed at: a proposal
|
||||
// is answered again only when either moved (issue 1126, 2026-10-28: shanks answered probe/1126-seed-7 at revision 84
|
||||
// every 13-17 s, woken by its own modules' haves whose health words changed at every look).
|
||||
answered map[string]answeredAt
|
||||
// moved counts, by input (inputWords), the changes of each of the node's own inputs, and retry names the proposals
|
||||
// answered again after a read the bus did not answer: what woke each answer is said with it (issue 1126,
|
||||
// 2026-10-28, seed-9's probe: a safety check could not tell a loop from inputs moving).
|
||||
moved [len(inputWords)]uint64
|
||||
retry map[string]bool
|
||||
// owed says an answer met a read the bus did not answer: it is no answer to keep, and its proposal is answered again
|
||||
// after a bound (laterAgain), whether or not an input moves (issue 1126, 2026-10-28: ace kept "the vault's own: not
|
||||
// yet ... no responders" in its answer to seed-7).
|
||||
owed bool
|
||||
}
|
||||
|
||||
// answeredAt is what one proposal's answer was composed at: its keys' revisions, by node, and the changes of each input.
|
||||
type answeredAt struct {
|
||||
revs string
|
||||
moved [len(inputWords)]uint64
|
||||
}
|
||||
|
||||
// inputWords name the shadow's inputs, in Run's order, as an answer says which of them woke it.
|
||||
var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves", "wants"}
|
||||
|
||||
// wokenBy says what moved between the answer a proposal stands at (had, when done) and the one about to be composed (at):
|
||||
// its first answer, a retry, its candidate's revisions, or the inputs that changed, by their words.
|
||||
func wokenBy(had answeredAt, done, retry bool, at answeredAt) string {
|
||||
switch {
|
||||
case retry:
|
||||
return "a retry after a read the bus did not answer"
|
||||
case !done:
|
||||
return "its first answer since the shadow began"
|
||||
}
|
||||
var words []string
|
||||
if had.revs != at.revs {
|
||||
words = append(words, inputWords[0])
|
||||
}
|
||||
for i := 1; i < len(inputWords); i++ {
|
||||
if had.moved[i] != at.moved[i] {
|
||||
words = append(words, inputWords[i])
|
||||
}
|
||||
}
|
||||
if len(words) == 0 {
|
||||
return "nothing that moved"
|
||||
}
|
||||
return strings.Join(words, ", ")
|
||||
}
|
||||
|
||||
// candidateKey is one candidate key as last read: its value and revision.
|
||||
@@ -42,7 +102,7 @@ type candidateKey struct {
|
||||
// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
|
||||
func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
|
||||
return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
|
||||
answers: map[string]delivery.CandidateAnswer{}}
|
||||
answers: map[string]delivery.CandidateAnswer{}, wants: map[string]delivery.Want{}, ran: map[string]string{}}
|
||||
}
|
||||
|
||||
// Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch
|
||||
@@ -84,10 +144,16 @@ func (s *Shadow) Run(ctx context.Context) error {
|
||||
defer cancel()
|
||||
s.mu.Lock()
|
||||
s.candidates = map[string]map[string]candidateKey{}
|
||||
s.wants = map[string]delivery.Want{}
|
||||
s.ran = map[string]string{}
|
||||
s.answered = map[string]answeredAt{}
|
||||
s.moved = [len(inputWords)]uint64{}
|
||||
s.retry = map[string]bool{}
|
||||
s.c = New(s.c.env)
|
||||
s.mu.Unlock()
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
// In inputWords' order.
|
||||
ins := []input{
|
||||
{delivery.PlacementCandidatesBucket, ">", nil, nil},
|
||||
{delivery.PlacementBucket, ">", (*Composer).takePlacement, nil},
|
||||
@@ -96,6 +162,8 @@ func (s *Shadow) Run(ctx context.Context) error {
|
||||
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
||||
// What this node runs moved: an apply ends in its module's have.
|
||||
{delivery.HaveBucket, node + ".>", nil, nil},
|
||||
// Its wants: a proposal deleting its placement returns it to the controller's compositions at them.
|
||||
{delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil},
|
||||
}
|
||||
type tagged struct {
|
||||
in int
|
||||
@@ -124,26 +192,51 @@ func (s *Shadow) Run(ctx context.Context) error {
|
||||
}
|
||||
caught := make([]bool, len(ins))
|
||||
ready, dirty := false, false
|
||||
// Not ready after a minute is said, naming each input not read yet, in the journal and in an answer to each proposal it
|
||||
// knows: a shadow that waits is never silent (issue 1126: ace, g14 and shanks answered nothing to seed #531).
|
||||
waiting := time.NewTimer(notReadyAfter)
|
||||
defer waiting.Stop()
|
||||
var again <-chan time.Time
|
||||
tries := 0
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-again:
|
||||
again = nil
|
||||
dirty = true
|
||||
case <-waiting.C:
|
||||
if !ready {
|
||||
s.notReady(ctx, ins, caught)
|
||||
}
|
||||
continue
|
||||
case t := <-merged:
|
||||
if !t.ok {
|
||||
return fmt.Errorf("the watch of %s ended; the node answers no placement candidate until its agent starts again", ins[t.in].bucket)
|
||||
}
|
||||
// Only a change of what an answer is composed from answers again: a key written again with the value it held,
|
||||
// this node's own key (its answers), or a module's have whose health or words moved but not what it runs, is
|
||||
// none (issue 1126, 2026-10-28: each look rewrote shanks' unhealthy modules' haves, and each woke a full answer).
|
||||
changed := false
|
||||
switch {
|
||||
case t.u.Caught:
|
||||
caught[t.in] = true
|
||||
case t.in == 0:
|
||||
dirty = s.takeCandidate(t.u) || dirty
|
||||
changed = s.takeCandidate(t.u)
|
||||
case t.in == 2:
|
||||
dirty = s.takeRecord(t.u) || dirty
|
||||
changed = s.takeRecord(t.u)
|
||||
case t.in == 5:
|
||||
c.takeModuleHave(t.u)
|
||||
dirty = !isNodeKey(t.u.Key) || dirty
|
||||
folded := c.takeModuleHave(t.u)
|
||||
changed = s.takeRan(t.u) || folded
|
||||
case t.in == 6:
|
||||
changed = s.takeWant(t.u)
|
||||
default:
|
||||
ins[t.in].take(c, t.u)
|
||||
changed = s.takeChanged(ins[t.in], t.u)
|
||||
}
|
||||
if changed {
|
||||
if t.in != 0 {
|
||||
s.moved[t.in]++ // the candidates' own changes are their revisions
|
||||
}
|
||||
dirty = true
|
||||
}
|
||||
if len(merged) > 0 {
|
||||
@@ -159,46 +252,110 @@ func (s *Shadow) Run(ctx context.Context) error {
|
||||
if dirty {
|
||||
dirty = false
|
||||
s.answerAll(ctx)
|
||||
if s.owed {
|
||||
wait := LaterWait(tries)
|
||||
tries++
|
||||
c.env.Say(fmt.Sprintf("this node answers again in %s the placement candidates whose answer met a bus that did not answer", wait))
|
||||
again = time.After(wait)
|
||||
} else {
|
||||
tries, again = 0, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isNodeKey(key string) bool {
|
||||
_, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
return err == nil && isNode
|
||||
// takeChanged takes u through in.take and says whether the entry it names changed: a placement, a module or a config
|
||||
// value written again with the value it held changes nothing composed.
|
||||
func (s *Shadow) takeChanged(in input, u agent.Update) bool {
|
||||
before, had, known := s.entry(in.bucket, u.Key)
|
||||
in.take(s.c, u)
|
||||
if !known {
|
||||
return true
|
||||
}
|
||||
after, has, _ := s.entry(in.bucket, u.Key)
|
||||
return had != has || !reflect.DeepEqual(before, after)
|
||||
}
|
||||
|
||||
// entry is what the composer holds of one key of bucket, whether it holds it, and whether the bucket is one it knows.
|
||||
func (s *Shadow) entry(bucket, key string) (v any, has, known bool) {
|
||||
c := s.c
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
switch bucket {
|
||||
case delivery.PlacementBucket:
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
return nil, false, true
|
||||
}
|
||||
p, ok := c.placements[node]
|
||||
return p, ok, true
|
||||
case delivery.ModulesBucket:
|
||||
name, err := delivery.ParseModulesKey(key)
|
||||
if err != nil {
|
||||
return nil, false, true
|
||||
}
|
||||
m, ok := c.modules[name]
|
||||
return m, ok, true
|
||||
case ResolvedBucket:
|
||||
rest, ok := strings.CutPrefix(key, c.env.Node+".")
|
||||
if !ok {
|
||||
return nil, false, true
|
||||
}
|
||||
module, k, ok := strings.Cut(rest, ".")
|
||||
if !ok {
|
||||
return nil, false, true
|
||||
}
|
||||
val, ok := c.values[module][k]
|
||||
return val, ok, true
|
||||
}
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
// revsOf is a proposal's keys' revisions, by node, as one word.
|
||||
func revsOf(keys map[string]candidateKey) string {
|
||||
nodes := make([]string, 0, len(keys))
|
||||
for n := range keys {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var b strings.Builder
|
||||
for _, n := range nodes {
|
||||
fmt.Fprintf(&b, "%s=%d;", n, keys[n].rev)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// takeCandidate takes one candidate key, and says whether it changed anything.
|
||||
func (s *Shadow) takeCandidate(u agent.Update) bool {
|
||||
group, node, err := delivery.ParsePlacementCandidateKey(u.Key)
|
||||
proposal, node, err := delivery.ParsePlacementCandidateKey(u.Key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u.Deleted {
|
||||
if _, ok := s.candidates[group][node]; !ok {
|
||||
if _, ok := s.candidates[proposal][node]; !ok {
|
||||
return false
|
||||
}
|
||||
delete(s.candidates[group], node)
|
||||
if len(s.candidates[group]) == 0 {
|
||||
delete(s.candidates, group)
|
||||
delete(s.candidates[proposal], node)
|
||||
if len(s.candidates[proposal]) == 0 {
|
||||
delete(s.candidates, proposal)
|
||||
}
|
||||
return true
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(u.Value)
|
||||
if err != nil || c.Group != group || (c.Placement != nil && c.Placement.Node != node) {
|
||||
if err != nil || c.Proposal != proposal || (c.Placement != nil && c.Placement.Node != node) {
|
||||
// Not one: read as gone.
|
||||
if _, ok := s.candidates[group][node]; ok {
|
||||
delete(s.candidates[group], node)
|
||||
if _, ok := s.candidates[proposal][node]; ok {
|
||||
delete(s.candidates[proposal], node)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if s.candidates[group] == nil {
|
||||
s.candidates[group] = map[string]candidateKey{}
|
||||
if s.candidates[proposal] == nil {
|
||||
s.candidates[proposal] = map[string]candidateKey{}
|
||||
}
|
||||
s.candidates[group][node] = candidateKey{c: c, rev: u.Revision}
|
||||
s.candidates[proposal][node] = candidateKey{c: c, rev: u.Revision}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -220,70 +377,199 @@ func (s *Shadow) takeRecord(u agent.Update) bool {
|
||||
return had != has || (has && !delivery.SameNodeRecord(before, after))
|
||||
}
|
||||
|
||||
// answerAll answers every group and hands the answers when any changed.
|
||||
// How long the shadow waits before it says it is not ready, and how long a proposal's answer runs before it is said slow.
|
||||
var notReadyAfter, slowAfter = time.Minute, time.Minute
|
||||
|
||||
// notReady says which inputs the shadow has not read yet, and answers each proposal it knows refused for it.
|
||||
func (s *Shadow) notReady(ctx context.Context, ins []input, caught []bool) {
|
||||
var waits []string
|
||||
for i, in := range ins {
|
||||
if !caught[i] {
|
||||
waits = append(waits, in.bucket+" ("+in.pattern+")")
|
||||
}
|
||||
}
|
||||
why := fmt.Sprintf("the node has not read %s after %s; it answers once it has", strings.Join(waits, ", "), notReadyAfter)
|
||||
s.c.env.Say("this node answers no placement candidate yet: " + why)
|
||||
s.mu.Lock()
|
||||
next := map[string]delivery.CandidateAnswer{}
|
||||
for proposal, keys := range s.candidates {
|
||||
own, ok := keys[s.c.env.Node]
|
||||
if !ok {
|
||||
continue // no key of this node's: nothing asks it to answer
|
||||
}
|
||||
next[proposal] = delivery.CandidateAnswer{Proposal: proposal, Revision: own.rev, Refused: why, Verdict: delivery.CandidateRefused}
|
||||
}
|
||||
s.answers = next
|
||||
s.mu.Unlock()
|
||||
if len(next) > 0 && s.answer != nil {
|
||||
s.answer(ctx, next)
|
||||
}
|
||||
}
|
||||
|
||||
// answerAll answers every proposal, the newest candidate first, and hands the answers each time one changes (issue 1126,
|
||||
// T3: ace answered a closed pull request's group for 22 minutes before the newer one, all handed at the end). Newest
|
||||
// is the highest revision of the proposal's keys: a withdrawn proposal's candidate is the older and waits; the seat
|
||||
// drops a closed one's keys, which ends its answer here (a node does not read the forge). Each answer says when it
|
||||
// starts, every progressEvery while it runs what it waits on, and when it ends (T2: the node said nothing for 22
|
||||
// minutes).
|
||||
func (s *Shadow) answerAll(ctx context.Context) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
s.mu.Lock()
|
||||
groups := make([]string, 0, len(s.candidates))
|
||||
for g := range s.candidates {
|
||||
groups = append(groups, g)
|
||||
type listed struct {
|
||||
proposal string
|
||||
rev uint64
|
||||
at answeredAt
|
||||
}
|
||||
s.mu.Unlock()
|
||||
sort.Strings(groups)
|
||||
var proposals []listed
|
||||
for g, keys := range s.candidates {
|
||||
l := listed{proposal: g, at: answeredAt{revs: revsOf(keys), moved: s.moved}}
|
||||
for _, k := range keys {
|
||||
l.rev = max(l.rev, k.rev)
|
||||
}
|
||||
proposals = append(proposals, l)
|
||||
}
|
||||
// A proposal whose keys' revisions and the node's inputs are those its answer stands at is not answered again.
|
||||
for g := range s.answered {
|
||||
if _, ok := s.candidates[g]; !ok {
|
||||
delete(s.answered, g)
|
||||
}
|
||||
}
|
||||
// Each proposal's last answer stands until it is answered again; a proposal whose keys went is dropped.
|
||||
next := map[string]delivery.CandidateAnswer{}
|
||||
for _, group := range groups {
|
||||
if a, ok := s.answerGroup(ctx, group); ok {
|
||||
next[group] = a
|
||||
for _, l := range proposals {
|
||||
if a, ok := s.answers[l.proposal]; ok {
|
||||
next[l.proposal] = a
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
same := len(next) == len(s.answers)
|
||||
for g, a := range next {
|
||||
if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) {
|
||||
same = false
|
||||
}
|
||||
}
|
||||
if !same {
|
||||
s.answers = next
|
||||
}
|
||||
s.owed = false
|
||||
s.mu.Unlock()
|
||||
if !same && s.answer != nil {
|
||||
sort.Slice(proposals, func(i, j int) bool {
|
||||
if proposals[i].rev != proposals[j].rev {
|
||||
return proposals[i].rev > proposals[j].rev
|
||||
}
|
||||
return proposals[i].proposal < proposals[j].proposal
|
||||
})
|
||||
hand := func() {
|
||||
s.mu.Lock()
|
||||
same := len(next) == len(s.answers)
|
||||
for g, a := range next {
|
||||
if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) {
|
||||
same = false
|
||||
}
|
||||
}
|
||||
out := make(map[string]delivery.CandidateAnswer, len(next))
|
||||
for g, a := range next {
|
||||
out[g] = a
|
||||
}
|
||||
s.answer(ctx, out)
|
||||
c.env.Say(fmt.Sprintf("%s answered %d placement candidate group(s)", node, len(out)))
|
||||
if !same {
|
||||
s.answers = out
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !same && s.answer != nil {
|
||||
given := make(map[string]delivery.CandidateAnswer, len(out))
|
||||
for g, a := range out {
|
||||
given[g] = a
|
||||
}
|
||||
s.answer(ctx, given)
|
||||
c.env.Say(fmt.Sprintf("%s answered %d placement candidate proposal(s)", node, len(given)))
|
||||
}
|
||||
}
|
||||
for _, l := range proposals {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
proposal := l.proposal
|
||||
s.mu.Lock()
|
||||
at, done := s.answered[proposal]
|
||||
retry := s.retry[proposal]
|
||||
s.mu.Unlock()
|
||||
if done && at == l.at {
|
||||
continue
|
||||
}
|
||||
began := time.Now()
|
||||
c.env.Say(fmt.Sprintf("answering placement candidate %s (revision %d; woken by %s)", proposal, l.rev, wokenBy(at, done, retry, l.at)))
|
||||
stop := make(chan struct{})
|
||||
go func() {
|
||||
tick := time.NewTicker(progressEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-tick.C:
|
||||
c.env.Say(fmt.Sprintf("the answer to %s has run %s and waits on %s", proposal,
|
||||
time.Since(began).Round(time.Second), c.waitingOn()))
|
||||
}
|
||||
}
|
||||
}()
|
||||
missed := c.missed.Load()
|
||||
a, ok := s.answerProposal(ctx, proposal)
|
||||
close(stop)
|
||||
provisional := c.missed.Load() > missed && ctx.Err() == nil
|
||||
took := time.Since(began)
|
||||
if took > slowAfter {
|
||||
c.env.Say(fmt.Sprintf("the answer to %s took %s", proposal, took.Round(time.Second)))
|
||||
if ok {
|
||||
// Words that do not move with the time it took, so an answer the same as before is not written again.
|
||||
a.Note = strings.TrimPrefix(a.Note+"; ", "; ") + "this answer took over " + slowAfter.String() + " to compose"
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
a = delivery.FitAnswer(a, delivery.MaxAnswerBytes)
|
||||
next[proposal] = a
|
||||
c.env.Say(fmt.Sprintf("answered placement candidate %s in %s: %s", proposal, took.Round(time.Millisecond), a.Verdict))
|
||||
} else {
|
||||
delete(next, proposal)
|
||||
c.env.Say(fmt.Sprintf("placement candidate %s does not place this node: nothing to answer", proposal))
|
||||
}
|
||||
s.mu.Lock()
|
||||
if provisional {
|
||||
// What the bus did not answer is no answer: handed as it stands, and composed again after a bound.
|
||||
s.owed = true
|
||||
s.retry[proposal] = true
|
||||
delete(s.answered, proposal)
|
||||
c.env.Say(fmt.Sprintf("the answer to %s met a read on the bus that did not answer: it is answered again", proposal))
|
||||
} else {
|
||||
s.answered[proposal] = l.at
|
||||
delete(s.retry, proposal)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
hand()
|
||||
}
|
||||
hand() // a proposal whose keys went, when no proposal is left to answer
|
||||
}
|
||||
|
||||
// answerGroup is the node's answer to one group, and false when the node has nothing to answer: neither the trunk nor
|
||||
// the group places it.
|
||||
func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.CandidateAnswer, bool) {
|
||||
// How often a running answer says what it waits on.
|
||||
var progressEvery = time.Minute
|
||||
|
||||
// answerProposal is the node's answer to one proposal, and false when the node has nothing to answer: neither the trunk nor
|
||||
// the proposal places it.
|
||||
func (s *Shadow) answerProposal(ctx context.Context, proposal string) (delivery.CandidateAnswer, bool) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
defer c.waiting("composing")()
|
||||
s.mu.Lock()
|
||||
keys := s.candidates[group]
|
||||
keys := s.candidates[proposal]
|
||||
given := make(map[string]*delivery.Placement, len(keys))
|
||||
var head string
|
||||
for n, k := range keys {
|
||||
if k.c.Reached && k.c.Placement == nil {
|
||||
continue // reached, with no key of its own: nothing laid over it
|
||||
}
|
||||
given[n] = k.c.Placement
|
||||
head = k.c.Pull.Head
|
||||
}
|
||||
// The revision answered is the node's own key's, 0 when the group has none for it (judged by the head alone).
|
||||
// The revision answered is the node's own candidate key's; a node with none is not asked.
|
||||
own, mine := keys[node]
|
||||
rev := own.rev
|
||||
s.mu.Unlock()
|
||||
|
||||
c.mu.Lock()
|
||||
_, placed := c.placements[node]
|
||||
modules := make(map[string]delivery.Module, len(c.modules))
|
||||
for k, v := range c.modules {
|
||||
modules[k] = v
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if !placed && !mine {
|
||||
if !mine {
|
||||
return delivery.CandidateAnswer{}, false
|
||||
}
|
||||
|
||||
@@ -291,19 +577,17 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid
|
||||
if c.env.Running != nil {
|
||||
running = c.env.Running()
|
||||
}
|
||||
if p, ok := given[node]; ok && p == nil {
|
||||
// The head removes this node's placement file: everything it runs is removed.
|
||||
a := delivery.CandidateAnswer{Head: head, Revision: rev}
|
||||
for _, r := range running {
|
||||
if r.Module == "" {
|
||||
a.Removed = append(a.Removed, delivery.HaveNodeWord)
|
||||
} else {
|
||||
a.Removed = append(a.Removed, r.Module)
|
||||
}
|
||||
if p, ok := given[node]; ok && p == nil && !own.c.Reached {
|
||||
// The proposal deletes this node's placement key: the node then returns to the controller's compositions, as one the
|
||||
// catalogue does not place (endWhenUnplaced). Those at its wants are compared with what runs (issue 1126, the
|
||||
// seed's revert, whose check read "would stop running node").
|
||||
s.mu.Lock()
|
||||
wants := make(map[string]delivery.Want, len(s.wants))
|
||||
for m, w := range s.wants {
|
||||
wants[m] = w
|
||||
}
|
||||
sort.Strings(a.Removed)
|
||||
a.Verdict = delivery.VerdictOf(a)
|
||||
return a, true
|
||||
s.mu.Unlock()
|
||||
return returning(ctx, c.get, node, wants, running, proposal, rev), true
|
||||
}
|
||||
|
||||
placement := given[node]
|
||||
@@ -323,12 +607,7 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid
|
||||
if mod, ok := modules[module]; ok && mod.FP2 == fp {
|
||||
continue
|
||||
}
|
||||
digest, err := c.builtManifest(ctx, module, fp)
|
||||
if err != nil {
|
||||
return delivery.CandidateAnswer{Head: head, Revision: rev, Verdict: delivery.CandidateRefused,
|
||||
Refused: fmt.Sprintf("%s runs at %s, whose manifest cannot be read: %v", module, fp, err)}, true
|
||||
}
|
||||
kept[module] = digest
|
||||
kept[module] = fp // composed from that build (frombus.Scope.KeptAt, issue 1126)
|
||||
}
|
||||
// The boxes it runs, by what each module's have and its own key say after a restart (Held), are reused for the
|
||||
// modules it runs: a value sealed again is no difference.
|
||||
@@ -339,8 +618,93 @@ func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.Candid
|
||||
c.mu.Lock()
|
||||
c.wants = held
|
||||
c.mu.Unlock()
|
||||
a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given, InOwn: inOwn}, fps, running, head, rev)
|
||||
return a, true
|
||||
a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, KeptAt: kept, Placements: given, InOwn: inOwn,
|
||||
SameValue: c.sameValue, HeldToday: s.heldToday()}, fps, running, proposal, rev)
|
||||
s.mu.Lock()
|
||||
ran := make(map[string]string, len(s.ran))
|
||||
for m, d := range s.ran {
|
||||
ran[m] = d
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return asHavesSay(a, ran), true
|
||||
}
|
||||
|
||||
// heldToday is what the node holds today: each module whose have's phase is held, and what its own composer holds, the
|
||||
// node's own composition ("node") among them when it composes none of its own.
|
||||
func (s *Shadow) heldToday() map[string]string {
|
||||
out := map[string]string{}
|
||||
s.mu.Lock()
|
||||
for m, how := range s.heldNow {
|
||||
if how != "" {
|
||||
out[m] = how
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for _, m := range s.c.heldNow() {
|
||||
if _, said := out[m]; !said {
|
||||
out[m] = "held"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// takeRan keeps the digest one of this node's module haves says runs, and says whether it changed: a have whose health
|
||||
// or words alone moved runs the same.
|
||||
func (s *Shadow) takeRan(u agent.Update) bool {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(u.Key)
|
||||
if err != nil || isNode || node != s.c.env.Node {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
before, had := s.ran[module]
|
||||
h, err := delivery.DecodeHave(u.Value)
|
||||
if s.heldNow == nil {
|
||||
s.heldNow = map[string]string{}
|
||||
}
|
||||
wasHeld := s.heldNow[module]
|
||||
s.heldNow[module] = ""
|
||||
if !u.Deleted && err == nil {
|
||||
switch {
|
||||
case h.Phase == delivery.PhaseHeld:
|
||||
s.heldNow[module] = "held"
|
||||
case h.Phase == delivery.PhaseFailed:
|
||||
s.heldNow[module] = "failed"
|
||||
case h.Phase != delivery.PhaseRemoved && h.FP == "" && h.Build != "":
|
||||
// A build of the old form: its have names a `build` and no `fp`, which no registered build of the trunk
|
||||
// names, so the controller keeps it as it runs (issue 1126: novox's de-spiegel at build-…); a seed keeps it so.
|
||||
s.heldNow[module] = "runs a build of the old form"
|
||||
}
|
||||
}
|
||||
holdMoved := wasHeld != s.heldNow[module] // a hold's start or end changes the answer too
|
||||
if u.Deleted || err != nil || h.Composed == "" || h.Phase == delivery.PhaseRemoved {
|
||||
delete(s.ran, module)
|
||||
return had || holdMoved
|
||||
}
|
||||
s.ran[module] = h.Composed
|
||||
return !had || before != h.Composed || holdMoved
|
||||
}
|
||||
|
||||
// asHavesSay is a with each module that differs only from the copy the agent keeps of the declaration before the switch
|
||||
// counted the same where the composed digest is the one its have says runs: what runs is the composition at that
|
||||
// digest, never the kept copy (issue 1126, probe #532: shanks' xterm, zsh, zsh-autosuggestions and
|
||||
// zsh-syntax-highlighting).
|
||||
func asHavesSay(a delivery.CandidateAnswer, ran map[string]string) delivery.CandidateAnswer {
|
||||
if a.Refused != "" || len(a.Differs) == 0 {
|
||||
return a
|
||||
}
|
||||
kept := a.Differs[:0:0]
|
||||
for _, d := range a.Differs {
|
||||
if d.Composed != "" && ran[d.Module] == d.Composed {
|
||||
a.Same = append(a.Same, d.Module)
|
||||
continue
|
||||
}
|
||||
kept = append(kept, d)
|
||||
}
|
||||
a.Differs = kept
|
||||
sort.Strings(a.Same)
|
||||
a.Verdict = delivery.VerdictOf(a)
|
||||
return a
|
||||
}
|
||||
|
||||
// shadowBuilds is the build each module placed is composed at in shadow: the one it runs, and for one the candidate
|
||||
@@ -374,3 +738,102 @@ func sameAnswer(a, b delivery.CandidateAnswer) bool {
|
||||
y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}})
|
||||
return errA == nil && errB == nil && string(x) == string(y)
|
||||
}
|
||||
|
||||
// takeWant takes one of the node's wants, and says whether it changed.
|
||||
func (s *Shadow) takeWant(u agent.Update) bool {
|
||||
module := ""
|
||||
var w delivery.Want
|
||||
if _, err := delivery.ParseWantNodeKey(u.Key); err == nil {
|
||||
module = agent.NodeModule
|
||||
} else if node, m, err := delivery.ParseWantKey(u.Key); err == nil && node == s.c.env.Node {
|
||||
module = m
|
||||
if !u.Deleted {
|
||||
if w, err = delivery.DecodeWant(u.Value); err != nil || w.Removed || w.FP == "" {
|
||||
u.Deleted = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
old, had := s.wants[module]
|
||||
if u.Deleted {
|
||||
delete(s.wants, module)
|
||||
return had
|
||||
}
|
||||
s.wants[module] = w
|
||||
return !had || old.FP != w.FP
|
||||
}
|
||||
|
||||
// returning is the node's answer to a proposal deleting its placement: it returns to the controller's compositions, each
|
||||
// read at the fingerprint its want names in the node's composition bucket and compared with what runs
|
||||
// (delivery.CompareComposed). A want the controller holds no composition for, or one that cannot be read, refuses the
|
||||
// answer, naming each: returning then would leave that module with nothing to apply.
|
||||
func returning(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
|
||||
wants map[string]delivery.Want, running []delivery.Composed, proposal string, rev uint64) delivery.CandidateAnswer {
|
||||
a := delivery.CandidateAnswer{Proposal: proposal, Revision: rev}
|
||||
bucket, err := delivery.ComposedBucketFor(node)
|
||||
if err != nil {
|
||||
a.Refused, a.Verdict = err.Error(), delivery.CandidateRefused
|
||||
return a
|
||||
}
|
||||
modules := make([]string, 0, len(wants))
|
||||
for m := range wants {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
comps := map[string]delivery.Composed{}
|
||||
var missing []string
|
||||
for _, m := range modules {
|
||||
var key string
|
||||
if m == agent.NodeModule {
|
||||
key, err = delivery.ComposedNodeKey(node)
|
||||
} else {
|
||||
key, err = delivery.ComposedKey(node, m, wants[m].FP)
|
||||
}
|
||||
if err != nil {
|
||||
missing = append(missing, m+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
raw, err := get(ctx, bucket, key)
|
||||
switch {
|
||||
case err != nil:
|
||||
missing = append(missing, fmt.Sprintf("%s: the controller's composition %s cannot be read: %v", m, key, err))
|
||||
continue
|
||||
case raw == nil:
|
||||
missing = append(missing, fmt.Sprintf("%s: the controller holds no composition %s, which its want names", m, key))
|
||||
continue
|
||||
}
|
||||
comp, err := delivery.DecodeComposed(raw)
|
||||
if err != nil {
|
||||
missing = append(missing, fmt.Sprintf("%s: the controller's composition %s: %v", m, key, err))
|
||||
continue
|
||||
}
|
||||
comps[key] = comp
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
a.Refused = "returning to the controller's compositions, " + strings.Join(missing, "; ")
|
||||
a.Verdict = delivery.CandidateRefused
|
||||
return a
|
||||
}
|
||||
cmp := delivery.CompareComposed(comps, running)
|
||||
a.Returns = delivery.ReturnsToController
|
||||
a.Same, a.Differs, a.Added, a.Removed = cmp.Same, cmp.Differs, cmp.Added, cmp.Removed
|
||||
a.Verdict = delivery.VerdictOf(a)
|
||||
return a
|
||||
}
|
||||
|
||||
// sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not
|
||||
// open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work).
|
||||
func (c *Composer) sameValue(a, b string) (same, known bool) {
|
||||
if c.env.Unseal == nil {
|
||||
return false, false
|
||||
}
|
||||
x, errA := c.env.Unseal(a)
|
||||
y, errB := c.env.Unseal(b)
|
||||
if errA != nil || errB != nil {
|
||||
return false, false
|
||||
}
|
||||
return bytes.Equal(x, y), true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"module": "de-spiegel",
|
||||
"version": "1",
|
||||
"slug": "spiegel",
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"requires": [
|
||||
"route",
|
||||
"smtp"
|
||||
],
|
||||
"contributes": {
|
||||
"route": {
|
||||
"site": {
|
||||
"label": "de-spiegel",
|
||||
"port": 35621
|
||||
},
|
||||
"praktijk": {
|
||||
"name": "${config:praktijk-name}",
|
||||
"port": 35621
|
||||
},
|
||||
"praktijk-www": {
|
||||
"name": "${config:praktijk-www-name}",
|
||||
"redirect": "https://${config:praktijk-name}"
|
||||
},
|
||||
"psychologie": {
|
||||
"name": "${config:psychologie-name}",
|
||||
"port": 35621
|
||||
},
|
||||
"psychologie-www": {
|
||||
"name": "${config:psychologie-www-name}",
|
||||
"redirect": "https://${config:psychologie-name}"
|
||||
}
|
||||
},
|
||||
"smtp": {
|
||||
"account": "de-spiegel"
|
||||
}
|
||||
},
|
||||
"binds": {
|
||||
"route": "/var/lib/de-spiegel/route.json",
|
||||
"smtp": "/var/lib/de-spiegel/smtp.json"
|
||||
},
|
||||
"config": {
|
||||
"praktijk-name": {
|
||||
"type": "string",
|
||||
"description": "The public name the practice's site is served on; the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
},
|
||||
"praktijk-www-name": {
|
||||
"type": "string",
|
||||
"description": "The public name that redirects to the practice's site (its www form); the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
},
|
||||
"psychologie-name": {
|
||||
"type": "string",
|
||||
"description": "The public name the psychology practice's site is served on; the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
},
|
||||
"psychologie-www-name": {
|
||||
"type": "string",
|
||||
"description": "The public name that redirects to the psychology practice's site (its www form); the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"listens": [
|
||||
{
|
||||
"port": 35621,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"why": "the de-spiegel site and its /contact endpoint over http; the practice's three public names are route grants, and the proxy reaches every one of them here"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/de-spiegel",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "server-env",
|
||||
"type": "file",
|
||||
"path": "/var/lib/de-spiegel/server.env",
|
||||
"mode": "0600",
|
||||
"content": "SMTP_HOST=${bound:smtp:at}\nSMTP_PORT=${bound:smtp:port}\nSMTP_AUTH_USER=de-spiegel@${bound:smtp:domain}\nSMTP_AUTH_PASS=${secret:smtp}\n"
|
||||
},
|
||||
{
|
||||
"id": "net",
|
||||
"type": "network",
|
||||
"name": "de-spiegel"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
"name": "de-spiegel",
|
||||
"artifact": "site",
|
||||
"network": "de-spiegel",
|
||||
"env-file": [
|
||||
"/var/lib/de-spiegel/server.env"
|
||||
],
|
||||
"ports": [
|
||||
"35621"
|
||||
],
|
||||
"secrets-in-environment": "the application's own code reads SMTP_AUTH_USER/PASS from the environment (server/index.js); converting is this repository's change"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"on": [
|
||||
{
|
||||
"arg": "NODE_BASE",
|
||||
"image": "node@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9"
|
||||
}
|
||||
],
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "site",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile"
|
||||
}
|
||||
]
|
||||
},
|
||||
"secrets": {
|
||||
"smtp": "/var/lib/de-spiegel/smtp.secret"
|
||||
}
|
||||
}
|
||||
+73
-36
@@ -187,38 +187,11 @@ func describeAge(when, now time.Time) string {
|
||||
// so LoadState is what is read — and it is the thing an interface spanning systemd and OpenRC
|
||||
// would have had to drop.
|
||||
func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string, error) {
|
||||
out, asked := run(ctx, "systemctl", "show",
|
||||
"--property=LoadState", "--property=ActiveState", "--property=Type",
|
||||
"--property=RemainAfterExit", "--property=ExecMainStatus", "--", unit)
|
||||
|
||||
var load, active, kind, remains, exited string
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
key, value, found := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case "LoadState":
|
||||
load = value
|
||||
case "ActiveState":
|
||||
active = value
|
||||
case "Type":
|
||||
kind = value
|
||||
case "RemainAfterExit":
|
||||
remains = value
|
||||
case "ExecMainStatus":
|
||||
exited = value
|
||||
}
|
||||
u, err := showUnit(ctx, run, unit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch load {
|
||||
case "":
|
||||
// With why, where it said why: an account's manager that could not be reached says so
|
||||
// here, and nowhere else (novox/hq ADR 0177).
|
||||
if asked != nil {
|
||||
return "", fmt.Errorf("the service manager said nothing about %s: %w", unit, asked)
|
||||
}
|
||||
return "", fmt.Errorf("the service manager said nothing about %s", unit)
|
||||
switch u.load {
|
||||
case "not-found":
|
||||
return "", fmt.Errorf(
|
||||
"%s does not exist on this machine. A declaration naming a unit that is not "+
|
||||
@@ -227,7 +200,7 @@ func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string,
|
||||
case "masked":
|
||||
return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit)
|
||||
case "error", "bad-setting":
|
||||
return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load)
|
||||
return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, u.load)
|
||||
}
|
||||
|
||||
// **A one-shot that finished is not stopped.** A unit whose whole job is to apply something
|
||||
@@ -243,24 +216,88 @@ func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string,
|
||||
// wrong with.
|
||||
//
|
||||
// So for that shape, what "running" means is "it ran, and it worked".
|
||||
if kind == "oneshot" && remains != "yes" && active == "inactive" {
|
||||
if exited == "0" || exited == "" {
|
||||
//
|
||||
// It cannot be told from one that never ran: systemd says ExecMainStatus=0 and a start timestamp
|
||||
// of 0 for a one-shot that never ran, and also for one that ran and was unloaded because nothing
|
||||
// referenced it, then loaded again. So a one-shot declared running that must be seen to have run
|
||||
// lingers (RemainAfterExit) or is enabled at boot; one declared stopped is judged at rest by the
|
||||
// apply (ServiceAtRest, novox/hq issue 1126).
|
||||
if u.oneShotAtRest() {
|
||||
if u.exited == "0" || u.exited == "" {
|
||||
return "running", nil
|
||||
}
|
||||
return "stopped", nil
|
||||
}
|
||||
|
||||
switch active {
|
||||
switch u.active {
|
||||
case "active", "activating", "reloading":
|
||||
return "running", nil
|
||||
case "inactive", "failed", "deactivating":
|
||||
return "stopped", nil
|
||||
default:
|
||||
return "", fmt.Errorf(
|
||||
"the service manager reports %s as %q, which is neither running nor stopped", unit, active)
|
||||
"the service manager reports %s as %q, which is neither running nor stopped", unit, u.active)
|
||||
}
|
||||
}
|
||||
|
||||
// ServiceAtRest is whether unit is a one-shot that does not linger and has nothing running now
|
||||
// (novox/hq issue 1126). Such a unit has no state in which a process is up between its runs, so a
|
||||
// declaration that it be stopped is satisfied whatever its last run did: there is nothing to stop,
|
||||
// and asking the manager to stop it changes nothing. A boot-time cleanup a package ships is declared
|
||||
// this way — enabled at boot, never started by an apply.
|
||||
func (arch) ServiceAtRest(ctx context.Context, run Runner, unit string) (bool, error) {
|
||||
u, err := showUnit(ctx, run, unit)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return u.oneShotAtRest(), nil
|
||||
}
|
||||
|
||||
// shownUnit is what `systemctl show` says of a unit, as far as its state is read.
|
||||
type shownUnit struct {
|
||||
load, active, kind, remains, exited string
|
||||
}
|
||||
|
||||
// oneShotAtRest is a one-shot that does not linger and is not running now.
|
||||
func (u shownUnit) oneShotAtRest() bool {
|
||||
return u.kind == "oneshot" && u.remains != "yes" && u.active == "inactive"
|
||||
}
|
||||
|
||||
// showUnit reads a unit's state properties. A manager that said nothing is an error, with why
|
||||
// where it said why: an account's manager that could not be reached says so here, and nowhere
|
||||
// else (novox/hq ADR 0177).
|
||||
func showUnit(ctx context.Context, run Runner, unit string) (shownUnit, error) {
|
||||
out, asked := run(ctx, "systemctl", "show",
|
||||
"--property=LoadState", "--property=ActiveState", "--property=Type",
|
||||
"--property=RemainAfterExit", "--property=ExecMainStatus", "--", unit)
|
||||
var u shownUnit
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
key, value, found := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case "LoadState":
|
||||
u.load = value
|
||||
case "ActiveState":
|
||||
u.active = value
|
||||
case "Type":
|
||||
u.kind = value
|
||||
case "RemainAfterExit":
|
||||
u.remains = value
|
||||
case "ExecMainStatus":
|
||||
u.exited = value
|
||||
}
|
||||
}
|
||||
if u.load == "" {
|
||||
if asked != nil {
|
||||
return u, fmt.Errorf("the service manager said nothing about %s: %w", unit, asked)
|
||||
}
|
||||
return u, fmt.Errorf("the service manager said nothing about %s", unit)
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (arch) SetServiceState(ctx context.Context, run Runner, unit, state string) error {
|
||||
verb := "start"
|
||||
if state == "stopped" {
|
||||
|
||||
@@ -5,47 +5,51 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A one-shot unit that did its work and exited is satisfied, not stopped.
|
||||
// How a one-shot unit's state is read: what it did, not whether a process is up.
|
||||
//
|
||||
// **This made the firewall permanently unsatisfiable.** Its unit loads a rule set and exits, so it
|
||||
// is inactive the instant it succeeds — the host started it, it worked, the host read back
|
||||
// "stopped" and reported the machine as not doing what it was told. On every apply, for ever, with
|
||||
// the rules correctly in place the whole time.
|
||||
func TestAOneShotThatFinishedIsRunning(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\nActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=0\n", nil
|
||||
}
|
||||
state, err := arch{}.ServiceState(context.Background(), run, "nftables.service")
|
||||
if err != nil {
|
||||
t.Fatalf("a one-shot that succeeded was an error: %v", err)
|
||||
}
|
||||
if state != "running" {
|
||||
t.Fatalf("a one-shot that did its work reads as %q, so it can never be satisfied", state)
|
||||
}
|
||||
}
|
||||
|
||||
// And one that failed is still stopped, or the host would report success for work that did not
|
||||
// happen — which is the opposite mistake and the worse one.
|
||||
func TestAOneShotThatFailedIsStopped(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\nActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=1\n", nil
|
||||
}
|
||||
state, err := arch{}.ServiceState(context.Background(), run, "nftables.service")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if state != "stopped" {
|
||||
t.Fatalf("a one-shot that failed reads as %q, so a broken firewall reports success", state)
|
||||
}
|
||||
}
|
||||
|
||||
// A unit that lingers deliberately is unaffected: it says so, and its active state is the answer.
|
||||
func TestAOneShotThatRemainsIsJudgedByItsActiveState(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\nExecMainStatus=0\n", nil
|
||||
}
|
||||
state, _ := arch{}.ServiceState(context.Background(), run, "thing.service")
|
||||
if state != "running" {
|
||||
t.Fatalf("a lingering one-shot reads as %q", state)
|
||||
// **A one-shot that did its work and exited is satisfied, not stopped.** Its unit loads a rule set
|
||||
// and exits, so it is inactive the instant it succeeds; reading that as stopped made the firewall
|
||||
// permanently unsatisfiable, on every apply, with the rules correctly in place the whole time.
|
||||
//
|
||||
// **One that failed is still stopped**, or the host would report success for work that did not
|
||||
// happen — the opposite mistake and the worse one.
|
||||
//
|
||||
// **At rest** (novox/hq issue 1126, the kernel's boot-time cleanup on ace) is whether a one-shot that does not linger has nothing running now: a declaration
|
||||
// that it be stopped is then satisfied, whatever it last did, because there is nothing to stop.
|
||||
func TestAOneShotIsReadByWhatItDid(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, show, want string
|
||||
atRest bool
|
||||
}{
|
||||
{"finished", "ActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=0\n",
|
||||
"running", true},
|
||||
{"failed", "ActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=1\n",
|
||||
"stopped", true},
|
||||
{"running now", "ActiveState=activating\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=0\n",
|
||||
"running", false},
|
||||
{"lingers", "ActiveState=active\nType=oneshot\nRemainAfterExit=yes\nExecMainStatus=0\n",
|
||||
"running", false},
|
||||
{"a daemon, stopped", "ActiveState=inactive\nType=simple\nRemainAfterExit=no\nExecMainStatus=0\n",
|
||||
"stopped", false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\n" + c.show, nil
|
||||
}
|
||||
state, err := arch{}.ServiceState(context.Background(), run, "thing.service")
|
||||
if err != nil {
|
||||
t.Fatalf("read as an error: %v", err)
|
||||
}
|
||||
if state != c.want {
|
||||
t.Errorf("reads as %q, want %q", state, c.want)
|
||||
}
|
||||
rest, err := arch{}.ServiceAtRest(context.Background(), run, "thing.service")
|
||||
if err != nil {
|
||||
t.Fatalf("at rest read as an error: %v", err)
|
||||
}
|
||||
if rest != c.atRest {
|
||||
t.Errorf("at rest is %v, want %v", rest, c.atRest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,12 @@
|
||||
"consumes": [
|
||||
"restic.backed-up"
|
||||
],
|
||||
"state": [
|
||||
{
|
||||
"name": "acts",
|
||||
"ttl-seconds": 7776000
|
||||
}
|
||||
],
|
||||
"config": {
|
||||
"operator-signers": {
|
||||
"type": "string",
|
||||
|
||||
@@ -68,4 +68,13 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
|
||||
if len(m.Consumes) != 1 || m.Consumes[0] != "restic.backed-up" {
|
||||
t.Fatalf("consumes %v", m.Consumes)
|
||||
}
|
||||
// 774 (ADR 0563 §2, §3): the acts of the node seat and the engine's own asks are its declared state acts, the bucket
|
||||
// mesh-host_acts the grants part creates and each node's engine writes under its own keys.
|
||||
parsed, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(parsed.State) != 1 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 {
|
||||
t.Fatalf("the manifest declares the state %+v", parsed.State)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user