Compare commits
79
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc960c51db | ||
|
|
07066fe7a0 | ||
|
|
974177a074 | ||
|
|
0422893873 | ||
|
|
1d3029fa1e | ||
|
|
af66f72931 | ||
|
|
233bb23bbc | ||
|
|
e031758e4a | ||
|
|
122f4dd727 | ||
|
|
e0261cdfc1 | ||
|
|
6c1f7cc7f5 | ||
|
|
71afdf3ad2 | ||
|
|
ffa63d25a0 | ||
|
|
76c7ddfc52 | ||
|
|
39365c62c9 | ||
|
|
a4713e2d28 | ||
|
|
8bdfddab4a | ||
|
|
a4c0cee251 | ||
|
|
bb8383b065 | ||
|
|
da0f8f1cfd | ||
|
|
62b211bb60 | ||
|
|
3d1465606b | ||
|
|
1d8d726758 | ||
|
|
b0d6684d37 | ||
|
|
56eb0b792c | ||
|
|
9c15610f70 | ||
|
|
91fb323ce0 | ||
|
|
8da56ead81 | ||
|
|
7bbba2e39e | ||
|
|
08e9d15817 | ||
|
|
1b624c3ec7 | ||
|
|
7e2d947f5f | ||
|
|
1e833feb39 | ||
|
|
62cfb45789 | ||
|
|
16b8f591ed | ||
|
|
15361c661f | ||
|
|
cc3f7e8e01 | ||
|
|
d5e8458340 | ||
|
|
b133a8949d | ||
|
|
33131dcbfe | ||
|
|
2ae7c53376 | ||
|
|
4055e0094b | ||
|
|
ee71ba4dea | ||
|
|
b9c24a7b42 | ||
|
|
744bfec4a0 | ||
|
|
8bff438e5e | ||
|
|
7ecf7ba356 | ||
|
|
737bce9c94 | ||
|
|
ec191247a0 | ||
|
|
a9edd2951c | ||
|
|
015334020a | ||
|
|
a3c098a5c6 | ||
|
|
2fd165f5af | ||
|
|
db1151c773 | ||
|
|
02f78c9949 | ||
|
|
192633edb8 | ||
|
|
e5e8ff5783 | ||
|
|
c3ff18de4e | ||
|
|
3ed9abf658 | ||
|
|
552f97c919 | ||
|
|
65fbc1e6f8 | ||
|
|
cbe8528f91 | ||
|
|
5497d362dd | ||
|
|
6b32e07164 | ||
|
|
f7814853f9 | ||
|
|
58aacaad2d | ||
|
|
81672b183d | ||
|
|
76e214a8b9 | ||
|
|
acf02d52ff | ||
|
|
a7acee3eed | ||
|
|
d2645a27e9 | ||
|
|
7d39510450 | ||
|
|
f38ac87ac9 | ||
|
|
d8de0a8853 | ||
|
|
45d0348044 | ||
|
|
8c5af3e4a3 | ||
|
|
ece607e215 | ||
|
|
86a05babed | ||
|
|
1ec73915b8 |
@@ -0,0 +1,107 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the
|
||||||
|
// directory `applied` beside the node's state (agent.Env.Applied, issue 774). It is the machine's own record of what runs:
|
||||||
|
// read at the agent's start before anything is watched, so the node never composes its own as if a module it runs ran
|
||||||
|
// nothing because the bucket no longer keeps that composition or the kept declaration is retired. Files are 0600 in a
|
||||||
|
// 0700 directory: a composition carries values sealed to this machine.
|
||||||
|
type appliedStore struct{ dir string }
|
||||||
|
|
||||||
|
func appliedStoreFor(statePath string) appliedStore {
|
||||||
|
return appliedStore{dir: filepath.Join(filepath.Dir(statePath), "applied")}
|
||||||
|
}
|
||||||
|
|
||||||
|
// appliedName is a module name the store takes as a file name: a catalogue module's or the node's own, never a path.
|
||||||
|
var appliedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`)
|
||||||
|
|
||||||
|
func (s appliedStore) path(module string) (string, error) {
|
||||||
|
if !appliedName.MatchString(module) || strings.Contains(module, "..") {
|
||||||
|
return "", fmt.Errorf("%q is not a module name the applied store takes", module)
|
||||||
|
}
|
||||||
|
return filepath.Join(s.dir, module+".json"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// holds says whether the store keeps the module's composition.
|
||||||
|
func (s appliedStore) holds(module string) bool {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, err = os.Stat(path)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Load() (map[string][]byte, error) {
|
||||||
|
entries, err := os.ReadDir(s.dir)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
var errs []error
|
||||||
|
for _, e := range entries {
|
||||||
|
module, ok := strings.CutSuffix(e.Name(), ".json")
|
||||||
|
if !ok || e.IsDir() || !appliedName.MatchString(module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(s.dir, e.Name()))
|
||||||
|
if err != nil {
|
||||||
|
errs = append(errs, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[module] = raw
|
||||||
|
}
|
||||||
|
return out, errors.Join(errs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Keep(module string, raw []byte) error {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if was, err := os.ReadFile(path); err == nil && string(was) == string(raw) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(s.dir, 0o700); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(s.dir, "."+module+".*")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(tmp.Name())
|
||||||
|
if _, err := tmp.Write(raw); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
tmp.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp.Name(), path)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s appliedStore) Drop(module string) error {
|
||||||
|
path, err := s.path(module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -1,16 +1,27 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/x509"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"encoding/pem"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
||||||
// record's keys read it; a composition carrying none changes nothing.
|
// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what
|
||||||
|
// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it.
|
||||||
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||||
state := filepath.Join(t.TempDir(), "state", "state.json")
|
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||||
res := func(id, content string) json.RawMessage {
|
res := func(id, content string) json.RawMessage {
|
||||||
@@ -26,7 +37,137 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
|||||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got := recordKeys(state)(); got.Authority != "CA-1" {
|
if got := recordKeys(state, nil)(); got.Authority != "CA-1" {
|
||||||
t.Errorf("the record's keys read the authority %q", got.Authority)
|
t.Errorf("the record's keys read the authority %q", got.Authority)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the
|
||||||
|
// vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused
|
||||||
|
// and the record names the vault's alone. Each change tells the record's writer.
|
||||||
|
controllerPub, controller, _ := ed25519.GenerateKey(nil)
|
||||||
|
vaultPub, vault, _ := ed25519.GenerateKey(nil)
|
||||||
|
trust := newMeshTrust(controllerPub)
|
||||||
|
told := 0
|
||||||
|
trust.onChange(func() { told++ })
|
||||||
|
keys := recordKeys(state, trust.Trusts)
|
||||||
|
ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub)
|
||||||
|
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) {
|
||||||
|
t.Errorf("before the signers are read the record trusts %v", got)
|
||||||
|
}
|
||||||
|
enrol, _ := delivery.SignContext(delivery.KindEnrolment)
|
||||||
|
msg := []byte("a reply")
|
||||||
|
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil {
|
||||||
|
t.Error("a vault-signed message is accepted before the vault's key is read")
|
||||||
|
}
|
||||||
|
vaultCA := testAuthority(t, vaultPub, vault)
|
||||||
|
keyValue, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now()))
|
||||||
|
fp, _ := delivery.AuthorityFingerprint(vaultCA)
|
||||||
|
authValue, _ := json.Marshal(delivery.SignerAuthority{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()})
|
||||||
|
trust.heardSigner(delivery.SignersKeyKey, keyValue, false, func(string) {})
|
||||||
|
trust.heardSigner(delivery.SignersAuthorityKey, authValue, false, func(string) {})
|
||||||
|
if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault {
|
||||||
|
t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err)
|
||||||
|
}
|
||||||
|
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
|
||||||
|
trust.VaultAuthority() != vaultCA || told != 2 {
|
||||||
|
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
|
||||||
|
}
|
||||||
|
handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"})
|
||||||
|
trust.heardLease(handed, true, nil)
|
||||||
|
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil {
|
||||||
|
t.Error("after the lease is handed away the controller's key is still trusted")
|
||||||
|
}
|
||||||
|
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 {
|
||||||
|
t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told)
|
||||||
|
}
|
||||||
|
trust.heardLease(handed, true, nil)
|
||||||
|
if told != 3 {
|
||||||
|
t.Error("a lease heard again unchanged wrote the record again")
|
||||||
|
}
|
||||||
|
// A signers' key naming another fingerprint is not trusted.
|
||||||
|
bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP})
|
||||||
|
trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {})
|
||||||
|
if !delivery.TrustsKey(keys().Trusts, delivery.SignerVault, vaultFP) {
|
||||||
|
t.Error("a signers' key that does not check replaced the vault's")
|
||||||
|
}
|
||||||
|
|
||||||
|
// L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads
|
||||||
|
// it, and the record is written again; one signed by another authority is refused and nothing is kept.
|
||||||
|
serving, err := identity.GenerateServingKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
servingPub, _ := base64.StdEncoding.DecodeString(serving.Public)
|
||||||
|
signedBy := func(ca string, key ed25519.PrivateKey) string {
|
||||||
|
parent, _ := delivery.ParseCertificate(ca)
|
||||||
|
der, err := x509.CreateCertificate(nil, delivery.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)),
|
||||||
|
parent, ed25519.PublicKey(servingPub), key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||||
|
}
|
||||||
|
var asked []any
|
||||||
|
answer := signedBy(vaultCA, vault)
|
||||||
|
ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) {
|
||||||
|
asked = append(asked, verb, args)
|
||||||
|
return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer})
|
||||||
|
}
|
||||||
|
written := 0
|
||||||
|
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb {
|
||||||
|
t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked)
|
||||||
|
}
|
||||||
|
answer = signedBy(testAuthority(t, controllerPub, controller), controller)
|
||||||
|
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 {
|
||||||
|
t.Errorf("a certificate of another authority was kept: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 774: what the engine applied is kept beside the state too, one file per module, 0600 in a 0700 directory,
|
||||||
|
// written by a rename so a crash leaves the one before; a name that is no module's is refused, never a path.
|
||||||
|
t.Run("the compositions the engine applied are kept beside the state", func(t *testing.T) {
|
||||||
|
applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json"))
|
||||||
|
if m, err := applied.Load(); err != nil || len(m) != 0 || applied.holds("node") {
|
||||||
|
t.Fatalf("an empty store loads %v, %v", m, err)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
module string
|
||||||
|
takes bool
|
||||||
|
}{{"node", true}, {"nats", true}, {"mesh-delivery-view", true}, {"../state", false}, {"a/b", false}, {"..", false}, {"", false}} {
|
||||||
|
if err := applied.Keep(tc.module, []byte(`{"v":1}`)); (err == nil) != tc.takes {
|
||||||
|
t.Errorf("keeping %q: %v, want taken %v", tc.module, err, tc.takes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := applied.Keep("node", []byte(`{"v":2}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entries, _ := os.ReadDir(applied.dir)
|
||||||
|
dirInfo, _ := os.Stat(applied.dir)
|
||||||
|
fileInfo, _ := os.Stat(filepath.Join(applied.dir, "node.json"))
|
||||||
|
if len(entries) != 3 || dirInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 {
|
||||||
|
t.Fatalf("the store holds %d files (want 3, no temporary one left), directory %v, file %v", len(entries), dirInfo.Mode(), fileInfo.Mode())
|
||||||
|
}
|
||||||
|
if err := applied.Drop("nats"); err != nil || applied.Drop("nats") != nil || applied.holds("nats") {
|
||||||
|
t.Fatalf("a dropped module is still held, or dropping twice fails: %v", err)
|
||||||
|
}
|
||||||
|
if m, err := applied.Load(); err != nil || string(m["node"]) != `{"v":2}` || len(m) != 2 {
|
||||||
|
t.Fatalf("the store loads %v, %v", m, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// testAuthority is a self-signed authority certificate for key, as the vault makes its own.
|
||||||
|
func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string {
|
||||||
|
t.Helper()
|
||||||
|
tpl := delivery.AuthorityTemplate(pub, time.Now().Add(-time.Hour))
|
||||||
|
der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,9 +3,14 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
||||||
@@ -39,4 +44,45 @@ func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
|
|||||||
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
||||||
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
||||||
}
|
}
|
||||||
|
// The node agent stopping while it waits is no failure of the backup: said as interrupted, so its next start backs
|
||||||
|
// the module up again (postgres on novox, 2026-10-31).
|
||||||
|
stopped, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if got := awaitBackup(stopped, make(chan []byte), "n1", "db", since, time.Second); got.Done || !got.Interrupted ||
|
||||||
|
got.Words != agent.BackupInterrupted {
|
||||||
|
t.Errorf("a stopped node agent: %+v", got)
|
||||||
|
}
|
||||||
|
if got := awaitBackup(context.Background(), make(chan []byte), "n1", "db", since, 10*time.Millisecond); got.Interrupted {
|
||||||
|
t.Errorf("silence within the bound taken as interrupted: %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it
|
||||||
|
// while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have
|
||||||
|
// reached it is never asked twice.
|
||||||
|
notAsked := fmt.Errorf("node-backup.now could not be asked: %w: nothing on this machine answers "+
|
||||||
|
"mesh.seat.node-backup.tool.now.ace", link.ErrNotAsked)
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
answers []error
|
||||||
|
within time.Duration
|
||||||
|
asked int
|
||||||
|
words string
|
||||||
|
}{
|
||||||
|
{"heard after the tool runner is back", []error{notAsked, notAsked, nil}, time.Second, 3, ""},
|
||||||
|
// Asked again until the bound, however slow the machine: at least twice, and fewer than it would answer.
|
||||||
|
{"never heard within the bound", []error{notAsked, notAsked, notAsked, notAsked, notAsked, nil}, 25 * time.Millisecond,
|
||||||
|
0, "times over"},
|
||||||
|
{"a timeout may have started it", []error{errors.New("no answer from x: context deadline exceeded"), nil},
|
||||||
|
time.Second, 1, "deadline exceeded"},
|
||||||
|
{"a refusal in words", []error{errors.New("restic is not on this machine"), nil}, time.Second, 1, "not on this"},
|
||||||
|
} {
|
||||||
|
asked := 0
|
||||||
|
_, err := askUntilHeard(context.Background(), func(context.Context) (json.RawMessage, error) {
|
||||||
|
asked++
|
||||||
|
return json.RawMessage(`{}`), row.answers[asked-1]
|
||||||
|
}, row.within, func(int) time.Duration { return 10 * time.Millisecond })
|
||||||
|
if row.asked == 0 && (asked < 2 || asked >= len(row.answers)) || row.asked != 0 && asked != row.asked || (row.words == "") != (err == nil) || err != nil && !strings.Contains(err.Error(), row.words) {
|
||||||
|
t.Errorf("%s: asked %d times, %v", row.name, asked, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,331 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"github.com/novox/mesh-host/internal/accounts"
|
||||||
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
"github.com/novox/mesh-host/internal/selfcompose"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node seat node-engine's judgements on this machine (novox/hq ADR 0563): what a hand-over and a library's take
|
||||||
|
// would set and what the disk holds now, and whether this machine is root-free.
|
||||||
|
|
||||||
|
// diskOf is what is on disk at a directory, as an act compares it when the operator's yes arrives: its owner and
|
||||||
|
// group, its mode, and which file it is.
|
||||||
|
func diskOf(info os.FileInfo) string {
|
||||||
|
owner := "unknown"
|
||||||
|
if uid, gid, ok := apply.OwnerOf(info); ok {
|
||||||
|
owner = fmt.Sprintf("%d:%d", uid, gid)
|
||||||
|
}
|
||||||
|
which := ""
|
||||||
|
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||||
|
which = fmt.Sprintf(" %v:%v", st.Dev, st.Ino)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s %o%s", owner, info.Mode().Perm(), which)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOverJudgeFrom judges a hand-over through the node seat as the controller's ask is judged (judgeFound): the
|
||||||
|
// declared owner and mode it will set, and the disk as it is.
|
||||||
|
func handOverJudgeFrom(statePath string) func(string) (link.Judged, error) {
|
||||||
|
return func(path string) (link.Judged, error) {
|
||||||
|
found, info, err := judgeFound(statePath, path)
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, err
|
||||||
|
}
|
||||||
|
return link.Judged{Subject: path, Path: path, Sets: orUnsaid(found.Declared), Disk: diskOf(info),
|
||||||
|
Words: "A directory of " + moduleOf(found.ID) + " is used as found: the mesh left its owner and mode."}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleOf(id string) string {
|
||||||
|
if at := strings.LastIndex(id, "."); at > 0 {
|
||||||
|
return id[:at]
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessJudgeFrom judges the take act on a library through the node seat (ADR 0328 §7, ADR 0563 §4): declared on this
|
||||||
|
// machine, its home, and its root holding data the mesh has not taken; what the take sets, and the disk as it is.
|
||||||
|
func accessJudgeFrom(statePath string) func(string, string) (link.Judged, error) {
|
||||||
|
return func(act, access string) (link.Judged, error) {
|
||||||
|
d, err := declaredNow(statePath, nil)
|
||||||
|
if errors.Is(err, store.ErrNothingDeclared) {
|
||||||
|
return link.Judged{}, errors.New("this node keeps no declaration, so it is declared no library")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, fmt.Errorf("what runs on this node does not read: %w", err)
|
||||||
|
}
|
||||||
|
for _, l := range d.Libraries {
|
||||||
|
if apply.LibraryAccess(l.Library) != access {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !l.Home {
|
||||||
|
return link.Judged{}, fmt.Errorf("this machine is not the home of the library %s, and a library is "+
|
||||||
|
"taken on its home", l.Library)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), lookBound)
|
||||||
|
defer cancel()
|
||||||
|
if v := apply.JudgeLibrary(ctx, l); v.Kind != apply.VerdictWaitKeep {
|
||||||
|
return link.Judged{}, fmt.Errorf("the library %s does not wait for the operator's word to be kept "+
|
||||||
|
"(it is judged %s), so there is nothing to take", l.Library, v.Kind)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(l.Root)
|
||||||
|
if err != nil {
|
||||||
|
return link.Judged{}, fmt.Errorf("the library %s's root cannot be read: %v", l.Library, err)
|
||||||
|
}
|
||||||
|
return link.Judged{Subject: access, Path: filepath.Clean(l.Root),
|
||||||
|
Sets: "the group " + l.Group + ", group write and setgid on everything under its root",
|
||||||
|
Disk: diskOf(info), Words: "The library " + l.Library + " holds data the mesh has not taken."}, nil
|
||||||
|
}
|
||||||
|
return link.Judged{}, fmt.Errorf("the access %s names no library declared on this machine", access)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// askTakes asks the take of every library the look found waiting for the operator's word, once per library (ADR 0563
|
||||||
|
// §4): asked by the engine itself, never a second time while one is open or after one was asked; access-act asks again
|
||||||
|
// by hand.
|
||||||
|
func askTakes(acts *link.HandActs, waiting []string, say func(string)) {
|
||||||
|
if acts == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, access := range waiting {
|
||||||
|
if acts.AskedFor(link.ActAccess, access) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := acts.AskAccess(link.EngineSeat+"-"+acts.Node, link.ActTake, access,
|
||||||
|
"its root holds data outside its group", true); err != nil && say != nil {
|
||||||
|
say("the take of " + access + " could not be asked: " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastAccounts is the newest look at every account (the health loop's), which root-free judges from.
|
||||||
|
var lastAccounts atomic.Pointer[accounts.Statement]
|
||||||
|
|
||||||
|
// rootFreshFor is how old the verdict root-free judges from may be (ADR 0563 §6, the controller's verdictFreshFor).
|
||||||
|
const rootFreshFor = 15 * time.Minute
|
||||||
|
|
||||||
|
// judgeRootFree is this machine's root-free (ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most
|
||||||
|
// rootFreshFor old, that every account declared never to become root without a person cannot, at least one such
|
||||||
|
// account declared, and the login shell's execute not served here. Anything else is no pass, and says why.
|
||||||
|
func judgeRootFree(node string, st *accounts.Statement, now time.Time, served bool, servedWhy string) link.RootVerdict {
|
||||||
|
v := link.RootVerdict{Machine: node, Judged: now.UTC()}
|
||||||
|
var not []string
|
||||||
|
confined := ""
|
||||||
|
switch {
|
||||||
|
case st == nil:
|
||||||
|
not = append(not, "this machine's engine has judged no account yet")
|
||||||
|
case now.Sub(st.At) > rootFreshFor:
|
||||||
|
not = append(not, fmt.Sprintf("the engine's newest judgement of its accounts was made at %s, more than %d "+
|
||||||
|
"minutes ago, and a verdict that old is not a verdict on now", st.At.UTC().Format(time.RFC3339),
|
||||||
|
int(rootFreshFor.Minutes())))
|
||||||
|
default:
|
||||||
|
var named []string
|
||||||
|
for _, a := range st.Accounts {
|
||||||
|
if !a.Root {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
named = append(named, a.Name)
|
||||||
|
if a.State != accounts.Healthy {
|
||||||
|
not = append(not, "the account "+a.Name+" "+orNoneKnownWord(a.Reason))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(named) == 0 {
|
||||||
|
not = append(not, "no account here is declared never to become root, so agents run as the operator's "+
|
||||||
|
"account, which may become root")
|
||||||
|
} else if len(not) == 0 {
|
||||||
|
confined = "the account " + strings.Join(named, ", ") + " cannot become root without a person (judged " +
|
||||||
|
st.At.UTC().Format(time.RFC3339) + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if served {
|
||||||
|
not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, which "+
|
||||||
|
"can become root ("+servedWhy+")")
|
||||||
|
}
|
||||||
|
if len(not) > 0 {
|
||||||
|
v.Why = strings.Join(not, "; ")
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
v.Free, v.Why = true, confined+"; the login shell's execute is not served here"
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNoneKnownWord(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "is not judged"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingFileSuffix is the resource the login shell's holder renders its setting execute into (ADR 0268: zsh's
|
||||||
|
// tools-conf, `execute=serve` or `execute=withhold`), read beside the bus's discovery.
|
||||||
|
const settingFileSuffix = ".tools-conf"
|
||||||
|
|
||||||
|
// executeServedHere is whether the login shell's execute is served on this machine (ADR 0268), failing closed: served
|
||||||
|
// while the bus's discovery hears it answered here or could not be asked, or while the holder's setting file applied
|
||||||
|
// here says anything but withhold or cannot be read. why says which.
|
||||||
|
func executeServedHere(statePath string, heard func() (bool, error)) (bool, string) {
|
||||||
|
var why []string
|
||||||
|
if heard == nil {
|
||||||
|
why = append(why, "the bus could not be asked whether execute is answered here")
|
||||||
|
} else if h, err := heard(); err != nil {
|
||||||
|
why = append(why, "the bus could not be asked whether execute is answered here: "+err.Error())
|
||||||
|
} else if h {
|
||||||
|
why = append(why, "the bus hears execute answered here")
|
||||||
|
}
|
||||||
|
known, err := store.Load(statePath)
|
||||||
|
if err != nil {
|
||||||
|
why = append(why, "what this machine applied could not be read: "+err.Error())
|
||||||
|
}
|
||||||
|
for _, r := range known.Resources {
|
||||||
|
if r.Type != "file" || !strings.HasSuffix(r.ID, settingFileSuffix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if value, err := settingIn(r.Target); err != nil {
|
||||||
|
why = append(why, moduleOf(r.ID)+"'s setting execute cannot be read: "+err.Error())
|
||||||
|
} else if value != "withhold" {
|
||||||
|
why = append(why, moduleOf(r.ID)+"'s setting execute here is "+value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(why) > 0, strings.Join(why, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// settingIn is the value of execute in a setting file, "" when it says none.
|
||||||
|
func settingIn(path string) (string, error) {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
value := ""
|
||||||
|
lines := bufio.NewScanner(f)
|
||||||
|
for lines.Scan() {
|
||||||
|
t := strings.TrimSpace(lines.Text())
|
||||||
|
if key, v, ok := strings.Cut(t, "="); ok && !strings.HasPrefix(t, "#") && strings.TrimSpace(key) == "execute" {
|
||||||
|
value = strings.TrimSpace(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if value == "" {
|
||||||
|
value = "not said"
|
||||||
|
}
|
||||||
|
return value, lines.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepWaiting is every library a look found waiting for the operator's word to keep it (ADR 0328 §8).
|
||||||
|
func keepWaiting(vs []link.ResourceHealth) []string {
|
||||||
|
var out []string
|
||||||
|
for _, v := range vs {
|
||||||
|
for _, w := range v.AccessWaits {
|
||||||
|
if w.Reason == link.WaitKeep {
|
||||||
|
out = append(out, w.Access)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// engineActs is the node seat this engine holds (ADR 0563): judged and performed by the same code as the controller's
|
||||||
|
// signed asks, its asks to the operator over the session's link, its acts kept beside the state.
|
||||||
|
func engineActs(opts options, node string, queue *link.Session, search *accounts.SetuidCache) *link.HandActs {
|
||||||
|
state := link.BusState{Session: queue, Node: node}
|
||||||
|
h := &link.HandActs{Node: node, State: state,
|
||||||
|
JudgeHandOver: handOverJudgeFrom(opts.state), HandOver: handOverFrom(opts.state),
|
||||||
|
JudgeAccess: accessJudgeFrom(opts.state), AccessAct: accessActFrom(opts.state, opts.timeout),
|
||||||
|
JudgeDirectories: judgeDirectoriesFrom(opts.state), SetuidSearch: queue.SetuidSearch, Searching: search.Running,
|
||||||
|
RootFree: func() link.RootVerdict {
|
||||||
|
served, why := executeServedHere(opts.state, func() (bool, error) {
|
||||||
|
return queue.HeardServed("node-login-shell", "execute")
|
||||||
|
})
|
||||||
|
return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why)
|
||||||
|
},
|
||||||
|
Say: queue.SayEngine}
|
||||||
|
asker := asks.EngineAsker(node)
|
||||||
|
h.Asker = &meshasker.Client{Module: asker, State: state, Publish: queue.PublishAsk,
|
||||||
|
Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcileEvery is how often the engine catches up its open asks from the router's records (a warrant heard while no
|
||||||
|
// link was open, an ask past its expiry): besides each link's opening.
|
||||||
|
const reconcileEvery = 5 * time.Minute
|
||||||
|
|
||||||
|
func reconcileActs(ctx context.Context, h *link.HandActs) {
|
||||||
|
t := time.NewTicker(reconcileEvery)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
h.Reconcile()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// compositionReadFor bounds one read of the verb composition: the revisions of one key from the bucket.
|
||||||
|
const compositionReadFor = 20 * time.Second
|
||||||
|
|
||||||
|
// compositionFrom answers the verb composition from the node agent that runs now (issue 774); refused in words while
|
||||||
|
// none does.
|
||||||
|
func compositionFrom(current *agentNow) func(module, digest string, diff bool) (any, error) {
|
||||||
|
return func(module, digest string, diff bool) (any, error) {
|
||||||
|
ag := current.get()
|
||||||
|
if ag == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so it holds no composition yet")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), compositionReadFor)
|
||||||
|
defer cancel()
|
||||||
|
return ag.Composition(ctx, module, digest, diff)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composeReadFor bounds the verb compose's wait for its pass: a pass that waits on a seat longer says what it waits on.
|
||||||
|
const composeReadFor = 2 * time.Minute
|
||||||
|
|
||||||
|
// composeFrom answers the verb compose (issue 774): a pass of the composer that runs now, and with module that module's
|
||||||
|
// apply tried again by the node agent; refused in words while the node composes nothing of its own.
|
||||||
|
func composeFrom(current *agentNow, composer *atomic.Pointer[selfcompose.Composer]) func(module string) (any, error) {
|
||||||
|
return func(module string) (any, error) {
|
||||||
|
c := composer.Load()
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("this node composes none of its own modules now (it applies the controller's " +
|
||||||
|
"compositions, or its node agent is starting), so there is no pass to make")
|
||||||
|
}
|
||||||
|
out := map[string]any{}
|
||||||
|
if module != "" {
|
||||||
|
// Only a failed apply that is not held is tried again; what runs is never taken away (review round 1: the
|
||||||
|
// verb has no warrant, ADR 0563 §2, and a hold for a failed backup is the operator's to release, ADR 0568 §4).
|
||||||
|
ag := current.get()
|
||||||
|
if ag == nil {
|
||||||
|
return nil, errors.New("this engine's node agent has not started, so no apply is tried again")
|
||||||
|
}
|
||||||
|
if err := ag.RetryFailed(module); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out["retried"] = module
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), composeReadFor)
|
||||||
|
defer cancel()
|
||||||
|
pass, err := c.ComposeNow(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out["pass"] = pass
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -10,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
)
|
)
|
||||||
@@ -82,6 +84,77 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
|
|||||||
if err != nil || string(got) != string(body) {
|
if err != nil || string(got) != string(body) {
|
||||||
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
|
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named,
|
||||||
|
// and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked.
|
||||||
|
gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)}
|
||||||
|
never := []byte(strings.Repeat("never had here\n", 6000))
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
raw []byte
|
||||||
|
onDisk []byte // nil removes the file
|
||||||
|
refused string // "" applies
|
||||||
|
}{
|
||||||
|
{"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""},
|
||||||
|
{"the file here is gone, its object deleted", raw, nil, ""},
|
||||||
|
{"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"},
|
||||||
|
} {
|
||||||
|
if row.onDisk == nil {
|
||||||
|
_ = os.Remove(target)
|
||||||
|
} else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil)
|
||||||
|
if row.refused == "" {
|
||||||
|
got, _ := os.ReadFile(target)
|
||||||
|
if report.Refused != "" || string(got) != string(body) {
|
||||||
|
t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep",
|
||||||
|
row.name, report.Refused, len(got))
|
||||||
|
}
|
||||||
|
} else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") {
|
||||||
|
t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Issue 924, the live case: what runs came from the kept declaration, whose export names a digest swept long ago,
|
||||||
|
// while the node wrote a newer export since. Composed as it runs, the file names what the node wrote, and applies with
|
||||||
|
// no object; a file someone else changed since is left as named.
|
||||||
|
newer := []byte(strings.Repeat("the export the node wrote since\n", 3000))
|
||||||
|
newerName, _ := declaration.ObjectName(declaration.DigestOf(newer))
|
||||||
|
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{newerName: newer}})
|
||||||
|
if report = applyAndKeep(ctx, opts, referenceDeclaration(target, newer), nil, nil, nil); report.Refused != "" {
|
||||||
|
t.Fatalf("924: the newer export: %s", report.Refused)
|
||||||
|
}
|
||||||
|
stale := json.RawMessage(`{"id":"m.big","type":"file","path":"` + target + `","digest":"` + declaration.DigestOf(body) +
|
||||||
|
`","size":` + strconv.Itoa(len(body)) + `}`)
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
edit []byte // what someone else wrote there since; nil for nobody
|
||||||
|
digest string
|
||||||
|
}{
|
||||||
|
{"the file is what the node wrote", nil, declaration.DigestOf(newer)},
|
||||||
|
{"someone changed the file since", []byte("a hand edit"), declaration.DigestOf(body)},
|
||||||
|
} {
|
||||||
|
if row.edit != nil {
|
||||||
|
if err := os.WriteFile(target, row.edit, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
runs := runsAsWritten([]delivery.Composed{{Node: "n", Module: "m", Resources: []json.RawMessage{stale}}}, opts.state)
|
||||||
|
var got struct {
|
||||||
|
Digest string `json:"digest"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(runs[0].Resources[0], &got)
|
||||||
|
if got.Digest != row.digest {
|
||||||
|
t.Errorf("924: %s: what runs names %s, want %s", row.name, got.Digest, row.digest)
|
||||||
|
}
|
||||||
|
if row.edit == nil {
|
||||||
|
raw := []byte(`{"declaration":1,"resources":[` + string(runs[0].Resources[0]) + `]}`)
|
||||||
|
if report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, raw, nil, nil, nil); report.Refused != "" {
|
||||||
|
t.Errorf("924: %s: composed as it runs, refused %q; want applied with no object", row.name, report.Refused)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
|
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
|
||||||
|
|||||||
+76
-10
@@ -3,22 +3,25 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
"github.com/novox/mesh-host/internal/filling"
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/system"
|
"github.com/novox/mesh-host/internal/system"
|
||||||
)
|
)
|
||||||
|
|
||||||
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
|
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
|
||||||
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
|
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
|
||||||
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
|
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) {
|
||||||
e := fillingEngine(opts, signer, say)
|
e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say)
|
||||||
fillingNow.Store(e)
|
fillingNow.Store(e)
|
||||||
defer fillingNow.CompareAndSwap(e, nil)
|
defer fillingNow.CompareAndSwap(e, nil)
|
||||||
if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
|
if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
|
||||||
@@ -30,9 +33,10 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s
|
|||||||
var fillingNow atomic.Pointer[filling.Engine]
|
var fillingNow atomic.Pointer[filling.Engine]
|
||||||
|
|
||||||
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
|
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
|
||||||
// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled).
|
// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled
|
||||||
// What it told, or why it could not, is said.
|
// (agent.Env.Settled, "<module> settled"), the haves were read at the start, or a module was taken as running
|
||||||
func knockFilling(module string, say func(string)) {
|
// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said.
|
||||||
|
func knockFilling(after string, say func(string)) {
|
||||||
e := fillingNow.Load()
|
e := fillingNow.Load()
|
||||||
if e == nil {
|
if e == nil {
|
||||||
return
|
return
|
||||||
@@ -40,16 +44,19 @@ func knockFilling(module string, say func(string)) {
|
|||||||
told, err := e.Knock(filling.Socket)
|
told, err := e.Knock(filling.Socket)
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case err != nil:
|
||||||
say(fmt.Sprintf("after %s settled: %v", module, err))
|
say(fmt.Sprintf("after %s: %v", after, err))
|
||||||
case told != "":
|
case told != "":
|
||||||
say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told))
|
say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
|
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
|
||||||
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
|
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
|
||||||
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
|
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
|
||||||
func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
|
//
|
||||||
|
// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a
|
||||||
|
// file so filled is written 0600 when the declaration gives no mode.
|
||||||
|
func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine {
|
||||||
return &filling.Engine{
|
return &filling.Engine{
|
||||||
Declared: func() (*declaration.Declaration, error) {
|
Declared: func() (*declaration.Declaration, error) {
|
||||||
return declaredNow(opts.state, signer)
|
return declaredNow(opts.state, signer)
|
||||||
@@ -73,8 +80,13 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", err
|
return "", "", err
|
||||||
}
|
}
|
||||||
report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
|
content, secret, err := secrets.fill(ctx, filledModule(d, path), content)
|
||||||
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
|
if err != nil {
|
||||||
|
return "", "", fmt.Errorf("%s was not written: %w", path, err)
|
||||||
|
}
|
||||||
|
report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner,
|
||||||
|
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)),
|
||||||
|
apply.Filled{Secret: secret})
|
||||||
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
||||||
if err := store.Save(opts.state, updated); err != nil {
|
if err := store.Save(opts.state, updated); err != nil {
|
||||||
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
|
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
|
||||||
@@ -93,3 +105,57 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// filledModule is the module whose file the declaration declares at path, or "" when none or several do.
|
||||||
|
func filledModule(d *declaration.Declaration, path string) string {
|
||||||
|
module := ""
|
||||||
|
for _, r := range d.Resources {
|
||||||
|
if f, ok := r.(*declaration.File); ok && f.Path == path {
|
||||||
|
if module != "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
module, _ = d.ModuleOfID(f.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return module
|
||||||
|
}
|
||||||
|
|
||||||
|
// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key.
|
||||||
|
func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets {
|
||||||
|
p := provisionSecrets{node: node, open: sealOpener(statePath)}
|
||||||
|
if queue == nil {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
// What the bus holds under prefix, read by a watch to its caught-up mark: no poll, no list of every key.
|
||||||
|
p.list = func(ctx context.Context, prefix string) (map[string][]byte, error) {
|
||||||
|
wctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
updates, err := queue.KVWatch(wctx, provisions.ProvisionsBucket, prefix+">")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case u, ok := <-updates:
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("the watch of %s ended before it caught up", provisions.ProvisionsBucket)
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case u.Caught:
|
||||||
|
return out, nil
|
||||||
|
case u.Deleted:
|
||||||
|
delete(out, u.Key)
|
||||||
|
default:
|
||||||
|
out[u.Key] = u.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||||
|
return queue.AskVault(ctx, "own", args)
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,9 +7,12 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/user"
|
"os/user"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/filling"
|
"github.com/novox/mesh-host/internal/filling"
|
||||||
"github.com/novox/mesh-host/internal/store"
|
"github.com/novox/mesh-host/internal/store"
|
||||||
)
|
)
|
||||||
@@ -45,7 +48,34 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
|
|||||||
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
|
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
e := fillingEngine(opts, public, func(string) {})
|
// The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked:
|
||||||
|
// the opener reads the box's "v").
|
||||||
|
provision := map[string]string{}
|
||||||
|
vaultName := "provision/n2/db"
|
||||||
|
secrets := provisionSecrets{node: "n1",
|
||||||
|
list: func(_ context.Context, prefix string) (map[string][]byte, error) {
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for k, v := range provision {
|
||||||
|
if strings.HasPrefix(k, prefix) {
|
||||||
|
out[k] = []byte(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
},
|
||||||
|
own: func(_ context.Context, args map[string]any) (json.RawMessage, error) {
|
||||||
|
raw, _ := json.Marshal(args)
|
||||||
|
var a struct{ Credentials []map[string]any }
|
||||||
|
_ = json.Unmarshal(raw, &a)
|
||||||
|
c := a.Credentials[0]
|
||||||
|
c["name"], c["generation"], c["sealed"] = vaultName, 2, map[string]string{"v": "s3cret"}
|
||||||
|
return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}})
|
||||||
|
},
|
||||||
|
open: func(sealed string) ([]byte, error) {
|
||||||
|
var b struct{ V string }
|
||||||
|
err := json.Unmarshal([]byte(sealed), &b)
|
||||||
|
return []byte(b.V), err
|
||||||
|
}}
|
||||||
|
e := fillingEngine(opts, public, secrets, func(string) {})
|
||||||
uid := uint32(os.Getuid())
|
uid := uint32(os.Getuid())
|
||||||
if err := e.Accepts(uid); err != nil {
|
if err := e.Accepts(uid); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -102,4 +132,75 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
|
|||||||
if a := <-done; a.Action != "updated" {
|
if a := <-done; a.Action != "updated" {
|
||||||
t.Fatalf("the second write: %+v", a)
|
t.Fatalf("the second write: %+v", a)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write
|
||||||
|
// from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it.
|
||||||
|
keyed := func(state string, ref bool, id, local string) string {
|
||||||
|
p := provisions.Provision{Provision: "db", Local: local, Consumer: provisions.End{Module: "m", Node: "n1"},
|
||||||
|
Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()}
|
||||||
|
if state != provisions.Provided {
|
||||||
|
p.Why = "the provider has not answered"
|
||||||
|
}
|
||||||
|
if ref {
|
||||||
|
p.Reference = &provisions.Reference{ID: id, Generation: 2}
|
||||||
|
}
|
||||||
|
raw, err := provisions.Encode(p)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, state string
|
||||||
|
ref bool
|
||||||
|
content string
|
||||||
|
want string // the file after; "" keeps what was there
|
||||||
|
mode os.FileMode
|
||||||
|
vault string // the vault's credential name, when not the one the provision names
|
||||||
|
refID string // the reference's id, when not the provision's shared value
|
||||||
|
local string // the provision's local name (ADR 0094): the placeholder names it
|
||||||
|
}{
|
||||||
|
{name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"},
|
||||||
|
{name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"},
|
||||||
|
{name: "a reference to no credential of the provision is refused", state: provisions.Provided, ref: true,
|
||||||
|
refID: "provision/n9/db", content: "password=${secret:db}\n"},
|
||||||
|
{name: "a name no provision goes by is refused", state: provisions.Provided, ref: true, content: "password=${secret:other}\n"},
|
||||||
|
{name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true,
|
||||||
|
vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"},
|
||||||
|
{name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true,
|
||||||
|
content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600},
|
||||||
|
{name: "a local name is the provision's by its key", state: provisions.Provided, ref: true, local: "admin",
|
||||||
|
content: "admin=${secret:admin}\n", want: "admin=s3cret\n", mode: 0o600},
|
||||||
|
{name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644},
|
||||||
|
} {
|
||||||
|
before, _ := os.ReadFile(path)
|
||||||
|
if tc.state != "" {
|
||||||
|
id := "provision/n2/db"
|
||||||
|
if tc.refID != "" {
|
||||||
|
id = tc.refID
|
||||||
|
}
|
||||||
|
clear(provision)
|
||||||
|
key := "n1.m.db"
|
||||||
|
if tc.local != "" {
|
||||||
|
key += "." + tc.local
|
||||||
|
}
|
||||||
|
provision[key] = keyed(tc.state, tc.ref, id, tc.local)
|
||||||
|
}
|
||||||
|
vaultName = "provision/n2/db"
|
||||||
|
if tc.vault != "" {
|
||||||
|
vaultName = tc.vault
|
||||||
|
}
|
||||||
|
a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)})
|
||||||
|
got, _ := os.ReadFile(path)
|
||||||
|
if tc.want == "" {
|
||||||
|
if a.Failed == "" && a.Refused == "" || string(got) != string(before) {
|
||||||
|
t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
info, _ := os.Stat(path)
|
||||||
|
if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode {
|
||||||
|
t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode())
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+33
-14
@@ -68,14 +68,23 @@ func foundDirectories(known store.State) []store.Applied {
|
|||||||
// known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's
|
// known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's
|
||||||
// path is the resource's target beside the reason, never in it: the controller says a reason on the
|
// path is the resource's target beside the reason, never in it: the controller says a reason on the
|
||||||
// operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence.
|
// operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence.
|
||||||
|
//
|
||||||
|
// From the lease write, heard as the engine hears it (meshTrust), the line has no answerer: the way out is the node
|
||||||
|
// seat's hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5).
|
||||||
func handOverLine() string {
|
func handOverLine() string {
|
||||||
node, _ := foundNode.Load().(string)
|
node, _ := foundNode.Load().(string)
|
||||||
if node == "" {
|
if node == "" {
|
||||||
node = "<node>"
|
node = "<node>"
|
||||||
}
|
}
|
||||||
|
if t := foundTrust.Load(); t != nil && t.HandedAway() {
|
||||||
|
return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node)
|
||||||
|
}
|
||||||
return fmt.Sprintf("`nox node hand-over %s <directory>` on the control-node", node)
|
return fmt.Sprintf("`nox node hand-over %s <directory>` on the control-node", node)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// foundTrust is what the engine trusts, whose lease decides the way out a directory used as found names.
|
||||||
|
var foundTrust atomic.Pointer[meshTrust]
|
||||||
|
|
||||||
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
|
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
|
||||||
// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line
|
// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line
|
||||||
// that hands it over. The path is the target, which the controller keeps as evidence.
|
// that hands it over. The path is the target, which the controller keeps as evidence.
|
||||||
@@ -129,16 +138,30 @@ func foundNow() []store.Applied {
|
|||||||
// never follows a path it did not state: a hand-over is root giving a directory to an account at the next
|
// never follows a path it did not state: a hand-over is root giving a directory to an account at the next
|
||||||
// apply, and a path that resolves elsewhere would give that elsewhere.
|
// apply, and a path that resolves elsewhere would give that elsewhere.
|
||||||
func handOverFound(statePath, path, by string) (string, error) {
|
func handOverFound(statePath, path, by string) (string, error) {
|
||||||
|
found, _, err := judgeFound(statePath, path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
|
||||||
|
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeFound is the judgement of a hand-over's path, recording nothing: the directory this machine uses as found at
|
||||||
|
// that path, as the engine states it, and what is on disk there.
|
||||||
|
func judgeFound(statePath, path string) (store.Applied, os.FileInfo, error) {
|
||||||
if !filepath.IsAbs(path) {
|
if !filepath.IsAbs(path) {
|
||||||
return "", fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
|
return store.Applied{}, nil, fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
|
||||||
}
|
}
|
||||||
if filepath.Clean(path) != path {
|
if filepath.Clean(path) != path {
|
||||||
return "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
|
return store.Applied{}, nil, fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
|
||||||
"separator); nothing was handed over", path)
|
"separator); nothing was handed over", path)
|
||||||
}
|
}
|
||||||
known, err := store.Load(statePath)
|
known, err := store.Load(statePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return store.Applied{}, nil, err
|
||||||
}
|
}
|
||||||
var found *store.Applied
|
var found *store.Applied
|
||||||
for _, r := range foundDirectories(known) {
|
for _, r := range foundDirectories(known) {
|
||||||
@@ -149,31 +172,27 @@ func handOverFound(statePath, path, by string) (string, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if found == nil {
|
if found == nil {
|
||||||
return "", errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
|
return store.Applied{}, nil, errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
|
||||||
"the ones that are); nothing was handed over")
|
"the ones that are); nothing was handed over")
|
||||||
}
|
}
|
||||||
info, err := os.Lstat(path)
|
info, err := os.Lstat(path)
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return "", fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
|
return store.Applied{}, nil, fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
|
||||||
case info.Mode()&os.ModeSymlink != 0:
|
case info.Mode()&os.ModeSymlink != 0:
|
||||||
return "", fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
|
return store.Applied{}, nil, fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
|
||||||
case !info.IsDir():
|
case !info.IsDir():
|
||||||
return "", fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
|
return store.Applied{}, nil, fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
|
||||||
}
|
}
|
||||||
resolved, err := filepath.EvalSymlinks(path)
|
resolved, err := filepath.EvalSymlinks(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
|
return store.Applied{}, nil, fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
|
||||||
}
|
}
|
||||||
if resolved != path {
|
if resolved != path {
|
||||||
return "", fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
|
return store.Applied{}, nil, fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
|
||||||
"(novox/hq issue 339); nothing was handed over", path, resolved)
|
"(novox/hq issue 339); nothing was handed over", path, resolved)
|
||||||
}
|
}
|
||||||
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
|
return *found, info, nil
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
|
|
||||||
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked.
|
// handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked.
|
||||||
|
|||||||
@@ -40,6 +40,21 @@ func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
|
|||||||
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
|
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
|
||||||
t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason)
|
t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason)
|
||||||
}
|
}
|
||||||
|
// ADR 0563 §5: from the lease write the remedy is the node seat's hand-over, answered on the operator's channel, and
|
||||||
|
// no longer the controller's terminal line; the lease is the one the engine hears.
|
||||||
|
handed := newMeshTrust(nil)
|
||||||
|
handed.handedAway = true
|
||||||
|
foundTrust.Store(handed)
|
||||||
|
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||||
|
"`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") || strings.Contains(r.Reason, dir) {
|
||||||
|
t.Fatalf("after the lease write the way out is %q", r.Reason)
|
||||||
|
}
|
||||||
|
foundTrust.Store(newMeshTrust(nil))
|
||||||
|
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||||
|
"`nox node hand-over laptop <directory>`") {
|
||||||
|
t.Fatalf("while the controller holds its lease the way out is %q", r.Reason)
|
||||||
|
}
|
||||||
|
foundTrust.Store((*meshTrust)(nil))
|
||||||
foundNode.Store("")
|
foundNode.Store("")
|
||||||
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
|
||||||
"`nox node hand-over <node> <directory>`") {
|
"`nox node hand-over <node> <directory>`") {
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"github.com/novox/mesh-host/internal/apply"
|
"github.com/novox/mesh-host/internal/apply"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
@@ -53,4 +55,42 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
|||||||
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
||||||
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
||||||
}
|
}
|
||||||
|
// 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is
|
||||||
|
// open, nor after it ended; and a library waiting for its disk is not asked.
|
||||||
|
t.Run("the home engine asks the take once", func(t *testing.T) {
|
||||||
|
var asked []asks.Ask
|
||||||
|
state := &link.MemState{}
|
||||||
|
h := &link.HandActs{Node: "n1", State: state,
|
||||||
|
JudgeAccess: func(act, access string) (link.Judged, error) {
|
||||||
|
return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil
|
||||||
|
},
|
||||||
|
AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }}
|
||||||
|
h.Asker = &meshasker.Client{Module: asks.EngineAsker("n1"), State: state,
|
||||||
|
Publish: func(subject string, body any, id string) error {
|
||||||
|
if a, ok := body.(asks.Ask); ok {
|
||||||
|
asked = append(asked, a)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}}
|
||||||
|
waiting := keepWaiting(vs)
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 {
|
||||||
|
t.Fatalf("waiting %v asked %d times", waiting, len(asked))
|
||||||
|
}
|
||||||
|
h.Heard(asks.Warrant{Ask: asked[0].ID, Asker: asks.EngineAsker("n1"), Outcome: asks.OutcomeExpired, At: time.Now()})
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Fatalf("the take was asked again after its ask ended: %d", len(asked))
|
||||||
|
}
|
||||||
|
// Kept in a key of its own (rev-l3 round 1): the acts gone from the record never make it ask again.
|
||||||
|
keys, _ := state.Keys("act.")
|
||||||
|
for _, k := range keys {
|
||||||
|
_ = state.Delete(k)
|
||||||
|
}
|
||||||
|
askTakes(h, waiting, nil)
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Fatalf("the take was asked again with the acts gone: %d", len(asked))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
+115
-13
@@ -516,7 +516,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
|
|||||||
|
|
||||||
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
|
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
|
||||||
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
|
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
|
||||||
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
|
if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
|
||||||
return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
|
return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1105,10 +1105,17 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
|
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
|
||||||
// controller asks.
|
// controller asks.
|
||||||
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
||||||
|
// What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's
|
||||||
|
// once read from its signers (novox/hq ADR 0550 §2).
|
||||||
|
trust := newMeshTrust(mine.Membership.Signer)
|
||||||
|
// And the way out a directory used as found names follows the lease it hears (novox/hq ADR 0563 §5).
|
||||||
|
foundTrust.Store(trust)
|
||||||
queue := &link.Session{
|
queue := &link.Session{
|
||||||
Membership: membership,
|
Membership: membership,
|
||||||
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
||||||
Keys: recordKeys(opts.state),
|
Keys: recordKeys(opts.state, trust.Trusts),
|
||||||
|
// And the controller's key, read per signed ask: none once its lease is handed away (novox/hq ADR 0563 §7).
|
||||||
|
ControllerKey: trust.Controller,
|
||||||
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
||||||
HaveWritten: proof.haveWritten,
|
HaveWritten: proof.haveWritten,
|
||||||
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
||||||
@@ -1129,7 +1136,13 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
Say: say,
|
Say: say,
|
||||||
Timeout: opts.timeout,
|
Timeout: opts.timeout,
|
||||||
}
|
}
|
||||||
|
// And the node seat node-engine this engine holds (novox/hq ADR 0563): its verbs on every link, its own asks to the
|
||||||
|
// operator under node-engine-<node>, and its acts kept beside its state.
|
||||||
|
queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch)
|
||||||
|
go reconcileActs(ctx, queue.Acts)
|
||||||
current := &agentNow{}
|
current := &agentNow{}
|
||||||
|
queue.Acts.Composition = compositionFrom(current)
|
||||||
|
queue.Acts.Compose = composeFrom(current, &composerNow)
|
||||||
setRunsHere(runsFromAgent(current))
|
setRunsHere(runsFromAgent(current))
|
||||||
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
|
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
|
||||||
sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag)
|
sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag)
|
||||||
@@ -1173,7 +1186,10 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
}
|
}
|
||||||
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
|
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
|
||||||
// busy containers are asked for again soon (ADR 0360 §11).
|
// busy containers are asked for again soon (ADR 0360 §11).
|
||||||
go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow)
|
// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
|
||||||
|
trust.onChange(func() { factsOfNow(aside, current.get()) })
|
||||||
|
go followTrust(aside, queue, trust, say)
|
||||||
|
go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow)
|
||||||
go busyPasses(aside, func() {
|
go busyPasses(aside, func() {
|
||||||
if ag := current.get(); ag != nil {
|
if ag := current.get(); ag != nil {
|
||||||
ag.Retry(busyModules()...)
|
ag.Retry(busyModules()...)
|
||||||
@@ -1186,7 +1202,7 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
go serveMeshCLI(aside, queue, say)
|
go serveMeshCLI(aside, queue, say)
|
||||||
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
|
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
|
||||||
// 0349 §4).
|
// 0349 §4).
|
||||||
go serveFilling(aside, opts, mine.Membership.Signer, say)
|
go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say)
|
||||||
|
|
||||||
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
|
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
|
||||||
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
|
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
|
||||||
@@ -1339,6 +1355,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
var lastSaid time.Time
|
var lastSaid time.Time
|
||||||
owed := true
|
owed := true
|
||||||
spaced := 1.0
|
spaced := 1.0
|
||||||
|
// healthUnwritten is why the health report was last not written, said once each time it changes.
|
||||||
|
var healthUnwritten string
|
||||||
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
|
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
|
||||||
// long-running is then ok within a look of its apply, not at the five-minute statement.
|
// long-running is then ok within a look of its apply, not at the five-minute statement.
|
||||||
var nudged <-chan struct{}
|
var nudged <-chan struct{}
|
||||||
@@ -1370,6 +1388,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
if a := accountJudge; a != nil {
|
if a := accountJudge; a != nil {
|
||||||
as, accountsChanged := a.Look(ctx)
|
as, accountsChanged := a.Look(ctx)
|
||||||
accountSt = &as
|
accountSt = &as
|
||||||
|
lastAccounts.Store(&as)
|
||||||
owed = owed || accountsChanged
|
owed = owed || accountsChanged
|
||||||
if accountsChanged {
|
if accountsChanged {
|
||||||
for _, v := range as.Accounts {
|
for _, v := range as.Accounts {
|
||||||
@@ -1384,6 +1403,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
accessSt, accessesChanged := accesses.look(time.Now())
|
accessSt, accessesChanged := accesses.look(time.Now())
|
||||||
keeping.poke(ctx)
|
keeping.poke(ctx)
|
||||||
owed = owed || accessesChanged
|
owed = owed || accessesChanged
|
||||||
|
// And a library whose root holds data outside its group is asked of the operator by this engine, once (novox/hq
|
||||||
|
// ADR 0563 §4).
|
||||||
|
askTakes(queue.Acts, keepWaiting(accessSt), say)
|
||||||
if accessesChanged {
|
if accessesChanged {
|
||||||
for _, v := range accessSt {
|
for _, v := range accessSt {
|
||||||
if v.State != link.StateHealthy {
|
if v.State != link.StateHealthy {
|
||||||
@@ -1418,6 +1440,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
||||||
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
||||||
accessesHealthy(accessSt)
|
accessesHealthy(accessSt)
|
||||||
|
h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||||
|
unitSt), accountSt), found), accessSt)
|
||||||
|
// The node's own health report in its key on the bus, compared on every look and written only when what it says
|
||||||
|
// changed (issue 774): what the watchman raises the machine's alerts from, without the controller.
|
||||||
|
writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say)
|
||||||
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -1444,13 +1471,40 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
|||||||
reconcile(inactive...)
|
reconcile(inactive...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
if queue.SayHealth(ctx, h) {
|
||||||
unitSt), accountSt), found), accessSt)) {
|
|
||||||
lastSaid, owed = time.Now(), false
|
lastSaid, owed = time.Now(), false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could
|
||||||
|
// not is said once per reason, and said again when it is written after all.
|
||||||
|
func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string,
|
||||||
|
say link.Announce) {
|
||||||
|
if queue == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var peers []link.PeerSeen
|
||||||
|
if ns != nil {
|
||||||
|
for _, p := range ns.Peers {
|
||||||
|
peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var machines link.Machines
|
||||||
|
if c := machinesNow.Load(); c != nil {
|
||||||
|
machines.ByAddress, machines.Hub = c.Machines()
|
||||||
|
}
|
||||||
|
_, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines))
|
||||||
|
switch {
|
||||||
|
case err != nil && err.Error() != *unwritten:
|
||||||
|
*unwritten = err.Error()
|
||||||
|
say("this machine's health report is not written to the bus: " + err.Error())
|
||||||
|
case err == nil && *unwritten != "":
|
||||||
|
*unwritten = ""
|
||||||
|
say("this machine's health report is written to the bus again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// healthAsReported is a statement as the report and the event carry it.
|
// healthAsReported is a statement as the report and the event carry it.
|
||||||
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
||||||
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||||
@@ -1646,6 +1700,15 @@ func announceOr(say link.Announce) func(string) {
|
|||||||
// run by hand beside this service — the lock beside the state does the same (store.Lock).
|
// run by hand beside this service — the lock beside the state does the same (store.Lock).
|
||||||
var applying sync.Mutex
|
var applying sync.Mutex
|
||||||
|
|
||||||
|
// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep,
|
||||||
|
// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files.
|
||||||
|
func objectKeep(opts options) *apply.ObjectKeep {
|
||||||
|
if opts.state == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects"))
|
||||||
|
}
|
||||||
|
|
||||||
// appliedModule is how the apply of one module's declaration came out.
|
// appliedModule is how the apply of one module's declaration came out.
|
||||||
type appliedModule struct {
|
type appliedModule struct {
|
||||||
// Refused is why nothing was applied, when nothing was.
|
// Refused is why nothing was applied, when nothing was.
|
||||||
@@ -1685,7 +1748,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De
|
|||||||
|
|
||||||
// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
|
// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
|
||||||
// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
|
// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
|
||||||
if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
|
if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
|
||||||
return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
|
return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1779,9 +1842,8 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
if j := judging; j != nil {
|
if j := judging; j != nil {
|
||||||
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
||||||
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
||||||
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared))))
|
judgeWhatIsDeclared(declared, held)
|
||||||
st, _ := j.Look(ctx)
|
st, _ := j.Look(ctx)
|
||||||
netJudge.declare(declaredResolvConf(declared))
|
|
||||||
var netSt *network.Statement
|
var netSt *network.Statement
|
||||||
if n := netJudge.get(); n != nil {
|
if n := netJudge.get(); n != nil {
|
||||||
ns := n.Last()
|
ns := n.Last()
|
||||||
@@ -1789,7 +1851,6 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
}
|
}
|
||||||
var unitSt *units.Statement
|
var unitSt *units.Statement
|
||||||
if u := unitJudge; u != nil {
|
if u := unitJudge; u != nil {
|
||||||
u.Set(units.OwnedBy(declared, held))
|
|
||||||
us := u.Last()
|
us := u.Last()
|
||||||
unitSt = &us
|
unitSt = &us
|
||||||
}
|
}
|
||||||
@@ -1797,11 +1858,9 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
// said with it, relogin needed included, in the report that says the apply.
|
// said with it, relogin needed included, in the report that says the apply.
|
||||||
var accountSt *accounts.Statement
|
var accountSt *accounts.Statement
|
||||||
if a := accountJudge; a != nil {
|
if a := accountJudge; a != nil {
|
||||||
a.Set(accounts.Of(declared, held))
|
|
||||||
as, _ := a.Look(ctx)
|
as, _ := a.Look(ctx)
|
||||||
accountSt = &as
|
accountSt = &as
|
||||||
}
|
}
|
||||||
accesses.set(declared, held)
|
|
||||||
// A take act stopped at its bound goes on, beside (ADR 0328 §7).
|
// A take act stopped at its bound goes on, beside (ADR 0328 §7).
|
||||||
keeping.poke(ctx)
|
keeping.poke(ctx)
|
||||||
accessSt, _ := accesses.look(time.Now())
|
accessSt, _ := accesses.look(time.Now())
|
||||||
@@ -1896,6 +1955,46 @@ func factsOf(ctx context.Context, opts options, declared *declaration.Declaratio
|
|||||||
return report
|
return report
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// judgeWhatIsDeclared hands every judge of the health loop what is declared here now: the long-running resources, the
|
||||||
|
// resolver file, the units, the accounts and the libraries. A resource, unit, account or library no longer declared is
|
||||||
|
// forgotten by its judge, so a module whose composition drops one is no longer watched for it.
|
||||||
|
func judgeWhatIsDeclared(declared *declaration.Declaration, held map[string]bool) {
|
||||||
|
if j := judging; j != nil {
|
||||||
|
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
||||||
|
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
||||||
|
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared))))
|
||||||
|
}
|
||||||
|
netJudge.declare(declaredResolvConf(declared))
|
||||||
|
if u := unitJudge; u != nil {
|
||||||
|
u.Set(units.OwnedBy(declared, held))
|
||||||
|
}
|
||||||
|
if a := accountJudge; a != nil {
|
||||||
|
a.Set(accounts.Of(declared, held))
|
||||||
|
}
|
||||||
|
accesses.set(declared, held)
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgeWhatRunsNow hands the judges what runs now, as the node agent holds it once a module settled (applied or
|
||||||
|
// removed) or was taken as running: the facts said inside an apply read the module's composition from before it, so a
|
||||||
|
// resource its new composition dropped stayed watched until the next facts (g14 and shanks, 2026-10-30 16:14: restic's
|
||||||
|
// process restic.measure, removed, then said down every look).
|
||||||
|
func judgeWhatRunsNow(statePath string, signer ed25519.PublicKey) {
|
||||||
|
if judging == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
declared, err := declaredNow(statePath, signer)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
held := map[string]bool{}
|
||||||
|
if known, err := store.Load(statePath); err == nil {
|
||||||
|
for _, h := range known.Held {
|
||||||
|
held[h.ID] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
judgeWhatIsDeclared(declared, held)
|
||||||
|
}
|
||||||
|
|
||||||
// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs
|
// sayFacts says this node's facts on the link open now (novox/hq ADR 0539 §9): what the node is made of, how what runs
|
||||||
// here is, and its filters and ports. What runs is the compositions the node agent holds running (ag), or, before it
|
// here is, and its filters and ports. What runs is the compositions the node agent holds running (ag), or, before it
|
||||||
// has run, the declaration the node kept before the switch. It applies nothing; the haves are the agent's.
|
// has run, the declaration the node kept before the switch. It applies nothing; the haves are the agent's.
|
||||||
@@ -2167,9 +2266,12 @@ func withMissing(st liveness.Statement) liveness.Statement {
|
|||||||
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
|
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
|
||||||
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
|
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
|
||||||
// empty, never invented; the certificate is there only after the first declaration that carried it.
|
// empty, never invented; the certificate is there only after the first declaration that carried it.
|
||||||
func recordKeys(state string) func() link.RecordKeys {
|
func recordKeys(state string, trusts func() []string) func() link.RecordKeys {
|
||||||
return func() link.RecordKeys {
|
return func() link.RecordKeys {
|
||||||
var k link.RecordKeys
|
var k link.RecordKeys
|
||||||
|
if trusts != nil {
|
||||||
|
k.Trusts = trusts()
|
||||||
|
}
|
||||||
if id, err := identity.Load(identity.Path(state)); err == nil {
|
if id, err := identity.Load(identity.Path(state)); err == nil {
|
||||||
k.TunnelKey = id.Overlay.Public
|
k.TunnelKey = id.Overlay.Public
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/crypto/nacl/box"
|
"golang.org/x/crypto/nacl/box"
|
||||||
@@ -40,6 +41,11 @@ func moveSecretFrom(statePath string, queue *link.Session) link.MoveSecretFunc {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
|
fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
|
||||||
}
|
}
|
||||||
|
if moved.Superseded != "" {
|
||||||
|
if _, said := supersededSaid.LoadOrStore(moved.Secret, true); !said {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-host: %s\n", moved.Superseded)
|
||||||
|
}
|
||||||
|
}
|
||||||
return moved, err
|
return moved, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -104,8 +110,30 @@ func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret)
|
|||||||
Generation int `json:"generation"`
|
Generation int `json:"generation"`
|
||||||
Unchanged bool `json:"unchanged"`
|
Unchanged bool `json:"unchanged"`
|
||||||
ReadBack string `json:"read_back"`
|
ReadBack string `json:"read_back"`
|
||||||
|
// Refused is the vault's refusal as fields (mesh-vault's MoveRefused, mesh-catalog f3b13de2): nothing was taken.
|
||||||
|
Refused *struct {
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
Held *heldValue `json:"held"`
|
||||||
|
} `json:"refused"`
|
||||||
|
Why string `json:"why"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(raw, &kept); err != nil || kept.Name == "" || kept.ReadBack == "" {
|
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault's answer for %s does not read: %v", opened.Secret, err)
|
||||||
|
}
|
||||||
|
if r := kept.Refused; r != nil {
|
||||||
|
// The vault holds another value for the name, and the move never replaces one: the vault's value is the live
|
||||||
|
// one, and this machine's older value is superseded, never asked again (the sweep of 2026-10-30: it failed
|
||||||
|
// "answered with no read-back" on every ask).
|
||||||
|
if r.Reason == vaultHoldsAnother && r.Held != nil && r.Held.Name != "" {
|
||||||
|
return link.MovedSecret{Secret: r.Held.Name, Generation: r.Held.Generation, Superseded: r.Held.words()}, nil
|
||||||
|
}
|
||||||
|
why := kept.Why
|
||||||
|
if why == "" {
|
||||||
|
why = r.Reason
|
||||||
|
}
|
||||||
|
return link.MovedSecret{}, fmt.Errorf("the vault refused %s: %s", opened.Secret, why)
|
||||||
|
}
|
||||||
|
if kept.Name == "" || kept.ReadBack == "" {
|
||||||
return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
|
return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
|
||||||
}
|
}
|
||||||
blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
|
blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
|
||||||
@@ -124,6 +152,31 @@ func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret)
|
|||||||
return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
|
return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// vaultHoldsAnother is the vault's reason when it holds another value for the name (mesh-vault RefusedHeldAnother).
|
||||||
|
const vaultHoldsAnother = "holds-another-value"
|
||||||
|
|
||||||
|
// heldValue is the record of the value the vault holds instead, never the value (mesh-vault HeldAnother).
|
||||||
|
type heldValue struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
ChangedBy string `json:"changed_by"`
|
||||||
|
ChangedAt time.Time `json:"changed_at"`
|
||||||
|
Origin string `json:"origin"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// words say the held value's record, for the journal and the controller.
|
||||||
|
func (h heldValue) words() string {
|
||||||
|
w := fmt.Sprintf("the vault holds another value for %s, generation %d, changed by %s at %s", h.Name, h.Generation,
|
||||||
|
h.ChangedBy, h.ChangedAt.UTC().Format(time.RFC3339))
|
||||||
|
if h.Origin != "" {
|
||||||
|
w += ", origin " + h.Origin
|
||||||
|
}
|
||||||
|
return w + ": that value is the live one, and this machine's older value is superseded and not moved"
|
||||||
|
}
|
||||||
|
|
||||||
|
// supersededSaid holds the names a superseded move was said of, so the journal says each once.
|
||||||
|
var supersededSaid sync.Map
|
||||||
|
|
||||||
func wipe(b []byte) {
|
func wipe(b []byte) {
|
||||||
for i := range b {
|
for i := range b {
|
||||||
b[i] = 0
|
b[i] = 0
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ type standInVault struct {
|
|||||||
kept string
|
kept string
|
||||||
lie string
|
lie string
|
||||||
refuse string
|
refuse string
|
||||||
|
// answer replaces the second step's answer: the vault's refusal as fields (mesh-catalog f3b13de2).
|
||||||
|
answer map[string]any
|
||||||
carried []string
|
carried []string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,6 +53,9 @@ func (v *standInVault) ask(verb string, args any) (json.RawMessage, error) {
|
|||||||
v.t.Fatal("what the engine sealed does not open with the call's key")
|
v.t.Fatal("what the engine sealed does not open with the call's key")
|
||||||
}
|
}
|
||||||
v.kept = string(value)
|
v.kept = string(value)
|
||||||
|
if v.answer != nil {
|
||||||
|
return json.Marshal(v.answer)
|
||||||
|
}
|
||||||
back := v.kept
|
back := v.kept
|
||||||
if v.lie != "" {
|
if v.lie != "" {
|
||||||
back = v.lie
|
back = v.lie
|
||||||
@@ -119,6 +124,28 @@ func TestTheEngineSaysAMoveThatDidNotHold(t *testing.T) {
|
|||||||
!strings.Contains(err.Error(), "never replaces") {
|
!strings.Contains(err.Error(), "never replaces") {
|
||||||
t.Fatalf("the vault's refusal: %v", err)
|
t.Fatalf("the vault's refusal: %v", err)
|
||||||
}
|
}
|
||||||
|
// The vault's refusal as fields (mesh-catalog f3b13de2, no read-back): for another value held, the vault's is the
|
||||||
|
// live one and the move is done, superseded, at the held generation; any other reason fails in the vault's words.
|
||||||
|
held := map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 3,
|
||||||
|
"changed_by": "mesh-vault@novox", "changed_at": "2026-10-30T15:50:00Z", "origin": "made"}
|
||||||
|
for _, row := range []struct {
|
||||||
|
refused map[string]any
|
||||||
|
gen int
|
||||||
|
fails string
|
||||||
|
}{
|
||||||
|
{map[string]any{"reason": "holds-another-value", "held": held}, 3, ""},
|
||||||
|
{map[string]any{"reason": "locked"}, 0, "the vault refused"},
|
||||||
|
} {
|
||||||
|
v := &standInVault{t: t, answer: map[string]any{"refused": row.refused, "why": "in words"}}
|
||||||
|
moved, err := moveIntoTheVault(v.ask, key, aMoveOf(t, key))
|
||||||
|
switch {
|
||||||
|
case row.fails != "" && (err == nil || !strings.Contains(err.Error(), row.fails)):
|
||||||
|
t.Errorf("refused %v: %v", row.refused["reason"], err)
|
||||||
|
case row.fails == "" && (err != nil || moved.Generation != row.gen || moved.Secret != held["name"] ||
|
||||||
|
!strings.Contains(moved.Superseded, "changed by mesh-vault@novox")):
|
||||||
|
t.Errorf("refused %v: moved %+v, %v", row.refused["reason"], moved, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
// A pair credential names both its ends.
|
// A pair credential names both its ends.
|
||||||
pair := aMoveOf(t, key)
|
pair := aMoveOf(t, key)
|
||||||
pair.Module, pair.Name = "", ""
|
pair.Module, pair.Name = "", ""
|
||||||
|
|||||||
+173
-36
@@ -5,12 +5,14 @@ import (
|
|||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||||
@@ -163,19 +165,54 @@ func backupNode(session *link.Session) func(ctx context.Context, module string)
|
|||||||
defer stop()
|
defer stop()
|
||||||
// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
|
// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
|
||||||
// 1945). An answer that cannot be had or read asks for the backup as before.
|
// 1945). An answer that cannot be had or read asks for the backup as before.
|
||||||
if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil {
|
ask := func(verb string) func(context.Context) (json.RawMessage, error) {
|
||||||
|
return func(ctx context.Context) (json.RawMessage, error) {
|
||||||
|
return session.AskSeat(ctx, "node-backup", verb, map[string]any{"module": module}, 30*time.Second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if said, err := askUntilHeard(ctx, ask("backed-up"), backupAskFor, link.Backoff); err == nil {
|
||||||
if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
|
if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
|
||||||
return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
|
return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
asked := time.Now()
|
asked := time.Now()
|
||||||
if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil {
|
if _, err := askUntilHeard(ctx, ask("now"), backupAskFor, link.Backoff); err != nil {
|
||||||
return agent.Backup{Words: err.Error()}
|
return agent.Backup{Words: err.Error()}
|
||||||
}
|
}
|
||||||
return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin)
|
return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backupAskFor is how long the backup's ask is asked again while it is known never to have reached the holder of
|
||||||
|
// node-backup: nothing on this machine answers it (the tool runner restarting, its subscriptions not made yet) or no
|
||||||
|
// link is open. The sweep of 2026-10-30 met it on ace's redis, postgres and restic (14:03), step-ca (13:55) and
|
||||||
|
// mesh-vault on novox (15:22): each was held on one unanswered ask. Past the bound the backup failed, said with how
|
||||||
|
// often and how long it was asked: the module is held, its have says so, and the assigner asks the operator.
|
||||||
|
var backupAskFor = 3 * time.Minute
|
||||||
|
|
||||||
|
// askUntilHeard asks, and asks again after wait(try) while the answer says the ask was never delivered (link.ErrNotAsked),
|
||||||
|
// for at most within: an ask that may have reached its holder (a timeout, a refusal in words) is never asked twice, so a
|
||||||
|
// backup is never started twice.
|
||||||
|
func askUntilHeard(ctx context.Context, ask func(context.Context) (json.RawMessage, error), within time.Duration,
|
||||||
|
wait func(int) time.Duration) (json.RawMessage, error) {
|
||||||
|
began := time.Now()
|
||||||
|
for try := 1; ; try++ {
|
||||||
|
said, err := ask(ctx)
|
||||||
|
if err == nil || !errors.Is(err, link.ErrNotAsked) {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
pause := wait(try)
|
||||||
|
if time.Since(began)+pause > within {
|
||||||
|
return nil, fmt.Errorf("asked %d times over %s and never heard: %w", try, time.Since(began).Round(time.Second), err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, fmt.Errorf("the node agent was stopped while it asked again: %w", err)
|
||||||
|
case <-time.After(pause):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it
|
// awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it
|
||||||
// says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node
|
// says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node
|
||||||
// or module, or from before the ask, is not the answer.
|
// or module, or from before the ask, is not the answer.
|
||||||
@@ -185,7 +222,7 @@ func awaitBackup(ctx context.Context, events <-chan []byte, node, module string,
|
|||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return agent.Backup{Words: "the node agent was stopped while the backup ran"}
|
return agent.Backup{Words: agent.BackupInterrupted, Interrupted: true}
|
||||||
case <-bound.C:
|
case <-bound.C:
|
||||||
return agent.Backup{Words: fmt.Sprintf("the backup did not say it was done within %s", within)}
|
return agent.Backup{Words: fmt.Sprintf("the backup did not say it was done within %s", within)}
|
||||||
case raw := <-events:
|
case raw := <-events:
|
||||||
@@ -343,13 +380,14 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
|
|||||||
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
|
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
|
||||||
const operatorKeysFile = "operator-keys"
|
const operatorKeysFile = "operator-keys"
|
||||||
|
|
||||||
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the
|
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the
|
||||||
// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at
|
// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from
|
||||||
// the next composition. A file that cannot be read trusts the controller's alone, and says so.
|
// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition.
|
||||||
func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey {
|
// A file that cannot be read trusts the controller's alone, and says so.
|
||||||
|
func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey {
|
||||||
return func() []ed25519.PublicKey {
|
return func() []ed25519.PublicKey {
|
||||||
keys := []ed25519.PublicKey{}
|
keys := []ed25519.PublicKey{}
|
||||||
if len(controller) == ed25519.PublicKeySize {
|
if controller := controllerNow(); len(controller) == ed25519.PublicKeySize {
|
||||||
keys = append(keys, controller)
|
keys = append(keys, controller)
|
||||||
}
|
}
|
||||||
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
|
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
|
||||||
@@ -373,13 +411,16 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu
|
|||||||
}
|
}
|
||||||
|
|
||||||
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
||||||
// none, ADR 0539 §9) and its filled paths.
|
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
|
||||||
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
var by map[string][]json.RawMessage
|
var by map[string][]json.RawMessage
|
||||||
var filled map[string][]string
|
var filled map[string][]string
|
||||||
return func() (map[string][]json.RawMessage, map[string][]string) {
|
return func() (map[string][]json.RawMessage, map[string][]string) {
|
||||||
once.Do(func() {
|
once.Do(func() {
|
||||||
|
if keptRetired.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
|
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
@@ -429,7 +470,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) {
|
|||||||
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
|
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
|
||||||
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
|
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
|
||||||
// account may not read) is said, and tried again with a growing wait, as the link is.
|
// account may not read) is said, and tried again with a growing wait, as the link is.
|
||||||
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler,
|
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler,
|
||||||
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
|
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
|
||||||
wait := 2 * time.Second
|
wait := 2 * time.Second
|
||||||
for ctx.Err() == nil {
|
for ctx.Err() == nil {
|
||||||
@@ -446,6 +487,18 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
var ag *agent.Agent
|
var ag *agent.Agent
|
||||||
|
// The kept release-5 declaration is no longer read once every module the seat's placement puts here has a have of its
|
||||||
|
// own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write).
|
||||||
|
// Nor before this engine keeps the node's own as it runs (applied/node.json): until then the kept declaration is the one
|
||||||
|
// record of what the node's own composition carries, its folded modules among them (issue 774, review of the
|
||||||
|
// re-land, round 1: novox's first start would compose its own with nats running nothing).
|
||||||
|
if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Load() {
|
||||||
|
if !appliedStoreFor(opts.state).holds(agent.NodeModule) {
|
||||||
|
say("the kept declaration is still read: " + why + ", but this engine keeps no node's own composition yet")
|
||||||
|
} else if !keptRetired.Swap(true) {
|
||||||
|
say("the kept declaration is no longer read: " + why)
|
||||||
|
}
|
||||||
|
}
|
||||||
kept := keptOnce(opts, signer)
|
kept := keptOnce(opts, signer)
|
||||||
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
|
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
|
||||||
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
|
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
|
||||||
@@ -458,7 +511,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
}
|
}
|
||||||
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
|
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
|
||||||
// node, and a node whose compositions carry no list writes nothing.
|
// node, and a node whose compositions carry no list writes nothing.
|
||||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
|
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() {
|
||||||
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
||||||
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
||||||
time.AfterFunc(reloadSettles, func() {
|
time.AfterFunc(reloadSettles, func() {
|
||||||
@@ -483,6 +536,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
|
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
|
||||||
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
|
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
|
||||||
Recorded: recordedIn(opts.state, say),
|
Recorded: recordedIn(opts.state, say),
|
||||||
|
// What this engine applied, kept on the machine and read at the start, so what runs is known before the node
|
||||||
|
// composes its own (issue 774).
|
||||||
|
Applied: appliedStoreFor(opts.state),
|
||||||
Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
|
Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
|
||||||
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
||||||
got, err := session.KVHistory(ctx, composedBucket, key)
|
got, err := session.KVHistory(ctx, composedBucket, key)
|
||||||
@@ -495,11 +551,20 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
},
|
},
|
||||||
Trusted: trustedKeys(opts, signer),
|
Trusted: trustedKeys(opts, trust.Controller),
|
||||||
Backup: backupNode(session),
|
Backup: backupNode(session),
|
||||||
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
|
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
|
||||||
// again (ace, 2026-10-25 21:17).
|
// again (ace, 2026-10-25 21:17).
|
||||||
Settled: func(module string) { knockFilling(module, say) },
|
Settled: func(module string) {
|
||||||
|
knockFilling(module+" settled", say)
|
||||||
|
judgeWhatRunsNow(opts.state, signer)
|
||||||
|
},
|
||||||
|
// And once the haves are read at the start, and whenever a module is taken as running without an apply, so
|
||||||
|
// a refusal from the engine's start does not wait for some module to settle (issue 924).
|
||||||
|
RunsChanged: func(why string) {
|
||||||
|
knockFilling(why, say)
|
||||||
|
judgeWhatRunsNow(opts.state, signer)
|
||||||
|
},
|
||||||
Say: say,
|
Say: say,
|
||||||
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
||||||
Backoff: link.Backoff,
|
Backoff: link.Backoff,
|
||||||
@@ -512,7 +577,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
if herr != nil {
|
if herr != nil {
|
||||||
res = failedWith(herr.Error())
|
res = failedWith(herr.Error())
|
||||||
} else {
|
} else {
|
||||||
res = applyComposition(actx, opts, held, others, sched, say)
|
res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), ag.RemovedByWant(), sched, say)
|
||||||
}
|
}
|
||||||
done()
|
done()
|
||||||
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
|
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
|
||||||
@@ -531,7 +596,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
},
|
},
|
||||||
Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result {
|
Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result {
|
||||||
actx, done := applyContext(session.WithLink(ctx))
|
actx, done := applyContext(session.WithLink(ctx))
|
||||||
res := removeComposition(actx, opts, module, others, sched, say)
|
res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
|
||||||
done()
|
done()
|
||||||
afterApply()
|
afterApply()
|
||||||
sayFacts(ctx, ag)
|
sayFacts(ctx, ag)
|
||||||
@@ -549,7 +614,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
hooks := composerHooks{
|
hooks := composerHooks{
|
||||||
running: func() []delivery.Composed {
|
running: func() []delivery.Composed {
|
||||||
if now := own.get(); now != nil {
|
if now := own.get(); now != nil {
|
||||||
return now.Everything()
|
return runsAsWritten(now.Everything(), opts.state)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
@@ -572,11 +637,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126)
|
now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
// And the vault's authority beside the controller's, and this node's certificate asked of the vault when a
|
||||||
|
// module here asks for one (ADR 0550 §2).
|
||||||
|
vaultAuthority: trust.VaultAuthority,
|
||||||
|
certify: func(ctx context.Context) error {
|
||||||
|
return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) })
|
||||||
|
},
|
||||||
}
|
}
|
||||||
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
||||||
// its own key, `<node>.node`, under candidates.
|
// its own key, `<node>.node`, under candidates.
|
||||||
shadowHooks := hooks
|
shadowHooks := hooks
|
||||||
shadowHooks.refused = nil
|
shadowHooks.refused, shadowHooks.certify = nil, nil
|
||||||
shadowEnv := composerEnv(session, node, say, shadowHooks)
|
shadowEnv := composerEnv(session, node, say, shadowHooks)
|
||||||
// And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to
|
// And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to
|
||||||
// the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync).
|
// the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync).
|
||||||
@@ -596,8 +667,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if composer != nil {
|
if composer != nil {
|
||||||
|
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
|
||||||
|
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
|
||||||
composes.set(delivery.ComposesOwn)
|
composes.set(delivery.ComposesOwn)
|
||||||
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
||||||
|
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
||||||
session.NoteComposes(linked, delivery.ComposesOwn, why)
|
session.NoteComposes(linked, delivery.ComposesOwn, why)
|
||||||
// And once the catalogue no longer places it, the agent begins again on the controller's compositions: the
|
// And once the catalogue no longer places it, the agent begins again on the controller's compositions: the
|
||||||
// placement key deleted ends this run, the way back (issue 1126, the seed's revert).
|
// placement key deleted ends this run, the way back (issue 1126, the seed's revert).
|
||||||
@@ -619,6 +693,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
|||||||
stopRun()
|
stopRun()
|
||||||
cancel()
|
cancel()
|
||||||
current.set(nil)
|
current.set(nil)
|
||||||
|
composerNow.Store(nil)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -663,7 +738,13 @@ func (c *agentNow) get() *agent.Agent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// applyComposition applies one composition: a module's, or the node's own.
|
// applyComposition applies one composition: a module's, or the node's own.
|
||||||
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed,
|
//
|
||||||
|
// placed is the modules a want still places here besides this one: none of their resources is removed by this apply,
|
||||||
|
// which is refused instead (issue 774).
|
||||||
|
//
|
||||||
|
// removed is the modules whose want says removed: in the node's own apply, a resource of any other module goes only when
|
||||||
|
// its module is among them (issue 774).
|
||||||
|
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed, removed map[string]bool,
|
||||||
sched *apply.Scheduler, say link.Announce) agent.Result {
|
sched *apply.Scheduler, say link.Announce) agent.Result {
|
||||||
name := scopeName(c)
|
name := scopeName(c)
|
||||||
if c.Module == "" {
|
if c.Module == "" {
|
||||||
@@ -679,7 +760,13 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
|
|||||||
if err := refuseEmptyWrites(d); err != nil {
|
if err := refuseEmptyWrites(d); err != nil {
|
||||||
return failedWith(err.Error())
|
return failedWith(err.Error())
|
||||||
}
|
}
|
||||||
scope := apply.Scope{Module: name, Others: resourcesOf(others)}
|
scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed}
|
||||||
|
if name == agent.NodeModule {
|
||||||
|
scope.RemovedByWant = removed
|
||||||
|
if scope.RemovedByWant == nil {
|
||||||
|
scope.RemovedByWant = map[string]bool{}
|
||||||
|
}
|
||||||
|
}
|
||||||
got := applyModuleHeld(ctx, opts, d, &scope, sched, say)
|
got := applyModuleHeld(ctx, opts, d, &scope, sched, say)
|
||||||
if got.Refused != "" {
|
if got.Refused != "" {
|
||||||
return failedWith(got.Refused)
|
return failedWith(got.Refused)
|
||||||
@@ -698,9 +785,9 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
|
|||||||
}
|
}
|
||||||
|
|
||||||
// removeComposition removes one module's own resources and nothing of another's.
|
// removeComposition removes one module's own resources and nothing of another's.
|
||||||
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed,
|
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed map[string]bool,
|
||||||
sched *apply.Scheduler, say link.Announce) agent.Result {
|
sched *apply.Scheduler, say link.Announce) agent.Result {
|
||||||
scope := apply.Scope{Module: module, Others: resourcesOf(others)}
|
scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed}
|
||||||
got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
|
got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
|
||||||
switch {
|
switch {
|
||||||
case got.Refused != "":
|
case got.Refused != "":
|
||||||
@@ -725,8 +812,27 @@ func busyModules() []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given.
|
// machinesNow is the composer whose placements name the mesh's machines by address, for the health report's targets
|
||||||
|
// (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target.
|
||||||
|
var machinesNow atomic.Pointer[selfcompose.Composer]
|
||||||
|
|
||||||
|
// composerNow is the composer of the run that composes the node's own now, nil while the node applies the controller's
|
||||||
|
// compositions or no run is going: what the verb compose asks a pass of (issue 774).
|
||||||
|
var composerNow atomic.Pointer[selfcompose.Composer]
|
||||||
|
|
||||||
|
// keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of
|
||||||
|
// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9).
|
||||||
|
var keptRetired atomic.Bool
|
||||||
|
|
||||||
|
// errKeptRetired is keptDeclaration's answer once the read is retired.
|
||||||
|
var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own")
|
||||||
|
|
||||||
|
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given;
|
||||||
|
// once the read is retired, nothing is read.
|
||||||
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||||
|
if keptRetired.Load() {
|
||||||
|
return nil, errKeptRetired
|
||||||
|
}
|
||||||
var raw []byte
|
var raw []byte
|
||||||
if len(signer) > 0 {
|
if len(signer) > 0 {
|
||||||
var err error
|
var err error
|
||||||
@@ -903,15 +1009,21 @@ type composerHooks struct {
|
|||||||
unseal func(string) ([]byte, error)
|
unseal func(string) ([]byte, error)
|
||||||
refused func(why string, held map[string]string)
|
refused func(why string, held map[string]string)
|
||||||
held func(module string) (delivery.Seals, []string, error)
|
held func(module string) (delivery.Seals, []string, error)
|
||||||
|
// vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify.
|
||||||
|
vaultAuthority func() string
|
||||||
|
certify func(ctx context.Context) error
|
||||||
}
|
}
|
||||||
|
|
||||||
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
|
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
|
||||||
key, err := delivery.PlacementKey(node)
|
key, err := delivery.PlacementNodeKey(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
say("this node applies the controller's compositions: " + err.Error())
|
say("this node applies the controller's compositions: " + err.Error())
|
||||||
return nil, err.Error()
|
return nil, err.Error()
|
||||||
}
|
}
|
||||||
env := composerEnv(session, node, say, hooks)
|
env := composerEnv(session, node, say, hooks)
|
||||||
|
// And it keeps each placed module's data items in its own state data, which the node's disk-load watches (novox/hq
|
||||||
|
// ADR 0577 clause 1); the shadow composes for candidates and keeps none.
|
||||||
|
env.Data = link.EngineData{Session: session}
|
||||||
unreadable := ""
|
unreadable := ""
|
||||||
placed, err := selfcompose.Placed(ctx, env.Get, node, link.Backoff, func(m string) {
|
placed, err := selfcompose.Placed(ctx, env.Get, node, link.Backoff, func(m string) {
|
||||||
if strings.Contains(m, "cannot be read") {
|
if strings.Contains(m, "cannot be read") {
|
||||||
@@ -966,7 +1078,30 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
|
|||||||
Hear: session.Hear,
|
Hear: session.Hear,
|
||||||
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
|
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
|
||||||
Fetch: apply.Fetch,
|
Fetch: apply.Fetch,
|
||||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
|
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held,
|
||||||
|
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
|
||||||
|
}
|
||||||
|
|
||||||
|
// placementKeysOf is a node's node key and the pattern of the keys that say it is placed: its node key and its legacy
|
||||||
|
// key `placement.<node>`, read while the holder may not have converted it (research 072 row 37). Its module keys say
|
||||||
|
// nothing of whether it is placed.
|
||||||
|
func placementKeysOf(node string) (key, keys string, err error) {
|
||||||
|
key, err = delivery.PlacementNodeKey(node)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
legacy, _ := delivery.LegacyPlacementKey(node)
|
||||||
|
return key, link.WatchPatterns(key, legacy), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// anyHeld says some key holds a value.
|
||||||
|
func anyHeld(held map[string]bool) bool {
|
||||||
|
for _, h := range held {
|
||||||
|
if h {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch
|
// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch
|
||||||
@@ -974,26 +1109,28 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
|
|||||||
// catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be
|
// catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be
|
||||||
// made is said; the node then returns at its next start.
|
// made is said; the node then returns at its next start.
|
||||||
func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
|
func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
|
||||||
key, err := delivery.PlacementKey(node)
|
key, keys, err := placementKeysOf(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, key)
|
updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, keys)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
say("whether this node's key in the seat's placement state still exists is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start")
|
say("whether this node's key in the seat's placement state still exists is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
placed := false
|
// Its node key or, before the holder converted it, its legacy key: placed while either holds a value (research
|
||||||
|
// 072 row 37); the holder deletes the legacy key before the node key, so the node key's delete is the last.
|
||||||
|
held, caught := map[string]bool{}, false
|
||||||
for u := range updates {
|
for u := range updates {
|
||||||
switch {
|
if u.Caught {
|
||||||
case u.Caught && placed:
|
caught = true
|
||||||
continue
|
} else {
|
||||||
case u.Caught || u.Deleted || len(u.Value) == 0:
|
held[u.Key] = !u.Deleted && len(u.Value) > 0
|
||||||
|
}
|
||||||
|
if caught && !anyHeld(held) {
|
||||||
say("this node's key in the seat's placement state is gone (" + key + "): the node agent begins again, applying the controller's compositions")
|
say("this node's key in the seat's placement state is gone (" + key + "): the node agent begins again, applying the controller's compositions")
|
||||||
stop()
|
stop()
|
||||||
return
|
return
|
||||||
default:
|
|
||||||
placed = true
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ctx.Err() == nil {
|
if ctx.Err() == nil {
|
||||||
@@ -1005,7 +1142,7 @@ func endWhenUnplaced(ctx context.Context, session *link.Session, node string, st
|
|||||||
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
|
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
|
||||||
// the node then switches at its next start.
|
// the node then switches at its next start.
|
||||||
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce, note func(string)) {
|
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce, note func(string)) {
|
||||||
key, err := delivery.PlacementKey(node)
|
key, keys, err := placementKeysOf(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1014,7 +1151,7 @@ func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop
|
|||||||
// (no timer), says so once in the journal and in the node's own have, where the watchman reads it, and stays on the
|
// (no timer), says so once in the journal and in the node's own have, where the watchman reads it, and stays on the
|
||||||
// controller's compositions until its key is written. A watch the account may not make is said the same way, and the
|
// controller's compositions until its key is written. A watch the account may not make is said the same way, and the
|
||||||
// node stays until its next start.
|
// node stays until its next start.
|
||||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, key, func(err error) {
|
updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, keys, func(err error) {
|
||||||
why := "this node's key in the seat's placement state is waited for, the state is not on the bus yet (" + err.Error() + "): it stays on the controller's compositions and is watched when the bus says the state was created"
|
why := "this node's key in the seat's placement state is waited for, the state is not on the bus yet (" + err.Error() + "): it stays on the controller's compositions and is watched when the bus says the state was created"
|
||||||
say(why)
|
say(why)
|
||||||
note(why)
|
note(why)
|
||||||
@@ -1027,7 +1164,7 @@ func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop
|
|||||||
}
|
}
|
||||||
for u := range updates {
|
for u := range updates {
|
||||||
if !u.Caught && !u.Deleted && len(u.Value) > 0 {
|
if !u.Caught && !u.Deleted && len(u.Value) > 0 {
|
||||||
say("this node's key in the seat's placement state is written now (" + key + "): the node agent begins again, composing its own modules")
|
say("this node's key in the seat's placement state is written now (" + u.Key + "): the node agent begins again, composing its own modules")
|
||||||
stop()
|
stop()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -159,7 +159,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) {
|
|||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
go func() {
|
go func() {
|
||||||
defer close(done)
|
defer close(done)
|
||||||
nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {},
|
controller := b.controll.Public().(ed25519.PublicKey)
|
||||||
|
nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {},
|
||||||
&agentNow{}, func(context.Context, *agent.Agent) {})
|
&agentNow{}, func(context.Context, *agent.Agent) {})
|
||||||
}()
|
}()
|
||||||
return func() { cancel(); <-done }
|
return func() { cancel(); <-done }
|
||||||
@@ -497,7 +498,7 @@ func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValu
|
|||||||
strings.Contains(a.Refused, "web: the controller holds no composition")
|
strings.Contains(a.Refused, "web: the controller holds no composition")
|
||||||
})
|
})
|
||||||
ran := b.have("web").Composed
|
ran := b.have("web").Composed
|
||||||
pkey, _ := delivery.PlacementKey(itNode)
|
pkey, _ := delivery.PlacementNodeKey(itNode)
|
||||||
// A pass in flight at the switch (issue 1126, 2026-10-28: the last pass of the own run, failing on its run's end,
|
// A pass in flight at the switch (issue 1126, 2026-10-28: the last pass of the own run, failing on its run's end,
|
||||||
// marked the next agent's modules failed): a config value changed starts one, and the key goes at once.
|
// marked the next agent's modules failed): a config value changed starts one, and the key goes at once.
|
||||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
||||||
@@ -586,7 +587,10 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
|
|||||||
modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)})
|
modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)})
|
||||||
place, _ := delivery.EncodePlacement(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
place, _ := delivery.EncodePlacement(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}})
|
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}})
|
||||||
pkey, _ := delivery.PlacementKey(itNode)
|
// it1 in one key per module and node (research 072 row 37), it2 in the form before: a node reads both.
|
||||||
|
pkey, _ := delivery.PlacementNodeKey(itNode)
|
||||||
|
places, _ := delivery.PlacementValues(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||||
|
place = places[pkey]
|
||||||
hkey, _ := delivery.HaveNodeKey(itNode)
|
hkey, _ := delivery.HaveNodeKey(itNode)
|
||||||
// A provider on it2 serving from its own config value: a mail domain, which it1's composition settles.
|
// A provider on it2 serving from its own config value: a mail domain, which it1's composition settles.
|
||||||
mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` +
|
mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` +
|
||||||
@@ -597,7 +601,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
|
|||||||
mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)})
|
mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)})
|
||||||
place2, _ := delivery.EncodePlacement(delivery.Placement{Node: "it2", Modules: []delivery.PlacedModule{{Module: "mailer"}}})
|
place2, _ := delivery.EncodePlacement(delivery.Placement{Node: "it2", Modules: []delivery.PlacedModule{{Module: "mailer"}}})
|
||||||
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}})
|
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}})
|
||||||
pkey2, _ := delivery.PlacementKey("it2")
|
pkey2, _ := delivery.LegacyPlacementKey("it2")
|
||||||
hkey2, _ := delivery.HaveNodeKey("it2")
|
hkey2, _ := delivery.HaveNodeKey("it2")
|
||||||
buildKey, _ := delivery.BuilderBuildKey("web", fp)
|
buildKey, _ := delivery.BuilderBuildKey("web", fp)
|
||||||
done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest,
|
done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest,
|
||||||
@@ -631,6 +635,9 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
|
|||||||
if b.file("web") != "" {
|
if b.file("web") != "" {
|
||||||
t.Fatal("a node the catalogue does not place composed its own")
|
t.Fatal("a node the catalogue does not place composed its own")
|
||||||
}
|
}
|
||||||
|
if _, err := placement.Put(itNode+".web", places[itNode+".web"]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if _, err := placement.Put(pkey, place); err != nil {
|
if _, err := placement.Put(pkey, place); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||||
|
)
|
||||||
|
|
||||||
|
// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write
|
||||||
|
// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:<provision>}`, and the node-engine
|
||||||
|
// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the
|
||||||
|
// placeholder alone.
|
||||||
|
//
|
||||||
|
// A placeholder names the credential as the module calls it, `${secret:<name>}` (catalogue.SecretLocal, ADR 0094): the
|
||||||
|
// local name where the module keeps several values of one provision, else the provision. For each, it finds the one
|
||||||
|
// provision key `<node>.<consumer>.<provision>[.<local>]` of the seat mesh-provisioner's state `provisions` (clause 1)
|
||||||
|
// going by that name, which must be `provided` and carry the vault's reference; asks the vault for that pair
|
||||||
|
// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client,
|
||||||
|
// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the
|
||||||
|
// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot
|
||||||
|
// connect.
|
||||||
|
type provisionSecrets struct {
|
||||||
|
node string
|
||||||
|
// list is every key of mesh-provisioner's `provisions` under prefix (`<node>.<consumer>.`), by key: what this
|
||||||
|
// engine may read, its own node's (`<node>.>`).
|
||||||
|
list func(ctx context.Context, prefix string) (map[string][]byte, error)
|
||||||
|
// own asks the vault's verb `own` in this engine's name.
|
||||||
|
own func(ctx context.Context, args map[string]any) (json.RawMessage, error)
|
||||||
|
// open opens a value sealed to this machine.
|
||||||
|
open func(sealed string) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// secretPlaceholder is `${secret:<name>}`.
|
||||||
|
var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)\}`)
|
||||||
|
|
||||||
|
// fill is content with each provision placeholder of module's filled, and whether any was.
|
||||||
|
func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) {
|
||||||
|
found := secretPlaceholder.FindAllSubmatch(content, -1)
|
||||||
|
if len(found) == 0 {
|
||||||
|
return content, false, nil
|
||||||
|
}
|
||||||
|
if module == "" {
|
||||||
|
return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found))
|
||||||
|
}
|
||||||
|
if p.list == nil || p.own == nil || p.open == nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's provisions cannot be read on this machine: the node-engine is not linked to the mesh", module)
|
||||||
|
}
|
||||||
|
prefix := p.node + "." + module + "."
|
||||||
|
keys, err := p.list(ctx, prefix)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's provisions cannot be read: %w", module, err)
|
||||||
|
}
|
||||||
|
out := string(content)
|
||||||
|
done := map[string]bool{}
|
||||||
|
for _, f := range found {
|
||||||
|
whole, name := string(f[0]), string(f[1])
|
||||||
|
if done[whole] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
done[whole] = true
|
||||||
|
pr, key, err := byName(keys, name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("%s's secret %q: %w", module, name, err)
|
||||||
|
}
|
||||||
|
value, err := p.value(ctx, key, pr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
out = strings.ReplaceAll(out, whole, string(value))
|
||||||
|
}
|
||||||
|
return []byte(out), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// byName is the one provision of keys the module calls name (catalogue.SecretLocal), and its key.
|
||||||
|
func byName(keys map[string][]byte, name string) (provisions.Provision, string, error) {
|
||||||
|
var hit []string
|
||||||
|
var got provisions.Provision
|
||||||
|
for key, raw := range keys {
|
||||||
|
pr, err := provisions.Decode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("the provision %s: %w", key, err)
|
||||||
|
}
|
||||||
|
if catalogue.SecretLocal(pr.Provision, pr.Local) == name {
|
||||||
|
hit, got = append(hit, key), pr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch len(hit) {
|
||||||
|
case 0:
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("no provision of %s goes by that name: nothing asks for it", provisions.ProvisionsRead)
|
||||||
|
case 1:
|
||||||
|
return got, hit[0], nil
|
||||||
|
}
|
||||||
|
sort.Strings(hit)
|
||||||
|
return provisions.Provision{}, "", fmt.Errorf("the provisions %s all go by that name", strings.Join(hit, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// value is one provision's value on this node, opened.
|
||||||
|
func (p provisionSecrets) value(ctx context.Context, key string, pr provisions.Provision) ([]byte, error) {
|
||||||
|
if pr.State != provisions.Provided || pr.Reference == nil {
|
||||||
|
why := pr.State
|
||||||
|
if pr.Why != "" {
|
||||||
|
why += ": " + pr.Why
|
||||||
|
}
|
||||||
|
if pr.State == provisions.Provided {
|
||||||
|
why = "provided with no reference to the vault's value"
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why)
|
||||||
|
}
|
||||||
|
// The reference names a credential of this provision, as the vault names it (provisions.VaultName).
|
||||||
|
ours := false
|
||||||
|
for _, form := range []string{provisions.FormPair, provisions.FormShared, provisions.FormIssued} {
|
||||||
|
ours = ours || provisions.VaultName(form, pr) == pr.Reference.ID
|
||||||
|
}
|
||||||
|
if !ours {
|
||||||
|
return nil, fmt.Errorf("the provision %s names the vault's %s, which is no credential of it", key, pr.Reference.ID)
|
||||||
|
}
|
||||||
|
module, provision, local := pr.Consumer.Module, pr.Provision, pr.Local
|
||||||
|
type asked struct {
|
||||||
|
Role string `json:"role"`
|
||||||
|
Provision string `json:"provision"`
|
||||||
|
Consumer string `json:"consumer"`
|
||||||
|
ConsumerModule string `json:"consumer_module"`
|
||||||
|
Provider string `json:"provider"`
|
||||||
|
Local string `json:"local,omitempty"`
|
||||||
|
}
|
||||||
|
ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local}
|
||||||
|
answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's value of %s: %w", key, err)
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Credentials []struct {
|
||||||
|
asked
|
||||||
|
Name string `json:"name"`
|
||||||
|
Generation int `json:"generation"`
|
||||||
|
Sealed json.RawMessage `json:"sealed"`
|
||||||
|
} `json:"credentials"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(answer, &a); err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err)
|
||||||
|
}
|
||||||
|
if a.Node != p.node {
|
||||||
|
return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node)
|
||||||
|
}
|
||||||
|
for _, c := range a.Credentials {
|
||||||
|
if c.asked != ask {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// The value the provision names, by the vault's credential name and its generation (clause 6): another is refused.
|
||||||
|
if c.Name != pr.Reference.ID || c.Generation != pr.Reference.Generation {
|
||||||
|
return nil, fmt.Errorf("the vault answered %s generation %d for %s, and the provision names %s generation %d: "+
|
||||||
|
"nothing is written until they agree", c.Name, c.Generation, key, pr.Reference.ID, pr.Reference.Generation)
|
||||||
|
}
|
||||||
|
value, err := p.open(string(c.Sealed))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err)
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the vault answered no value of %s for this machine", key)
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// runsAsWritten is what the node runs as the composer reads it (selfcompose.Env.Running), with every file given by digest
|
||||||
|
// named as this node last wrote it, where the two differ (issue 924, novox 2026-10-29 01:16). What runs of a module the
|
||||||
|
// agent has not applied since its start can come from the kept declaration, older than the compositions applied since:
|
||||||
|
// its export (mesh-vault.kept) then named a digest the controller's sweep had removed, while the file on disk was the
|
||||||
|
// newer export the node wrote. A node composing its own carries such a file as it runs (mesh-sdk frombus keptOf), so it
|
||||||
|
// names what is on disk: **only when the node's record says it wrote exactly that content there** (store Applied.Wrote,
|
||||||
|
// Target), never a file someone else changed. A file that cannot be read, or that the record does not vouch for, is
|
||||||
|
// left as the composition names it.
|
||||||
|
func runsAsWritten(running []delivery.Composed, statePath string) []delivery.Composed {
|
||||||
|
if len(running) == 0 || statePath == "" {
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
known, err := store.Load(statePath)
|
||||||
|
if err != nil {
|
||||||
|
return running
|
||||||
|
}
|
||||||
|
out := make([]delivery.Composed, len(running))
|
||||||
|
for i, c := range running {
|
||||||
|
out[i] = c
|
||||||
|
var changed []json.RawMessage
|
||||||
|
for j, raw := range c.Resources {
|
||||||
|
if fixed, ok := asWritten(raw, known); ok {
|
||||||
|
if changed == nil {
|
||||||
|
changed = append([]json.RawMessage(nil), c.Resources...)
|
||||||
|
}
|
||||||
|
changed[j] = fixed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if changed != nil {
|
||||||
|
out[i].Resources = changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// asWritten is the file resource raw named by the digest and size of what this node last wrote at its path, when raw is a
|
||||||
|
// file given by another digest and the file there is what the record says the node wrote.
|
||||||
|
func asWritten(raw json.RawMessage, known store.State) (json.RawMessage, bool) {
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if json.Unmarshal(raw, &fields) != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
var kind, id, path, digest string
|
||||||
|
_ = json.Unmarshal(fields["type"], &kind)
|
||||||
|
_ = json.Unmarshal(fields["id"], &id)
|
||||||
|
_ = json.Unmarshal(fields["path"], &path)
|
||||||
|
_ = json.Unmarshal(fields["digest"], &digest)
|
||||||
|
if kind != string(declaration.TypeFile) || digest == "" || path == "" {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
rec, ok := known.Find(id)
|
||||||
|
if !ok || rec.Wrote == "" || rec.Target != path {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
if hex.EncodeToString(sum[:]) != rec.Wrote {
|
||||||
|
return nil, false // changed since the node wrote it: not the node's to vouch for
|
||||||
|
}
|
||||||
|
if now := declaration.DigestOf(body); now == digest {
|
||||||
|
return nil, false
|
||||||
|
} else {
|
||||||
|
fields["digest"], _ = json.Marshal(now)
|
||||||
|
}
|
||||||
|
fields["size"], _ = json.Marshal(len(body))
|
||||||
|
fixed, err := json.Marshal(fields)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return fixed, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,353 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/identity"
|
||||||
|
"github.com/novox/mesh-host/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the
|
||||||
|
// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's
|
||||||
|
// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the
|
||||||
|
// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key
|
||||||
|
// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its
|
||||||
|
// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus.
|
||||||
|
type meshTrust struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
controller ed25519.PublicKey
|
||||||
|
handedAway bool
|
||||||
|
vault ed25519.PublicKey
|
||||||
|
authority string
|
||||||
|
// changed is told when what it trusts changed, so the record is written again.
|
||||||
|
changed func()
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMeshTrust trusts the controller's key alone until the vault's is read.
|
||||||
|
func newMeshTrust(controller ed25519.PublicKey) *meshTrust {
|
||||||
|
t := &meshTrust{}
|
||||||
|
if len(controller) == ed25519.PublicKeySize {
|
||||||
|
t.controller = controller
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
// onChange names who is told when what it trusts changed.
|
||||||
|
func (t *meshTrust) onChange(f func()) {
|
||||||
|
t.mu.Lock()
|
||||||
|
t.changed = f
|
||||||
|
t.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away.
|
||||||
|
func (t *meshTrust) Controller() ed25519.PublicKey {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if t.handedAway {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return t.controller
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandedAway says the controller's lease was handed away, as heard on the lease key.
|
||||||
|
func (t *meshTrust) HandedAway() bool {
|
||||||
|
if t == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return t.handedAway
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the
|
||||||
|
// vault's once read.
|
||||||
|
func (t *meshTrust) Signers() []delivery.Signer {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
var out []delivery.Signer
|
||||||
|
if t.controller != nil && !t.handedAway {
|
||||||
|
out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller})
|
||||||
|
}
|
||||||
|
if t.vault != nil {
|
||||||
|
out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify is the signer that signed msg as kind, or why none it trusts did.
|
||||||
|
func (t *meshTrust) Verify(kind string, msg, sig []byte) (delivery.Signer, error) {
|
||||||
|
context, err := delivery.SignContext(kind)
|
||||||
|
if err != nil {
|
||||||
|
return delivery.Signer{}, err
|
||||||
|
}
|
||||||
|
return delivery.VerifyBy(t.Signers(), context, msg, sig)
|
||||||
|
}
|
||||||
|
|
||||||
|
// VaultAuthority is the vault's authority certificate as read from `signers`, or "".
|
||||||
|
func (t *meshTrust) VaultAuthority() string {
|
||||||
|
if t == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
return t.authority
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint.
|
||||||
|
func (t *meshTrust) Trusts() []string {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
authorities := map[string]string{}
|
||||||
|
if a := t.VaultAuthority(); a != "" {
|
||||||
|
authorities[delivery.SignerVault] = a
|
||||||
|
}
|
||||||
|
return delivery.Trusts(t.Signers(), authorities)
|
||||||
|
}
|
||||||
|
|
||||||
|
// change applies f under the lock and tells who follows when Trusts differs after it.
|
||||||
|
func (t *meshTrust) change(f func()) {
|
||||||
|
before := strings.Join(t.Trusts(), "\n")
|
||||||
|
t.mu.Lock()
|
||||||
|
f()
|
||||||
|
changed := t.changed
|
||||||
|
t.mu.Unlock()
|
||||||
|
if strings.Join(t.Trusts(), "\n") != before && changed != nil {
|
||||||
|
changed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not
|
||||||
|
// check; a deleted key is no longer trusted.
|
||||||
|
func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) {
|
||||||
|
switch key {
|
||||||
|
case delivery.SignersKeyKey:
|
||||||
|
var pub ed25519.PublicKey
|
||||||
|
if !deleted && len(value) > 0 {
|
||||||
|
k, err := delivery.DecodeSignerKey(value)
|
||||||
|
if err == nil {
|
||||||
|
pub, err = k.PublicKey()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
say("the vault's signing key in " + delivery.SignersRead + " is not trusted: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.change(func() { t.vault = pub })
|
||||||
|
case delivery.SignersAuthorityKey:
|
||||||
|
pem := ""
|
||||||
|
if !deleted && len(value) > 0 {
|
||||||
|
a, err := delivery.DecodeSignerAuthority(value)
|
||||||
|
if err != nil {
|
||||||
|
say("the vault's authority in " + delivery.SignersRead + " is not trusted: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pem = a.Certificate
|
||||||
|
}
|
||||||
|
t.change(func() { t.authority = pem })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the
|
||||||
|
// controller's key is trusted no more. A lease that cannot be read changes nothing.
|
||||||
|
func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) {
|
||||||
|
if readErr != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
handed, _ := buscompose.LeaseHandedAway(raw, found, nil)
|
||||||
|
t.change(func() { t.handedAway = handed })
|
||||||
|
}
|
||||||
|
|
||||||
|
// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may
|
||||||
|
// not serve `sign` yet, and until it does this node is not granted the reads.
|
||||||
|
var trustFollowAgain = 2 * time.Minute
|
||||||
|
|
||||||
|
// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as
|
||||||
|
// long as ctx lasts. A watch that cannot be made is said once per reason and tried again.
|
||||||
|
func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) {
|
||||||
|
said := ""
|
||||||
|
sayOnce := func(line string) {
|
||||||
|
if line != said {
|
||||||
|
said = line
|
||||||
|
if line != "" {
|
||||||
|
say(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
linked, cancel, err := session.WhenLinked(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
watching, stop := context.WithCancel(linked)
|
||||||
|
signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">")
|
||||||
|
lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
|
||||||
|
if serr != nil || lerr != nil {
|
||||||
|
stop()
|
||||||
|
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+
|
||||||
|
"the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain))
|
||||||
|
signers, lease = nil, nil
|
||||||
|
} else {
|
||||||
|
sayOnce("")
|
||||||
|
}
|
||||||
|
for signers != nil || lease != nil {
|
||||||
|
select {
|
||||||
|
case u, ok := <-signers:
|
||||||
|
if !ok {
|
||||||
|
signers = nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !u.Caught {
|
||||||
|
trust.heardSigner(u.Key, u.Value, u.Deleted, say)
|
||||||
|
}
|
||||||
|
case u, ok := <-lease:
|
||||||
|
if !ok {
|
||||||
|
lease = nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !u.Caught {
|
||||||
|
trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
select {
|
||||||
|
case <-linked.Done():
|
||||||
|
case <-time.After(trustFollowAgain):
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedAllHave says whether every module the seat mesh-placement places on node has a have of its own, read from the
|
||||||
|
// bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose placement or haves
|
||||||
|
// cannot be read, keeps reading it.
|
||||||
|
func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) {
|
||||||
|
key, err := delivery.PlacementNodeKey(node)
|
||||||
|
if err != nil {
|
||||||
|
return false, err.Error()
|
||||||
|
}
|
||||||
|
legacy, _ := delivery.LegacyPlacementKey(node)
|
||||||
|
// Its node key names every module placed (research 072 row 37); before the holder converted it, its legacy key does.
|
||||||
|
values := map[string][]byte{}
|
||||||
|
for _, k := range []string{key, legacy} {
|
||||||
|
if h, err := session.KVHistory(ctx, delivery.PlacementBucket, k); err == nil && len(h) > 0 && !h[len(h)-1].Deleted {
|
||||||
|
values[k] = h[len(h)-1].Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p, err := placedModules(node, values)
|
||||||
|
if err != nil {
|
||||||
|
return false, err.Error()
|
||||||
|
}
|
||||||
|
return allHave(p, func(module string) (delivery.Have, bool, error) {
|
||||||
|
return session.ReadHaveErr(ctx, module)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedModules is the modules node's node key names (a placement of their names alone), or its legacy key's placement
|
||||||
|
// where it has no node key; an error when neither reads.
|
||||||
|
func placedModules(node string, values map[string][]byte) (delivery.Placement, error) {
|
||||||
|
key, _ := delivery.PlacementNodeKey(node)
|
||||||
|
if raw, ok := values[key]; ok {
|
||||||
|
n, err := delivery.DecodePlacementNode(raw)
|
||||||
|
if err != nil {
|
||||||
|
return delivery.Placement{}, err
|
||||||
|
}
|
||||||
|
p := delivery.Placement{Node: node}
|
||||||
|
for _, r := range n.Modules {
|
||||||
|
p.Modules = append(p.Modules, delivery.PlacedModule{Module: r.Module})
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
legacy, _ := delivery.LegacyPlacementKey(node)
|
||||||
|
raw, ok := values[legacy]
|
||||||
|
if !ok {
|
||||||
|
return delivery.Placement{}, errors.New("its placement is not read")
|
||||||
|
}
|
||||||
|
return delivery.DecodePlacement(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// allHave says whether has finds, for every module p places, a have that names a fingerprint of the new form, and why
|
||||||
|
// not. A module whose have names only a build of the old form runs inside the node's own composition, and the kept
|
||||||
|
// declaration is the one record of what runs of it: retired then, the node's own composed it as nothing and refused to
|
||||||
|
// remove what runs (issue 774, 2026-10-30 19:43: systemd-resolved on shanks, nats and restic on novox, each node's own
|
||||||
|
// apply refused from then on).
|
||||||
|
func allHave(p delivery.Placement, has func(module string) (delivery.Have, bool, error)) (bool, string) {
|
||||||
|
if len(p.Modules) == 0 {
|
||||||
|
return false, "the seat mesh-placement places no module here"
|
||||||
|
}
|
||||||
|
for _, m := range p.Modules {
|
||||||
|
h, ok, err := has(m.Module)
|
||||||
|
if err != nil {
|
||||||
|
return false, "the have of " + m.Module + " is not read: " + err.Error()
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return false, m.Module + " has no have yet"
|
||||||
|
}
|
||||||
|
// A module the mesh provides (mesh-wireguard) never has a fingerprint: the node's own composes it from the manifest
|
||||||
|
// mesh-sdk provides (delivery.ProvidedManifest), never from the kept declaration, so it keeps no read.
|
||||||
|
if h.Phase != delivery.PhaseRemoved && !strings.HasPrefix(h.FP, delivery.SourcePrefix2) && !delivery.ProvidedByMesh(m.Module) {
|
||||||
|
return false, m.Module + " runs at a build of the old form (" + h.Build + "), which only the kept declaration records"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules))
|
||||||
|
}
|
||||||
|
|
||||||
|
// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks
|
||||||
|
// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the
|
||||||
|
// vault's this node trusts, keeps it where the record reads it, and writes the record again (written).
|
||||||
|
func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust,
|
||||||
|
ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error {
|
||||||
|
serving, err := identity.LoadServingKey(identity.ServingKeyPath(state))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
key, err := base64.StdEncoding.DecodeString(serving.Public)
|
||||||
|
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||||
|
return fmt.Errorf("this node's serving key is not an ed25519 key")
|
||||||
|
}
|
||||||
|
raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var a delivery.SignAnswer
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return fmt.Errorf("the vault's answer is not a signing: %w", err)
|
||||||
|
}
|
||||||
|
trusted := trust.VaultAuthority()
|
||||||
|
if trusted == "" {
|
||||||
|
return fmt.Errorf("this node has read no authority of the vault's to check the certificate against")
|
||||||
|
}
|
||||||
|
if err := delivery.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node),
|
||||||
|
ed25519.PublicKey(key), time.Now()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
path := identity.CertificatePath(state)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if written != nil {
|
||||||
|
written()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -9,6 +12,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/accounts"
|
"github.com/novox/mesh-host/internal/accounts"
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
"github.com/novox/mesh-host/internal/liveness"
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
"github.com/novox/mesh-host/internal/units"
|
"github.com/novox/mesh-host/internal/units"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -103,6 +107,61 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
|||||||
if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" {
|
if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" {
|
||||||
t.Fatalf("root said: %v", got)
|
t.Fatalf("root said: %v", got)
|
||||||
}
|
}
|
||||||
|
// 774 (ADR 0563 §6): the node seat's root-free passes only on the engine's own positive verdict at most 15 minutes old
|
||||||
|
// with execute not served here; a stale, missing or negative verdict, or execute served, is no pass and says why.
|
||||||
|
t.Run("this machine's root-free", func(t *testing.T) {
|
||||||
|
agent := accounts.Verdict{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true},
|
||||||
|
State: accounts.Healthy, Since: at}
|
||||||
|
free := &accounts.Statement{At: at, Accounts: []accounts.Verdict{agent}}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
st *accounts.Statement
|
||||||
|
after time.Duration
|
||||||
|
served bool
|
||||||
|
free bool
|
||||||
|
why string
|
||||||
|
}{
|
||||||
|
{name: "a positive verdict", st: free, after: 14 * time.Minute, free: true, why: "the account agent cannot become root"},
|
||||||
|
{name: "a stale verdict", st: free, after: 16 * time.Minute, why: "more than 15 minutes ago"},
|
||||||
|
{name: "no verdict", st: nil, why: "judged no account yet"},
|
||||||
|
{name: "a way to root", st: as, why: "in the group docker, which grants root"},
|
||||||
|
{name: "no agent account", st: &accounts.Statement{At: at}, why: "agents run as the operator's account"},
|
||||||
|
{name: "execute served", st: free, served: true, why: "the login shell runs any command"},
|
||||||
|
} {
|
||||||
|
v := judgeRootFree("ace", row.st, at.Add(row.after), row.served, "the bus hears execute answered here")
|
||||||
|
if v.Free != row.free || !strings.Contains(v.Why, row.why) || v.Machine != "ace" {
|
||||||
|
t.Errorf("%s: %+v", row.name, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// execute is served while the holder's setting file says anything but withhold, or the bus hears it.
|
||||||
|
dir := t.TempDir()
|
||||||
|
statePath, conf := filepath.Join(dir, "state.json"), filepath.Join(dir, "tools.conf")
|
||||||
|
if err := store.Save(statePath, store.State{Resources: []store.Applied{{ID: "zsh.tools-conf", Type: "file", Target: conf}}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
quiet := func() (bool, error) { return false, nil }
|
||||||
|
for _, row := range []struct {
|
||||||
|
name, setting string
|
||||||
|
heard func() (bool, error)
|
||||||
|
served bool
|
||||||
|
}{
|
||||||
|
{name: "withheld and not heard", setting: "# the mesh's\nexecute=withhold\n", heard: quiet},
|
||||||
|
{name: "served", setting: "execute=serve\n", heard: quiet, served: true},
|
||||||
|
{name: "withheld and heard", setting: "execute=withhold\n", heard: func() (bool, error) { return true, nil }, served: true},
|
||||||
|
{name: "the bus not asked", setting: "execute=withhold\n", heard: func() (bool, error) { return false, errors.New("no link") }, served: true},
|
||||||
|
{name: "no setting file", heard: quiet, served: true},
|
||||||
|
} {
|
||||||
|
_ = os.Remove(conf)
|
||||||
|
if row.setting != "" {
|
||||||
|
if err := os.WriteFile(conf, []byte(row.setting), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if served, why := executeServedHere(statePath, row.heard); served != row.served {
|
||||||
|
t.Errorf("%s: served %v (%s)", row.name, served, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as
|
// A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as
|
||||||
|
|||||||
@@ -1,16 +1,22 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/agent"
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
"github.com/novox/mesh-host/internal/liveness"
|
||||||
|
"github.com/novox/mesh-host/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
func aFileOf(module, id string) json.RawMessage {
|
func aFileOf(module, id string) json.RawMessage {
|
||||||
@@ -59,6 +65,96 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
|
|||||||
if err != nil || d.Provisioner != "from-agent" {
|
if err != nil || d.Provisioner != "from-agent" {
|
||||||
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
|
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// And the health loop's judges are handed what runs once a module settles (g14 and shanks, 2026-10-30 16:14):
|
||||||
|
// restic's composition dropped its process restic.measure, which stayed watched and said down on every look.
|
||||||
|
j, err := liveness.Open(filepath.Join(t.TempDir(), "liveness.json"), fakeRuntime{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
judging = j
|
||||||
|
defer func() { judging = nil }()
|
||||||
|
process := func(name string) json.RawMessage {
|
||||||
|
raw, _ := json.Marshal(map[string]any{"id": "restic." + name, "type": "process", "name": "restic-" + name,
|
||||||
|
"source": "https://store.invalid/restic", "digest": "sha256:" + strings.Repeat("a", 64),
|
||||||
|
"run": []string{"./restic"}, "module": "restic"})
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
for _, row := range []struct {
|
||||||
|
runs []string
|
||||||
|
watch string
|
||||||
|
}{{[]string{"night", "measure"}, "restic.measure restic.night"}, {[]string{"night"}, "restic.night"}} {
|
||||||
|
var raws []json.RawMessage
|
||||||
|
for _, name := range row.runs {
|
||||||
|
raws = append(raws, process(name))
|
||||||
|
}
|
||||||
|
runs, err := whatRuns([]delivery.Composed{{Node: "n1", Module: "restic", Resources: raws}}, false,
|
||||||
|
func() (*declaration.Declaration, error) { return nil, errors.New("none kept") })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) { return runs, nil })
|
||||||
|
judgeWhatRunsNow(filepath.Join(t.TempDir(), "state.json"), nil)
|
||||||
|
st, _ := j.Look(context.Background())
|
||||||
|
var watched []string
|
||||||
|
for _, r := range st.Resources {
|
||||||
|
watched = append(watched, r.ID)
|
||||||
|
}
|
||||||
|
sort.Strings(watched)
|
||||||
|
if got := strings.Join(watched, " "); got != row.watch {
|
||||||
|
t.Errorf("running %v, the judge watches %q, want %q", row.runs, got, row.watch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the
|
||||||
|
// file is there and readable and the controller's key is dropped; until then it reads it.
|
||||||
|
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||||
|
if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` +
|
||||||
|
`"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 {
|
||||||
|
t.Fatalf("before the read is retired: %+v, %v", d, err)
|
||||||
|
}
|
||||||
|
placed := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}}
|
||||||
|
// Issue 774, 2026-10-30 19:43: a have naming only a build of the old form (systemd-resolved on shanks, folded in the
|
||||||
|
// node's own) keeps the kept declaration read, since it is the one record of what runs of it.
|
||||||
|
newForm := delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning}
|
||||||
|
oldForm := delivery.Have{Build: "build-1791404191702719760", Phase: delivery.PhaseRunning}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
has map[string]delivery.Have
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"every placed module has a have of the new form", map[string]delivery.Have{"zsh": newForm, "lemurs": newForm}, true},
|
||||||
|
{"one has none yet", map[string]delivery.Have{"zsh": newForm}, false},
|
||||||
|
{"one runs at a build of the old form", map[string]delivery.Have{"zsh": newForm, "lemurs": oldForm}, false},
|
||||||
|
} {
|
||||||
|
if got, why := allHave(placed, func(m string) (delivery.Have, bool, error) { h, ok := row.has[m]; return h, ok, nil }); got != row.want {
|
||||||
|
t.Errorf("%s: %v (%s)", row.name, got, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The module the mesh provides never has a fingerprint, and keeps no read (review of issue 774).
|
||||||
|
withWireguard := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: overlay.Name}}}
|
||||||
|
if got, why := allHave(withWireguard, func(m string) (delivery.Have, bool, error) {
|
||||||
|
if m == overlay.Name {
|
||||||
|
return oldForm, true, nil
|
||||||
|
}
|
||||||
|
return newForm, true, nil
|
||||||
|
}); !got {
|
||||||
|
t.Errorf("%s without a fingerprint beside zsh at the new form keeps the read: %s", overlay.Name, why)
|
||||||
|
}
|
||||||
|
if got, _ := allHave(delivery.Placement{Node: "n1"}, nil); got {
|
||||||
|
t.Error("a node with nothing placed retires the read")
|
||||||
|
}
|
||||||
|
keptRetired.Store(true)
|
||||||
|
defer keptRetired.Store(false)
|
||||||
|
if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) {
|
||||||
|
t.Errorf("after the read is retired: %v", err)
|
||||||
|
}
|
||||||
|
if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil {
|
||||||
|
t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
|
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028234900-c2999a4e13cc
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028234900-c2999a4e13cc h1:mOk2tAKTiuvnap/BUIYz91lLVbHi/EB6JPAD6KxTPvc=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028234900-c2999a4e13cc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -240,6 +240,17 @@ func (c *SetuidCache) SearchAgain() {
|
|||||||
c.forgetProgress()
|
c.forgetProgress()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Running says whether a search walks now (novox/hq ADR 0563 §2: the node seat starts no second while one does).
|
||||||
|
// Nil c runs none.
|
||||||
|
func (c *SetuidCache) Running() bool {
|
||||||
|
if c == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.running
|
||||||
|
}
|
||||||
|
|
||||||
// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
|
// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
|
||||||
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
|
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
|
||||||
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
|
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
|
||||||
|
|||||||
+431
-44
@@ -84,9 +84,15 @@ type Result struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Backup is the answer of the backup asked before a module moves.
|
// Backup is the answer of the backup asked before a module moves.
|
||||||
|
// BackupInterrupted is the words of a backup the node agent itself stopped waiting for (its context ended): no failure, so
|
||||||
|
// the next start backs the module up again rather than holding it for the operator.
|
||||||
|
const BackupInterrupted = "the node agent was stopped while the backup ran"
|
||||||
|
|
||||||
type Backup struct {
|
type Backup struct {
|
||||||
Done bool
|
Done bool
|
||||||
Words string
|
Words string
|
||||||
|
// Interrupted says the node agent stopped waiting for it (its context ended): not a failure of the backup.
|
||||||
|
Interrupted bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Env is everything the agent needs of the node.
|
// Env is everything the agent needs of the node.
|
||||||
@@ -109,7 +115,9 @@ type Env struct {
|
|||||||
Kept func() map[string][]json.RawMessage
|
Kept func() map[string][]json.RawMessage
|
||||||
// KeptFilled is the paths the kept declaration names as filled by the provisioner, by the module whose file is at
|
// KeptFilled is the paths the kept declaration names as filled by the provisioner, by the module whose file is at
|
||||||
// each (the node's own under NodeModule). A module that still runs from the kept declaration keeps its filled files
|
// each (the node's own under NodeModule). A module that still runs from the kept declaration keeps its filled files
|
||||||
// filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none.
|
// filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. A module
|
||||||
|
// whose have names a composition the bucket keeps runs from that composition, never from the kept declaration
|
||||||
|
// (issue 924): the kept declaration is older than every composition applied since the switch.
|
||||||
KeptFilled func() map[string][]string
|
KeptFilled func() map[string][]string
|
||||||
// Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A
|
// Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A
|
||||||
// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
|
// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
|
||||||
@@ -131,6 +139,12 @@ type Env struct {
|
|||||||
// them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host
|
// them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host
|
||||||
// internal/filling Knock; ace, 2026-10-25 21:17). Nil tells nothing.
|
// internal/filling Knock; ace, 2026-10-25 21:17). Nil tells nothing.
|
||||||
Settled func(module string)
|
Settled func(module string)
|
||||||
|
// RunsChanged is told, outside the agent's lock, that what runs (Everything) changed other than by an apply or a
|
||||||
|
// removal: once at the start, when the haves and the compositions they name are read, and each time a module is
|
||||||
|
// taken as running from its have or the kept declaration. Like Settled, the filling then tells the provisioner's
|
||||||
|
// client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09
|
||||||
|
// UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing.
|
||||||
|
RunsChanged func(why string)
|
||||||
// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
|
// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
|
||||||
// transient failure only on RetryTransient or a change of either key.
|
// transient failure only on RetryTransient or a change of either key.
|
||||||
Backoff func(try int) time.Duration
|
Backoff func(try int) time.Duration
|
||||||
@@ -143,6 +157,27 @@ type Env struct {
|
|||||||
// move of a module of policy record that recreates a container or moves its data waits for a person's yes in its
|
// move of a module of policy record that recreates a container or moves its data waits for a person's yes in its
|
||||||
// want (ADR 0547 §3). Nil judges no move.
|
// want (ADR 0547 §3). Nil judges no move.
|
||||||
Policy func(ctx context.Context, module string) string
|
Policy func(ctx context.Context, module string) string
|
||||||
|
// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), and whether its manifest
|
||||||
|
// could be read: what a composition's Data is, for what runs of a module from the kept declaration, which carries no
|
||||||
|
// Data of its own. A composition says Data, so without it a data module from the kept declaration is judged to move
|
||||||
|
// its data against its first composition after a restart (mongodb and mssql on novox, issue 924). Nil reads none.
|
||||||
|
HoldsData func(ctx context.Context, module string) (data, known bool)
|
||||||
|
// Applied keeps every composition the agent applied, by module (the node's own under NodeModule), beside the engine's
|
||||||
|
// state on the machine: read at the start before anything is watched, so what runs is known before the node composes
|
||||||
|
// its own for the first time, whatever the bucket and the kept declaration still hold (issue 774: on 2026-10-29 the
|
||||||
|
// bus node, its kept declaration retired and its own composition in no bucket, composed its own as if its folded
|
||||||
|
// modules ran nothing and removed the bus server and four held modules). Nil keeps none.
|
||||||
|
Applied AppliedStore
|
||||||
|
}
|
||||||
|
|
||||||
|
// AppliedStore is where the engine keeps the compositions it applied (Env.Applied): each module's last, as its bytes.
|
||||||
|
type AppliedStore interface {
|
||||||
|
// Load is every composition kept, by module; a store that cannot be read says so and loads none.
|
||||||
|
Load() (map[string][]byte, error)
|
||||||
|
// Keep keeps the composition the module was applied from, in place of the one before.
|
||||||
|
Keep(module string, raw []byte) error
|
||||||
|
// Drop forgets the module's, once it is removed.
|
||||||
|
Drop(module string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has
|
// transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has
|
||||||
@@ -160,6 +195,13 @@ type Agent struct {
|
|||||||
// applied is the composition each module was last applied from: a revision a want named from the key's history is
|
// applied is the composition each module was last applied from: a revision a want named from the key's history is
|
||||||
// not the key's latest value, and what it holds is still held out of the others' applies.
|
// not the key's latest value, and what it holds is still held out of the others' applies.
|
||||||
applied map[string]compNow
|
applied map[string]compNow
|
||||||
|
// had is, for each module whose have names a composition the bucket keeps (its latest value or a revision), that
|
||||||
|
// composition, read once at the start (loadHad): what runs of a module the agent has not taken yet, ahead of the kept
|
||||||
|
// declaration (issue 924).
|
||||||
|
had map[string]compNow
|
||||||
|
// keptData is, for each module of the kept declaration whose manifest could be read, whether it declares data
|
||||||
|
// (Env.HoldsData): the Data of what runs of it from the kept declaration.
|
||||||
|
keptData map[string]bool
|
||||||
// attempt is the last apply of a module that failed or is held, with the revisions it was made from.
|
// attempt is the last apply of a module that failed or is held, with the revisions it was made from.
|
||||||
attempt map[string]attempt
|
attempt map[string]attempt
|
||||||
checked map[string]bool
|
checked map[string]bool
|
||||||
@@ -174,6 +216,8 @@ type Agent struct {
|
|||||||
retry chan []string
|
retry chan []string
|
||||||
// reconcile carries the judge's edges: a resource found not active (ADR 0568 §2).
|
// reconcile carries the judge's edges: a resource found not active (ADR 0568 §2).
|
||||||
reconcile chan []string
|
reconcile chan []string
|
||||||
|
// failedAgain carries the modules whose failed apply a person asked to try again (RetryFailed).
|
||||||
|
failedAgain chan string
|
||||||
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
|
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
|
||||||
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
|
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
|
||||||
tries map[string]int
|
tries map[string]int
|
||||||
@@ -218,9 +262,9 @@ func New(env Env) *Agent {
|
|||||||
if env.Say == nil {
|
if env.Say == nil {
|
||||||
env.Say = func(string) {}
|
env.Say = func(string) {}
|
||||||
}
|
}
|
||||||
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{},
|
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
|
||||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
|
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
|
||||||
retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64),
|
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||||
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
|
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,6 +272,9 @@ func New(env Env) *Agent {
|
|||||||
// before any module's, then each change as it comes.
|
// before any module's, then each change as it comes.
|
||||||
func (a *Agent) Run(ctx context.Context) error {
|
func (a *Agent) Run(ctx context.Context) error {
|
||||||
defer close(a.stopped)
|
defer close(a.stopped)
|
||||||
|
// What this engine last applied, before any composition is watched: the node composing its own reads what runs
|
||||||
|
// (Everything) from its first pass on, and that pass comes before the haves are read below (issue 774).
|
||||||
|
a.loadApplied(ctx)
|
||||||
wants, err := a.env.Wants.Watch(ctx, "want."+a.env.Node+".>")
|
wants, err := a.env.Wants.Watch(ctx, "want."+a.env.Node+".>")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the agent cannot watch its wants: %w", err)
|
return fmt.Errorf("the agent cannot watch its wants: %w", err)
|
||||||
@@ -236,7 +283,7 @@ func (a *Agent) Run(ctx context.Context) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the agent cannot watch its compositions: %w", err)
|
return fmt.Errorf("the agent cannot watch its compositions: %w", err)
|
||||||
}
|
}
|
||||||
var wantsCaught, compsCaught bool
|
var wantsCaught, compsCaught, hadRead bool
|
||||||
dirty := map[string]bool{}
|
dirty := map[string]bool{}
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
@@ -263,6 +310,15 @@ func (a *Agent) Run(ctx context.Context) error {
|
|||||||
dirty[m] = true
|
dirty[m] = true
|
||||||
}
|
}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
case m := <-a.failedAgain:
|
||||||
|
// A failed apply a person asked to try again: only one still failed and not held (RetryFailed); what runs is
|
||||||
|
// left as it is.
|
||||||
|
a.mu.Lock()
|
||||||
|
if att, ok := a.attempt[m]; ok && !att.Held {
|
||||||
|
delete(a.attempt, m)
|
||||||
|
dirty[m] = true
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
case ask := <-a.again:
|
case ask := <-a.again:
|
||||||
// The wait of a transiently failed apply ran out: tried again from the keys it has, if nothing tried it since.
|
// The wait of a transiently failed apply ran out: tried again from the keys it has, if nothing tried it since.
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -303,7 +359,17 @@ func (a *Agent) Run(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
// Nothing is applied until what was there at the start is all in: a module's composition may arrive after its
|
// Nothing is applied until what was there at the start is all in: a module's composition may arrive after its
|
||||||
// want, and the node's own composition comes before any module's.
|
// want, and the node's own composition comes before any module's.
|
||||||
if !(wantsCaught && compsCaught) || len(dirty) == 0 {
|
if !(wantsCaught && compsCaught) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !hadRead {
|
||||||
|
// Before the first settle: what runs of each module with a have, from the composition it names, so the
|
||||||
|
// filling judges by it at once and not by the kept declaration until some module settles (issue 924).
|
||||||
|
hadRead = true
|
||||||
|
a.loadHad(ctx)
|
||||||
|
a.runsChanged("the haves were read at the start")
|
||||||
|
}
|
||||||
|
if len(dirty) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
a.settle(ctx, dirty)
|
a.settle(ctx, dirty)
|
||||||
@@ -693,44 +759,21 @@ func (a *Agent) compositionFor(module, fp string) (compNow, bool) {
|
|||||||
// revisionNamed is the revision of a module's composition key whose digest is the one given, signed by a trusted key,
|
// revisionNamed is the revision of a module's composition key whose digest is the one given, signed by a trusted key,
|
||||||
// from the revisions the bucket keeps; false when none is.
|
// from the revisions the bucket keeps; false when none is.
|
||||||
func (a *Agent) revisionNamed(ctx context.Context, module, fp, digest string) (compNow, bool) {
|
func (a *Agent) revisionNamed(ctx context.Context, module, fp, digest string) (compNow, bool) {
|
||||||
if a.env.Revisions == nil {
|
key, err := a.compositionKey(module, fp)
|
||||||
return compNow{}, false
|
|
||||||
}
|
|
||||||
var key string
|
|
||||||
var err error
|
|
||||||
if module == NodeModule {
|
|
||||||
key, err = delivery.ComposedNodeKey(a.env.Node)
|
|
||||||
} else {
|
|
||||||
key, err = delivery.ComposedKey(a.env.Node, module, fp)
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return compNow{}, false
|
return compNow{}, false
|
||||||
}
|
}
|
||||||
revs, err := a.env.Revisions(ctx, key)
|
revs, err := a.revisionsOf(ctx, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
a.sayOnce("history:"+key, fmt.Sprintf("the revisions of %s cannot be read: %v", key, err))
|
a.sayOnce("history:"+key, fmt.Sprintf("the revisions of %s cannot be read: %v", key, err))
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
return compNow{}, false
|
return compNow{}, false
|
||||||
}
|
}
|
||||||
var trusted []ed25519.PublicKey
|
|
||||||
if a.env.Trusted != nil {
|
|
||||||
trusted = a.env.Trusted()
|
|
||||||
}
|
|
||||||
for i := len(revs) - 1; i >= 0; i-- {
|
for i := len(revs) - 1; i >= 0; i-- {
|
||||||
u := revs[i]
|
if revs[i].Err == nil && revs[i].Dig == digest {
|
||||||
if u.Deleted || u.Key != key {
|
return revs[i], true
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
c, err := delivery.DecodeComposed(u.Value)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
dig, err := delivery.VerifyComposed(c, trusted)
|
|
||||||
if err != nil || dig != digest {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return compNow{Raw: u.Value, C: c, Dig: dig, Rev: u.Revision}, true
|
|
||||||
}
|
}
|
||||||
return compNow{}, false
|
return compNow{}, false
|
||||||
}
|
}
|
||||||
@@ -755,22 +798,33 @@ func (a *Agent) others(except string) []delivery.Composed {
|
|||||||
out = append(out, c.C)
|
out = append(out, c.C)
|
||||||
} else if c, ok := a.applied[m]; ok && c.Dig == h.Composed {
|
} else if c, ok := a.applied[m]; ok && c.Dig == h.Composed {
|
||||||
out = append(out, c.C)
|
out = append(out, c.C)
|
||||||
|
} else if c, ok := a.had[m]; ok && c.Dig == h.Composed {
|
||||||
|
out = append(out, c.C) // a revision its have named, replaced since under the same key
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
hadNames := make([]string, 0, len(a.had))
|
||||||
|
for m := range a.had {
|
||||||
|
if _, held := a.running[m]; !held && m != except {
|
||||||
|
hadNames = append(hadNames, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(hadNames)
|
||||||
|
for _, m := range hadNames {
|
||||||
|
out = append(out, a.had[m].C)
|
||||||
|
}
|
||||||
if a.env.Kept != nil {
|
if a.env.Kept != nil {
|
||||||
kept := a.env.Kept()
|
kept := a.env.Kept()
|
||||||
names := make([]string, 0, len(kept))
|
names := make([]string, 0, len(kept))
|
||||||
for m := range kept {
|
for m := range kept {
|
||||||
if _, held := a.running[m]; !held && m != except {
|
_, held := a.running[m]
|
||||||
|
_, had := a.had[m]
|
||||||
|
if !held && !had && m != except {
|
||||||
names = append(names, m)
|
names = append(names, m)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Strings(names)
|
sort.Strings(names)
|
||||||
for _, m := range names {
|
for _, m := range names {
|
||||||
c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]}
|
c, _ := a.keptComposed(m, kept[m])
|
||||||
if a.env.KeptFilled != nil {
|
|
||||||
c.Filled = a.env.KeptFilled()[m]
|
|
||||||
}
|
|
||||||
if m == NodeModule {
|
if m == NodeModule {
|
||||||
c.Module = ""
|
c.Module = ""
|
||||||
}
|
}
|
||||||
@@ -780,6 +834,188 @@ func (a *Agent) others(except string) []delivery.Composed {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// loadHad reads, once at the start, the have of every module with a want or in the kept declaration and, where it names
|
||||||
|
// a composition, that composition: the bucket's value at its key when its digest is the have's, else the revision the
|
||||||
|
// bucket keeps with that digest. A have naming none, or one the bucket no longer keeps, leaves the module to the kept
|
||||||
|
// declaration, said once (issue 924).
|
||||||
|
func (a *Agent) loadHad(ctx context.Context) {
|
||||||
|
a.mu.Lock()
|
||||||
|
seen := map[string]bool{}
|
||||||
|
for m, w := range a.wants {
|
||||||
|
if m != NodeModule && w.Present && !w.Want.Removed {
|
||||||
|
seen[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.env.Kept != nil {
|
||||||
|
for m := range a.env.Kept() {
|
||||||
|
if m != NodeModule {
|
||||||
|
seen[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
names := make([]string, 0, len(seen))
|
||||||
|
for m := range seen {
|
||||||
|
names = append(names, m)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
a.readKeptData(ctx, names...)
|
||||||
|
for _, m := range names {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h, ok := a.env.Haves.Read(ctx, m)
|
||||||
|
if !ok || h.Composed == "" || !haveRuns(h) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
if c, kept := a.had[m]; kept && c.Dig == h.Composed {
|
||||||
|
a.mu.Unlock()
|
||||||
|
continue // what this engine applied and kept itself (loadApplied)
|
||||||
|
}
|
||||||
|
c, found := a.compositionFor(m, h.FP)
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !found || c.Err != nil || c.Dig != h.Composed {
|
||||||
|
c, found = a.revisionNamed(ctx, m, h.FP, h.Composed)
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
if found {
|
||||||
|
a.had[m] = c
|
||||||
|
} else if _, applied := a.had[m]; applied {
|
||||||
|
// Kept as this engine applied it (loadApplied), never composed as nothing.
|
||||||
|
} else if _, kept := a.keptHas(m); kept {
|
||||||
|
a.sayOnce("had:"+m, fmt.Sprintf("%s's have names the composition %s, which the bucket no longer keeps: it runs "+
|
||||||
|
"from the kept declaration until it is applied or taken", m, short(h.Composed)))
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadApplied takes, once at the start, what this engine last applied of each module as what runs of it, ahead of the
|
||||||
|
// bucket and the kept declaration (Env.Applied, issue 774): it is what the machine holds, whatever the bus still keeps.
|
||||||
|
// One whose have names another composition runs as applied until loadHad reads that one from the bucket; one whose
|
||||||
|
// have cannot be read, or that has none, is kept as applied, so a module is never composed as running nothing.
|
||||||
|
func (a *Agent) loadApplied(ctx context.Context) {
|
||||||
|
if a.env.Applied == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
kept, err := a.env.Applied.Load()
|
||||||
|
if err != nil {
|
||||||
|
a.env.Say(fmt.Sprintf("the compositions this engine applied cannot be read, so what runs is read from the haves alone: %v", err))
|
||||||
|
}
|
||||||
|
// At an engine's first start nothing is kept yet: the node's own runs as the kept declaration has it, which the store
|
||||||
|
// keeps now, so the node never composes its own as if its folded modules ran nothing once the declaration retires
|
||||||
|
// (issue 774, review of the re-land, round 1). Taken whatever digest the node's have names: the composition that names
|
||||||
|
// is in no bucket, and the declaration is the one record of what runs on disk.
|
||||||
|
seeded := false
|
||||||
|
if _, own := kept[NodeModule]; !own && err == nil && a.env.Kept != nil {
|
||||||
|
if rs, ok := a.env.Kept()[NodeModule]; ok && len(rs) > 0 {
|
||||||
|
c, _ := a.keptComposed(NodeModule, rs)
|
||||||
|
c.Module = ""
|
||||||
|
raw, eerr := delivery.EncodeComposed(c)
|
||||||
|
if eerr == nil {
|
||||||
|
eerr = a.env.Applied.Keep(NodeModule, raw)
|
||||||
|
}
|
||||||
|
if eerr != nil {
|
||||||
|
a.env.Say(fmt.Sprintf("the node's own as the kept declaration has it cannot be kept: %v", eerr))
|
||||||
|
} else {
|
||||||
|
if kept == nil {
|
||||||
|
kept = map[string][]byte{}
|
||||||
|
}
|
||||||
|
kept[NodeModule], seeded = raw, true
|
||||||
|
a.env.Say("the node's own runs as the kept declaration has it, kept now as what this engine runs")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(kept))
|
||||||
|
for m := range kept {
|
||||||
|
names = append(names, m)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, m := range names {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c, err := delivery.DecodeComposed(kept[m])
|
||||||
|
var dig string
|
||||||
|
if err == nil {
|
||||||
|
dig, err = delivery.ComposedDigestOf(c)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
a.env.Say(fmt.Sprintf("the composition this engine kept of %s cannot be read, so what runs of it is read from its have: %v", m, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
h, found, unread := a.readHave(ctx, m)
|
||||||
|
if seeded && m == NodeModule {
|
||||||
|
found, unread = true, false // what the kept declaration has runs, whatever the have names
|
||||||
|
} else if found && h.Composed != "" && h.Composed != dig && haveRuns(h) {
|
||||||
|
// The composition the have names replaces this once loadHad finds it in the bucket; until then, and when
|
||||||
|
// the bucket keeps it no more, what this engine applied runs (issue 774, review of the re-land, round 1).
|
||||||
|
a.env.Say(fmt.Sprintf("%s's have names the composition %s, not %s this engine applied last: it runs as applied "+
|
||||||
|
"until the one its have names is read", m, short(h.Composed), short(dig)))
|
||||||
|
}
|
||||||
|
if m == NodeModule {
|
||||||
|
c.Module, c.FP = "", ""
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
a.had[m] = compNow{Raw: kept[m], C: c, Dig: dig}
|
||||||
|
a.mu.Unlock()
|
||||||
|
switch {
|
||||||
|
case unread:
|
||||||
|
a.env.Say(fmt.Sprintf("%s's have cannot be read: it runs as this engine applied it, %s", m, short(dig)))
|
||||||
|
case !found:
|
||||||
|
a.env.Say(fmt.Sprintf("%s has no have: it runs as this engine applied it, %s", m, short(dig)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepApplied keeps the composition a module was applied from (Env.Applied); one that cannot be kept is said, and the
|
||||||
|
// apply stands.
|
||||||
|
func (a *Agent) keepApplied(module string, comp compNow) {
|
||||||
|
if a.env.Applied == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
raw := comp.Raw
|
||||||
|
if len(raw) == 0 {
|
||||||
|
var err error
|
||||||
|
if raw, err = delivery.EncodeComposed(comp.C); err != nil {
|
||||||
|
a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := a.env.Applied.Keep(module, raw); err != nil {
|
||||||
|
a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// haveRuns says a have names what runs, in the phases baseline takes as running: running (or empty), held for a failed
|
||||||
|
// backup, or a composer's mark over what still runs. A have backing up, applying, checking, failed or waiting names a
|
||||||
|
// composition not applied, or not yet (issue 924, review of 031e106).
|
||||||
|
func haveRuns(h Have) bool {
|
||||||
|
switch delivery.Phase(h.Phase) {
|
||||||
|
case "", delivery.PhaseRunning, delivery.PhaseHeld:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return composerMark(h)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptHas says whether the kept declaration has resources for the module. Called holding a.mu or not: Kept takes no lock
|
||||||
|
// of the agent's.
|
||||||
|
func (a *Agent) keptHas(module string) ([]json.RawMessage, bool) {
|
||||||
|
if a.env.Kept == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
rs, ok := a.env.Kept()[module]
|
||||||
|
return rs, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// runsChanged tells RunsChanged, outside the agent's lock.
|
||||||
|
func (a *Agent) runsChanged(why string) {
|
||||||
|
if a.env.RunsChanged != nil {
|
||||||
|
a.env.RunsChanged(why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// unreadableModules are the keys of the module compositions in the node's bucket that cannot be read or are refused, of a
|
// unreadableModules are the keys of the module compositions in the node's bucket that cannot be read or are refused, of a
|
||||||
// module the assigner has not removed: for each module only the composition its want names, else its newest, since a
|
// module the assigner has not removed: for each module only the composition its want names, else its newest, since a
|
||||||
// stale one nobody applies (written before a rule the SDK now holds) must not hold the node's own back. Called holding
|
// stale one nobody applies (written before a rule the SDK now holds) must not hold the node's own back. Called holding
|
||||||
@@ -901,7 +1137,8 @@ func (a *Agent) heldOut(except string) []delivery.Composed {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(rs) > 0 {
|
if len(rs) > 0 {
|
||||||
out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs})
|
// Held out, never judged: its Data from its manifest where known, as every other composition says it.
|
||||||
|
out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs, Data: a.keptData[m]})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
@@ -916,6 +1153,62 @@ func (a *Agent) Everything() []delivery.Composed {
|
|||||||
return a.others("\x00")
|
return a.others("\x00")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Placed is the modules a want still places on the node (present and not removed), held ones included, but except:
|
||||||
|
// what no apply but the module's own may remove (apply.Scope.Placed, issue 774). Called without a.mu held.
|
||||||
|
func (a *Agent) Placed(except string) map[string]bool {
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
out := map[string]bool{}
|
||||||
|
for m, w := range a.wants {
|
||||||
|
if m != NodeModule && m != except && w.Present && !w.Want.Removed {
|
||||||
|
out[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetryFailed asks a module's failed apply to be tried again (the node-engine's verb compose, issue 774): only an apply
|
||||||
|
// that failed and is not held for a failed backup, whose release is the operator's answer (ADR 0568 §4); what runs is
|
||||||
|
// never taken away, so nothing is applied that the engine would not apply on its own at the next change. Refused in
|
||||||
|
// words otherwise.
|
||||||
|
func (a *Agent) RetryFailed(module string) error {
|
||||||
|
a.mu.Lock()
|
||||||
|
w := a.wants[module]
|
||||||
|
att, tried := a.attempt[module]
|
||||||
|
r, ran := a.running[module]
|
||||||
|
a.mu.Unlock()
|
||||||
|
switch {
|
||||||
|
case !w.Present || w.Want.Removed:
|
||||||
|
return fmt.Errorf("%s has no want on this machine, so there is no apply of it to try again", module)
|
||||||
|
case tried && att.Held:
|
||||||
|
return fmt.Errorf("%s is held for a failed backup: the operator's answer releases it", module)
|
||||||
|
case !tried && ran && r.FP == w.Want.FP && r.Phase != delivery.PhaseRemoved:
|
||||||
|
return fmt.Errorf("%s runs what its want names, nothing to retry", module)
|
||||||
|
case !tried:
|
||||||
|
return fmt.Errorf("%s has no failed apply to try again: it waits for its composition, or settles on its own", module)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case a.failedAgain <- module:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s's retry is not taken now: the engine has too many asked; ask again in a moment", module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemovedByWant is the modules whose want says removed (apply.Scope.RemovedByWant, issue 774): in the node's own apply,
|
||||||
|
// only their resources may go. Called without a.mu held.
|
||||||
|
func (a *Agent) RemovedByWant() map[string]bool {
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
out := map[string]bool{}
|
||||||
|
for m, w := range a.wants {
|
||||||
|
if m != NodeModule && w.Present && w.Want.Removed {
|
||||||
|
out[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept
|
// HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept
|
||||||
// declaration's.
|
// declaration's.
|
||||||
func (a *Agent) HoldsNode() bool {
|
func (a *Agent) HoldsNode() bool {
|
||||||
@@ -1038,6 +1331,15 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
|||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
a.prior[module] = true
|
a.prior[module] = true
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
if h.Phase == string(delivery.PhaseHeld) && h.Detail == BackupInterrupted && w.Want.Backup != delivery.BackupSkip {
|
||||||
|
// Held only because this node agent stopped while the backup ran (postgres on novox, 2026-10-31): no
|
||||||
|
// failure and nothing for the operator to answer. It is backed up again and applied (a.prior makes the
|
||||||
|
// apply back it up first); the want still asking a backup is all the answer needed.
|
||||||
|
a.mu.Lock()
|
||||||
|
a.sayOnce("interrupted:"+module, fmt.Sprintf("%s's backup was interrupted by this node agent's stop: it is backed up again", module))
|
||||||
|
a.mu.Unlock()
|
||||||
|
return false
|
||||||
|
}
|
||||||
if h.Phase == string(delivery.PhaseHeld) {
|
if h.Phase == string(delivery.PhaseHeld) {
|
||||||
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
|
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
|
||||||
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4).
|
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4).
|
||||||
@@ -1048,6 +1350,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
|||||||
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
||||||
}
|
}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
a.runsChanged(module + " was taken as held")
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
if h.Composed != "" {
|
if h.Composed != "" {
|
||||||
@@ -1068,6 +1371,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
|||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
a.runsChanged(module + " was taken from its have")
|
||||||
// Said again, so the session holds the module and its key carries the health this engine judges,
|
// Said again, so the session holds the module and its key carries the health this engine judges,
|
||||||
// not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25).
|
// not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25).
|
||||||
a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
|
a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
|
||||||
@@ -1096,6 +1400,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
|||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning}
|
a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
a.runsChanged(module + " was taken from the kept declaration")
|
||||||
a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
|
a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -1124,16 +1429,33 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
|||||||
// A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on
|
// A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on
|
||||||
// without one (ADR 0539 §16).
|
// without one (ADR 0539 §16).
|
||||||
if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != delivery.BackupSkip {
|
if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != delivery.BackupSkip {
|
||||||
|
// What runs, read before the have says backing-up: a module known only from its have (a.prior) has no running
|
||||||
|
// entry, and its held have kept naming nothing (postgres on novox, 2026-10-31), as the not-composed mark keeps it.
|
||||||
|
a.mu.Lock()
|
||||||
|
prev, runs := a.running[module]
|
||||||
|
a.mu.Unlock()
|
||||||
|
if !runs {
|
||||||
|
if h, found, _ := a.readHave(ctx, module); found {
|
||||||
|
prev = delivery.Have{FP: h.FP, Composed: h.Composed}
|
||||||
|
}
|
||||||
|
}
|
||||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)})
|
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)})
|
||||||
if got := a.env.Backup(ctx, module); !got.Done {
|
if got := a.env.Backup(ctx, module); !got.Done {
|
||||||
a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, got.Words))
|
words := got.Words
|
||||||
|
if got.Interrupted || ctx.Err() != nil {
|
||||||
|
// The node agent stopped while it waited: no failure of the backup, so the next start backs it up again
|
||||||
|
// (baseline) rather than holding it for the operator.
|
||||||
|
words = BackupInterrupted
|
||||||
|
a.env.Say(fmt.Sprintf("%s waits: %s; it is backed up again at the next start", module, words))
|
||||||
|
} else {
|
||||||
|
a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, words))
|
||||||
|
}
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
|
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev}
|
||||||
prev := a.running[module]
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
// It keeps running what it has: the have says what it holds and why it waits.
|
// It keeps running what it has: the have says what it holds and why it waits.
|
||||||
a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld),
|
a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld),
|
||||||
Detail: got.Words})
|
Detail: words})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1184,6 +1506,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
|||||||
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
|
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
|
||||||
a.applied[module] = comp
|
a.applied[module] = comp
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
a.keepApplied(module, comp)
|
||||||
if a.env.Settled != nil {
|
if a.env.Settled != nil {
|
||||||
a.env.Settled(module)
|
a.env.Settled(module)
|
||||||
}
|
}
|
||||||
@@ -1213,8 +1536,14 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
delete(a.attempt, module)
|
delete(a.attempt, module)
|
||||||
|
delete(a.had, module)
|
||||||
a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved}
|
a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved}
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
if a.env.Applied != nil {
|
||||||
|
if err := a.env.Applied.Drop(module); err != nil {
|
||||||
|
a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be forgotten now it is removed: %v", module, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
if a.env.Settled != nil {
|
if a.env.Settled != nil {
|
||||||
a.env.Settled(module)
|
a.env.Settled(module)
|
||||||
}
|
}
|
||||||
@@ -1286,10 +1615,21 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
|
|||||||
if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
|
if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
a.readKeptData(ctx, module) // the manifest may read now though it did not at the start
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
r, ran := a.running[module]
|
r, ran := a.running[module]
|
||||||
if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved {
|
if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved {
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
// Composed back to what runs: a have still waiting for a yes for a move since gone says running again, or it waits
|
||||||
|
// for a yes nobody needs (issue 774, 2026-10-31: four providers stayed waiting-yes once their consumers composed).
|
||||||
|
if h, ok := a.env.Haves.Read(ctx, module); ok && h.Phase == string(delivery.PhaseWaitingYes) {
|
||||||
|
// Its sealed values only when they are of what runs; else writeHave reads them (sealedOf).
|
||||||
|
var sealed delivery.Seals
|
||||||
|
if h.Composed == r.Composed {
|
||||||
|
sealed = h.Sealed
|
||||||
|
}
|
||||||
|
a.writeHave(ctx, Have{Module: module, FP: r.FP, Composed: r.Composed, Phase: string(delivery.PhaseRunning), Sealed: sealed})
|
||||||
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
from, known := a.runningComposition(module)
|
from, known := a.runningComposition(module)
|
||||||
@@ -1331,14 +1671,61 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
|
|||||||
|
|
||||||
// runningComposition is what the module runs now: the composition last applied, else its resources in the declaration
|
// runningComposition is what the module runs now: the composition last applied, else its resources in the declaration
|
||||||
// the node kept from before the switch. Called with a.mu held.
|
// the node kept from before the switch. Called with a.mu held.
|
||||||
|
//
|
||||||
|
// The composition its have named at the start comes before the kept declaration (issue 924). From the kept declaration,
|
||||||
|
// what runs carries Data as its manifest declares it; a module whose manifest could not be read cannot be judged against
|
||||||
|
// it (false), and that is said, rather than taking it to hold no data and judging a false move of its data.
|
||||||
func (a *Agent) runningComposition(module string) (delivery.Composed, bool) {
|
func (a *Agent) runningComposition(module string) (delivery.Composed, bool) {
|
||||||
if c, ok := a.applied[module]; ok && c.Err == nil {
|
if c, ok := a.applied[module]; ok && c.Err == nil {
|
||||||
return c.C, true
|
return c.C, true
|
||||||
}
|
}
|
||||||
if a.env.Kept != nil {
|
if c, ok := a.had[module]; ok {
|
||||||
if rs, ok := a.env.Kept()[module]; ok {
|
return c.C, true
|
||||||
return delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}, true
|
|
||||||
}
|
}
|
||||||
|
if rs, ok := a.keptHas(module); ok {
|
||||||
|
c, known := a.keptComposed(module, rs)
|
||||||
|
if !known {
|
||||||
|
a.sayOnce("kept-data:"+module, fmt.Sprintf("%s runs from the kept declaration, and whether it holds data cannot be "+
|
||||||
|
"read from its manifest: a move of it cannot be judged", module))
|
||||||
|
}
|
||||||
|
return c, known
|
||||||
}
|
}
|
||||||
return delivery.Composed{}, false
|
return delivery.Composed{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// keptComposed is what runs of a module from the kept declaration: its resources there, the paths it names as filled,
|
||||||
|
// and Data as the module's manifest declares it; false when the manifest could not be read, so Data is not known. Called
|
||||||
|
// with a.mu held.
|
||||||
|
func (a *Agent) keptComposed(module string, rs []json.RawMessage) (delivery.Composed, bool) {
|
||||||
|
c := delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}
|
||||||
|
if a.env.KeptFilled != nil {
|
||||||
|
c.Filled = a.env.KeptFilled()[module]
|
||||||
|
}
|
||||||
|
data, known := a.keptData[module]
|
||||||
|
c.Data = data
|
||||||
|
return c, known || module == NodeModule
|
||||||
|
}
|
||||||
|
|
||||||
|
// readKeptData reads, outside the agent's lock, whether each module of the kept declaration not known yet declares data
|
||||||
|
// (Env.HoldsData), and keeps the answers it could read.
|
||||||
|
func (a *Agent) readKeptData(ctx context.Context, modules ...string) {
|
||||||
|
if a.env.HoldsData == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, m := range modules {
|
||||||
|
if m == NodeModule {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
_, done := a.keptData[m]
|
||||||
|
a.mu.Unlock()
|
||||||
|
if _, kept := a.keptHas(m); done || !kept {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if data, known := a.env.HoldsData(ctx, m); known {
|
||||||
|
a.mu.Lock()
|
||||||
|
a.keptData[m] = data
|
||||||
|
a.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+391
-14
@@ -457,13 +457,27 @@ func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
|
|||||||
if n := len(b.appliedList()); n != 1 {
|
if n := len(b.appliedList()); n != 1 {
|
||||||
t.Fatalf("a failed apply was retried by an unrelated key or the clock (%d applies)", n)
|
t.Fatalf("a failed apply was retried by an unrelated key or the clock (%d applies)", n)
|
||||||
}
|
}
|
||||||
// The want is written again: the retry.
|
// A person asks it again (the node-engine's verb compose, issue 774): the failed apply is tried again with no key
|
||||||
|
// changed; once it runs, nothing is left to retry, and a module with no want has none.
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
b.failWith = ""
|
b.failWith = ""
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
b.wants.put(wk, wraw)
|
if err := b.ag.RetryFailed("web"); err != nil {
|
||||||
eventually(t, "the change of the want did not retry", func() bool { return len(b.appliedList()) == 2 })
|
t.Fatalf("a failed apply was refused a retry: %v", err)
|
||||||
|
}
|
||||||
|
eventually(t, "the asked retry did not apply", func() bool { return len(b.appliedList()) == 2 })
|
||||||
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
eventually(t, "not running", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
||||||
|
for module, words := range map[string]string{"web": "runs what its want names", "ghost": "has no want"} {
|
||||||
|
if err := b.ag.RetryFailed(module); err == nil || !strings.Contains(err.Error(), words) {
|
||||||
|
t.Errorf("a retry of %s: %v, want refused saying %q", module, err, words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The want is written again: no apply, it runs what the want names.
|
||||||
|
b.wants.put(wk, wraw)
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if n := len(b.appliedList()); n != 2 {
|
||||||
|
t.Errorf("a module running its want was applied again (%d applies)", n)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
|
// The bus did not answer for a moment (mesh-vault on the bus node, 2026-10-25 23:04:24): tried again with no new
|
||||||
// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
|
// revision of either key, by the backoff or by RetryTransient once the engine knows the bus is back; the have says
|
||||||
@@ -540,6 +554,17 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
|||||||
if got := b.appliedList(); len(got) != 0 {
|
if got := b.appliedList(); len(got) != 0 {
|
||||||
t.Fatalf("something was applied behind a failed backup: %v", got)
|
t.Fatalf("something was applied behind a failed backup: %v", got)
|
||||||
}
|
}
|
||||||
|
// A person's retry (the verb compose, issue 774) never releases it: the operator's answer does (ADR 0568 §4).
|
||||||
|
if err := b.ag.RetryFailed("db"); err == nil || !strings.Contains(err.Error(), "the operator's answer releases it") {
|
||||||
|
t.Errorf("a retry of db held for a failed backup: %v", err)
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
b.mu.Lock()
|
||||||
|
asked := len(b.backups)
|
||||||
|
b.mu.Unlock()
|
||||||
|
if got := b.appliedList(); len(got) != 0 || asked != 1 {
|
||||||
|
t.Fatalf("a retry released db held for a failed backup: applied %v, %d backups asked", got, asked)
|
||||||
|
}
|
||||||
|
|
||||||
// The answer arrives in the module's own want: back up again (it fails again), then skip.
|
// The answer arrives in the module's own want: back up again (it fails again), then skip.
|
||||||
wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain)
|
wk, again := wantFor(t, "db", fp2, dig2, delivery.BackupAgain)
|
||||||
@@ -565,6 +590,65 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) {
|
|||||||
h := b.haves.last("db")
|
h := b.haves.last("db")
|
||||||
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2
|
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp2 && h.Composed == dig2
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// The node agent stopped while the backup ran, the module known only from its have: the held have names what runs
|
||||||
|
// and says it was interrupted (postgres on novox, 2026-10-31, held naming nothing).
|
||||||
|
ib := newBed(t)
|
||||||
|
ib.comps.before(ck1, craw1)
|
||||||
|
ib.comps.before(ck2, craw2)
|
||||||
|
iwk, iwraw := wantFor(t, "db", fp2, dig2, delivery.BackupAgain)
|
||||||
|
ib.wants.before(iwk, iwraw)
|
||||||
|
ib.haves.stored = map[string]Have{"db": {Module: "db", FP: fp1, Composed: dig1, Phase: string(delivery.PhaseRunning)}}
|
||||||
|
began := make(chan struct{})
|
||||||
|
ib.env.Backup = func(ctx context.Context, _ string) Backup {
|
||||||
|
close(began)
|
||||||
|
<-ctx.Done()
|
||||||
|
return Backup{Words: "the backup's ask was cancelled"}
|
||||||
|
}
|
||||||
|
stopI := ib.run()
|
||||||
|
<-began
|
||||||
|
stopI()
|
||||||
|
if h := ib.haves.last("db"); h.Phase != string(delivery.PhaseHeld) || h.FP != fp1 || h.Composed != dig1 || h.Detail != BackupInterrupted {
|
||||||
|
t.Errorf("held after an interrupted backup: %+v", h)
|
||||||
|
}
|
||||||
|
|
||||||
|
// At a start: a backup the node agent's own stop interrupted (postgres on novox, 2026-10-31) is no failure, so it is
|
||||||
|
// backed up again and applied while the want asks a backup; a backup that failed stays held for the operator.
|
||||||
|
for _, row := range []struct {
|
||||||
|
detail, backup string
|
||||||
|
backups, applied int
|
||||||
|
phase delivery.Phase
|
||||||
|
namesNothing bool // the have names no fp and no composition
|
||||||
|
}{
|
||||||
|
{BackupInterrupted, delivery.BackupAgain, 1, 1, delivery.PhaseRunning, false},
|
||||||
|
// novox's postgres: its held have, written before mesh-host #189, names nothing it runs.
|
||||||
|
{BackupInterrupted, "", 1, 1, delivery.PhaseRunning, true},
|
||||||
|
{"restic: the repository is locked", delivery.BackupAgain, 0, 0, delivery.PhaseHeld, false},
|
||||||
|
} {
|
||||||
|
r := newBed(t)
|
||||||
|
r.comps.before(ck1, craw1)
|
||||||
|
r.comps.before(ck2, craw2)
|
||||||
|
wk, wraw := wantFor(t, "db", fp2, dig2, row.backup)
|
||||||
|
r.wants.before(wk, wraw)
|
||||||
|
held := Have{Module: "db", FP: fp1, Composed: dig1, Phase: string(delivery.PhaseHeld), Detail: row.detail}
|
||||||
|
if row.namesNothing {
|
||||||
|
held.FP, held.Composed = "", ""
|
||||||
|
}
|
||||||
|
r.haves.stored = map[string]Have{"db": held}
|
||||||
|
stopR := r.run()
|
||||||
|
if row.applied > 0 {
|
||||||
|
eventually(t, row.detail+": not applied after its backup", func() bool { return len(r.appliedList()) == row.applied })
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
r.mu.Lock()
|
||||||
|
n := len(r.backups)
|
||||||
|
r.mu.Unlock()
|
||||||
|
h := r.haves.last("db")
|
||||||
|
if n != row.backups || len(r.appliedList()) != row.applied || h.Phase != string(row.phase) {
|
||||||
|
t.Errorf("%s, want backup %q: %d backups, applied %v, have %+v", row.detail, row.backup, n, r.appliedList(), h)
|
||||||
|
}
|
||||||
|
stopR()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
|
// A data module with a good backup is applied after it; one that never ran has nothing to back up.
|
||||||
@@ -769,6 +853,162 @@ func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) {
|
|||||||
if a.HoldsNode() {
|
if a.HoldsNode() {
|
||||||
t.Fatal("the agent holds no node composition")
|
t.Fatal("the agent holds no node composition")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 774 (novox, 2026-10-29 07:54 UTC): the kept declaration retired and the node's own composition in no bucket, a
|
||||||
|
// fresh engine's first own composition was composed from what runs with the node's own missing, so its folded modules
|
||||||
|
// (the bus server among them) were composed as nothing and removed. What this engine applied is kept on the machine
|
||||||
|
// and is what runs from before the first composition is watched; an apply keeps, a removal forgets.
|
||||||
|
t.Run("what this engine applied is what runs before the node composes its own", func(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
b.env.SelfComposed, b.env.Trusted, b.kept = true, nil, nil
|
||||||
|
store := &memApplied{kept: map[string][]byte{}}
|
||||||
|
b.env.Applied = store
|
||||||
|
// A folded module's resource in the node's own composition: its id names the module, as the controller folds it.
|
||||||
|
natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"})
|
||||||
|
own := delivery.Composed{Node: node, Resources: []json.RawMessage{natsConf}}
|
||||||
|
ownRaw, err := delivery.EncodeComposed(own)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ownDig, _ := delivery.ComposedDigestOf(own)
|
||||||
|
web := delivery.Composed{Node: node, Module: "web", FP: fpOf("a"), Resources: []json.RawMessage{file("web", "x", "1")}}
|
||||||
|
webRaw, _ := delivery.EncodeComposed(web)
|
||||||
|
webDig, _ := delivery.ComposedDigestOf(web)
|
||||||
|
store.kept[NodeModule], store.kept["web"] = ownRaw, webRaw
|
||||||
|
b.haves.Write(context.Background(), Have{Module: NodeModule, Composed: ownDig})
|
||||||
|
b.haves.Write(context.Background(), Have{Module: "web", FP: fpOf("a"), Composed: webDig, Phase: string(delivery.PhaseRunning)})
|
||||||
|
// The composer's first pass reads what runs when its watch begins, before the agent reads any have.
|
||||||
|
firstCh := make(chan []delivery.Composed, 1)
|
||||||
|
comps := b.comps
|
||||||
|
var ag *Agent
|
||||||
|
b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) {
|
||||||
|
firstCh <- ag.Everything()
|
||||||
|
return comps.Watch(ctx, p)
|
||||||
|
})
|
||||||
|
ag = New(b.env)
|
||||||
|
b.ag = ag
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); _ = ag.Run(ctx) }()
|
||||||
|
defer func() { cancel(); <-done }()
|
||||||
|
var first []delivery.Composed
|
||||||
|
select {
|
||||||
|
case first = <-firstCh:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the composer was not watched")
|
||||||
|
}
|
||||||
|
var ranOwn, ranWeb bool
|
||||||
|
for _, c := range first {
|
||||||
|
ranOwn = ranOwn || (c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222"))
|
||||||
|
ranWeb = ranWeb || c.Module == "web"
|
||||||
|
}
|
||||||
|
if !ranOwn || !ranWeb {
|
||||||
|
t.Fatalf("what runs at the node's first own composition: own %v, web %v (%d compositions); want both as this engine applied them",
|
||||||
|
ranOwn, ranWeb, len(first))
|
||||||
|
}
|
||||||
|
// An apply keeps the composition it applied; a removal forgets it.
|
||||||
|
web2 := delivery.Composed{Node: node, Module: "web", FP: fpOf("b"), Resources: []json.RawMessage{file("web", "x", "2")}}
|
||||||
|
ck, _ := delivery.ComposedKey(node, "web", fpOf("b"))
|
||||||
|
raw2, _ := delivery.EncodeComposed(web2)
|
||||||
|
comps.put(ck, raw2)
|
||||||
|
wk, _ := delivery.WantKey(node, "web")
|
||||||
|
wraw, _ := delivery.EncodeWant(delivery.Want{FP: fpOf("b"), Self: true})
|
||||||
|
b.wants.put(wk, wraw)
|
||||||
|
eventually(t, "the applied composition was not kept", func() bool { return string(store.get("web")) == string(raw2) })
|
||||||
|
rraw, _ := delivery.EncodeWant(delivery.Want{Removed: true})
|
||||||
|
b.wants.put(wk, rraw)
|
||||||
|
eventually(t, "the removed module's composition was not forgotten", func() bool { return store.get("web") == nil })
|
||||||
|
})
|
||||||
|
|
||||||
|
// Issue 774, review of the re-land, round 1: at the re-landed engine's first start nothing is kept yet, and the node's
|
||||||
|
// have names a composition the node composed itself, in no bucket. The node's own runs as the kept declaration has it
|
||||||
|
// and is kept then, so the next start, the declaration retired, still runs nats.
|
||||||
|
t.Run("at the first start nothing is kept: the node's own runs as the kept declaration has it", func(t *testing.T) {
|
||||||
|
natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"})
|
||||||
|
store := &memApplied{kept: map[string][]byte{}}
|
||||||
|
h := &haves{}
|
||||||
|
h.Write(context.Background(), Have{Module: NodeModule, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||||
|
firstPass := func(kept map[string][]json.RawMessage) []delivery.Composed {
|
||||||
|
b := newBed(t)
|
||||||
|
b.env.SelfComposed, b.env.Trusted, b.kept, b.env.Applied, b.env.Haves = true, nil, kept, store, h
|
||||||
|
firstCh := make(chan []delivery.Composed, 1)
|
||||||
|
comps := b.comps
|
||||||
|
var ag *Agent
|
||||||
|
b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) {
|
||||||
|
firstCh <- ag.Everything()
|
||||||
|
return comps.Watch(ctx, p)
|
||||||
|
})
|
||||||
|
ag = New(b.env)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); _ = ag.Run(ctx) }()
|
||||||
|
defer func() { cancel(); <-done }()
|
||||||
|
select {
|
||||||
|
case got := <-firstCh:
|
||||||
|
return got
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the composer was not watched")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
runsNats := func(cs []delivery.Composed) bool {
|
||||||
|
for _, c := range cs {
|
||||||
|
if c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if got := firstPass(map[string][]json.RawMessage{NodeModule: {natsConf}}); !runsNats(got) {
|
||||||
|
t.Fatalf("the first start does not run nats in the node's own (%d compositions)", len(got))
|
||||||
|
}
|
||||||
|
if store.get(NodeModule) == nil {
|
||||||
|
t.Fatal("the first start kept no node's own composition")
|
||||||
|
}
|
||||||
|
if got := firstPass(nil); !runsNats(got) {
|
||||||
|
t.Fatalf("the next start, the kept declaration retired, does not run nats in the node's own (%d compositions)", len(got))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchFunc is a Watcher from a function.
|
||||||
|
type watchFunc func(ctx context.Context, pattern string) (<-chan Update, error)
|
||||||
|
|
||||||
|
func (f watchFunc) Watch(ctx context.Context, pattern string) (<-chan Update, error) {
|
||||||
|
return f(ctx, pattern)
|
||||||
|
}
|
||||||
|
|
||||||
|
// memApplied is an AppliedStore in memory.
|
||||||
|
type memApplied struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
kept map[string][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memApplied) Load() (map[string][]byte, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
out := map[string][]byte{}
|
||||||
|
for k, v := range m.kept {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
func (m *memApplied) Keep(module string, raw []byte) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.kept[module] = raw
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (m *memApplied) Drop(module string) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
delete(m.kept, module)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (m *memApplied) get(module string) []byte {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
return m.kept[module]
|
||||||
}
|
}
|
||||||
|
|
||||||
// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this
|
// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this
|
||||||
@@ -1026,18 +1266,102 @@ func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1945/AC18 (unit): a module that still runs from the kept declaration carries the kept declaration's filled paths of
|
// 1945/AC18 and issue 924 (unit): what runs of a module the agent has not taken yet carries the filled paths the
|
||||||
// its files, so the filling writes them ("not named as filled" for claude-code and screen-lock, 2026-10-25).
|
// filling judges by. From the kept declaration while the module has no have naming a composition ("not named as filled"
|
||||||
func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) {
|
// for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept
|
||||||
b := newBed(t)
|
// declaration being older, read at the start before any module settles and told to the filling then, so a file the
|
||||||
b.kept = map[string][]json.RawMessage{"claude-code": {file("claude-code", "settings", "")}}
|
// newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29).
|
||||||
b.env.KeptFilled = func() map[string][]string {
|
// Only a have in a phase baseline takes as running names what runs; and what runs carries Data, from the composition
|
||||||
return map[string][]string{"claude-code": {"/tmp/r6/claude-code/settings"}}
|
// or, from the kept declaration, from the module's manifest, so a data module is not judged to move its data against
|
||||||
|
// its first composition after a restart (mongodb and mssql on novox).
|
||||||
|
func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) {
|
||||||
|
const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound"
|
||||||
|
filledFile := func(id string) json.RawMessage {
|
||||||
|
return json.RawMessage(`{"id":"power.` + id + `","type":"file","module":"power","path":"/tmp/r6/power/` + id +
|
||||||
|
`","mode":"0644","owner-at-runtime":"owned"}`)
|
||||||
}
|
}
|
||||||
|
fp := fpOf("a")
|
||||||
|
ck, _ := delivery.ComposedKey(node, "power", fp)
|
||||||
|
applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, Data: true,
|
||||||
|
Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}}
|
||||||
|
_, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey)
|
||||||
|
newer := applied
|
||||||
|
newer.Resources = append([]json.RawMessage{}, applied.Resources...)
|
||||||
|
newer.Resources = append(newer.Resources, file("power", "later", "1"))
|
||||||
|
_, newerRaw, _ := signedBy(t, newer, ck, controllerKey)
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
// have is the module's have; bucket the value its composition key holds now, history the revisions kept.
|
||||||
|
have *Have
|
||||||
|
bucket []byte
|
||||||
|
history []Update
|
||||||
|
filled, told string
|
||||||
|
// data is what runs' Data; holdsData the manifest's answer (nil: none read).
|
||||||
|
data bool
|
||||||
|
holdsData func(context.Context, string) (bool, bool)
|
||||||
|
}{
|
||||||
|
{name: "no have: the kept declaration's", filled: old},
|
||||||
|
{name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
||||||
|
bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start", data: true},
|
||||||
|
{name: "a have applying names nothing that runs yet", have: &Have{Module: "power", FP: fp, Composed: appliedDig,
|
||||||
|
Phase: string(delivery.PhaseApplying)}, bucket: appliedRaw, filled: old, told: "the haves were read at the start"},
|
||||||
|
{name: "a data module from the kept declaration says Data as its manifest does", filled: old,
|
||||||
|
told: "the haves were read at the start", data: true,
|
||||||
|
holdsData: func(context.Context, string) (bool, bool) { return true, true }},
|
||||||
|
{name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
||||||
|
bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}},
|
||||||
|
filled: old + "," + added, told: "the haves were read at the start", data: true},
|
||||||
|
{name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
|
||||||
|
bucket: newerRaw, filled: old, told: "the haves were read at the start"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
b.kept = map[string][]json.RawMessage{"power": {filledFile("settings")}}
|
||||||
|
b.env.KeptFilled = func() map[string][]string { return map[string][]string{"power": {old}} }
|
||||||
|
if tc.have != nil {
|
||||||
|
b.haves.stored = map[string]Have{"power": *tc.have}
|
||||||
|
}
|
||||||
|
if tc.bucket != nil {
|
||||||
|
b.comps.before(ck, tc.bucket)
|
||||||
|
}
|
||||||
|
b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil }
|
||||||
|
b.env.HoldsData = tc.holdsData
|
||||||
|
var told []string
|
||||||
|
var mu sync.Mutex
|
||||||
|
b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() }
|
||||||
|
filledNow := func() string {
|
||||||
|
for _, c := range b.ag.Everything() {
|
||||||
|
if c.Module == "power" {
|
||||||
|
if c.Data != tc.data {
|
||||||
|
t.Errorf("what runs says Data %v, want %v", c.Data, tc.data)
|
||||||
|
}
|
||||||
|
return strings.Join(c.Filled, ",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if tc.told == "" { // no want, no have: nothing to start, and the kept declaration is what runs
|
||||||
a := New(b.env)
|
a := New(b.env)
|
||||||
got := a.Everything()
|
b.ag = a
|
||||||
if len(got) != 1 || got[0].Module != "claude-code" || strings.Join(got[0].Filled, ",") != "/tmp/r6/claude-code/settings" {
|
if got := filledNow(); got != tc.filled {
|
||||||
t.Fatalf("what runs from the kept declaration: %+v", got)
|
t.Fatalf("filled %q, want %q", got, tc.filled)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stop := b.run()
|
||||||
|
defer stop()
|
||||||
|
eventually(t, "the filling was not told what runs at the start", func() bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return len(told) > 0 && told[0] == tc.told
|
||||||
|
})
|
||||||
|
if got := filledNow(); got != tc.filled {
|
||||||
|
t.Fatalf("filled %q, want %q", got, tc.filled)
|
||||||
|
}
|
||||||
|
if got := b.appliedList(); len(got) != 0 {
|
||||||
|
t.Fatalf("applied %v: nothing has a want", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1054,6 +1378,19 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
|||||||
}
|
}
|
||||||
return catalogue.PolicyRoll
|
return catalogue.PolicyRoll
|
||||||
}
|
}
|
||||||
|
// The bucket's revisions of each key, as the verb composition reads them (issue 774).
|
||||||
|
var historyMu sync.Mutex
|
||||||
|
history := map[string][]Update{}
|
||||||
|
b.env.Revisions = func(_ context.Context, key string) ([]Update, error) {
|
||||||
|
historyMu.Lock()
|
||||||
|
defer historyMu.Unlock()
|
||||||
|
return append([]Update(nil), history[key]...), nil
|
||||||
|
}
|
||||||
|
keep := func(key string, raw []byte) {
|
||||||
|
historyMu.Lock()
|
||||||
|
defer historyMu.Unlock()
|
||||||
|
history[key] = append(history[key], Update{Key: key, Value: raw, Revision: uint64(len(history[key]) + 1)})
|
||||||
|
}
|
||||||
stop := b.run()
|
stop := b.run()
|
||||||
defer stop()
|
defer stop()
|
||||||
unsigned := func(module, fp string, rs ...json.RawMessage) (string, []byte, string) {
|
unsigned := func(module, fp string, rs ...json.RawMessage) (string, []byte, string) {
|
||||||
@@ -1192,10 +1529,45 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
|
|||||||
raw, _ := json.Marshal(map[string]any{"id": module + ".server", "type": "container", "module": module, "name": module, "image": image})
|
raw, _ := json.Marshal(map[string]any{"id": module + ".server", "type": "container", "module": module, "name": module, "image": image})
|
||||||
return raw
|
return raw
|
||||||
}
|
}
|
||||||
dk, draw, _ := unsigned("db", fp, container("db@sha256:"+strings.Repeat("1", 64)))
|
dk, draw, ddig := unsigned("db", fp, container("db@sha256:"+strings.Repeat("1", 64)))
|
||||||
b.comps.put(dk, draw)
|
b.comps.put(dk, draw)
|
||||||
want("db", fp, "")
|
want("db", fp, "")
|
||||||
eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 4 })
|
eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 4 })
|
||||||
|
// A revision at the same build that recreates the container waits for a yes; composed back to what runs, the have
|
||||||
|
// says running again, not waiting-yes for a move nobody needs (issue 774, 2026-10-31: ace.postgres, novox.postgres,
|
||||||
|
// novox.keycloak and novox.mongodb stayed waiting-yes after their providers' received files went back).
|
||||||
|
_, drawMoved, ddigMoved := unsigned("db", fp, container("db@sha256:"+strings.Repeat("7", 64)))
|
||||||
|
b.comps.put(dk, drawMoved)
|
||||||
|
eventually(t, "db's revision did not wait for a yes", func() bool {
|
||||||
|
h := b.haves.last("db")
|
||||||
|
return h.Phase == string(delivery.PhaseWaitingYes) && h.Waits == ddigMoved
|
||||||
|
})
|
||||||
|
// The verb composition says why it waits: the revision that waits, what runs, the judgement, and with diff the
|
||||||
|
// container it recreates; the revision that runs, asked by its digest, is no move (issue 774).
|
||||||
|
keep(dk, draw)
|
||||||
|
keep(dk, drawMoved)
|
||||||
|
view, err := b.ag.Composition(context.Background(), "db", "", true)
|
||||||
|
if err != nil || view.Have == nil || view.Have.Phase != string(delivery.PhaseWaitingYes) || view.Composition.Digest != ddigMoved ||
|
||||||
|
view.Runs == nil || view.Runs.Composed != ddig || view.Judged.Judgement != string(delivery.JudgedRecreates) ||
|
||||||
|
!view.Judged.WaitsForYes || len(view.Diff) != 1 || view.Diff[0].ID != "db.server" || view.Diff[0].Change != "changed" ||
|
||||||
|
len(view.Revisions) != 2 {
|
||||||
|
t.Errorf("the composition of db waiting for a yes reads %+v (%v)", view, err)
|
||||||
|
}
|
||||||
|
if view, err := b.ag.Composition(context.Background(), "db", ddig, false); err != nil || view.Composition.Digest != ddig ||
|
||||||
|
view.Judged.Judgement != string(delivery.JudgedUnchanged) || view.Judged.WaitsForYes || view.Diff != nil {
|
||||||
|
t.Errorf("db's running revision, asked by its digest, reads %+v (%v)", view, err)
|
||||||
|
}
|
||||||
|
if _, err := b.ag.Composition(context.Background(), "db", "sha256:"+strings.Repeat("0", 64), false); err == nil {
|
||||||
|
t.Error("a digest no revision has was answered")
|
||||||
|
}
|
||||||
|
b.comps.put(dk, draw)
|
||||||
|
eventually(t, "db's have stays waiting-yes after its composition went back to what runs", func() bool {
|
||||||
|
h := b.haves.last("db")
|
||||||
|
return h.Phase == string(delivery.PhaseRunning) && h.Composed == ddig && h.Waits == ""
|
||||||
|
})
|
||||||
|
if got := len(b.appliedList()); got != 4 {
|
||||||
|
t.Fatalf("a composition equal to what runs was applied again: %d applies", got)
|
||||||
|
}
|
||||||
fpB := fpOf("b")
|
fpB := fpOf("b")
|
||||||
dk2, draw2, ddig2 := unsigned("db", fpB, container("db@sha256:"+strings.Repeat("2", 64)))
|
dk2, draw2, ddig2 := unsigned("db", fpB, container("db@sha256:"+strings.Repeat("2", 64)))
|
||||||
// rec is a record module this agent has run nothing of since its start (the switch and every delivery of the
|
// rec is a record module this agent has run nothing of since its start (the switch and every delivery of the
|
||||||
@@ -1327,6 +1699,11 @@ func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone(
|
|||||||
if got := b.appliedList(); len(got) != 0 {
|
if got := b.appliedList(); len(got) != 0 {
|
||||||
t.Fatalf("applied at the start: %v", got)
|
t.Fatalf("applied at the start: %v", got)
|
||||||
}
|
}
|
||||||
|
// Held, it keeps naming what runs, which this agent knows only from the have (postgres on novox, 2026-10-31): not
|
||||||
|
// the build it was to move to, and never nothing.
|
||||||
|
if h := b.haves.last("db"); h.FP != fpOf("0") || h.Composed != "sha256:"+strings.Repeat("0", 64) {
|
||||||
|
t.Fatalf("the held have does not name what runs: %+v", h)
|
||||||
|
}
|
||||||
|
|
||||||
b.ag.Reconcile("db", "ghost")
|
b.ag.Reconcile("db", "ghost")
|
||||||
time.Sleep(50 * time.Millisecond)
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node-engine seat's read verb composition (issue 774): one module's composition beside what runs of it, so why a
|
||||||
|
// record module waits for a yes, or what its next apply would change, is read from the engine and not guessed from its
|
||||||
|
// have's words. It changes nothing.
|
||||||
|
|
||||||
|
// CompositionView is what the verb composition answers.
|
||||||
|
type CompositionView struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
// Want is the module's want as the agent holds it; nil when it holds none.
|
||||||
|
Want *WantView `json:"want,omitempty"`
|
||||||
|
// Have is what the module's have key says now; nil when it says nothing.
|
||||||
|
Have *HaveView `json:"have,omitempty"`
|
||||||
|
// Composition is the revision read: the latest of the want's build, or the one whose digest was asked.
|
||||||
|
Composition RevisionView `json:"composition"`
|
||||||
|
// Revisions are the revisions of its key the bucket keeps, oldest first, each with its digest.
|
||||||
|
Revisions []RevisionView `json:"revisions,omitempty"`
|
||||||
|
// Runs is what runs of the module, and where that is known from; nil when nothing of it runs.
|
||||||
|
Runs *RunsView `json:"runs,omitempty"`
|
||||||
|
// Judged is the move from what runs to the composition read, as the agent judges it before an apply.
|
||||||
|
Judged JudgedView `json:"judged"`
|
||||||
|
// Diff is every resource that differs from what runs, when asked.
|
||||||
|
Diff []delivery.ResourceChange `json:"diff,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// WantView is a want as the verb shows it.
|
||||||
|
type WantView struct {
|
||||||
|
FP string `json:"fp"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
Yes string `json:"yes,omitempty"`
|
||||||
|
Removed bool `json:"removed,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HaveView is a have as the verb shows it.
|
||||||
|
type HaveView struct {
|
||||||
|
FP string `json:"fp,omitempty"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
Phase string `json:"phase"`
|
||||||
|
Detail string `json:"detail,omitempty"`
|
||||||
|
Judgement string `json:"judgement,omitempty"`
|
||||||
|
Waits string `json:"waits,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RevisionView is one revision of a composition key.
|
||||||
|
type RevisionView struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Revision uint64 `json:"revision,omitempty"`
|
||||||
|
Digest string `json:"digest,omitempty"`
|
||||||
|
Resources int `json:"resources,omitempty"`
|
||||||
|
Latest bool `json:"latest,omitempty"`
|
||||||
|
// Refused is why the revision cannot be read or trusted.
|
||||||
|
Refused string `json:"refused,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunsView is what runs of a module.
|
||||||
|
type RunsView struct {
|
||||||
|
FP string `json:"fp,omitempty"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
Phase string `json:"phase,omitempty"`
|
||||||
|
// From is applied (what this engine applied), have (what its have named at the start) or kept (the declaration the
|
||||||
|
// node kept before the switch).
|
||||||
|
From string `json:"from"`
|
||||||
|
Resources int `json:"resources"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// JudgedView is a move's judgement.
|
||||||
|
type JudgedView struct {
|
||||||
|
Judgement string `json:"judgement"`
|
||||||
|
Words string `json:"words"`
|
||||||
|
// WaitsForYes says the agent would wait for a person's yes before applying it: a module of policy record, a move that
|
||||||
|
// recreates a container, moves data or cannot be judged, and no yes in its want for this digest.
|
||||||
|
WaitsForYes bool `json:"waits_for_yes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Composition reads one module's composition beside what runs of it: the want's build (else the have's), its latest
|
||||||
|
// revision or the one whose digest is given, and the move's judgement; with diff, the resources that differ.
|
||||||
|
func (a *Agent) Composition(ctx context.Context, module, digest string, diff bool) (CompositionView, error) {
|
||||||
|
if module == "" {
|
||||||
|
return CompositionView{}, errors.New("name the module, or node for the node's own composition")
|
||||||
|
}
|
||||||
|
view := CompositionView{Node: a.env.Node, Module: module}
|
||||||
|
var have *Have
|
||||||
|
if a.env.Haves != nil {
|
||||||
|
if h, ok := a.env.Haves.Read(ctx, module); ok {
|
||||||
|
have = &h
|
||||||
|
phase := h.Phase
|
||||||
|
if phase == "" {
|
||||||
|
phase = string(delivery.PhaseRunning)
|
||||||
|
}
|
||||||
|
view.Have = &HaveView{FP: h.FP, Composed: h.Composed, Phase: phase, Detail: h.Detail, Judgement: h.Judgement, Waits: h.Waits}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
w := a.wants[module]
|
||||||
|
fp := ""
|
||||||
|
if w.Present {
|
||||||
|
fp = w.Want.FP
|
||||||
|
view.Want = &WantView{FP: w.Want.FP, Composed: w.Want.Composed, Yes: w.Want.Yes, Removed: w.Want.Removed}
|
||||||
|
} else if have != nil {
|
||||||
|
fp = have.FP
|
||||||
|
}
|
||||||
|
latest, haveLatest := a.compositionFor(module, fp)
|
||||||
|
r, ran := a.running[module]
|
||||||
|
from, known := a.runningComposition(module)
|
||||||
|
source := "applied"
|
||||||
|
if _, ok := a.applied[module]; !ok {
|
||||||
|
if _, ok := a.had[module]; ok {
|
||||||
|
source = "have"
|
||||||
|
} else {
|
||||||
|
source = "kept"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.mu.Unlock()
|
||||||
|
if module != NodeModule && fp == "" {
|
||||||
|
return view, fmt.Errorf("%s has no want and no have on %s, so no build of it is composed here", module, a.env.Node)
|
||||||
|
}
|
||||||
|
key, err := a.compositionKey(module, fp)
|
||||||
|
if err != nil {
|
||||||
|
return view, err
|
||||||
|
}
|
||||||
|
revs, err := a.revisionsOf(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return view, fmt.Errorf("the revisions of %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
var read *compNow
|
||||||
|
for i := range revs {
|
||||||
|
c := revs[i]
|
||||||
|
rv := RevisionView{Key: key, Revision: c.Rev, Digest: c.Dig, Resources: len(c.C.Resources),
|
||||||
|
Latest: haveLatest && c.Rev == latest.Rev}
|
||||||
|
if c.Err != nil {
|
||||||
|
rv.Refused = c.Err.Error()
|
||||||
|
}
|
||||||
|
view.Revisions = append(view.Revisions, rv)
|
||||||
|
if digest != "" && c.Err == nil && c.Dig == digest {
|
||||||
|
read = &revs[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case digest != "" && read == nil && haveLatest && latest.Err == nil && latest.Dig == digest:
|
||||||
|
read = &latest
|
||||||
|
case digest != "" && read == nil:
|
||||||
|
return view, fmt.Errorf("no revision of %s the bucket keeps has the digest %s; revisions lists those it keeps", key, digest)
|
||||||
|
case read == nil && !haveLatest:
|
||||||
|
return view, fmt.Errorf("the bucket holds no composition of %s at %s", module, fp)
|
||||||
|
case read == nil:
|
||||||
|
read = &latest
|
||||||
|
}
|
||||||
|
view.Composition = RevisionView{Key: key, Revision: read.Rev, Digest: read.Dig, Resources: len(read.C.Resources),
|
||||||
|
Latest: haveLatest && read.Rev == latest.Rev}
|
||||||
|
if read.Err != nil {
|
||||||
|
view.Composition.Refused = read.Err.Error()
|
||||||
|
return view, nil
|
||||||
|
}
|
||||||
|
var running *delivery.Composed
|
||||||
|
if known {
|
||||||
|
running = &from
|
||||||
|
rv := &RunsView{From: source, Resources: len(from.Resources), FP: from.FP}
|
||||||
|
if ran {
|
||||||
|
rv.FP, rv.Composed, rv.Phase = r.FP, r.Composed, string(r.Phase)
|
||||||
|
}
|
||||||
|
view.Runs = rv
|
||||||
|
}
|
||||||
|
j := delivery.JudgeMove(running, read.C)
|
||||||
|
view.Judged = JudgedView{Judgement: string(j.Judgement), Words: j.Words}
|
||||||
|
if a.env.SelfComposed && a.env.Policy != nil && module != NodeModule && (!w.Present || w.Want.Yes != read.Dig) {
|
||||||
|
switch j.Judgement {
|
||||||
|
case delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot:
|
||||||
|
view.Judged.WaitsForYes = a.env.Policy(ctx, module) == catalogue.PolicyRecord
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if diff {
|
||||||
|
view.Diff = delivery.ChangedResources(running, read.C)
|
||||||
|
}
|
||||||
|
return view, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// compositionKey is a module's composition key at a build, or the node's own.
|
||||||
|
func (a *Agent) compositionKey(module, fp string) (string, error) {
|
||||||
|
if module == NodeModule {
|
||||||
|
return delivery.ComposedNodeKey(a.env.Node)
|
||||||
|
}
|
||||||
|
return delivery.ComposedKey(a.env.Node, module, fp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// revisionsOf is every revision of a composition key the bucket keeps, oldest first, each with its digest as this node
|
||||||
|
// trusts it (its own unsigned with SelfComposed, else signed by a trusted key), or why it is refused. Nil Revisions
|
||||||
|
// reads none.
|
||||||
|
func (a *Agent) revisionsOf(ctx context.Context, key string) ([]compNow, error) {
|
||||||
|
if a.env.Revisions == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
revs, err := a.env.Revisions(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var trusted []ed25519.PublicKey
|
||||||
|
if a.env.Trusted != nil {
|
||||||
|
trusted = a.env.Trusted()
|
||||||
|
}
|
||||||
|
var out []compNow
|
||||||
|
for _, u := range revs {
|
||||||
|
if u.Deleted || u.Key != key {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c, err := delivery.DecodeComposed(u.Value)
|
||||||
|
if err != nil {
|
||||||
|
out = append(out, compNow{Raw: u.Value, Rev: u.Revision, Err: err})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var dig string
|
||||||
|
if a.env.SelfComposed {
|
||||||
|
dig, err = delivery.ComposedDigestOf(c) // the node's own: unsigned (ADR 0547 §2)
|
||||||
|
} else {
|
||||||
|
dig, err = delivery.VerifyComposed(c, trusted)
|
||||||
|
}
|
||||||
|
out = append(out, compNow{Raw: u.Value, C: c, Dig: dig, Rev: u.Revision, Err: err})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
+105
-1
@@ -312,11 +312,54 @@ type Scope struct {
|
|||||||
Module string
|
Module string
|
||||||
// Others are the resources of every other module the node has a composition for.
|
// Others are the resources of every other module the node has a composition for.
|
||||||
Others []declaration.Resource
|
Others []declaration.Resource
|
||||||
|
// Placed are the modules a want still places on the node (present, not removed), held ones included: no apply but
|
||||||
|
// the module's own removes a resource of one of them (issue 774: on 2026-10-29 the node's own apply on the bus node,
|
||||||
|
// composed without what its folded modules ran, removed the bus server and four held modules). Nil guards nothing.
|
||||||
|
Placed map[string]bool
|
||||||
|
// RemovedByWant are the modules whose want says removed. In the node's own apply (Module NodeModule) with this set
|
||||||
|
// non-nil, a resource of any other module is removed only when that module is here: a module the node runs folded
|
||||||
|
// in its own composition has no want, so Placed does not name it (issue 774, review of the re-land, round 1: nats on
|
||||||
|
// the bus node has none, and its nats.conf was guessed as nats's and removed). Nil guards by Placed alone.
|
||||||
|
RemovedByWant map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// placedOwner is the module a want still places on the node that the record belongs to, other than the scope's own:
|
||||||
|
// the module recorded with it, else the one its id names (a resource the node's own composition carried for a folded
|
||||||
|
// module is recorded as the node's, its id naming the module). Empty when there is none.
|
||||||
|
func (sc *Scope) placedOwner(a store.Applied) string {
|
||||||
|
if sc == nil || (len(sc.Placed) == 0 && sc.RemovedByWant == nil) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
m := a.Module
|
||||||
|
if m == "" || m == store.NodeModule {
|
||||||
|
m, _ = declaration.ModuleGuess(a.ID)
|
||||||
|
}
|
||||||
|
if m == "" || m == sc.Module || m == store.NodeModule {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if sc.Placed[m] {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
if sc.Module == store.NodeModule && sc.RemovedByWant != nil && !sc.RemovedByWant[m] {
|
||||||
|
return m // a module the node's own composition carried, which no want removed
|
||||||
|
}
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// owns reports whether a record belongs to the scope's module.
|
// owns reports whether a record belongs to the scope's module.
|
||||||
|
//
|
||||||
|
// In a module's own apply, a record the node's own composition made for the module while it was folded (recorded as the
|
||||||
|
// node's, its id naming the module) is the module's too: it is the only apply that may remove it once neither
|
||||||
|
// composition declares it (issue 774). One the node's own still declares stays, as another's held resource does.
|
||||||
func (sc *Scope) owns(a store.Applied) bool {
|
func (sc *Scope) owns(a store.Applied) bool {
|
||||||
return store.OwnerOf(a, declaration.ModuleGuess) == sc.Module
|
if store.OwnerOf(a, declaration.ModuleGuess) == sc.Module {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if sc.Module == store.NodeModule || a.Module != store.NodeModule {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
m, ok := declaration.ModuleGuess(a.ID)
|
||||||
|
return ok && m == sc.Module
|
||||||
}
|
}
|
||||||
|
|
||||||
// isNode is whether the scope is the node's own composition, the one that carries identities, groups, libraries and
|
// isNode is whether the scope is the node's own composition, the one that carries identities, groups, libraries and
|
||||||
@@ -441,6 +484,57 @@ func applyScoped(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **A resource the node's own carried for a module that now has a composition of its own is that module's to remove**
|
||||||
|
// (issue 774, 2026-10-30 on the bus node): recorded as the node's while the module was folded, it is no longer in the
|
||||||
|
// node's own composition and no longer in the module's own either (restic's measure, replaced by awake). The node's own
|
||||||
|
// apply leaves it, record and all, and says so; the module's own apply sees it as its own (Scope.owns) and removes it.
|
||||||
|
// Refused instead, the node's own applied nothing at all, every two minutes, for as long as the record stayed.
|
||||||
|
takenByItsModule := map[string]string{}
|
||||||
|
if scope != nil && scope.Module == store.NodeModule {
|
||||||
|
composes := map[string]bool{}
|
||||||
|
for _, r := range scope.Others {
|
||||||
|
if m := r.DeclaredModule(); m != "" && m != store.NodeModule {
|
||||||
|
composes[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, orphan := range known.Orphans(declared, origin) {
|
||||||
|
if orphan.Module != store.NodeModule {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m, ok := declaration.ModuleGuess(orphan.ID); ok && composes[m] {
|
||||||
|
takenByItsModule[orphan.ID] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A module a want still places here is never removed by another's apply** (issue 774): its resources go only with
|
||||||
|
// its own apply or its want's removal. One that would is refused whole, before anything is changed, naming them: a
|
||||||
|
// composition that leaves them out lost what runs, and removing what runs is the worst a node can do with that.
|
||||||
|
if scope != nil {
|
||||||
|
var refused []string
|
||||||
|
for _, orphan := range known.Orphans(declared, origin) {
|
||||||
|
if !scope.owns(orphan) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, kept := heldByOthers[orphan.ID]; kept {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, kept := takenByItsModule[orphan.ID]; kept {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m := scope.placedOwner(orphan); m != "" {
|
||||||
|
refused = append(refused, orphan.ID+" of "+m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(refused) > 0 {
|
||||||
|
sort.Strings(refused)
|
||||||
|
err := fmt.Errorf("the apply of %s would remove %s, of modules a want still places here; a module's resources "+
|
||||||
|
"go only with its own apply or its want's removal, so nothing was applied", scope.Module, strings.Join(refused, ", "))
|
||||||
|
log(" refused: " + err.Error())
|
||||||
|
return report, known, &Error{Resource: "the resources of modules placed here", Err: err, Done: report}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
removeOrphan := func(orphan store.Applied) error {
|
removeOrphan := func(orphan store.Applied) error {
|
||||||
var action, detail string
|
var action, detail string
|
||||||
var err error
|
var err error
|
||||||
@@ -456,6 +550,16 @@ func applyScoped(
|
|||||||
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, detail))
|
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, detail))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if module, taken := takenByItsModule[orphan.ID]; taken {
|
||||||
|
detail := "no longer declared here; " + module + " has a composition of its own here, whose apply removes it, " +
|
||||||
|
"so it was left as it is"
|
||||||
|
report.Outcomes = append(report.Outcomes, Outcome{
|
||||||
|
ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
|
||||||
|
Action: "kept", Detail: detail,
|
||||||
|
})
|
||||||
|
log(fmt.Sprintf(" kept %s (%s): %s", orphan.ID, orphan.Target, detail))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if by, held := heldFiles[fileKey(orphan)]; held && by != orphan.ID {
|
if by, held := heldFiles[fileKey(orphan)]; held && by != orphan.ID {
|
||||||
if scope != nil && !declared[by] {
|
if scope != nil && !declared[by] {
|
||||||
// Another module declares the path: it is that module's, so this record is let go and the file is
|
// Another module declares the path: it is that module's, so this record is let go and the file is
|
||||||
|
|||||||
@@ -45,6 +45,30 @@ func WriteFilled(
|
|||||||
log func(string),
|
log func(string),
|
||||||
keep Keep,
|
keep Keep,
|
||||||
windows *Windows,
|
windows *Windows,
|
||||||
|
) (report Report, _ store.State, _ error) {
|
||||||
|
return WriteFilledWith(ctx, sys, d, known, path, content, run, log, keep, windows, Filled{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filled is what the node-engine did to a filled file's content before its write.
|
||||||
|
type Filled struct {
|
||||||
|
// Secret says the node-engine filled a value into the content, a provision's `${secret:}` (novox/hq ADR 0570
|
||||||
|
// clause 6): the file is then written 0600 when the declaration gives no mode, as a file carrying a secret is.
|
||||||
|
Secret bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteFilledWith is WriteFilled for content the node-engine filled before the write (Filled).
|
||||||
|
func WriteFilledWith(
|
||||||
|
ctx context.Context,
|
||||||
|
sys system.System,
|
||||||
|
d *declaration.Declaration,
|
||||||
|
known store.State,
|
||||||
|
path string,
|
||||||
|
content []byte,
|
||||||
|
run Runner,
|
||||||
|
log func(string),
|
||||||
|
keep Keep,
|
||||||
|
windows *Windows,
|
||||||
|
filled Filled,
|
||||||
) (report Report, _ store.State, _ error) {
|
) (report Report, _ store.State, _ error) {
|
||||||
if log == nil {
|
if log == nil {
|
||||||
log = func(string) {}
|
log = func(string) {}
|
||||||
@@ -65,7 +89,7 @@ func WriteFilled(
|
|||||||
return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
|
return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
|
||||||
"is written for exactly one, so nothing was written", len(declared), path)}
|
"is written for exactly one, so nothing was written", len(declared), path)}
|
||||||
}
|
}
|
||||||
file := filledAs(declared[0], content)
|
file := filledAs(declared[0], content, filled.Secret)
|
||||||
if held, isHeld := known.HeldAt(file.ID); isHeld {
|
if held, isHeld := known.HeldAt(file.ID); isHeld {
|
||||||
return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
|
return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
|
||||||
"yet taken (%s), so nothing was written", path, held.Why)}
|
"yet taken (%s), so nothing was written", path, held.Why)}
|
||||||
@@ -163,10 +187,10 @@ func WriteFilled(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// filledAs is the declared file with the filled content in place of the declaration's.
|
// filledAs is the declared file with the filled content in place of the declaration's.
|
||||||
func filledAs(declared *declaration.File, content []byte) *declaration.File {
|
func filledAs(declared *declaration.File, content []byte, secret bool) *declaration.File {
|
||||||
f := *declared
|
f := *declared
|
||||||
f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
|
f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
|
||||||
if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) {
|
if f.Mode == "" && (secret || declared.Secret() || len(declared.Secrets) > 0) {
|
||||||
// Written as the declaration would have written it: a file carrying a credential is not world-readable.
|
// Written as the declaration would have written it: a file carrying a credential is not world-readable.
|
||||||
f.Mode = "0600"
|
f.Mode = "0600"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package apply
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
@@ -116,32 +117,107 @@ func TestARecordWithoutAnOwnerBelongsToTheModuleItsIdNames(t *testing.T) {
|
|||||||
// leaves it as it is, whatever its type, until the module's own apply takes it (issue 1945, live on 2026-10-25 at 02:32:
|
// leaves it as it is, whatever its type, until the module's own apply takes it (issue 1945, live on 2026-10-25 at 02:32:
|
||||||
// shanks and g14 lost systemd-resolved's guard, its resolver files and sudo's operator file, ace qbittorrent's server,
|
// shanks and g14 lost systemd-resolved's guard, its resolver files and sudo's operator file, ace qbittorrent's server,
|
||||||
// because the node's own apply removed every record no longer in it, the others holding the very same resources or not).
|
// because the node's own apply removed every record no longer in it, the others holding the very same resources or not).
|
||||||
|
//
|
||||||
|
// And a resource the node's own composition carried for a module a want still places here (a folded module of the old
|
||||||
|
// form, held) is never removed by the node's own apply when a composition leaves it out: the apply is refused whole and
|
||||||
|
// says why (issue 774, live on 2026-10-29 at 07:54 UTC on the bus node: the node's own composition, composed without what
|
||||||
|
// it ran once the kept declaration was read no more, removed the bus server, the proxy and three held modules). Only its
|
||||||
|
// want's removal lets it go.
|
||||||
func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *testing.T) {
|
func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
id string // the resource the node's own composition carried, and no longer does
|
||||||
|
moved bool // its module's own composition now declares it (Others)
|
||||||
|
// dropped: its module has a composition of its own here that no longer declares it (Others holds another of its
|
||||||
|
// resources); the node's own apply leaves it, and the module's own apply removes it.
|
||||||
|
dropped bool
|
||||||
|
placed bool // a want still places its module here
|
||||||
|
refused bool // the node's own apply is refused whole
|
||||||
|
stays bool // the resource is still on the machine and recorded
|
||||||
|
// wants names the modules whose want says removed (Scope.RemovedByWant); nil leaves the guard to Placed alone.
|
||||||
|
wants []string
|
||||||
|
}{
|
||||||
|
{name: "moved to its module's own composition", id: "systemd-resolved.fact-suffix", moved: true, placed: true, stays: true},
|
||||||
|
{name: "a held folded module, in no composition", id: "nats.server-conf", placed: true, refused: true, stays: true},
|
||||||
|
{name: "its module's want removed", id: "nats.server-conf"},
|
||||||
|
// Issue 774, review of the re-land, round 1: a module folded in the node's own has no want, so Placed does not name
|
||||||
|
// it; in the node's own apply only a module a want removed loses its resources.
|
||||||
|
{name: "a folded module with no want", id: "nats.server-conf", wants: []string{}, refused: true, stays: true},
|
||||||
|
{name: "a folded module a want removed", id: "nats.server-conf", wants: []string{"nats", "old"}},
|
||||||
|
// 2026-10-30 19:42 on the bus node: restic's manifest replaced its process measure by awake. Its record was the
|
||||||
|
// node's, carried when restic was folded, so restic's own apply never saw it and the node's own was refused
|
||||||
|
// whole every two minutes, applying nothing of the node's own.
|
||||||
|
{name: "its module's own composition dropped it", id: "restic.measure", dropped: true, placed: true,
|
||||||
|
wants: []string{"old"}, stays: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
fact := filepath.Join(dir, "suffix")
|
fact := filepath.Join(dir, "carried")
|
||||||
gone := filepath.Join(dir, "gone.conf")
|
gone := filepath.Join(dir, "gone.conf")
|
||||||
own := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n"},
|
own := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n"},
|
||||||
{"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`)
|
{"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`)
|
||||||
_, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule},
|
_, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule},
|
||||||
state0(), store.OriginDeclared, noServices, nil, nil, nil, nil)
|
state0(), store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// systemd-resolved now has a composition of its own, not applied yet; old.gone is no module's any more.
|
module, _ := declaration.ModuleGuess(tc.id)
|
||||||
moved := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n","module":"systemd-resolved"}`)
|
scope := Scope{Module: store.NodeModule}
|
||||||
report, state, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{},
|
if tc.moved {
|
||||||
Scope{Module: store.NodeModule, Others: moved.Resources}, state, store.OriginDeclared, noServices, nil, nil, nil, nil)
|
moved := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n","module":"`+module+`"}`)
|
||||||
|
scope.Others = moved.Resources
|
||||||
|
}
|
||||||
|
if tc.dropped {
|
||||||
|
later := declare(t, `{"id":"`+module+`.awake","type":"file","path":"`+filepath.Join(dir, "awake")+
|
||||||
|
`","content":"a\n","module":"`+module+`"}`)
|
||||||
|
scope.Others = later.Resources
|
||||||
|
}
|
||||||
|
if tc.placed {
|
||||||
|
scope.Placed = map[string]bool{module: true}
|
||||||
|
}
|
||||||
|
if tc.wants != nil {
|
||||||
|
scope.RemovedByWant = map[string]bool{}
|
||||||
|
for _, m := range tc.wants {
|
||||||
|
scope.RemovedByWant[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
report, after, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, state,
|
||||||
|
store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||||
|
if tc.refused != (err != nil) {
|
||||||
|
t.Fatalf("refused %v, want %v: %v", err != nil, tc.refused, err)
|
||||||
|
}
|
||||||
|
if tc.refused {
|
||||||
|
if !strings.Contains(err.Error(), tc.id+" of "+module) {
|
||||||
|
t.Fatalf("the refusal does not name what it kept: %v", err)
|
||||||
|
}
|
||||||
|
after = state
|
||||||
|
}
|
||||||
|
if exists(fact) != tc.stays {
|
||||||
|
t.Fatalf("the carried resource exists %v, want %v: %+v", exists(fact), tc.stays, outcomeOf(report, tc.id))
|
||||||
|
}
|
||||||
|
if _, still := after.Find(tc.id); still != tc.stays {
|
||||||
|
t.Fatalf("the carried resource's record kept %v, want %v", still, tc.stays)
|
||||||
|
}
|
||||||
|
if exists(gone) == !tc.refused {
|
||||||
|
t.Fatalf("a resource no composition holds any more exists %v, with the apply refused %v", exists(gone), tc.refused)
|
||||||
|
}
|
||||||
|
if tc.dropped {
|
||||||
|
if o := outcomeOf(report, tc.id); o.Action != "kept" {
|
||||||
|
t.Fatalf("the node's own apply did not say it left it: %+v", o)
|
||||||
|
}
|
||||||
|
later := declare(t, `{"id":"`+module+`.awake","type":"file","path":"`+filepath.Join(dir, "awake")+
|
||||||
|
`","content":"a\n","module":"`+module+`"}`)
|
||||||
|
_, last, err := ApplyModule(context.Background(), archHost(t), later, Scope{Module: module}, after,
|
||||||
|
store.OriginDeclared, noServices, nil, nil, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !exists(fact) {
|
if _, still := last.Find(tc.id); still || exists(fact) {
|
||||||
t.Fatalf("the moved resource was removed: %+v", outcomeOf(report, "systemd-resolved.fact-suffix"))
|
t.Fatalf("the module's own apply left what it no longer declares: recorded %v, on the machine %v",
|
||||||
|
still, exists(fact))
|
||||||
}
|
}
|
||||||
if exists(gone) {
|
|
||||||
t.Fatal("a resource no composition holds any more was kept")
|
|
||||||
}
|
}
|
||||||
if _, still := state.Find("systemd-resolved.fact-suffix"); !still {
|
})
|
||||||
t.Fatal("the moved resource's record was dropped, so its module's apply would not know it")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package apply
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one
|
||||||
|
// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for
|
||||||
|
// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the
|
||||||
|
// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account
|
||||||
|
// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back.
|
||||||
|
//
|
||||||
|
// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and
|
||||||
|
// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy.
|
||||||
|
type ObjectKeep struct {
|
||||||
|
Dir string
|
||||||
|
// Now is the clock; nil is time.Now.
|
||||||
|
Now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeptObjectsFor is how long an object of the keep is held after it was last used.
|
||||||
|
const KeptObjectsFor = 14 * 24 * time.Hour
|
||||||
|
|
||||||
|
// KeepObjectsIn is the keep at dir; an empty dir keeps nothing.
|
||||||
|
func KeepObjectsIn(dir string) *ObjectKeep {
|
||||||
|
if dir == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &ObjectKeep{Dir: dir}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *ObjectKeep) now() time.Time {
|
||||||
|
if k.Now != nil {
|
||||||
|
return k.Now()
|
||||||
|
}
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// get is the kept content of the object name when it is what f names, and marks it used.
|
||||||
|
func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) {
|
||||||
|
if k == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
path := filepath.Join(k.Dir, name)
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
now := k.now()
|
||||||
|
_ = os.Chtimes(path, now, now)
|
||||||
|
return body, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long.
|
||||||
|
func (k *ObjectKeep) put(name string, body []byte) {
|
||||||
|
if k == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
path := filepath.Join(k.Dir, name)
|
||||||
|
now := k.now()
|
||||||
|
if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) {
|
||||||
|
_ = os.Chtimes(path, now, now)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(k.Dir, 0o700); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
tmp, err := os.CreateTemp(k.Dir, ".put-*")
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, werr := tmp.Write(body)
|
||||||
|
cerr := tmp.Close()
|
||||||
|
if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil {
|
||||||
|
_ = os.Remove(tmp.Name())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = os.Chtimes(path, now, now)
|
||||||
|
k.prune(now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// prune lets go of every object unused for KeptObjectsFor.
|
||||||
|
func (k *ObjectKeep) prune(now time.Time) {
|
||||||
|
entries, err := os.ReadDir(k.Dir)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
info, err := e.Info()
|
||||||
|
if err != nil || !info.Mode().IsRegular() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if now.Sub(info.ModTime()) >= KeptObjectsFor {
|
||||||
|
_ = os.Remove(filepath.Join(k.Dir, e.Name()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own
|
||||||
|
// keep did not hold the object either.
|
||||||
|
func (k *ObjectKeep) never() string {
|
||||||
|
if k == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir)
|
||||||
|
}
|
||||||
@@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er
|
|||||||
// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
|
// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
|
||||||
// once however many files name its digest.
|
// once however many files name its digest.
|
||||||
func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
|
func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
|
||||||
|
return ResolveFileReferencesKept(ctx, d, fetch, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had
|
||||||
|
// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node
|
||||||
|
// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the
|
||||||
|
// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while
|
||||||
|
// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its
|
||||||
|
// own names objects the controller no longer knows of). A nil keep keeps nothing.
|
||||||
|
func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error {
|
||||||
fetched := map[string][]byte{}
|
fetched := map[string][]byte{}
|
||||||
for i, r := range d.Resources {
|
for i, r := range d.Resources {
|
||||||
f, ok := r.(*declaration.File)
|
f, ok := r.(*declaration.File)
|
||||||
if !ok || !f.IsReference() {
|
if !ok || !f.IsReference() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
body, err := referencedContent(ctx, f, fetch, fetched)
|
body, err := referencedContent(ctx, f, fetch, fetched, keep)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc
|
|||||||
}
|
}
|
||||||
|
|
||||||
func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
|
func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
|
||||||
fetched map[string][]byte) ([]byte, error) {
|
fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) {
|
||||||
if body, ok := fetched[f.Digest]; ok {
|
if body, ok := fetched[f.Digest]; ok {
|
||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
if body, ok := alreadyThere(f); ok {
|
|
||||||
fetched[f.Digest] = body
|
|
||||||
return body, nil
|
|
||||||
}
|
|
||||||
name, err := declaration.ObjectName(f.Digest)
|
name, err := declaration.ObjectName(f.Digest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
|
return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
|
||||||
}
|
}
|
||||||
|
if body, ok := alreadyThere(f); ok {
|
||||||
|
fetched[f.Digest] = body
|
||||||
|
keep.put(name, body)
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
if body, ok := keep.get(name, f); ok {
|
||||||
|
fetched[f.Digest] = body
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
if fetch == nil {
|
if fetch == nil {
|
||||||
return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
|
return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
|
||||||
"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
|
"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
|
||||||
}
|
}
|
||||||
body, err := fetch(ctx, name, f.Size)
|
body, err := fetch(ctx, name, f.Size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w",
|
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w",
|
||||||
f.ID, f.Path, name, err)
|
f.ID, f.Path, name, keep.never(), err)
|
||||||
}
|
}
|
||||||
if err := f.Verify(body); err != nil {
|
if err := f.Verify(body); err != nil {
|
||||||
return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
|
return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
|
||||||
}
|
}
|
||||||
fetched[f.Digest] = body
|
fetched[f.Digest] = body
|
||||||
|
keep.put(name, body)
|
||||||
return body, nil
|
return body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,10 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a
|
// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a
|
||||||
@@ -68,6 +72,112 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
|
|||||||
if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" {
|
if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" {
|
||||||
t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given)
|
t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given)
|
||||||
}
|
}
|
||||||
|
// 774 (ADR 0563 §3): a hand-over asked through the node seat is the engine's own ask, performed only on the
|
||||||
|
// operator's warrant to it: never on the call, not on a warrant for another digest, once on a warrant heard twice,
|
||||||
|
// and not when the disk changed since it was asked.
|
||||||
|
t.Run("the node seat's hand-over acts only on the operator's warrant", func(t *testing.T) {
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
warrant func(asked asks.Ask, w asks.Warrant) asks.Warrant
|
||||||
|
diskAfter string
|
||||||
|
twice bool
|
||||||
|
done int
|
||||||
|
state string
|
||||||
|
}{
|
||||||
|
{name: "no warrant", done: 0, state: HandAsked},
|
||||||
|
{name: "the operator's yes", warrant: yes, done: 1, state: HandDone},
|
||||||
|
{name: "the yes heard twice", warrant: yes, twice: true, done: 1, state: HandDone},
|
||||||
|
{name: "a warrant for another digest", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||||
|
w = yes(a, w)
|
||||||
|
w.AskDigest = "sha256:another"
|
||||||
|
return w
|
||||||
|
}, done: 0, state: HandAsked},
|
||||||
|
{name: "a no", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||||
|
w = yes(a, w)
|
||||||
|
w.Option, w.Label, w.Level = "no", "No, leave it", asks.Acknowledge
|
||||||
|
return w
|
||||||
|
}, done: 0, state: HandRefused},
|
||||||
|
{name: "the disk changed", warrant: yes, diskAfter: "0:0 700 9:2", done: 0, state: HandRefused},
|
||||||
|
{name: "expired", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||||
|
return asks.Warrant{Ask: a.ID, Asker: w.Asker, Outcome: asks.OutcomeExpired, At: w.At}
|
||||||
|
}, done: 0, state: HandRefused},
|
||||||
|
} {
|
||||||
|
t.Run(row.name, func(t *testing.T) {
|
||||||
|
h, asked, done := anEngine(t)
|
||||||
|
disk := "1000:1000 755 9:1"
|
||||||
|
h.JudgeHandOver = func(path string) (Judged, error) {
|
||||||
|
return Judged{Subject: path, Path: path, Sets: "owner jo, group jo, mode 0750", Disk: disk,
|
||||||
|
Words: "A directory used as found."}, nil
|
||||||
|
}
|
||||||
|
if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "shanks.node-tools",
|
||||||
|
Args: map[string]any{"path": "/srv/x", "why": "a test"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 || *done != 0 {
|
||||||
|
t.Fatalf("the call asked %d and performed %d", len(*asked), *done)
|
||||||
|
}
|
||||||
|
// The act is one key, and the open ask one key made by a create (ADR 0563 §3).
|
||||||
|
if acts, _ := h.State.Keys("act."); len(acts) != 1 || acts[0] != "act."+(*asked)[0].ID {
|
||||||
|
t.Fatalf("the act is kept as %v", acts)
|
||||||
|
}
|
||||||
|
if open, _ := h.State.Keys("open."); len(open) != 1 || !strings.HasPrefix(open[0], "open.hand-over.") {
|
||||||
|
t.Fatalf("the open ask is kept as %v", open)
|
||||||
|
}
|
||||||
|
if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "x", Args: map[string]any{"path": "/srv/x"}}); err == nil {
|
||||||
|
t.Fatal("a second ask of the same path was asked while one is open")
|
||||||
|
}
|
||||||
|
if row.diskAfter != "" {
|
||||||
|
disk = row.diskAfter
|
||||||
|
}
|
||||||
|
if row.warrant != nil {
|
||||||
|
w := row.warrant((*asked)[0], asks.Warrant{Asker: asks.EngineAsker("n1"), At: time.Now()})
|
||||||
|
h.Heard(w)
|
||||||
|
if row.twice {
|
||||||
|
h.Heard(w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, _ := h.Acts(0)
|
||||||
|
acts := got.(map[string]any)["acts"].([]HandAct)
|
||||||
|
if *done != row.done || len(acts) != 1 || acts[0].State != row.state {
|
||||||
|
t.Fatalf("performed %d (want %d), acts %+v (want %s)", *done, row.done, acts, row.state)
|
||||||
|
}
|
||||||
|
if open, _ := h.State.Keys("open."); (len(open) == 1) != (row.state == HandAsked) {
|
||||||
|
t.Fatalf("an ask %s leaves its open key as %v", row.state, open)
|
||||||
|
}
|
||||||
|
if row.state == HandDone && (acts[0].By != "operator" || acts[0].Via == "" || acts[0].Ask == "") {
|
||||||
|
t.Fatalf("the act does not record the warrant: %+v", acts[0])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// yes is the operator's yes to an ask, as the router writes it.
|
||||||
|
func yes(a asks.Ask, w asks.Warrant) asks.Warrant {
|
||||||
|
w.Ask, w.Outcome, w.Option, w.Label, w.Level = a.ID, asks.OutcomeChosen, "yes", "Yes", asks.Approve
|
||||||
|
w.By = &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"}
|
||||||
|
w.AskDigest, w.Proofs = a.Digest(), []string{"P1"}
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// anEngine is a node seat with its own state in a temporary folder, a router that keeps what it was asked, and a
|
||||||
|
// hand-over that counts what it performed.
|
||||||
|
func anEngine(t *testing.T) (*HandActs, *[]asks.Ask, *int) {
|
||||||
|
t.Helper()
|
||||||
|
var asked []asks.Ask
|
||||||
|
done := 0
|
||||||
|
state := &MemState{}
|
||||||
|
h := &HandActs{Node: "n1", State: state,
|
||||||
|
HandOver: func(path, by string) (string, error) { done++; return path + " handed over by " + by, nil }}
|
||||||
|
h.Asker = &asker.Client{Module: asks.EngineAsker("n1"), State: state,
|
||||||
|
Publish: func(subject string, body any, id string) error {
|
||||||
|
if a, ok := body.(asks.Ask); ok && subject == asks.AskSubject(asks.EngineAsker("n1")) {
|
||||||
|
asked = append(asked, a)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Record: func(string) (*stdio.StateEntry, error) { return nil, nil }}
|
||||||
|
return h, &asked, &done
|
||||||
}
|
}
|
||||||
|
|
||||||
// The controller names an access by its declared resource id exactly, which may hold a `/`: it reaches the act,
|
// The controller names an access by its declared resource id exactly, which may hold a `/`: it reaches the act,
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ---- the data items of the modules placed on this node, in the bus's own storage (novox/hq ADR 0577 clause 1) ------
|
||||||
|
|
||||||
|
// DataBucket is the node-engine's declared state data (delivery.EngineDataBucket), created by the grants part from the
|
||||||
|
// engine's manifest as its states acts and health are: one key `<node>.<module>` per module placed on the node that
|
||||||
|
// declares data items, written by that node's engine alone, read by that node's disk-load.
|
||||||
|
const DataBucket = delivery.EngineDataBucket
|
||||||
|
|
||||||
|
// dataStore binds DataBucket over the link open now: bound, never created.
|
||||||
|
func (l *natsLink) dataStore() (nats.KeyValue, error) {
|
||||||
|
kv, err := l.js.KeyValue(DataBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+
|
||||||
|
"manifest; written again at the next composition that changes): %w", DataBucket, err)
|
||||||
|
}
|
||||||
|
return kv, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineData is this node's keys of DataBucket over whichever link the session has open (selfcompose.DataStore); with
|
||||||
|
// none open every read and write is refused, and nothing is kept anywhere else.
|
||||||
|
type EngineData struct{ Session *Session }
|
||||||
|
|
||||||
|
func (d EngineData) kv() (nats.KeyValue, error) {
|
||||||
|
d.Session.init()
|
||||||
|
d.Session.mu.Lock()
|
||||||
|
bus := d.Session.bus
|
||||||
|
d.Session.mu.Unlock()
|
||||||
|
l, ok := bus.(interface{ dataStore() (nats.KeyValue, error) })
|
||||||
|
if bus == nil || !ok {
|
||||||
|
return nil, ErrNoLink
|
||||||
|
}
|
||||||
|
return l.dataStore()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get is a key's value, nil when it has none.
|
||||||
|
func (d EngineData) Get(_ context.Context, key string) ([]byte, error) {
|
||||||
|
kv, err := d.kv()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
e, err := kv.Get(key)
|
||||||
|
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return e.Value(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put writes a key.
|
||||||
|
func (d EngineData) Put(_ context.Context, key string, value []byte) error {
|
||||||
|
kv, err := d.kv()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = kv.Put(key, value)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes a key; one that was not there is no error.
|
||||||
|
func (d EngineData) Delete(_ context.Context, key string) error {
|
||||||
|
kv, err := d.kv()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := kv.Delete(key); err != nil && !errors.Is(err, nats.ErrKeyNotFound) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keys is every key matching pattern that holds a value: read by a watch of the pattern alone, the only read the
|
||||||
|
// engine's grant gives of its own node's keys.
|
||||||
|
func (d EngineData) Keys(_ context.Context, pattern string) ([]string, error) {
|
||||||
|
kv, err := d.kv()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
w, err := kv.WatchFiltered([]string{pattern}, nats.IgnoreDeletes(), nats.MetaOnly())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
var keys []string
|
||||||
|
for e := range w.Updates() {
|
||||||
|
if e == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
keys = append(keys, e.Key())
|
||||||
|
}
|
||||||
|
return keys, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,400 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node seat node-engine, served on the engine's own link (novox/hq ADR 0563 §1).
|
||||||
|
//
|
||||||
|
// Each verb is answered on this node's subject alone, `mesh.seat.node-engine.tool.<verb>.<node>`, and on the same
|
||||||
|
// address naming its caller, `mesh.seat.node-engine.call.<verb>.<node>.<caller>`, whose last token the bus lets each user
|
||||||
|
// publish with its own name alone (novox/hq issue 365): the caller is that token, and a call on the subject naming none is
|
||||||
|
// from an unproven caller, recorded so. The body is the call's arguments; the answer is {result, error, node}, as every
|
||||||
|
// tool's (mesh-tools node-tools). One service, `node-engine`, answers discovery with each verb's seat, tool and node.
|
||||||
|
|
||||||
|
// EngineSeat is the seat.
|
||||||
|
const EngineSeat = catalogue.EngineSeat
|
||||||
|
|
||||||
|
// EngineToolSubject is where a verb of this node's seat is asked naming no caller.
|
||||||
|
func EngineToolSubject(verb, node string) string {
|
||||||
|
return "mesh.seat." + EngineSeat + ".tool." + verb + "." + node
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineCallPattern is where a verb of this node's seat is asked naming its caller, any caller.
|
||||||
|
func EngineCallPattern(verb, node string) string {
|
||||||
|
return "mesh.seat." + EngineSeat + ".call." + verb + "." + node + ".*"
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineEventSubject is where the seat says one of its events about this node.
|
||||||
|
func EngineEventSubject(event, node string) string {
|
||||||
|
return "mesh.seat." + EngineSeat + ".event." + event + "." + node
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineCall is one call of the seat: its verb, its caller as the bus proved it, and its arguments.
|
||||||
|
type EngineCall struct {
|
||||||
|
Verb string
|
||||||
|
Caller string
|
||||||
|
Args map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineFunc answers one call: its result, or why it is refused.
|
||||||
|
type EngineFunc func(EngineCall) (any, error)
|
||||||
|
|
||||||
|
// CallerOfSubject is the caller a call's subject names for this node's verb: the last token of a caller-named subject,
|
||||||
|
// each `~` a dot, or stdio.Unproven on the subject that names none or on any other.
|
||||||
|
func CallerOfSubject(subject, verb, node string) string {
|
||||||
|
token, ok := strings.CutPrefix(subject, "mesh.seat."+EngineSeat+".call."+verb+"."+node+".")
|
||||||
|
if !ok || token == "" || strings.ContainsAny(token, ".*> ") {
|
||||||
|
return stdio.Unproven
|
||||||
|
}
|
||||||
|
return strings.ReplaceAll(token, "~", ".")
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxEngineCall bounds a call's arguments: a path, an access and a few words.
|
||||||
|
const MaxEngineCall = 16 << 10
|
||||||
|
|
||||||
|
// AnswerEngineCall is the envelope one call is answered with: the verb's result, or its refusal in words.
|
||||||
|
func AnswerEngineCall(handle EngineFunc, verb, node, subject string, body []byte) []byte {
|
||||||
|
envelope := map[string]any{"node": node}
|
||||||
|
say := func() []byte {
|
||||||
|
raw, err := json.Marshal(envelope)
|
||||||
|
if err != nil {
|
||||||
|
raw, _ = json.Marshal(map[string]any{"node": node, "error": "the answer could not be written: " + err.Error()})
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
args := map[string]any{}
|
||||||
|
switch {
|
||||||
|
case handle == nil:
|
||||||
|
envelope["error"] = "this engine serves no verb of " + EngineSeat
|
||||||
|
return say()
|
||||||
|
case len(body) > MaxEngineCall:
|
||||||
|
envelope["error"] = fmt.Sprintf("the call is %d bytes, more than a call of %s takes", len(body), EngineSeat)
|
||||||
|
return say()
|
||||||
|
case len(strings.TrimSpace(string(body))) > 0:
|
||||||
|
if err := json.Unmarshal(body, &args); err != nil {
|
||||||
|
envelope["error"] = "the call's arguments are not an object: " + err.Error()
|
||||||
|
return say()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result, err := handle(EngineCall{Verb: verb, Caller: CallerOfSubject(subject, verb, node), Args: args})
|
||||||
|
if err != nil {
|
||||||
|
envelope["error"] = err.Error()
|
||||||
|
} else {
|
||||||
|
envelope["result"] = result
|
||||||
|
}
|
||||||
|
return say()
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineSeats is a link that serves this node's seat. Optional, as Asked is.
|
||||||
|
type EngineSeats interface {
|
||||||
|
ServeEngineSeat(EngineFunc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeEngineSeat answers every verb of the seat on this node's subjects, as the service node-engine, which says each
|
||||||
|
// verb's seat, tool and node to discovery (novox/hq ADR 0197). A bus whose user list is older than the grant refuses the
|
||||||
|
// subscriptions, said by the client; a caller then hears nobody answer.
|
||||||
|
func (l *natsLink) ServeEngineSeat(handle EngineFunc) {
|
||||||
|
if handle == nil || l.conn == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
svc, err := micro.AddService(l.conn, micro.Config{Name: EngineSeat, Version: "1.0.0",
|
||||||
|
Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true,
|
||||||
|
Metadata: map[string]string{"seat": EngineSeat, "scope": "node", "node": l.node}})
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, v := range catalogue.EngineVerbs() {
|
||||||
|
verb := v.Name
|
||||||
|
answer := micro.HandlerFunc(func(r micro.Request) {
|
||||||
|
_ = r.Respond(AnswerEngineCall(handle, verb, l.node, r.Subject(), r.Data()))
|
||||||
|
})
|
||||||
|
meta := EngineEndpointMetadata(v, l.node)
|
||||||
|
_ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)),
|
||||||
|
micro.WithEndpointMetadata(meta))
|
||||||
|
_ = svc.AddEndpoint(verb+"-called", answer, micro.WithEndpointSubject(EngineCallPattern(verb, l.node)),
|
||||||
|
micro.WithEndpointMetadata(meta))
|
||||||
|
}
|
||||||
|
l.engine = svc
|
||||||
|
}
|
||||||
|
|
||||||
|
// EngineModule is the module whose code answers the seat, as discovery names it: the node-engine's manifest's.
|
||||||
|
const EngineModule = "mesh-host"
|
||||||
|
|
||||||
|
// EngineEndpointMetadata is what discovery reads of one verb's endpoint, in the shape every tool runner and the
|
||||||
|
// controller announce a seat's verb in (mesh-tools node-tools' announce package): its kind, the seat, its scope, the
|
||||||
|
// module whose code answers, the verb, the node, its description and its schema. Without the kind and the scope the
|
||||||
|
// console listed none of it, and `<node>/node-engine.<verb>` answered that nothing in the mesh is called node-engine
|
||||||
|
// (the sweep of 2026-10-30).
|
||||||
|
func EngineEndpointMetadata(v catalogue.Verb, node string) map[string]string {
|
||||||
|
schema, err := json.Marshal(v.Input)
|
||||||
|
if err != nil || v.Input == nil {
|
||||||
|
schema = []byte("{}")
|
||||||
|
}
|
||||||
|
return map[string]string{"kind": "seat", "seat": EngineSeat, "scope": "node", "module": EngineModule,
|
||||||
|
"tool": v.Name, "node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SayEngine says one of the seat's events about this node, on the events stream.
|
||||||
|
func (l *natsLink) SayEngine(event string, body any) error {
|
||||||
|
raw, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err = l.js.Publish(EngineEventSubject(event, l.node), raw, nats.Context(ctx))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The engine asks the operator under its own asker name (asks.EngineAsker) on the router's seat (ADR 0259 §3, as ADR
|
||||||
|
// 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and
|
||||||
|
// its asks' state read from the router's records under its own name.
|
||||||
|
|
||||||
|
// RouterRecords is the router's bucket of asks, which each asker reads under its own name: the state `asks` of the
|
||||||
|
// operator-channel's holder, the messenger (mesh-catalog modules/messenger).
|
||||||
|
const RouterRecords = "messenger_asks"
|
||||||
|
|
||||||
|
// OperatorAsks is a link that asks the operator as this node's engine. Optional.
|
||||||
|
type OperatorAsks interface {
|
||||||
|
PublishAsk(subject string, body any, id string) error
|
||||||
|
RouterRecord(key string) (*stdio.StateEntry, error)
|
||||||
|
HearWarrants(asker string, heard func(asks.Warrant)) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublishAsk publishes an ask or a cancel on the router's seat, once per id within the stream's duplicate window.
|
||||||
|
func (l *natsLink) PublishAsk(subject string, body any, id string) error {
|
||||||
|
raw, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_, err = l.js.Publish(subject, raw, nats.MsgId(id), nats.Context(ctx))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// RouterRecord reads one key of the router's records, `<asker>.<id>`; nil when it holds none.
|
||||||
|
func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+RouterRecords+".$KV."+RouterRecords+"."+key, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if status := msg.Header.Get("Status"); status == "404" {
|
||||||
|
return nil, nil
|
||||||
|
} else if status != "" {
|
||||||
|
return nil, fmt.Errorf("the router's record %s could not be read: %s %s", key, status, msg.Header.Get("Description"))
|
||||||
|
}
|
||||||
|
return &stdio.StateEntry{Key: key, Value: msg.Data}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// HearWarrants hears the warrants for asker on its decided subject, each handed to heard as it arrives.
|
||||||
|
func (l *natsLink) HearWarrants(asker string, heard func(asks.Warrant)) error {
|
||||||
|
sub, err := l.conn.Subscribe(asks.DecidedSubject(asker), func(msg *nats.Msg) {
|
||||||
|
var w asks.Warrant
|
||||||
|
if json.Unmarshal(msg.Data, &w) == nil {
|
||||||
|
heard(w)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
l.warrants = sub
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// errNoAsking is a link that cannot ask the operator.
|
||||||
|
var errNoAsking = errors.New("the link open now cannot ask the operator")
|
||||||
|
|
||||||
|
// discoveryQuiet is how long the engine waits after the last answer to the bus's discovery, and discoveryMost the
|
||||||
|
// longest it waits in all.
|
||||||
|
const (
|
||||||
|
discoveryQuiet = 300 * time.Millisecond
|
||||||
|
discoveryMost = 2 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// HeardServed asks the bus's discovery (novox/hq ADR 0197) whether a seat's verb is answered on this machine: an
|
||||||
|
// endpoint naming the seat, the verb as its tool and this node.
|
||||||
|
func (l *natsLink) HeardServed(seat, verb string) (bool, error) {
|
||||||
|
inbox := l.conn.NewRespInbox()
|
||||||
|
sub, err := l.conn.SubscribeSync(inbox)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if err := l.conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
|
||||||
|
return false, fmt.Errorf("asking the bus who serves what: %w", err)
|
||||||
|
}
|
||||||
|
heard := false
|
||||||
|
deadline := time.Now().Add(discoveryMost)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
msg, err := sub.NextMsg(discoveryQuiet)
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
var info micro.Info
|
||||||
|
if json.Unmarshal(msg.Data, &info) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, e := range info.Endpoints {
|
||||||
|
node := e.Metadata["node"]
|
||||||
|
if node == "" {
|
||||||
|
node = info.ID
|
||||||
|
}
|
||||||
|
if e.Metadata["seat"] == seat && e.Metadata["tool"] == verb && node == l.node {
|
||||||
|
heard = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return heard, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeardServed is the link open now asking the bus's discovery whether a seat's verb is answered on this machine.
|
||||||
|
func (q *Session) HeardServed(seat, verb string) (bool, error) {
|
||||||
|
q.init()
|
||||||
|
q.mu.Lock()
|
||||||
|
bus := q.bus
|
||||||
|
q.mu.Unlock()
|
||||||
|
asker, ok := bus.(interface {
|
||||||
|
HeardServed(string, string) (bool, error)
|
||||||
|
})
|
||||||
|
if bus == nil || !ok {
|
||||||
|
return false, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
|
||||||
|
}
|
||||||
|
return asker.HeardServed(seat, verb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the engine's acts in the bus's own storage (ADR 0563 §2, §3) -------------------------------------------------
|
||||||
|
|
||||||
|
// actsStore is this node's keys of ActsBucket over the link open now: bound, never created (the grants part creates
|
||||||
|
// it, as the node-engine's declared state).
|
||||||
|
func (l *natsLink) actsStore() (nats.KeyValue, error) {
|
||||||
|
kv, err := l.js.KeyValue(ActsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+
|
||||||
|
"manifest; asked again once it is): %w", ActsBucket, err)
|
||||||
|
}
|
||||||
|
return kv, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BusState is the engine's ActStore: its own node's keys of ActsBucket, `<node>.<key>`, over whichever link the session
|
||||||
|
// has open; with none open every read and write is refused, and nothing is kept anywhere else.
|
||||||
|
type BusState struct {
|
||||||
|
Session *Session
|
||||||
|
Node string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BusState) kv() (nats.KeyValue, error) {
|
||||||
|
b.Session.init()
|
||||||
|
b.Session.mu.Lock()
|
||||||
|
bus := b.Session.bus
|
||||||
|
b.Session.mu.Unlock()
|
||||||
|
l, ok := bus.(interface{ actsStore() (nats.KeyValue, error) })
|
||||||
|
if bus == nil || !ok {
|
||||||
|
return nil, fmt.Errorf("%w: this machine has no link to the bus open now, where its acts are kept", ErrNotAsked)
|
||||||
|
}
|
||||||
|
return l.actsStore()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BusState) key(k string) string { return b.Node + "." + k }
|
||||||
|
|
||||||
|
// changed is a write that lost its compare-and-set, as stdio.ErrChanged.
|
||||||
|
func changed(err error) error {
|
||||||
|
var api *nats.APIError
|
||||||
|
if errors.Is(err, nats.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == nats.JSErrCodeStreamWrongLastSequence) {
|
||||||
|
return stdio.ErrChanged
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BusState) Get(key string) (*stdio.StateEntry, error) {
|
||||||
|
kv, err := b.kv()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
e, err := kv.Get(b.key(key))
|
||||||
|
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BusState) Create(key string, value any) (uint64, error) {
|
||||||
|
kv, err := b.kv()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
rev, err := kv.Create(b.key(key), raw)
|
||||||
|
return rev, changed(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BusState) Update(key string, value any, revision uint64) (uint64, error) {
|
||||||
|
kv, err := b.kv()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
rev, err := kv.Update(b.key(key), raw, revision)
|
||||||
|
return rev, changed(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b BusState) Delete(key string) error {
|
||||||
|
kv, err := b.kv()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return kv.Delete(b.key(key))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keys are this node's keys under prefix, without the node: a watch of `<node>.>` alone, the one filter the node's
|
||||||
|
// grant names, its values left out.
|
||||||
|
func (b BusState) Keys(prefix string) ([]string, error) {
|
||||||
|
kv, err := b.kv()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
w, err := kv.Watch(b.Node+".>", nats.MetaOnly(), nats.IgnoreDeletes())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = w.Stop() }()
|
||||||
|
var out []string
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case e := <-w.Updates():
|
||||||
|
if e == nil {
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
if k := strings.TrimPrefix(e.Key(), b.Node+"."); strings.HasPrefix(k, prefix) {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
return nil, errors.New("the node-engine's acts could not be listed within 10s")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,793 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node seat node-engine's acts (novox/hq ADR 0563).
|
||||||
|
//
|
||||||
|
// A hand-over and an access act are the engine's own asks, performed only on the warrant (§3): a call performs nothing.
|
||||||
|
// The engine judges every value as it does for the controller's signed ask, then refuses in words or asks the operator
|
||||||
|
// under its own asker name (asks.EngineAsker) at the level approve, binding the digest of the verb, the node, every
|
||||||
|
// argument and the owner, group and mode it will set. A warrant is claimed once by compare-and-set in the engine's own
|
||||||
|
// state (asker.Client.Take), held to the ask it answers, and to the act bound and the disk as judged, judged again now;
|
||||||
|
// only then is the act performed and recorded with the warrant. A decline, an expiry or a changed disk says refused and
|
||||||
|
// changes nothing. One ask is open per path or access.
|
||||||
|
//
|
||||||
|
// The read-only verbs need no warrant (§2): judge-directories changes nothing; setuid-search is bounded here, one at a
|
||||||
|
// time and at most one started in an hour. Every act, with who asked as the runtime proved it, is kept in the engine's own
|
||||||
|
// declared state acts (ActsBucket, created by the grants part), under its own node's keys alone, read by the verb acts.
|
||||||
|
|
||||||
|
// The acts of the seat, as its verbs name them.
|
||||||
|
const (
|
||||||
|
ActHandOver = "hand-over"
|
||||||
|
ActAccess = "access-act"
|
||||||
|
ActJudgeDirectories = "judge-directories"
|
||||||
|
ActSetuidSearch = "setuid-search"
|
||||||
|
// ActComposition reads a module's composition beside what runs of it; read-only, no warrant (issue 774).
|
||||||
|
ActComposition = "composition"
|
||||||
|
// ActCompose composes the node now, the recovery of a failed composition (issue 774).
|
||||||
|
ActCompose = "compose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The states of an act.
|
||||||
|
const (
|
||||||
|
HandAsked = "asked"
|
||||||
|
HandDone = "done"
|
||||||
|
HandRefused = "refused"
|
||||||
|
HandAnswered = "answered"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SearchEvery is how often a setuid search may be started on a node (ADR 0563 §2): one in any hour.
|
||||||
|
const SearchEvery = time.Hour
|
||||||
|
|
||||||
|
// askMargin is how long past an ask's expiry the router is given to say it ended, before the engine closes it itself.
|
||||||
|
const askMargin = 5 * time.Minute
|
||||||
|
|
||||||
|
// ActsBucket is the node-engine's declared state acts (its manifest's state, ADR 0563 §2, §3), the bucket the grants
|
||||||
|
// part creates (mesh-sdk buscompose EngineActsBucket): keys `<node>.act.<id>`, `<node>.open.<act>.<digest>` and
|
||||||
|
// `<node>.ask.<id>`, each written by this node's engine alone.
|
||||||
|
const ActsBucket = "mesh-host_acts"
|
||||||
|
|
||||||
|
// Judged is what the engine judged of one act before it asks: what it is about, what it will set (the owner, group and
|
||||||
|
// mode, in words), and what the disk holds now (Disk), compared again when the warrant arrives.
|
||||||
|
type Judged struct {
|
||||||
|
// Subject is the path of a hand-over, or the access of an access act.
|
||||||
|
Subject string
|
||||||
|
// Path is the directory the act changes, as the engine states it.
|
||||||
|
Path string
|
||||||
|
// Sets is the owner, group and mode the act will set, in words; bound in the ask's digest.
|
||||||
|
Sets string
|
||||||
|
// Disk is what the engine read on disk when it judged: owner, group, mode and the file it is.
|
||||||
|
Disk string
|
||||||
|
// Words is what the operator is shown of it, without a path (ADR 0234 §6): the path is in the whole words alone.
|
||||||
|
Words string
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandAct is one act asked through the seat, as acts reads it.
|
||||||
|
type HandAct struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Act string `json:"act"`
|
||||||
|
Subject string `json:"subject,omitempty"`
|
||||||
|
Caller string `json:"caller"`
|
||||||
|
Why string `json:"why,omitempty"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Said string `json:"said,omitempty"`
|
||||||
|
Ended time.Time `json:"ended,omitempty"`
|
||||||
|
// Ask is the operator's ask of a hand-over or an access act; Sets and Disk what was judged when it was asked.
|
||||||
|
Ask string `json:"ask,omitempty"`
|
||||||
|
Sets string `json:"sets,omitempty"`
|
||||||
|
Disk string `json:"disk,omitempty"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
// Expires is when the ask ends unanswered.
|
||||||
|
Expires time.Time `json:"expires,omitempty"`
|
||||||
|
// By, Via, Proofs are the operator's warrant, recorded with the act (ADR 0563 §3).
|
||||||
|
By string `json:"by,omitempty"`
|
||||||
|
Via string `json:"via,omitempty"`
|
||||||
|
Proofs []string `json:"proofs,omitempty"`
|
||||||
|
// Auto says the engine asked it itself, on finding a library's root holding data outside its group (§4).
|
||||||
|
Auto bool `json:"auto,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RootVerdict is the engine's answer to root-free (ADR 0563 §6), in the controller's words.
|
||||||
|
type RootVerdict struct {
|
||||||
|
Machine string `json:"machine"`
|
||||||
|
Free bool `json:"free"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
Judged time.Time `json:"judged"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandActs is the engine's side of the seat. Every func is the engine's own judgement or act; nil refuses.
|
||||||
|
type HandActs struct {
|
||||||
|
Node string
|
||||||
|
// State is where the acts and the asks are kept: this node's keys of ActsBucket (BusState), each key one act.
|
||||||
|
State ActStore
|
||||||
|
// Asker asks the operator as asks.EngineAsker(Node), with State as its store.
|
||||||
|
Asker *asker.Client
|
||||||
|
// JudgeHandOver judges a hand-over's path, refusing in words; HandOver records it on the warrant.
|
||||||
|
JudgeHandOver func(path string) (Judged, error)
|
||||||
|
HandOver HandOverFunc
|
||||||
|
// JudgeAccess judges an access act on an access, refusing in words; AccessAct runs it on the warrant.
|
||||||
|
JudgeAccess func(act, access string) (Judged, error)
|
||||||
|
AccessAct AccessActFunc
|
||||||
|
// JudgeDirectories, SetuidSearch and Searching are the read-only verbs' and whether a search runs now.
|
||||||
|
JudgeDirectories JudgeDirectoriesFunc
|
||||||
|
SetuidSearch SetuidSearchFunc
|
||||||
|
Searching func() bool
|
||||||
|
// RootFree is this machine's root-free verdict now.
|
||||||
|
RootFree func() RootVerdict
|
||||||
|
// Composition reads one module's composition beside what runs of it, read-only (the verb composition, issue 774):
|
||||||
|
// the latest of its want's build or the one whose digest is given, and with diff the resources that differ.
|
||||||
|
Composition func(module, digest string, diff bool) (any, error)
|
||||||
|
// Compose makes a pass of the node's composition now and says how it went; with module, its apply is also tried
|
||||||
|
// again (the verb compose, issue 774).
|
||||||
|
Compose func(module string) (any, error)
|
||||||
|
// Say says one of the seat's events, asked, done or refused; nil says nothing.
|
||||||
|
Say func(event string, body any)
|
||||||
|
Now func() time.Time
|
||||||
|
// Nonce keeps two asks of one second apart; nil is random.
|
||||||
|
Nonce func() string
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HandActs) now() time.Time {
|
||||||
|
if h.Now != nil {
|
||||||
|
return h.Now()
|
||||||
|
}
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HandActs) nonce() string {
|
||||||
|
if h.Nonce != nil {
|
||||||
|
return h.Nonce()
|
||||||
|
}
|
||||||
|
var b [2]byte
|
||||||
|
_, _ = rand.Read(b[:])
|
||||||
|
return hex.EncodeToString(b[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HandActs) say(event string, a HandAct) {
|
||||||
|
if h.Say != nil {
|
||||||
|
h.Say(event, map[string]any{"act": a.Act, "subject": a.Subject, "caller": a.Caller, "ask": a.Ask,
|
||||||
|
"state": a.State, "said": a.Said, "node": h.Node})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call is the seat's one entry: a verb, the caller as the runtime proved it, and its arguments.
|
||||||
|
func (h *HandActs) Call(c EngineCall) (any, error) {
|
||||||
|
arg := func(k string) string { s, _ := c.Args[k].(string); return strings.TrimSpace(s) }
|
||||||
|
switch c.Verb {
|
||||||
|
case ActHandOver:
|
||||||
|
return h.AskHandOver(c.Caller, arg("path"), arg("why"), false)
|
||||||
|
case ActAccess:
|
||||||
|
return h.AskAccess(c.Caller, arg("act"), arg("access"), arg("why"), false)
|
||||||
|
case ActJudgeDirectories:
|
||||||
|
return h.Judge(c.Caller)
|
||||||
|
case ActSetuidSearch:
|
||||||
|
return h.Search(c.Caller, arg("why"))
|
||||||
|
case "root-free":
|
||||||
|
if h.RootFree == nil {
|
||||||
|
return nil, errors.New("this engine judges no machine root-free")
|
||||||
|
}
|
||||||
|
return map[string]any{"machines": []RootVerdict{h.RootFree()}}, nil
|
||||||
|
case ActComposition:
|
||||||
|
if h.Composition == nil {
|
||||||
|
return nil, errors.New("this engine reads no composition yet: its node agent has not started")
|
||||||
|
}
|
||||||
|
diff := arg("diff") == "true"
|
||||||
|
if b, ok := c.Args["diff"].(bool); ok {
|
||||||
|
diff = b
|
||||||
|
}
|
||||||
|
return h.Composition(arg("module"), arg("digest"), diff)
|
||||||
|
case ActCompose:
|
||||||
|
if h.Compose == nil {
|
||||||
|
return nil, errors.New("this engine composes nothing on a call yet: its node agent has not started")
|
||||||
|
}
|
||||||
|
return h.Compose(arg("module"))
|
||||||
|
case "acts":
|
||||||
|
if id := arg("id"); id != "" {
|
||||||
|
return h.Act(id)
|
||||||
|
}
|
||||||
|
limit, _ := strconv.Atoi(arg("limit"))
|
||||||
|
if f, ok := c.Args["limit"].(float64); ok {
|
||||||
|
limit = int(f)
|
||||||
|
}
|
||||||
|
return h.Acts(limit)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("the node seat %s serves no verb %q", EngineSeat, c.Verb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the record --------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// ActStore is the engine's own state, this node's keys alone (BusState prefixes them with the node): each act one key,
|
||||||
|
// written by compare-and-set on its revision, an open ask one key made by a create.
|
||||||
|
type ActStore interface {
|
||||||
|
Get(key string) (*stdio.StateEntry, error)
|
||||||
|
Create(key string, value any) (uint64, error)
|
||||||
|
Update(key string, value any, revision uint64) (uint64, error)
|
||||||
|
Delete(key string) error
|
||||||
|
// Keys are the keys under prefix.
|
||||||
|
Keys(prefix string) ([]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func actKey(id string) string { return "act." + id }
|
||||||
|
|
||||||
|
// openKey is the one key an open ask of act on subject holds: made by a create, so a second ask of it is refused.
|
||||||
|
func openKey(act, subject string) string {
|
||||||
|
sum := sha256.Sum256([]byte(subject))
|
||||||
|
return "open." + act + "." + hex.EncodeToString(sum[:12])
|
||||||
|
}
|
||||||
|
|
||||||
|
// read is every act kept, oldest first.
|
||||||
|
func (h *HandActs) read() ([]HandAct, error) {
|
||||||
|
keys, err := h.State.Keys("act.")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the engine's acts cannot be listed: %w", err)
|
||||||
|
}
|
||||||
|
var out []HandAct
|
||||||
|
for _, k := range keys {
|
||||||
|
a, _, err := h.readOne(k)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if a != nil {
|
||||||
|
out = append(out, *a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HandActs) readOne(key string) (*HandAct, uint64, error) {
|
||||||
|
e, err := h.State.Get(key)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
var a HandAct
|
||||||
|
if err := json.Unmarshal(e.Value, &a); err != nil {
|
||||||
|
return nil, 0, fmt.Errorf("the engine's act %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
return &a, e.Revision, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgedKey keeps the node's judgements and refused searches, the last keptJudged of them: any caller may ask them, so
|
||||||
|
// they roll in one key and never add one (ADR 0563 §2: a wrong caller gains nothing, and fills no bucket).
|
||||||
|
const (
|
||||||
|
judgedKey = "judged"
|
||||||
|
keptJudged = 50
|
||||||
|
)
|
||||||
|
|
||||||
|
// roll adds a to the rolling key by compare-and-set, the oldest falling out.
|
||||||
|
func (h *HandActs) roll(a HandAct) error {
|
||||||
|
for range 5 {
|
||||||
|
e, err := h.State.Get(judgedKey)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var kept []HandAct
|
||||||
|
if e != nil {
|
||||||
|
if err := json.Unmarshal(e.Value, &kept); err != nil {
|
||||||
|
kept = nil // an unreadable roll is started again, never a reason to refuse
|
||||||
|
}
|
||||||
|
}
|
||||||
|
kept = append(kept, a)
|
||||||
|
if len(kept) > keptJudged {
|
||||||
|
kept = kept[len(kept)-keptJudged:]
|
||||||
|
}
|
||||||
|
if e == nil {
|
||||||
|
_, err = h.State.Create(judgedKey, kept)
|
||||||
|
} else {
|
||||||
|
_, err = h.State.Update(judgedKey, kept, e.Revision)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, stdio.ErrChanged) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors.New("the engine's judgements kept changing under its write")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HandActs) rolled() ([]HandAct, error) {
|
||||||
|
e, err := h.State.Get(judgedKey)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var kept []HandAct
|
||||||
|
if err := json.Unmarshal(e.Value, &kept); err != nil {
|
||||||
|
return nil, fmt.Errorf("the engine's judgements cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return kept, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HandActs) record(a HandAct) error {
|
||||||
|
_, err := h.State.Create(actKey(a.ID), a)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// end marks the open act of an ask ended, once, by compare-and-set on its revision, and frees its open key: false when
|
||||||
|
// it was not open.
|
||||||
|
func (h *HandActs) end(ask string, set func(*HandAct)) (HandAct, bool, error) {
|
||||||
|
for range 5 {
|
||||||
|
a, rev, err := h.readOne(actKey(ask))
|
||||||
|
if err != nil || a == nil || a.State != HandAsked {
|
||||||
|
return HandAct{}, false, err
|
||||||
|
}
|
||||||
|
set(a)
|
||||||
|
a.Ended = h.now().UTC()
|
||||||
|
if _, err := h.State.Update(actKey(ask), *a, rev); errors.Is(err, stdio.ErrChanged) {
|
||||||
|
continue
|
||||||
|
} else if err != nil {
|
||||||
|
return HandAct{}, false, err
|
||||||
|
}
|
||||||
|
_ = h.State.Delete(openKey(a.Act, a.Subject))
|
||||||
|
return *a, true, nil
|
||||||
|
}
|
||||||
|
return HandAct{}, false, errors.New("the engine's act " + ask + " kept changing under its write")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Act is the verb acts with an id: that one act.
|
||||||
|
func (h *HandActs) Act(id string) (any, error) {
|
||||||
|
a, _, err := h.readOne(actKey(id))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if a == nil {
|
||||||
|
return nil, fmt.Errorf("this engine keeps no act %q", id)
|
||||||
|
}
|
||||||
|
return map[string]any{"node": h.Node, "acts": []HandAct{*a}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Acts is the verb acts: every act kept, newest first, at most limit when given.
|
||||||
|
func (h *HandActs) Acts(limit int) (any, error) {
|
||||||
|
acts, err := h.read()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
judged, err := h.rolled()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
acts = append(acts, judged...)
|
||||||
|
sort.SliceStable(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) })
|
||||||
|
out := make([]HandAct, 0, len(acts))
|
||||||
|
for i := len(acts) - 1; i >= 0; i-- {
|
||||||
|
out = append(out, acts[i])
|
||||||
|
}
|
||||||
|
if limit > 0 && len(out) > limit {
|
||||||
|
out = out[:limit]
|
||||||
|
}
|
||||||
|
return map[string]any{"node": h.Node, "acts": out}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the read-only verbs (§2) -------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// Judge is judge-directories: the engine's judgement, read-only, recorded with its caller.
|
||||||
|
func (h *HandActs) Judge(caller string) (any, error) {
|
||||||
|
if h.JudgeDirectories == nil {
|
||||||
|
return nil, errors.New("this engine judges no directory")
|
||||||
|
}
|
||||||
|
said, err := h.JudgeDirectories(caller)
|
||||||
|
a := HandAct{ID: h.id(ActJudgeDirectories), Act: ActJudgeDirectories, Caller: caller, At: h.now().UTC(), State: HandAnswered}
|
||||||
|
if err != nil {
|
||||||
|
a.State, a.Said = HandRefused, err.Error()
|
||||||
|
}
|
||||||
|
if rerr := h.roll(a); rerr != nil && err == nil {
|
||||||
|
return nil, rerr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"node": h.Node, "said": said}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search is setuid-search: refused while one runs and when one was started within the hour, naming when the next may
|
||||||
|
// start; otherwise the last search is thrown away and a full one starts, recorded with its caller.
|
||||||
|
func (h *HandActs) Search(caller, why string) (any, error) {
|
||||||
|
if h.SetuidSearch == nil {
|
||||||
|
return nil, errors.New("this engine keeps no search for setuid programs")
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
now := h.now().UTC()
|
||||||
|
if h.Searching != nil && h.Searching() {
|
||||||
|
return nil, errors.New("a search for setuid programs runs on this machine now; no second is started while it does")
|
||||||
|
}
|
||||||
|
// The last start is a key of its own, claimed by compare-and-set before the search starts: no number of other acts,
|
||||||
|
// nor the record of acts, moves it (ADR 0563 §2).
|
||||||
|
if err := h.claimSearch(now, caller); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
said, err := h.SetuidSearch(caller)
|
||||||
|
a := HandAct{ID: h.id(ActSetuidSearch), Act: ActSetuidSearch, Caller: caller, Why: why, At: now, State: HandDone, Said: said}
|
||||||
|
if err != nil {
|
||||||
|
a.State, a.Said = HandRefused, err.Error()
|
||||||
|
}
|
||||||
|
// A search started is bounded to one an hour and kept as an act of its own; a refused one, which any caller may
|
||||||
|
// cause, rolls with the judgements.
|
||||||
|
record := h.record
|
||||||
|
if err != nil {
|
||||||
|
record = h.roll
|
||||||
|
}
|
||||||
|
if rerr := record(a); rerr != nil {
|
||||||
|
return nil, rerr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"node": h.Node, "said": said}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// searchKey keeps the last search's start on this node, and libKey each library's own take ask: keys of their own, which
|
||||||
|
// no other act's key moves, kept for the bucket's age (ninety days, the node-engine's manifest).
|
||||||
|
const searchKey = "search.last"
|
||||||
|
|
||||||
|
func libKey(access string) string {
|
||||||
|
sum := sha256.Sum256([]byte(access))
|
||||||
|
return "lib." + hex.EncodeToString(sum[:12]) + ".asked"
|
||||||
|
}
|
||||||
|
|
||||||
|
// started is the last search's start as kept, and who asked it.
|
||||||
|
type started struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
By string `json:"by"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// claimSearch refuses a search within SearchEvery of the last start, naming when the next may start, and otherwise
|
||||||
|
// claims this one by compare-and-set: a second claim at the same moment loses.
|
||||||
|
func (h *HandActs) claimSearch(now time.Time, caller string) error {
|
||||||
|
e, err := h.State.Get(searchKey)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var rev uint64
|
||||||
|
if e != nil {
|
||||||
|
var last started
|
||||||
|
if err := json.Unmarshal(e.Value, &last); err != nil {
|
||||||
|
return fmt.Errorf("the last search's start cannot be read, so none is started: %w", err)
|
||||||
|
}
|
||||||
|
if now.Sub(last.At) < SearchEvery {
|
||||||
|
return fmt.Errorf("a search was started at %s by %s, and one is started in any hour: the next may start "+
|
||||||
|
"at %s", last.At.Format(time.RFC3339), last.By, last.At.Add(SearchEvery).Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
rev = e.Revision
|
||||||
|
}
|
||||||
|
claim := started{At: now, By: caller}
|
||||||
|
if rev == 0 {
|
||||||
|
_, err = h.State.Create(searchKey, claim)
|
||||||
|
} else {
|
||||||
|
_, err = h.State.Update(searchKey, claim, rev)
|
||||||
|
}
|
||||||
|
if errors.Is(err, stdio.ErrChanged) {
|
||||||
|
return errors.New("another search was started at this moment; no second is started")
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the asks (§3, §4) --------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func (h *HandActs) id(act string) string {
|
||||||
|
return act + "-" + h.now().UTC().Format("20060102t150405") + "-" + h.nonce()
|
||||||
|
}
|
||||||
|
|
||||||
|
// bound is the act an ask's yes binds: the verb, the node, every argument and what it will set (ADR 0563 §3).
|
||||||
|
func (h *HandActs) bound(a HandAct) asks.Act {
|
||||||
|
act := asks.Act{"verb": a.Act, "node": h.Node, "arg.subject": a.Subject, "arg.why": a.Why, "sets": a.Sets}
|
||||||
|
if a.Path != "" {
|
||||||
|
act["path"] = a.Path
|
||||||
|
}
|
||||||
|
return act
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskHandOver is hand-over: judged, refused in words or asked of the operator. auto is the engine asking by itself.
|
||||||
|
func (h *HandActs) AskHandOver(caller, path, why string, auto bool) (any, error) {
|
||||||
|
if h.JudgeHandOver == nil || h.HandOver == nil {
|
||||||
|
return nil, errors.New("this engine takes no hand-over")
|
||||||
|
}
|
||||||
|
j, err := h.JudgeHandOver(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return h.ask(ActHandOver, caller, why, auto, j)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskAccess is access-act: the act take on an access, judged on the library's home, refused or asked of the operator.
|
||||||
|
func (h *HandActs) AskAccess(caller, act, access, why string, auto bool) (any, error) {
|
||||||
|
if h.JudgeAccess == nil || h.AccessAct == nil {
|
||||||
|
return nil, errors.New("this engine runs no act on an accessed folder")
|
||||||
|
}
|
||||||
|
if act != ActTake {
|
||||||
|
return nil, fmt.Errorf("the act asked is %q, and the one act on an access is %q", act, ActTake)
|
||||||
|
}
|
||||||
|
j, err := h.JudgeAccess(act, access)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return h.ask(ActAccess, caller, why, auto, j)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskedFor says whether an act on subject was ever asked by the engine itself: a library's take is asked by itself once
|
||||||
|
// (ADR 0563 §4), and again only by hand.
|
||||||
|
func (h *HandActs) AskedFor(act, subject string) bool {
|
||||||
|
if act == ActAccess {
|
||||||
|
e, err := h.State.Get(libKey(subject))
|
||||||
|
return err != nil || e != nil // unread is never a reason to ask again
|
||||||
|
}
|
||||||
|
acts, err := h.read()
|
||||||
|
if err != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, a := range acts {
|
||||||
|
if a.Act == act && a.Subject == subject && (a.Auto || a.State == HandAsked) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error) {
|
||||||
|
if h.Asker == nil {
|
||||||
|
return nil, errors.New("this engine reaches no operator to ask: nothing was asked and nothing changed")
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
now := h.now().UTC()
|
||||||
|
a := HandAct{ID: h.id(act), Act: act, Subject: j.Subject, Path: j.Path, Caller: caller, Why: why, At: now,
|
||||||
|
State: HandAsked, Sets: j.Sets, Disk: j.Disk, Expires: now.Add(asks.ApproveLasts - time.Minute), Auto: auto}
|
||||||
|
a.Ask = a.ID
|
||||||
|
// One ask open per path or access: its open key made by a create, which a second ask cannot make.
|
||||||
|
if _, err := h.State.Create(openKey(act, j.Subject), map[string]string{"ask": a.Ask}); errors.Is(err, stdio.ErrChanged) {
|
||||||
|
held := "another"
|
||||||
|
if e, _ := h.State.Get(openKey(act, j.Subject)); e != nil {
|
||||||
|
var o struct {
|
||||||
|
Ask string `json:"ask"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(e.Value, &o) == nil && o.Ask != "" {
|
||||||
|
held = o.Ask
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%s of %s is asked already: the ask %s waits for the operator's answer", act, j.Subject, held)
|
||||||
|
} else if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := h.record(a); err != nil {
|
||||||
|
_ = h.State.Delete(openKey(act, j.Subject))
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
bind, err := asks.ActDigest(h.bound(a))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
headline, does, stays := "Hand a directory on "+h.Node+" to the mesh?", "The mesh gives it "+j.Sets+".",
|
||||||
|
"The directory stays as it was found."
|
||||||
|
if act == ActAccess {
|
||||||
|
headline, does, stays = "Keep the library "+strings.TrimPrefix(j.Subject, "library:")+" on "+h.Node+"?",
|
||||||
|
"The library's group, group write and setgid are set on everything under its root.",
|
||||||
|
"The library waits and nothing under its root changes."
|
||||||
|
}
|
||||||
|
if len([]rune(headline)) > asks.HeadlineLength {
|
||||||
|
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||||
|
}
|
||||||
|
asked := asks.Ask{ID: a.Ask, Headline: headline, Who: asks.Operator,
|
||||||
|
Explanation: j.Words + " Your yes sets " + j.Sets + "; nothing changes until you answer.",
|
||||||
|
Whole: j.Words + " (" + j.Path + ") Your yes sets " + j.Sets + "; nothing changes until you answer.",
|
||||||
|
Details: "Asked by " + caller + " on " + h.Node + orWhy(why),
|
||||||
|
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
|
||||||
|
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
|
||||||
|
Expires: a.Expires, OnExpiry: stays, About: EngineSeat + "-" + h.Node + "-" + act}
|
||||||
|
if err := h.Asker.Ask(asked); err != nil {
|
||||||
|
_, _, _ = h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the operator could not be asked: "+err.Error() })
|
||||||
|
return nil, fmt.Errorf("the operator could not be asked, and nothing changed: %w", err)
|
||||||
|
}
|
||||||
|
// Once asked, a library's own take ask is kept's own take ask: once, in a key of its own (ADR 0563 §4).
|
||||||
|
if auto && act == ActAccess {
|
||||||
|
_, _ = h.State.Create(libKey(j.Subject), map[string]any{"ask": a.Ask, "at": now})
|
||||||
|
}
|
||||||
|
h.say(HandAsked, a)
|
||||||
|
return map[string]any{"asked": a.Ask, "node": h.Node, "says": "the operator is asked on their channel; only " +
|
||||||
|
"their yes changes anything. acts shows where it stands."}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orWhy(why string) string {
|
||||||
|
if why == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return ": " + why
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warranted is the act on the operator's answer, handed once by asker.Client.Take after it held the warrant to the
|
||||||
|
// ask: a no is recorded refused; a yes is held to the act the option bound and the disk as judged, judged again now,
|
||||||
|
// and only then performed and recorded done with the warrant.
|
||||||
|
func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error {
|
||||||
|
a, _, err := h.readOne(actKey(w.Ask))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if a == nil || a.State != HandAsked {
|
||||||
|
return fmt.Errorf("the warrant answers the ask %s, which is not open here: nothing is done", w.Ask)
|
||||||
|
}
|
||||||
|
refuse := func(why string) error {
|
||||||
|
ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) })
|
||||||
|
if err == nil && ok {
|
||||||
|
h.say(HandRefused, ended)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if o.ID != "yes" {
|
||||||
|
return refuse("the operator said no")
|
||||||
|
}
|
||||||
|
var j Judged
|
||||||
|
switch a.Act {
|
||||||
|
case ActHandOver:
|
||||||
|
j, err = h.JudgeHandOver(a.Subject)
|
||||||
|
case ActAccess:
|
||||||
|
j, err = h.JudgeAccess(ActTake, a.Subject)
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("the act %s is not performed on a warrant", a.Act)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return refuse("judged again on the operator's yes, it is refused: " + err.Error())
|
||||||
|
}
|
||||||
|
now := *a
|
||||||
|
now.Sets, now.Path = j.Sets, j.Path
|
||||||
|
if err := o.Performs(h.bound(now)); err != nil {
|
||||||
|
return refuse("what the yes bound is not what would be done now: " + err.Error())
|
||||||
|
}
|
||||||
|
if j.Disk != a.Disk {
|
||||||
|
return refuse("the disk changed since the operator was asked (" + a.Disk + ", now " + j.Disk + "): ask again")
|
||||||
|
}
|
||||||
|
// Recorded with the way it came (ADR 0563 §5): the operator's yes to this engine's ask, not a terminal's line.
|
||||||
|
by := operatorOf(w) + ", to " + EngineSeat + "'s ask " + w.Ask
|
||||||
|
var said string
|
||||||
|
switch a.Act {
|
||||||
|
case ActHandOver:
|
||||||
|
said, err = h.HandOver(a.Subject, by)
|
||||||
|
case ActAccess:
|
||||||
|
said, err = h.AccessAct(AccessAct{Act: ActTake, Access: a.Subject, Path: j.Path, Warrant: w.Ask, By: by})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return refuse(err.Error())
|
||||||
|
}
|
||||||
|
ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandDone, said; warrantOn(x, w) })
|
||||||
|
if err == nil && ok {
|
||||||
|
h.say(HandDone, ended)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func warrantOn(a *HandAct, w asks.Warrant) {
|
||||||
|
a.Proofs = w.Proofs
|
||||||
|
if w.By != nil {
|
||||||
|
a.By, a.Via = w.By.Who, w.By.Kind
|
||||||
|
if w.By.Verified != "" {
|
||||||
|
a.Via += " (" + w.By.Verified + ")"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func operatorOf(w asks.Warrant) string {
|
||||||
|
if w.By == nil {
|
||||||
|
return "the operator"
|
||||||
|
}
|
||||||
|
return "the " + w.By.Who + " via " + w.By.Kind
|
||||||
|
}
|
||||||
|
|
||||||
|
// Heard is a warrant heard on the engine's decided subject: taken once (asker.Client.Take), and an ask that ended with
|
||||||
|
// no choice recorded refused.
|
||||||
|
func (h *HandActs) Heard(w asks.Warrant) {
|
||||||
|
if h.Asker == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// A warrant the ask kept here does not hold to (another digest, another option, too late) ends nothing: it is no
|
||||||
|
// answer to the ask, which stays open for the operator's.
|
||||||
|
if handled, _ := h.Asker.Take(w, h.Warranted); handled == asker.Recorded {
|
||||||
|
why := "the ask ended " + string(w.Outcome)
|
||||||
|
if w.Words != "" {
|
||||||
|
why += ": " + w.Words
|
||||||
|
}
|
||||||
|
if ended, ok, _ := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) }); ok {
|
||||||
|
h.say(HandRefused, ended)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reconcile catches up what the engine missed while it was away: a warrant the router holds for an open ask is taken,
|
||||||
|
// and an ask past its expiry and a margin is closed refused, expired.
|
||||||
|
func (h *HandActs) Reconcile() {
|
||||||
|
if h.Asker == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
acts, err := h.read()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, a := range acts {
|
||||||
|
if a.State != HandAsked || a.Ask == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, w, err := h.Asker.Where(a.Ask); err == nil && w != nil {
|
||||||
|
h.Heard(*w)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !a.Expires.IsZero() && h.now().After(a.Expires.Add(askMargin)) {
|
||||||
|
if ended, ok, _ := h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the ask expired unanswered" }); ok {
|
||||||
|
h.say(HandRefused, ended)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MemState is an ActStore in memory: an engine's acts while no bus is reached in a test.
|
||||||
|
type MemState struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
keys map[string]stdio.StateEntry
|
||||||
|
top uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MemState) Get(key string) (*stdio.StateEntry, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if e, ok := m.keys[key]; ok {
|
||||||
|
return &e, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MemState) Create(key string, value any) (uint64, error) { return m.put(key, value, 0, true) }
|
||||||
|
|
||||||
|
func (m *MemState) Update(key string, value any, revision uint64) (uint64, error) {
|
||||||
|
return m.put(key, value, revision, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MemState) put(key string, value any, revision uint64, create bool) (uint64, error) {
|
||||||
|
raw, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
was, kept := m.keys[key]
|
||||||
|
if (create && kept) || (!create && (!kept || was.Revision != revision)) {
|
||||||
|
return 0, stdio.ErrChanged
|
||||||
|
}
|
||||||
|
if m.keys == nil {
|
||||||
|
m.keys = map[string]stdio.StateEntry{}
|
||||||
|
}
|
||||||
|
m.top++
|
||||||
|
m.keys[key] = stdio.StateEntry{Key: key, Value: raw, Revision: m.top}
|
||||||
|
return m.top, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MemState) Delete(key string) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
delete(m.keys, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *MemState) Keys(prefix string) ([]string, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
var out []string
|
||||||
|
for k := range m.keys {
|
||||||
|
if strings.HasPrefix(k, prefix) {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -76,6 +76,10 @@ const MaxHandOverBy = 200
|
|||||||
// HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken.
|
// HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken.
|
||||||
type HandOverCheck struct {
|
type HandOverCheck struct {
|
||||||
Signer ed25519.PublicKey
|
Signer ed25519.PublicKey
|
||||||
|
// SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer:
|
||||||
|
// nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7,
|
||||||
|
// ADR 0550 §2).
|
||||||
|
SignerNow func() ed25519.PublicKey
|
||||||
Node string
|
Node string
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
Nonces *Nonces
|
Nonces *Nonces
|
||||||
@@ -157,8 +161,14 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod
|
|||||||
if err := json.Unmarshal(body, &signed); err != nil {
|
if err := json.Unmarshal(body, &signed); err != nil {
|
||||||
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
|
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
|
||||||
}
|
}
|
||||||
if len(check.Signer) != ed25519.PublicKeySize ||
|
signer := check.Signer
|
||||||
!ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) {
|
if check.SignerNow != nil {
|
||||||
|
if signer = check.SignerNow(); signer == nil {
|
||||||
|
return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(signer) != ed25519.PublicKeySize ||
|
||||||
|
!ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) {
|
||||||
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
|
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
|
||||||
}
|
}
|
||||||
var ask HandOverAsk
|
var ask HandOverAsk
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The hand-over's ask and answer hold these field names; the controller's side holds the same list
|
// The hand-over's ask and answer hold these field names; the controller's side holds the same list
|
||||||
@@ -145,6 +148,21 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing.
|
|||||||
if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" {
|
if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" {
|
||||||
t.Fatalf("the handler was given %v", given)
|
t.Fatalf("the handler was given %v", given)
|
||||||
}
|
}
|
||||||
|
// ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key
|
||||||
|
// is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over.
|
||||||
|
trusted := m.Signer
|
||||||
|
following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node,
|
||||||
|
Now: func() time.Time { return now }, Nonces: &Nonces{}}
|
||||||
|
before := len(given)
|
||||||
|
trusted = nil
|
||||||
|
late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) }))
|
||||||
|
if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before {
|
||||||
|
t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a)
|
||||||
|
}
|
||||||
|
trusted = m.Signer
|
||||||
|
if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 {
|
||||||
|
t.Fatalf("while the controller holds the lease its ask was answered %+v", a)
|
||||||
|
}
|
||||||
// A nonce is forgotten once its ask has expired, so the set does not grow for ever.
|
// A nonce is forgotten once its ask has expired, so the set does not grow for ever.
|
||||||
now = now.Add(HandOverGood + time.Second)
|
now = now.Add(HandOverGood + time.Second)
|
||||||
check.Nonces.take("ff", now.Add(time.Second), now)
|
check.Nonces.take("ff", now.Add(time.Second), now)
|
||||||
@@ -175,16 +193,22 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) {
|
|||||||
type handingLink struct {
|
type handingLink struct {
|
||||||
*quietLink
|
*quietLink
|
||||||
given chan HandOverFunc
|
given chan HandOverFunc
|
||||||
|
signer chan func() ed25519.PublicKey
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
|
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
|
||||||
|
|
||||||
|
// FollowSigner keeps what the link is told to verify the controller's asks with.
|
||||||
|
func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f }
|
||||||
|
|
||||||
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
|
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
|
||||||
// judgement, and nothing else.
|
// judgement, and nothing else.
|
||||||
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
||||||
m, _ := aMember(t)
|
m, _ := aMember(t)
|
||||||
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)}
|
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)}
|
||||||
q := &Session{Membership: m, Timeout: time.Second}
|
q := &Session{Membership: m, Timeout: time.Second}
|
||||||
|
// The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7).
|
||||||
|
q.ControllerKey = func() ed25519.PublicKey { return nil }
|
||||||
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
|
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
|
||||||
ctx, stop := context.WithCancel(context.Background())
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
served := make(chan error, 1)
|
served := make(chan error, 1)
|
||||||
@@ -197,8 +221,54 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
|||||||
case <-time.After(5 * time.Second):
|
case <-time.After(5 * time.Second):
|
||||||
t.Fatal("the link was never given the queue's hand-over")
|
t.Fatal("the link was never given the queue's hand-over")
|
||||||
}
|
}
|
||||||
|
select {
|
||||||
|
case f := <-l.signer:
|
||||||
|
if f() != nil {
|
||||||
|
t.Fatal("the link was given another key than the queue's")
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the link was never given the key the engine trusts now")
|
||||||
|
}
|
||||||
stop()
|
stop()
|
||||||
<-served
|
<-served
|
||||||
|
// 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on
|
||||||
|
// the subject that names none; a verb the seat does not serve is refused, and the answer is a tool's envelope.
|
||||||
|
t.Run("a call of the node seat reaches its verb with its proven caller", func(t *testing.T) {
|
||||||
|
h := &HandActs{Node: "n1", State: &MemState{},
|
||||||
|
RootFree: func() RootVerdict { return RootVerdict{Machine: "n1", Free: true, Why: "judged"} },
|
||||||
|
JudgeDirectories: func(by string) (string, error) { return "judged for " + by, nil },
|
||||||
|
Composition: func(module, digest string, diff bool) (any, error) {
|
||||||
|
return map[string]any{"module": module, "digest": digest, "diff": diff}, nil
|
||||||
|
},
|
||||||
|
Compose: func(module string) (any, error) { return map[string]any{"composed": true, "retried": module}, nil }}
|
||||||
|
for _, row := range []struct{ verb, subject, want string }{
|
||||||
|
{"judge-directories", "mesh.seat.node-engine.call.judge-directories.n1.shanks~node-tools", `"said":"judged for shanks.node-tools"`},
|
||||||
|
{"judge-directories", EngineToolSubject("judge-directories", "n1"), `"said":"judged for unproven"`},
|
||||||
|
{"judge-directories", "mesh.seat.node-engine.call.judge-directories.n2.shanks~node-tools", `"said":"judged for unproven"`},
|
||||||
|
{"root-free", "mesh.seat.node-engine.call.root-free.n1.ace~messenger", `"free":true`},
|
||||||
|
// The read verb composition (issue 774) takes its module, digest and diff, as a word or a boolean.
|
||||||
|
{"composition", EngineToolSubject("composition", "n1") + " " + `{"module":"postgres","digest":"sha256:a","diff":true}`, `"diff":true,"digest":"sha256:a","module":"postgres"`},
|
||||||
|
{"composition", EngineToolSubject("composition", "n1") + " " + `{"module":"postgres","diff":"true"}`, `"diff":true,"digest":"","module":"postgres"`},
|
||||||
|
// The recovery verb compose (issue 774) makes a pass now, with a module's apply tried again when named.
|
||||||
|
{"compose", EngineToolSubject("compose", "n1"), `"composed":true,"retried":""`},
|
||||||
|
{"compose", EngineToolSubject("compose", "n1") + " " + `{"module":"route-proxy"}`, `"retried":"route-proxy"`},
|
||||||
|
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
|
||||||
|
} {
|
||||||
|
subject, body, ok := strings.Cut(row.subject, " ")
|
||||||
|
if !ok {
|
||||||
|
body = `{}`
|
||||||
|
}
|
||||||
|
if got := string(AnswerEngineCall(h.Call, row.verb, "n1", subject, []byte(body))); !strings.Contains(got, row.want) ||
|
||||||
|
!strings.Contains(got, `"node":"n1"`) {
|
||||||
|
t.Errorf("%s on %s: %s", row.verb, row.subject, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
acts, _ := h.Acts(0)
|
||||||
|
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 3 || got[0].Caller != "unproven" ||
|
||||||
|
got[2].Caller != "shanks.node-tools" || got[2].Act != ActJudgeDirectories {
|
||||||
|
t.Errorf("the judgements are recorded as %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// **Against a real server**: a signed request on the node's own subject is answered; one on another node's is
|
// **Against a real server**: a signed request on the node's own subject is answered; one on another node's is
|
||||||
@@ -237,4 +307,92 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) {
|
|||||||
if len(handled) != 1 {
|
if len(handled) != 1 {
|
||||||
t.Fatalf("handled %d asks; want only the one with a reply", len(handled))
|
t.Fatalf("handled %d asks; want only the one with a reply", len(handled))
|
||||||
}
|
}
|
||||||
|
// 774 (ADR 0563 §2, §3): the engine's acts are its node's keys of its declared state on the bus, each written by
|
||||||
|
// compare-and-set: a second create and a stale revision lose, and another node's keys are not this node's.
|
||||||
|
t.Run("the acts are kept in the bus's own storage under this node's keys", func(t *testing.T) {
|
||||||
|
_ = js.DeleteKeyValue(ActsBucket)
|
||||||
|
if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: ActsBucket}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := &Session{Membership: m, Timeout: time.Second}
|
||||||
|
q.bus = l
|
||||||
|
mine, theirs := BusState{Session: q, Node: m.Node}, BusState{Session: q, Node: "another"}
|
||||||
|
rev, err := mine.Create("act.a-1", map[string]string{"state": "asked"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := mine.Create("act.a-1", map[string]string{}); !errors.Is(err, stdio.ErrChanged) {
|
||||||
|
t.Errorf("a second create: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev+7); !errors.Is(err, stdio.ErrChanged) {
|
||||||
|
t.Errorf("an update at a stale revision: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev); err != nil {
|
||||||
|
t.Errorf("an update at its revision: %v", err)
|
||||||
|
}
|
||||||
|
_, _ = theirs.Create("act.b-1", map[string]string{})
|
||||||
|
if keys, err := mine.Keys("act."); err != nil || len(keys) != 1 || keys[0] != "act.a-1" {
|
||||||
|
t.Errorf("this node's acts are %v, %v", keys, err)
|
||||||
|
}
|
||||||
|
if e, _ := js.KeyValue(ActsBucket); e != nil {
|
||||||
|
if got, err := e.Get(m.Node + ".act.a-1"); err != nil || !strings.Contains(string(got.Value()), "done") {
|
||||||
|
t.Errorf("the bucket holds %v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := mine.Delete("act.a-1"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if e, err := mine.Get("act.a-1"); err != nil || e != nil {
|
||||||
|
t.Errorf("a deleted act reads %v, %v", e, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
// 774 (ADR 0563 §1): the node seat is served on this node's subjects alone, naming its caller, and said to discovery.
|
||||||
|
t.Run("the node seat is served on this node and discovered", func(t *testing.T) {
|
||||||
|
l.ServeEngineSeat(func(c EngineCall) (any, error) { return c.Verb + " for " + c.Caller, nil })
|
||||||
|
reply, err := conn.Request("mesh.seat.node-engine.call.acts."+m.Node+".shanks~node-tools", []byte(`{}`), 5*time.Second)
|
||||||
|
if err != nil || !strings.Contains(string(reply.Data), `"result":"acts for shanks.node-tools"`) {
|
||||||
|
t.Fatalf("the call answered %v, %v", reply, err)
|
||||||
|
}
|
||||||
|
if _, err := conn.Request(EngineToolSubject("acts", "another"), []byte(`{}`), 300*time.Millisecond); err == nil {
|
||||||
|
t.Fatal("another node's verb was answered here")
|
||||||
|
}
|
||||||
|
// Announced in the shape the console reads a seat's verb in (sweep 2026-10-30): without kind and scope it listed
|
||||||
|
// none of it, and <node>/node-engine.root-free answered that nothing in the mesh is called node-engine.
|
||||||
|
info, err := conn.Request("$SRV.INFO."+EngineSeat, nil, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var said struct {
|
||||||
|
Endpoints []struct {
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
Metadata map[string]string `json:"metadata"`
|
||||||
|
} `json:"endpoints"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(info.Data, &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
announced := 0
|
||||||
|
for _, e := range said.Endpoints {
|
||||||
|
md := e.Metadata
|
||||||
|
if md["kind"] != "seat" || md["scope"] != "node" || md["seat"] != EngineSeat || md["module"] != "mesh-host" ||
|
||||||
|
md["node"] != m.Node || md["description"] == "" || !strings.HasPrefix(md["schema"], "{") {
|
||||||
|
t.Errorf("the endpoint on %s announces %v", e.Subject, md)
|
||||||
|
}
|
||||||
|
if e.Subject == EngineToolSubject("root-free", m.Node) {
|
||||||
|
announced++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if announced != 1 {
|
||||||
|
t.Errorf("root-free is announced on its tool subject %d times", announced)
|
||||||
|
}
|
||||||
|
for seat, want := range map[string]bool{EngineSeat: true, "node-login-shell": false} {
|
||||||
|
verb := "acts"
|
||||||
|
if !want {
|
||||||
|
verb = "execute"
|
||||||
|
}
|
||||||
|
if heard, err := l.HeardServed(seat, verb); err != nil || heard != want {
|
||||||
|
t.Errorf("discovery hears %s.%s here: %v, %v", seat, verb, heard, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -274,7 +274,8 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
|||||||
// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
|
// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
|
||||||
// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
|
// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
|
||||||
// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
|
// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
|
||||||
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"}
|
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert",
|
||||||
|
Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2)
|
||||||
q.Keys = func() RecordKeys { return keys }
|
q.Keys = func() RecordKeys { return keys }
|
||||||
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
|
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
|
||||||
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
|
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
|
||||||
@@ -286,7 +287,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
|||||||
t.Fatalf("the node key holds no record: %+v, %v", got, err)
|
t.Fatalf("the node key holds no record: %+v, %v", got, err)
|
||||||
}
|
}
|
||||||
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
|
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
|
||||||
Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
||||||
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
|
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
|
||||||
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
|
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
|
||||||
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
|
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
|
||||||
|
|||||||
+110
-15
@@ -3,9 +3,12 @@ package link
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A health statement is said on the link open now, as the machine's own word on its own subject, and
|
// A health statement is said on the link open now, as the machine's own word on its own subject, and
|
||||||
@@ -50,6 +53,101 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
|
|||||||
if HealthSubject("anchor") != "mesh.control.anchor.health" {
|
if HealthSubject("anchor") != "mesh.control.anchor.health" {
|
||||||
t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor"))
|
t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Issue 774: the node's own health report in its key of mesh-host_health, for the watchman's six alerts: written on
|
||||||
|
// a change only, the streaks and the time of the look not being one, and read first after a restart.
|
||||||
|
t.Run("the node's health report is written on a change only", func(t *testing.T) {
|
||||||
|
since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC)
|
||||||
|
look := func(streak int, writer string) Health {
|
||||||
|
return Health{Contract: 7, At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{
|
||||||
|
{Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}},
|
||||||
|
Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}},
|
||||||
|
Resources: []ResourceHealth{
|
||||||
|
{Module: "brother", Resource: "brother.scanner", Kind: KindAccount, Account: "jochen", State: StateUnhealthy,
|
||||||
|
Reason: "relogin needed: the session began before scanner", Since: since},
|
||||||
|
{Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy,
|
||||||
|
Reason: ReasonUsedAsFound + " owned by root", Since: since},
|
||||||
|
{Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since},
|
||||||
|
{Module: "claude-code", Resource: "claude-code.agent", Kind: KindAccount, Account: "agent", Root: "never",
|
||||||
|
State: StateHealthy, Since: since}}}
|
||||||
|
}
|
||||||
|
peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}}
|
||||||
|
r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers, Machines{})
|
||||||
|
if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" ||
|
||||||
|
len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" ||
|
||||||
|
len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" ||
|
||||||
|
len(r.Root) != 1 || r.Root[0].Account != "agent" || r.Root[0].State != StateHealthy || r.Contract != 7 {
|
||||||
|
t.Fatalf("the report is %+v", r)
|
||||||
|
}
|
||||||
|
// The machines an unhealthy part points at, as the controller holds it there; none for the machine's own.
|
||||||
|
machines := Machines{ByAddress: map[string]string{"10.10.0.1": "novox", "10.10.0.4": "ace", "10.10.0.9": "anchor"}, Hub: "novox"}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
toward []string
|
||||||
|
state string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"the tunnel to the hub", []string{"hub"}, StateUnhealthy, []string{"novox"}},
|
||||||
|
{"two mesh resolvers", []string{"10.10.0.1", "10.10.0.4"}, StateUnhealthy, []string{"novox", "ace"}},
|
||||||
|
{"a resolver a VPN client wrote in is the machine's own", []string{"10.10.0.1", "192.0.2.53"}, StateUnhealthy, nil},
|
||||||
|
{"its own resolver is its own", []string{"10.10.0.9"}, StateUnhealthy, nil},
|
||||||
|
{"a healthy part points at nothing", []string{"hub"}, StateHealthy, nil},
|
||||||
|
} {
|
||||||
|
h := Health{Network: &NetworkHealth{State: tc.state, Parts: []NetworkPart{{Part: "names", State: tc.state, Toward: tc.toward}}}}
|
||||||
|
got := NodeHealthOf("anchor", h, nil, machines).Network.Parts[0].Targets
|
||||||
|
if !slices.Equal(got, tc.want) {
|
||||||
|
t.Errorf("%s: targets %v, want %v", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
store := &memHealth{}
|
||||||
|
var last *NodeHealth
|
||||||
|
clock := func() time.Time { return since }
|
||||||
|
for i, step := range []struct {
|
||||||
|
h NodeHealth
|
||||||
|
fresh bool // a restart: nothing written in this run yet
|
||||||
|
wrote bool
|
||||||
|
}{
|
||||||
|
{h: r, wrote: true},
|
||||||
|
{h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers, Machines{})},
|
||||||
|
{h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers, Machines{}), fresh: true},
|
||||||
|
{h: NodeHealthOf("anchor", look(3, "openvpn"), peers, Machines{}), wrote: true},
|
||||||
|
{h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}, Machines{}), wrote: true},
|
||||||
|
} {
|
||||||
|
if step.fresh {
|
||||||
|
last = nil
|
||||||
|
}
|
||||||
|
wrote, err := writeNodeHealth(t.Context(), store, &last, step.h, clock)
|
||||||
|
if err != nil || wrote != step.wrote {
|
||||||
|
t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := delivery.DecodeNodeHealth(store.values["anchor"])
|
||||||
|
if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() {
|
||||||
|
t.Fatalf("the key holds %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
// The key's bytes are mesh-sdk's layout: decoded and encoded again by it, the same bytes.
|
||||||
|
if again, err := delivery.EncodeNodeHealth(got); err != nil || string(again) != string(store.values["anchor"]) {
|
||||||
|
t.Fatalf("the key's bytes are not the sdk's layout: %s\n%s (%v)", store.values["anchor"], again, err)
|
||||||
|
}
|
||||||
|
if store.puts != 3 {
|
||||||
|
t.Fatalf("%d writes, want 3: one per change", store.puts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// memHealth is a HealthStore in memory.
|
||||||
|
type memHealth struct {
|
||||||
|
values map[string][]byte
|
||||||
|
puts int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memHealth) Get(_ context.Context, key string) ([]byte, error) { return m.values[key], nil }
|
||||||
|
func (m *memHealth) Put(_ context.Context, key string, value []byte) error {
|
||||||
|
if m.values == nil {
|
||||||
|
m.values = map[string][]byte{}
|
||||||
|
}
|
||||||
|
m.values[key], m.puts = value, m.puts+1
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq
|
// A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq
|
||||||
@@ -70,6 +168,18 @@ func TestAToolsAnswerIsHealthyOnlyWhenItSaysSo(t *testing.T) {
|
|||||||
if ToolSubject("keycloak", "keycloak_admin_health", "anchor") != "mesh.mod.keycloak.tool.keycloak_admin_health.anchor" {
|
if ToolSubject("keycloak", "keycloak_admin_health", "anchor") != "mesh.mod.keycloak.tool.keycloak_admin_health.anchor" {
|
||||||
t.Errorf("asked on %s", ToolSubject("keycloak", "keycloak_admin_health", "anchor"))
|
t.Errorf("asked on %s", ToolSubject("keycloak", "keycloak_admin_health", "anchor"))
|
||||||
}
|
}
|
||||||
|
// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not
|
||||||
|
// healthy, so nothing reading `healthy` alone takes it for a pass.
|
||||||
|
got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` +
|
||||||
|
`"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`))
|
||||||
|
if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" {
|
||||||
|
t.Fatalf("read as %+v (%v)", got, err)
|
||||||
|
}
|
||||||
|
h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits}
|
||||||
|
b, _ := json.Marshal(h)
|
||||||
|
if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) {
|
||||||
|
t.Fatalf("said on the wire as %s", b)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A declared tool check is asked on the link the node-engine actually holds — the NATS link — and not
|
// A declared tool check is asked on the link the node-engine actually holds — the NATS link — and not
|
||||||
@@ -101,18 +211,3 @@ func TestAToolCheckIsAskedOnTheLinkTheEngineHolds(t *testing.T) {
|
|||||||
t.Fatalf("the NATS link was not asked: %v", err)
|
t.Fatalf("the NATS link was not asked: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A tool may answer that it waits for the operator (novox/hq ADR 0283): read with what it waits for, and not
|
|
||||||
// healthy, so nothing reading `healthy` alone takes it for a pass.
|
|
||||||
func TestAToolsWaitIsReadWithWhatItWaitsFor(t *testing.T) {
|
|
||||||
got, err := ReadToolAnswer([]byte(`{"result":{"state":"waits","healthy":false,"why":"the source games waits",` +
|
|
||||||
`"waits":[{"part":"the source games","secret":"smb-password-games","what":"the password of the source games"}]}}`))
|
|
||||||
if err != nil || got.Healthy || got.State != "waits" || len(got.Waits) != 1 || got.Waits[0].Secret != "smb-password-games" {
|
|
||||||
t.Fatalf("read as %+v (%v)", got, err)
|
|
||||||
}
|
|
||||||
h := ResourceHealth{Module: "mounts", Resource: "mounts.watch", State: StateWaiting, Waits: got.Waits}
|
|
||||||
b, _ := json.Marshal(h)
|
|
||||||
if !strings.Contains(string(b), `"state":"waiting"`) || !strings.Contains(string(b), `"waits":[{"part":"the source games","secret":"smb-password-games"`) {
|
|
||||||
t.Fatalf("said on the wire as %s", b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -5,10 +5,14 @@ import (
|
|||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
|
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
|
||||||
@@ -35,6 +39,9 @@ func natsURL(address string) string {
|
|||||||
|
|
||||||
// natsLink is this node's connection as a JetStream subscription.
|
// natsLink is this node's connection as a JetStream subscription.
|
||||||
type natsLink struct {
|
type natsLink struct {
|
||||||
|
// mu guards signerNow (FollowSigner).
|
||||||
|
mu sync.Mutex
|
||||||
|
signerNow func() ed25519.PublicKey
|
||||||
conn *nats.Conn
|
conn *nats.Conn
|
||||||
js nats.JetStreamContext
|
js nats.JetStreamContext
|
||||||
asking *nats.Subscription
|
asking *nats.Subscription
|
||||||
@@ -48,7 +55,13 @@ type natsLink struct {
|
|||||||
acting *nats.Subscription
|
acting *nats.Subscription
|
||||||
// moving hears the controller asking to move a value sealed to this machine into the vault (issue 676).
|
// moving hears the controller asking to move a value sealed to this machine into the vault (issue 676).
|
||||||
moving *nats.Subscription
|
moving *nats.Subscription
|
||||||
|
// engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563).
|
||||||
|
engine micro.Service
|
||||||
|
warrants *nats.Subscription
|
||||||
node string
|
node string
|
||||||
|
// refused carries each subject the bus refused this link a subscription to (RefusedSubscription), which serve drops
|
||||||
|
// the link for after a pause, so every subscription is asked again on the next.
|
||||||
|
refused chan string
|
||||||
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
|
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
|
||||||
signer ed25519.PublicKey
|
signer ed25519.PublicKey
|
||||||
lost chan error
|
lost chan error
|
||||||
@@ -102,7 +115,11 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e
|
|||||||
conn: conn, js: js, node: m.Node, signer: m.Signer,
|
conn: conn, js: js, node: m.Node, signer: m.Signer,
|
||||||
lost: make(chan error, 1),
|
lost: make(chan error, 1),
|
||||||
asked: make(chan struct{}, 1),
|
asked: make(chan struct{}, 1),
|
||||||
|
refused: make(chan string, 64),
|
||||||
}
|
}
|
||||||
|
// Set before anything subscribes, so the node seat's service (micro), which wraps the handler it finds, hands every
|
||||||
|
// refusal of its own subjects on to it too.
|
||||||
|
conn.SetErrorHandler(l.asyncError)
|
||||||
|
|
||||||
l.hearAsks()
|
l.hearAsks()
|
||||||
|
|
||||||
@@ -141,6 +158,21 @@ func (l *natsLink) hearAsks() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the
|
||||||
|
// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2).
|
||||||
|
func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) {
|
||||||
|
l.mu.Lock()
|
||||||
|
l.signerNow = f
|
||||||
|
l.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust.
|
||||||
|
func (l *natsLink) check() HandOverCheck {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node}
|
||||||
|
}
|
||||||
|
|
||||||
// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
|
// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
|
||||||
// (novox/hq issue 356): a request on this machine's own subject, answered once on the reply it carried. The
|
// (novox/hq issue 356): a request on this machine's own subject, answered once on the reply it carried. The
|
||||||
// ask is verified with the mesh's key before anything is read out of it (AnswerHandOver), because a responder
|
// ask is verified with the mesh's key before anything is read out of it (AnswerHandOver), because a responder
|
||||||
@@ -151,7 +183,7 @@ func (l *natsLink) HearHandOvers(handle HandOverFunc) {
|
|||||||
if handle == nil {
|
if handle == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
check := l.check()
|
||||||
handing, err := l.conn.Subscribe(AskHandOverSubject(l.node), func(msg *nats.Msg) {
|
handing, err := l.conn.Subscribe(AskHandOverSubject(l.node), func(msg *nats.Msg) {
|
||||||
if body, answer := HandOverReply(check, handle, msg.Reply, msg.Data); answer {
|
if body, answer := HandOverReply(check, handle, msg.Reply, msg.Data); answer {
|
||||||
_ = msg.Respond(body)
|
_ = msg.Respond(body)
|
||||||
@@ -170,7 +202,7 @@ func (l *natsLink) HearSetuidSearches(handle SetuidSearchFunc) {
|
|||||||
if handle == nil {
|
if handle == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
check := l.check()
|
||||||
searching, err := l.conn.Subscribe(AskSetuidSearchSubject(l.node), func(msg *nats.Msg) {
|
searching, err := l.conn.Subscribe(AskSetuidSearchSubject(l.node), func(msg *nats.Msg) {
|
||||||
if body, answer := SetuidSearchReply(check, handle, msg.Reply, msg.Data); answer {
|
if body, answer := SetuidSearchReply(check, handle, msg.Reply, msg.Data); answer {
|
||||||
_ = msg.Respond(body)
|
_ = msg.Respond(body)
|
||||||
@@ -189,7 +221,7 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) {
|
|||||||
if handle == nil {
|
if handle == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
check := l.check()
|
||||||
judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) {
|
judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) {
|
||||||
if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer {
|
if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer {
|
||||||
_ = msg.Respond(body)
|
_ = msg.Respond(body)
|
||||||
@@ -207,7 +239,7 @@ func (l *natsLink) HearAccessActs(handle AccessActFunc) {
|
|||||||
if handle == nil {
|
if handle == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
check := l.check()
|
||||||
acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) {
|
acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) {
|
||||||
if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer {
|
if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer {
|
||||||
_ = msg.Respond(body)
|
_ = msg.Respond(body)
|
||||||
@@ -225,7 +257,7 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) {
|
|||||||
if handle == nil {
|
if handle == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
check := l.check()
|
||||||
moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) {
|
moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) {
|
||||||
if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer {
|
if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer {
|
||||||
_ = msg.Respond(body)
|
_ = msg.Respond(body)
|
||||||
@@ -236,6 +268,23 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// asyncError is said by the client as before, and a subscription the bus refused is handed to serve (Refused).
|
||||||
|
func (l *natsLink) asyncError(_ *nats.Conn, _ *nats.Subscription, err error) {
|
||||||
|
if err == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||||
|
if subject, ok := RefusedSubscription(err); ok && l.refused != nil {
|
||||||
|
select {
|
||||||
|
case l.refused <- subject:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Refused says each subject the bus refused this link a subscription to.
|
||||||
|
func (l *natsLink) Refused() <-chan string { return l.refused }
|
||||||
|
|
||||||
func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
|
func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
|
||||||
func (l *natsLink) Lost() <-chan error { return l.lost }
|
func (l *natsLink) Lost() <-chan error { return l.lost }
|
||||||
|
|
||||||
@@ -258,6 +307,12 @@ func (l *natsLink) Close() {
|
|||||||
if l.judging != nil {
|
if l.judging != nil {
|
||||||
_ = l.judging.Unsubscribe()
|
_ = l.judging.Unsubscribe()
|
||||||
}
|
}
|
||||||
|
if l.engine != nil {
|
||||||
|
_ = l.engine.Stop()
|
||||||
|
}
|
||||||
|
if l.warrants != nil {
|
||||||
|
_ = l.warrants.Unsubscribe()
|
||||||
|
}
|
||||||
if l.conn != nil {
|
if l.conn != nil {
|
||||||
l.conn.Close()
|
l.conn.Close()
|
||||||
}
|
}
|
||||||
@@ -307,4 +362,22 @@ var (
|
|||||||
_ ToolBus = (*natsLink)(nil)
|
_ ToolBus = (*natsLink)(nil)
|
||||||
_ Asker = (*natsLink)(nil)
|
_ Asker = (*natsLink)(nil)
|
||||||
_ Asked = (*natsLink)(nil)
|
_ Asked = (*natsLink)(nil)
|
||||||
|
_ EngineSeats = (*natsLink)(nil)
|
||||||
|
_ OperatorAsks = (*natsLink)(nil)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// refusedSubscription is how the server words a subscription its user list does not grant.
|
||||||
|
var refusedSubscription = regexp.MustCompile(`Permissions Violation for Subscription to "([^"]+)"`)
|
||||||
|
|
||||||
|
// RefusedSubscription is the subject a permissions violation names when it refused a subscription; false for any other
|
||||||
|
// error, a refused publish among them (a publish is refused each time it is tried, and is said where it is tried).
|
||||||
|
func RefusedSubscription(err error) (string, bool) {
|
||||||
|
if err == nil || !errors.Is(err, nats.ErrPermissionViolation) && !strings.Contains(err.Error(), "permissions violation") {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
m := refusedSubscription.FindStringSubmatch(err.Error())
|
||||||
|
if m == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return m[1], true
|
||||||
|
}
|
||||||
|
|||||||
@@ -313,58 +313,14 @@ const KindRunner = "runner"
|
|||||||
// ReasonUsedAsFound starts the reason of a directory used as found.
|
// ReasonUsedAsFound starts the reason of a directory used as found.
|
||||||
const ReasonUsedAsFound = "used as found:"
|
const ReasonUsedAsFound = "used as found:"
|
||||||
|
|
||||||
// UnitsHealth is the machine's service managers in one statement (issue 315).
|
// The machine's networking and its service managers in a statement are the node's health report's parts (mesh-sdk
|
||||||
type UnitsHealth struct {
|
// delivery, issue 774): one layout for the message, the report and the watchman that reads it.
|
||||||
// State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no
|
type (
|
||||||
// manager could be read.
|
NetworkHealth = delivery.NetworkHealth
|
||||||
State string `json:"state"`
|
NetworkPart = delivery.NetworkPart
|
||||||
// Failed is every unit failed on two looks in a row that no module places: the machine's own.
|
UnitsHealth = delivery.UnitsHealth
|
||||||
Failed []FailedUnit `json:"failed"`
|
FailedUnit = delivery.FailedUnit
|
||||||
// Unread names a manager that could not be read, with why.
|
)
|
||||||
Unread []string `json:"unread,omitempty"`
|
|
||||||
// Away names an account's own manager not running, so not read: nothing in it is failed (issue 764).
|
|
||||||
Away []string `json:"away,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// FailedUnit is one failed unit no module places.
|
|
||||||
type FailedUnit struct {
|
|
||||||
Unit string `json:"unit"`
|
|
||||||
// Scope is system, or user: an account's own manager.
|
|
||||||
Scope string `json:"scope"`
|
|
||||||
// Load is loaded, not-found — a unit whose file is gone and whose failure its manager still holds…
|
|
||||||
Load string `json:"load,omitempty"`
|
|
||||||
// Result is how it failed: exit-code, timeout, start-limit-hit…
|
|
||||||
Result string `json:"result,omitempty"`
|
|
||||||
// Resource is the mesh's own resource that names it, when the mesh placed it in its own right.
|
|
||||||
Resource string `json:"resource,omitempty"`
|
|
||||||
Since time.Time `json:"since"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since
|
|
||||||
// when, and each part.
|
|
||||||
type NetworkHealth struct {
|
|
||||||
State string `json:"state"`
|
|
||||||
Since time.Time `json:"since"`
|
|
||||||
Parts []NetworkPart `json:"parts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// NetworkPart is one part of a machine's networking, as its engine judged it on its second look.
|
|
||||||
type NetworkPart struct {
|
|
||||||
// Part is resolv-conf, names, tunnel, bus or route.
|
|
||||||
Part string `json:"part"`
|
|
||||||
State string `json:"state"`
|
|
||||||
// Reason is why it is not healthy, in words with no address, path or domain; Said the detail.
|
|
||||||
Reason string `json:"reason,omitempty"`
|
|
||||||
Said string `json:"said,omitempty"`
|
|
||||||
// Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose
|
|
||||||
// file it is — the uplink's holder.
|
|
||||||
Writer string `json:"writer,omitempty"`
|
|
||||||
Owner string `json:"owner,omitempty"`
|
|
||||||
// Toward is what the failure points at: "hub", or each resolver's address that failed.
|
|
||||||
Toward []string `json:"toward,omitempty"`
|
|
||||||
Since time.Time `json:"since"`
|
|
||||||
Streak int `json:"streak,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// The states a long-running resource is said in (ADR 0240 §4).
|
// The states a long-running resource is said in (ADR 0240 §4).
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -54,6 +54,9 @@ type MoveSecretAnswer struct {
|
|||||||
Secret string `json:"secret,omitempty"`
|
Secret string `json:"secret,omitempty"`
|
||||||
Generation int `json:"generation,omitempty"`
|
Generation int `json:"generation,omitempty"`
|
||||||
Unchanged bool `json:"unchanged,omitempty"`
|
Unchanged bool `json:"unchanged,omitempty"`
|
||||||
|
// Superseded is set when the vault holds another value for the name, which is the live one: the row is done and
|
||||||
|
// is not asked again; Generation is the held value's, and Said its record in words.
|
||||||
|
Superseded bool `json:"superseded,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// MoveSecret is one move asked and believed, as the handler is given it: the row, this machine, and who asked.
|
// MoveSecret is one move asked and believed, as the handler is given it: the row, this machine, and who asked.
|
||||||
@@ -70,6 +73,9 @@ type MovedSecret struct {
|
|||||||
Secret string
|
Secret string
|
||||||
Generation int
|
Generation int
|
||||||
Unchanged bool
|
Unchanged bool
|
||||||
|
// Superseded says, in words, that the vault holds another value for the name, which is the live one: nothing was
|
||||||
|
// moved, and the move is done (the vault's record of the held value, its generation in Generation).
|
||||||
|
Superseded string
|
||||||
}
|
}
|
||||||
|
|
||||||
// MoveSecretFunc moves one value into the vault: what it kept, or why not.
|
// MoveSecretFunc moves one value into the vault: what it kept, or why not.
|
||||||
@@ -133,7 +139,11 @@ func AnswerMoveSecret(check HandOverCheck, handle MoveSecretFunc, body []byte) M
|
|||||||
if moved.Unchanged {
|
if moved.Unchanged {
|
||||||
said = fmt.Sprintf("the vault held %s already, the same value (generation %d)", moved.Secret, moved.Generation)
|
said = fmt.Sprintf("the vault held %s already, the same value (generation %d)", moved.Secret, moved.Generation)
|
||||||
}
|
}
|
||||||
return MoveSecretAnswer{Said: said, Secret: moved.Secret, Generation: moved.Generation, Unchanged: moved.Unchanged}
|
if moved.Superseded != "" {
|
||||||
|
said = moved.Superseded
|
||||||
|
}
|
||||||
|
return MoveSecretAnswer{Said: said, Secret: moved.Secret, Generation: moved.Generation, Unchanged: moved.Unchanged,
|
||||||
|
Superseded: moved.Superseded != ""}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MoveSecretReply is what a move request is answered with, and whether to answer, as HandOverReply.
|
// MoveSecretReply is what a move request is answered with, and whether to answer, as HandOverReply.
|
||||||
|
|||||||
@@ -28,8 +28,9 @@ func TestTheMovesAskKeepsItsFieldNamesAndContext(t *testing.T) {
|
|||||||
if MoveSecretContext != "novox-mesh move-secret v1\n" {
|
if MoveSecretContext != "novox-mesh move-secret v1\n" {
|
||||||
t.Fatalf("the signing context is %q", MoveSecretContext)
|
t.Fatalf("the signing context is %q", MoveSecretContext)
|
||||||
}
|
}
|
||||||
answer, _ := json.Marshal(MoveSecretAnswer{Said: "s", Refused: "r", Secret: "own/a/b/c", Generation: 1, Unchanged: true})
|
answer, _ := json.Marshal(MoveSecretAnswer{Said: "s", Refused: "r", Secret: "own/a/b/c", Generation: 1, Unchanged: true,
|
||||||
if got := keysOf(t, answer); got != "generation refused said secret unchanged" {
|
Superseded: true})
|
||||||
|
if got := keysOf(t, answer); got != "generation refused said secret superseded unchanged" {
|
||||||
t.Fatalf("the answer's fields are %q", got)
|
t.Fatalf("the answer's fields are %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,208 @@
|
|||||||
|
package link
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node-engine's own health report in the bus's own storage (issue 774, the operator's alert table, 2026-10-29): one
|
||||||
|
// key per node, `<node>`, of the engine's declared state `health` (HealthBucket, created by the grants part from the
|
||||||
|
// engine's manifest as its state `acts` is), written by this node's engine alone and only when what it says changed.
|
||||||
|
// The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the passing health message:
|
||||||
|
//
|
||||||
|
// - machine-network: Network, the machine's route, tunnel, name servers, resolver file and bus, each a part;
|
||||||
|
// - network-rewritten: the resolver file's part, unhealthy with Writer, the program that rewrote it, and Owner, the
|
||||||
|
// module whose file it is;
|
||||||
|
// - network-unreachable: Peers, each peer of the mesh's tunnel and whether this machine saw it within five minutes,
|
||||||
|
// read across every node's report;
|
||||||
|
// - machine-units: Units, the failed system units no module owns;
|
||||||
|
// - relogin-needed: Relogin, each account whose running session lacks a group it was given;
|
||||||
|
// - directory-used-as-found: Found, each directory a module declares that the engine uses as found.
|
||||||
|
//
|
||||||
|
|
||||||
|
// The report's layout is mesh-sdk's (delivery.NodeHealth): the bucket, its version and its parts.
|
||||||
|
const (
|
||||||
|
HealthBucket = delivery.NodeHealthBucket
|
||||||
|
NodeHealthVersion = delivery.NodeHealthVersion
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
NodeHealth = delivery.NodeHealth
|
||||||
|
PeerSeen = delivery.PeerSeen
|
||||||
|
NodeAccount = delivery.NodeAccount
|
||||||
|
NodeDirectory = delivery.NodeDirectory
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin).
|
||||||
|
const ReasonReloginNeeded = "relogin needed"
|
||||||
|
|
||||||
|
// Machines names the mesh's machines for a report's targets: each node by its address on the private network, and the
|
||||||
|
// hub's node. The zero value names none, and every part is then the machine's own.
|
||||||
|
type Machines struct {
|
||||||
|
ByAddress map[string]string
|
||||||
|
Hub string
|
||||||
|
}
|
||||||
|
|
||||||
|
// targets is the machines a part points at, as the controller holds it (mesh-controller machine_network.go): the hub's
|
||||||
|
// node for TowardHub, each resolver's node by its address; none when it points at nothing, or at this machine or an
|
||||||
|
// address no other mesh machine has, as one of them makes the part the machine's own.
|
||||||
|
func (m Machines) targets(node string, toward []string) []string {
|
||||||
|
if len(toward) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, t := range toward {
|
||||||
|
x := m.ByAddress[t]
|
||||||
|
if t == TowardHub {
|
||||||
|
x = m.Hub
|
||||||
|
}
|
||||||
|
if x == "" || x == node {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !slices.Contains(out, x) {
|
||||||
|
out = append(out, x)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TowardHub is what a part failing toward the hub names in Toward (network.TowardHub).
|
||||||
|
const TowardHub = "hub"
|
||||||
|
|
||||||
|
// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its
|
||||||
|
// streaks, each unhealthy part naming the machines it points at (machines), its machine units, and each account and
|
||||||
|
// directory of its resources that the six alerts read.
|
||||||
|
func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) NodeHealth {
|
||||||
|
out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units, Contract: h.Contract}
|
||||||
|
if len(peers) > 0 {
|
||||||
|
out.Peers = append([]PeerSeen(nil), peers...)
|
||||||
|
sort.Slice(out.Peers, func(i, k int) bool { return out.Peers[i].Key < out.Peers[k].Key })
|
||||||
|
}
|
||||||
|
if h.Network != nil {
|
||||||
|
n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))}
|
||||||
|
for _, p := range h.Network.Parts {
|
||||||
|
p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen
|
||||||
|
p.Targets = nil
|
||||||
|
if p.State == StateUnhealthy {
|
||||||
|
p.Targets = machines.targets(node, p.Toward)
|
||||||
|
}
|
||||||
|
n.Parts = append(n.Parts, p)
|
||||||
|
}
|
||||||
|
out.Network = &n
|
||||||
|
}
|
||||||
|
for _, r := range h.Resources {
|
||||||
|
// Each alert reads its own: an account may need a new login and be declared never to become root at once.
|
||||||
|
if r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded) {
|
||||||
|
out.Relogin = append(out.Relogin, NodeAccount{Account: r.Account, Module: r.Module, Resource: r.Resource,
|
||||||
|
Reason: r.Reason, Since: r.Since})
|
||||||
|
}
|
||||||
|
if r.Kind == KindAccount && r.Root == declaration.RootNever {
|
||||||
|
// The account declared never to become root without a person, judged (ADR 0266): what DA and D-root read.
|
||||||
|
out.Root = append(out.Root, delivery.RootVerdict{Account: r.Account, Module: r.Module, Resource: r.Resource,
|
||||||
|
State: r.State, Reason: r.Reason, Since: r.Since})
|
||||||
|
}
|
||||||
|
if r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound) {
|
||||||
|
out.Found = append(out.Found, NodeDirectory{Module: r.Module, Resource: r.Resource, Path: r.Target,
|
||||||
|
Reason: r.Reason, Since: r.Since})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(out.Relogin, func(i, k int) bool {
|
||||||
|
return out.Relogin[i].Account+out.Relogin[i].Module < out.Relogin[k].Account+out.Relogin[k].Module
|
||||||
|
})
|
||||||
|
sort.Slice(out.Found, func(i, k int) bool { return out.Found[i].Path < out.Found[k].Path })
|
||||||
|
sort.Slice(out.Root, func(i, k int) bool {
|
||||||
|
return out.Root[i].Module+out.Root[i].Resource < out.Root[k].Module+out.Root[k].Resource
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthStore is where a node's report is kept: its own key of HealthBucket.
|
||||||
|
type HealthStore interface {
|
||||||
|
// Get is the key's value, nil when it has none.
|
||||||
|
Get(ctx context.Context, key string) ([]byte, error)
|
||||||
|
Put(ctx context.Context, key string, value []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthStore binds HealthBucket over the link open now: bound, never created.
|
||||||
|
func (l *natsLink) healthStore() (HealthStore, error) {
|
||||||
|
kv, err := l.js.KeyValue(HealthBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+
|
||||||
|
"manifest; written again once it is): %w", HealthBucket, err)
|
||||||
|
}
|
||||||
|
return natsHealth{kv}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type natsHealth struct{ kv nats.KeyValue }
|
||||||
|
|
||||||
|
func (n natsHealth) Get(_ context.Context, key string) ([]byte, error) {
|
||||||
|
e, err := n.kv.Get(key)
|
||||||
|
if errors.Is(err, nats.ErrKeyNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return e.Value(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n natsHealth) Put(_ context.Context, key string, value []byte) error {
|
||||||
|
_, err := n.kv.Put(key, value)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteNodeHealth writes this node's report to its key when it says something other than the key holds, and says
|
||||||
|
// whether it wrote. The key is read before the first write of a run, so a restart that finds the same report writes
|
||||||
|
// nothing; a store that cannot be bound or written is an error, and the next look tries again.
|
||||||
|
func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, error) {
|
||||||
|
q.init()
|
||||||
|
q.mu.Lock()
|
||||||
|
bus := q.bus
|
||||||
|
q.mu.Unlock()
|
||||||
|
l, ok := bus.(interface{ healthStore() (HealthStore, error) })
|
||||||
|
if bus == nil || !ok {
|
||||||
|
return false, ErrNoLink
|
||||||
|
}
|
||||||
|
store, err := l.healthStore()
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeNodeHealth is WriteNodeHealth over a store: last is the report last written or read, nil before the first.
|
||||||
|
func writeNodeHealth(ctx context.Context, store HealthStore, last **NodeHealth, h NodeHealth, now func() time.Time) (bool, error) {
|
||||||
|
if *last == nil {
|
||||||
|
raw, err := store.Get(ctx, h.Node)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("the health report of %s cannot be read: %w", h.Node, err)
|
||||||
|
}
|
||||||
|
if raw != nil {
|
||||||
|
if was, err := delivery.DecodeNodeHealth(raw); err == nil {
|
||||||
|
*last = &was
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if *last != nil && delivery.SameNodeHealth(**last, h) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
h.At = now().UTC()
|
||||||
|
raw, err := delivery.EncodeNodeHealth(h)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := store.Put(ctx, h.Node, raw); err != nil {
|
||||||
|
return false, fmt.Errorf("the health report of %s cannot be written: %w", h.Node, err)
|
||||||
|
}
|
||||||
|
*last = &h
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
@@ -23,6 +23,8 @@ type RecordKeys struct {
|
|||||||
Certificate string
|
Certificate string
|
||||||
// Authority is the mesh's own certificate as this machine last applied it (public).
|
// Authority is the mesh's own certificate as this machine last applied it (public).
|
||||||
Authority string
|
Authority string
|
||||||
|
// Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2).
|
||||||
|
Trusts []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
|
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
|
||||||
@@ -45,7 +47,7 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
|
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
|
||||||
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority}
|
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts}
|
||||||
for _, c := range p.Capabilities {
|
for _, c := range p.Capabilities {
|
||||||
if c.Present && c.Name != "" {
|
if c.Present && c.Name != "" {
|
||||||
rec.Capabilities = append(rec.Capabilities, c.Name)
|
rec.Capabilities = append(rec.Capabilities, c.Name)
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
meshasks "git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Membership is what a node needs to reach its mesh again, held by the caller.
|
// Membership is what a node needs to reach its mesh again, held by the caller.
|
||||||
@@ -148,6 +150,15 @@ func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Refusals is a link that says each subject the bus refused it a subscription to. Optional.
|
||||||
|
type Refusals interface {
|
||||||
|
Refused() <-chan string
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusalsSettle is how long a link runs with no subscription refused before it counts as holding them: refusals arrive
|
||||||
|
// within moments of subscribing.
|
||||||
|
const refusalsSettle = 30 * time.Second
|
||||||
|
|
||||||
// Run holds the link open once, hearing what the controller asks and saying the node's facts.
|
// Run holds the link open once, hearing what the controller asks and saying the node's facts.
|
||||||
//
|
//
|
||||||
// Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason;
|
// Outbound only, and nothing listens on this machine. Returns when the link ends, for any reason;
|
||||||
@@ -185,6 +196,12 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
|||||||
}
|
}
|
||||||
// And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue
|
// And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue
|
||||||
// 356), answered on the link the question came over.
|
// 356), answered on the link the question came over.
|
||||||
|
// Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7).
|
||||||
|
if f, ok := link.(interface {
|
||||||
|
FollowSigner(func() ed25519.PublicKey)
|
||||||
|
}); ok && queue.ControllerKey != nil {
|
||||||
|
f.FollowSigner(queue.ControllerKey)
|
||||||
|
}
|
||||||
if h, ok := link.(HandOvers); ok {
|
if h, ok := link.(HandOvers); ok {
|
||||||
h.HearHandOvers(queue.HandOver)
|
h.HearHandOvers(queue.HandOver)
|
||||||
}
|
}
|
||||||
@@ -200,11 +217,39 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
|||||||
if s, ok := link.(AccessActs); ok {
|
if s, ok := link.(AccessActs); ok {
|
||||||
s.HearAccessActs(queue.AccessAct)
|
s.HearAccessActs(queue.AccessAct)
|
||||||
}
|
}
|
||||||
|
// And its node seat, node-engine, with the operator's answers to its own asks (novox/hq ADR 0563): served on this
|
||||||
|
// link, and what was answered while no link was open caught up from the router's records.
|
||||||
|
if queue.Acts != nil {
|
||||||
|
if s, ok := link.(EngineSeats); ok {
|
||||||
|
s.ServeEngineSeat(queue.Acts.Call)
|
||||||
|
}
|
||||||
|
if a, ok := link.(OperatorAsks); ok {
|
||||||
|
if err := a.HearWarrants(meshasks.EngineAsker(m.Node), queue.Acts.Heard); err != nil {
|
||||||
|
say("cannot hear the operator's answers to this engine's asks: " + err.Error())
|
||||||
|
}
|
||||||
|
go queue.Acts.Reconcile()
|
||||||
|
}
|
||||||
|
}
|
||||||
// And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676).
|
// And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676).
|
||||||
if s, ok := link.(MoveSecrets); ok {
|
if s, ok := link.(MoveSecrets); ok {
|
||||||
s.HearMoveSecrets(queue.MoveSecret)
|
s.HearMoveSecrets(queue.MoveSecret)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// And every subscription the bus refused, because the user list it loaded does not grant it (yet): nats asks a
|
||||||
|
// refused subscription again only on a new connection, and a reload of the user list does not make one, so the node
|
||||||
|
// seat, the hand-over and every other subject heard here stayed deaf until the engine restarted (the sweep of
|
||||||
|
// 2026-10-30: every engine was refused its node seat at 12:10 and answered nobody all afternoon). The link is kept
|
||||||
|
// for a pause, said, and then dropped, so Hold opens it again and every subscription is asked anew.
|
||||||
|
var refused <-chan string
|
||||||
|
if r, ok := link.(Refusals); ok {
|
||||||
|
refused = r.Refused()
|
||||||
|
}
|
||||||
|
var again <-chan time.Time
|
||||||
|
var first string
|
||||||
|
refusedCount := 0
|
||||||
|
held := time.NewTimer(refusalsSettle)
|
||||||
|
defer held.Stop()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -216,6 +261,23 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
|||||||
return reason
|
return reason
|
||||||
case <-asks:
|
case <-asks:
|
||||||
queue.FactsAsked(ctx)
|
queue.FactsAsked(ctx)
|
||||||
|
case subject := <-refused:
|
||||||
|
refusedCount++
|
||||||
|
if again == nil {
|
||||||
|
first = subject
|
||||||
|
pause := queue.refusedPause()
|
||||||
|
say(fmt.Sprintf("the bus refused this node a subscription to %s: the user list it loaded does not grant "+
|
||||||
|
"it, so nothing here answers there; the link is opened again in %s so every subscription is asked "+
|
||||||
|
"again", subject, pause))
|
||||||
|
again = time.After(pause)
|
||||||
|
}
|
||||||
|
case <-held.C:
|
||||||
|
if refusedCount == 0 {
|
||||||
|
queue.refusalsHeld()
|
||||||
|
}
|
||||||
|
case <-again:
|
||||||
|
return fmt.Errorf("the bus refused %d of this node's subscriptions, %s first; opened again so each is "+
|
||||||
|
"asked again", refusedCount, first)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,12 +2,14 @@ package link
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/health"
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -47,11 +49,23 @@ type Session struct {
|
|||||||
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
|
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
|
||||||
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
|
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
|
||||||
MoveSecret MoveSecretFunc
|
MoveSecret MoveSecretFunc
|
||||||
|
// ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq
|
||||||
|
// ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's.
|
||||||
|
ControllerKey func() ed25519.PublicKey
|
||||||
|
// Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the
|
||||||
|
// operator asked and heard over it. Nil serves no seat.
|
||||||
|
Acts *HandActs
|
||||||
// Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes
|
// Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes
|
||||||
// no record.
|
// no record.
|
||||||
Keys func() RecordKeys
|
Keys func() RecordKeys
|
||||||
Say Announce
|
Say Announce
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
|
// RefusedPause is how long a link keeps running after the bus refused it a subscription before it is dropped and
|
||||||
|
// opened again (zero is DefaultRefusedPause); doubled for each link in a row that is refused again, to RefusedPauseMost.
|
||||||
|
RefusedPause time.Duration
|
||||||
|
|
||||||
|
// refusedInARow is how many links in a row the bus refused a subscription to.
|
||||||
|
refusedInARow int
|
||||||
|
|
||||||
once sync.Once
|
once sync.Once
|
||||||
|
|
||||||
@@ -78,6 +92,39 @@ type Session struct {
|
|||||||
|
|
||||||
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
|
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
|
||||||
haves haveState
|
haves haveState
|
||||||
|
// healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone
|
||||||
|
// writes it, one look at a time.
|
||||||
|
healthWritten *NodeHealth
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultRefusedPause and RefusedPauseMost bound how soon a link refused a subscription is opened again: the grants
|
||||||
|
// part composes the user list again within a minute of what it is composed from changing, so the first pause waits for
|
||||||
|
// that; one that is still refused waits longer each time, never more than half an hour, and always says so.
|
||||||
|
const (
|
||||||
|
DefaultRefusedPause = time.Minute
|
||||||
|
RefusedPauseMost = 30 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// refusedPause is the pause before a link the bus refused a subscription to is opened again, counting it.
|
||||||
|
func (q *Session) refusedPause() time.Duration {
|
||||||
|
q.mu.Lock()
|
||||||
|
defer q.mu.Unlock()
|
||||||
|
pause := q.RefusedPause
|
||||||
|
if pause <= 0 {
|
||||||
|
pause = DefaultRefusedPause
|
||||||
|
}
|
||||||
|
for i := 0; i < q.refusedInARow && pause < RefusedPauseMost; i++ {
|
||||||
|
pause *= 2
|
||||||
|
}
|
||||||
|
q.refusedInARow++
|
||||||
|
return min(pause, RefusedPauseMost)
|
||||||
|
}
|
||||||
|
|
||||||
|
// refusalsHeld says a link has held its subscriptions: the next refusal starts from the first pause again.
|
||||||
|
func (q *Session) refusalsHeld() {
|
||||||
|
q.mu.Lock()
|
||||||
|
q.refusedInARow = 0
|
||||||
|
q.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (q *Session) init() {
|
func (q *Session) init() {
|
||||||
@@ -337,3 +384,49 @@ func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, with
|
|||||||
}
|
}
|
||||||
return env.Result, nil
|
return env.Result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PublishAsk publishes an ask or a cancel to the operator's router over the link open now (novox/hq ADR 0563 §3).
|
||||||
|
func (q *Session) PublishAsk(subject string, body any, id string) error {
|
||||||
|
asking, err := q.operatorAsks()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return asking.PublishAsk(subject, body, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RouterRecord reads one key of the router's records over the link open now.
|
||||||
|
func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) {
|
||||||
|
asking, err := q.operatorAsks()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return asking.RouterRecord(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SayEngine says one of the node seat's events over the link open now; with none open it is not said.
|
||||||
|
func (q *Session) SayEngine(event string, body any) {
|
||||||
|
q.init()
|
||||||
|
q.mu.Lock()
|
||||||
|
bus := q.bus
|
||||||
|
q.mu.Unlock()
|
||||||
|
if sayer, ok := bus.(interface{ SayEngine(string, any) error }); ok {
|
||||||
|
if err := sayer.SayEngine(event, body); err != nil {
|
||||||
|
q.Say("could not say the node seat's " + event + ": " + err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Session) operatorAsks() (OperatorAsks, error) {
|
||||||
|
q.init()
|
||||||
|
q.mu.Lock()
|
||||||
|
bus := q.bus
|
||||||
|
q.mu.Unlock()
|
||||||
|
if bus == nil {
|
||||||
|
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
|
||||||
|
}
|
||||||
|
asking, ok := bus.(OperatorAsks)
|
||||||
|
if !ok {
|
||||||
|
return nil, errNoAsking
|
||||||
|
}
|
||||||
|
return asking, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ import (
|
|||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -162,3 +164,78 @@ func eventuallyHeld(t *testing.T, what string, ok func() bool) {
|
|||||||
time.Sleep(5 * time.Millisecond)
|
time.Sleep(5 * time.Millisecond)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refusingLink is a quiet link the bus refused subscriptions to.
|
||||||
|
type refusingLink struct {
|
||||||
|
*quietLink
|
||||||
|
refused chan string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *refusingLink) Refused() <-chan string { return l.refused }
|
||||||
|
|
||||||
|
// The sweep of 2026-10-30: every engine was refused its node seat's subscriptions at 12:10, by a user list composed
|
||||||
|
// without them, and nats asks a refused subscription again only on a new connection; the seat answered nobody all
|
||||||
|
// afternoon. A refused subscription is read from the server's words, said, and the link opened again after a pause
|
||||||
|
// that grows while it is refused again; a refused publish is not one.
|
||||||
|
func TestASubscriptionTheBusRefusedIsAskedAgainOnANewLink(t *testing.T) {
|
||||||
|
// The server's words, as the client hands them on (journal of nox-mesh-host on novox, 2026-10-30 12:10:44).
|
||||||
|
for _, row := range []struct {
|
||||||
|
err error
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{fmt.Errorf("%w: %s", nats.ErrPermissionViolation,
|
||||||
|
`Permissions Violation for Subscription to "mesh.seat.node-engine.tool.root-free.novox"`),
|
||||||
|
"mesh.seat.node-engine.tool.root-free.novox"},
|
||||||
|
{errors.Join(fmt.Errorf("%w: %s", nats.ErrPermissionViolation,
|
||||||
|
`Permissions Violation for Subscription to "$SRV.PING.node-engine"`), errors.New("stopping service: x")),
|
||||||
|
"$SRV.PING.node-engine"},
|
||||||
|
{fmt.Errorf("%w: %s", nats.ErrPermissionViolation,
|
||||||
|
`Permissions Violation for Publish to "$JS.API.STREAM.INFO.KV_mesh-host_health"`), ""},
|
||||||
|
{errors.New("nats: timeout"), ""},
|
||||||
|
} {
|
||||||
|
got, ok := RefusedSubscription(row.err)
|
||||||
|
if got != row.want || ok != (row.want != "") {
|
||||||
|
t.Errorf("%v reads as %q, %v", row.err, got, ok)
|
||||||
|
}
|
||||||
|
l := &natsLink{refused: make(chan string, 1)}
|
||||||
|
l.asyncError(nil, nil, row.err)
|
||||||
|
if heard := len(l.refused); heard != map[bool]int{true: 1}[row.want != ""] {
|
||||||
|
t.Errorf("%v handed %d refusals on", row.err, heard)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
m, _ := aMember(t)
|
||||||
|
q := &Session{Membership: m, Timeout: time.Second, RefusedPause: 50 * time.Millisecond}
|
||||||
|
var said []string
|
||||||
|
var mu sync.Mutex
|
||||||
|
say := func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }
|
||||||
|
for _, want := range []time.Duration{50 * time.Millisecond, 100 * time.Millisecond} {
|
||||||
|
l := &refusingLink{quietLink: newQuietLink(), refused: make(chan string, 2)}
|
||||||
|
l.refused <- "mesh.seat.node-engine.tool.root-free.n1"
|
||||||
|
l.refused <- "$SRV.PING"
|
||||||
|
began := time.Now()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- serve(context.Background(), l, m, q, say, time.Second) }()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "refused 2 of this node's subscriptions, "+
|
||||||
|
"mesh.seat.node-engine.tool.root-free.n1 first") {
|
||||||
|
t.Fatalf("the link ended with %v", err)
|
||||||
|
}
|
||||||
|
if took := time.Since(began); took < want {
|
||||||
|
t.Errorf("the link was dropped after %s, before its pause of %s", took, want)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("a link the bus refused a subscription was kept, and the subscription never asked again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if n := strings.Count(strings.Join(said, "\n"), "the bus refused this node a subscription to "+
|
||||||
|
"mesh.seat.node-engine.tool.root-free.n1"); n != 2 {
|
||||||
|
t.Errorf("the refusal was said %d times in two links: %q", n, said)
|
||||||
|
}
|
||||||
|
if q.refusalsHeld(); q.refusedPause() != 50*time.Millisecond {
|
||||||
|
t.Error("a link that held its subscriptions did not start from the first pause again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -57,4 +57,82 @@ func TestASetuidSearchAskIsBelievedOnlySignedForThisNodeAndOnce(t *testing.T) {
|
|||||||
if a := AnswerSetuidSearch(check, nil, body); a.Refused == "" {
|
if a := AnswerSetuidSearch(check, nil, body); a.Refused == "" {
|
||||||
t.Fatalf("no handler: %+v", a)
|
t.Fatalf("no handler: %+v", a)
|
||||||
}
|
}
|
||||||
|
// 774 (ADR 0563 §2): the node seat's setuid-search starts none while one runs, at most one in any hour, naming when the
|
||||||
|
// next may start, and records who asked.
|
||||||
|
t.Run("the node seat's search is bounded", func(t *testing.T) {
|
||||||
|
at, running, started := now, false, 0
|
||||||
|
h := &HandActs{Node: "n1", State: &MemState{}, Now: func() time.Time { return at },
|
||||||
|
Searching: func() bool { return running },
|
||||||
|
SetuidSearch: func(by string) (string, error) { started++; return "a new search starts", nil }}
|
||||||
|
for _, row := range []struct {
|
||||||
|
name string
|
||||||
|
after time.Duration
|
||||||
|
running bool
|
||||||
|
refused string
|
||||||
|
started int
|
||||||
|
}{
|
||||||
|
{name: "one runs", running: true, refused: "runs on this machine now", started: 0},
|
||||||
|
{name: "the first", started: 1},
|
||||||
|
{name: "within the hour", after: 59 * time.Minute, refused: "the next may start at 2026-10-10T03:00:00Z", started: 1},
|
||||||
|
{name: "after the hour", after: time.Hour + time.Second, started: 2},
|
||||||
|
} {
|
||||||
|
at, running = now.Add(row.after), row.running
|
||||||
|
_, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"})
|
||||||
|
if (row.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.refused)) || started != row.started {
|
||||||
|
t.Errorf("%s: %v, %d started", row.name, err, started)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
acts, _ := h.Acts(1)
|
||||||
|
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 1 || got[0].Caller != "ace.node-tools" || got[0].State != HandDone {
|
||||||
|
t.Errorf("the last search is recorded as %+v", got)
|
||||||
|
}
|
||||||
|
// The bound is kept in a key of its own (rev-l3 round 1): two hundred judgements before a search, or the acts
|
||||||
|
// gone from the record, never let a second search start within the hour.
|
||||||
|
h.JudgeDirectories = func(string) (string, error) { return "judged", nil }
|
||||||
|
for range 200 {
|
||||||
|
if _, err := h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "ace.node-tools"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
at = now.Add(time.Hour + 2*time.Minute)
|
||||||
|
if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") {
|
||||||
|
t.Errorf("a search after two hundred judgements: %v", err)
|
||||||
|
}
|
||||||
|
keys, _ := h.State.Keys("act.")
|
||||||
|
for _, k := range keys {
|
||||||
|
_ = h.State.Delete(k)
|
||||||
|
}
|
||||||
|
if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") || started != 2 {
|
||||||
|
t.Errorf("a search with the acts gone: %v, %d started", err, started)
|
||||||
|
}
|
||||||
|
// rev-l3 round 2 (ADR 0563 §2, a wrong caller gains nothing): judgements and refused searches, which any caller
|
||||||
|
// may ask, keep one rolling key of the node's, so ten thousand of them write a bounded number of bytes and add no
|
||||||
|
// key to the bucket every node shares.
|
||||||
|
keysNow := func() (int, int) {
|
||||||
|
all, _ := h.State.Keys("")
|
||||||
|
size := 0
|
||||||
|
for _, k := range all {
|
||||||
|
if e, _ := h.State.Get(k); e != nil {
|
||||||
|
size += len(e.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(all), size
|
||||||
|
}
|
||||||
|
_, _ = h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "unproven"})
|
||||||
|
keysBefore, _ := keysNow()
|
||||||
|
for i := range 10000 {
|
||||||
|
verb := ActJudgeDirectories
|
||||||
|
if i%10 == 0 {
|
||||||
|
verb = ActSetuidSearch
|
||||||
|
}
|
||||||
|
_, _ = h.Call(EngineCall{Verb: verb, Caller: "unproven"})
|
||||||
|
}
|
||||||
|
if n, size := keysNow(); n != keysBefore || size > 64<<10 {
|
||||||
|
t.Errorf("ten thousand judgements and refused searches left %d keys (was %d) of %d bytes", n, keysBefore, size)
|
||||||
|
}
|
||||||
|
listed, _ := h.Acts(0)
|
||||||
|
if got := listed.(map[string]any)["acts"].([]HandAct); len(got) == 0 || got[0].Caller != "unproven" {
|
||||||
|
t.Errorf("acts does not show the latest judgement first: %d", len(got))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
@@ -23,7 +24,9 @@ type KVUpdate struct {
|
|||||||
// never polls.
|
// never polls.
|
||||||
type KVBus interface {
|
type KVBus interface {
|
||||||
// KVWatch yields the keys of the bucket matching the pattern now, a Caught update, then every change, until ctx ends
|
// KVWatch yields the keys of the bucket matching the pattern now, a Caught update, then every change, until ctx ends
|
||||||
// or the link does (the channel is then closed). A bucket the node's account may not read is an error.
|
// or the link does (the channel is then closed). A bucket the node's account may not read is an error. The pattern
|
||||||
|
// may be several, separated by spaces (WatchPatterns), watched as one: a node's placement in either form of its
|
||||||
|
// keys (research 072 row 37).
|
||||||
KVWatch(ctx context.Context, bucket, pattern string) (<-chan KVUpdate, error)
|
KVWatch(ctx context.Context, bucket, pattern string) (<-chan KVUpdate, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -41,7 +44,7 @@ func (b OverNATS) KVWatch(ctx context.Context, bucket, pattern string) (<-chan K
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", bucket, err)
|
return nil, fmt.Errorf("the bucket %s: %w", bucket, err)
|
||||||
}
|
}
|
||||||
w, err := kv.Watch(pattern, nats.Context(ctx))
|
w, err := kv.WatchFiltered(strings.Fields(pattern), nats.Context(ctx))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("watching %s in %s: %w", pattern, bucket, err)
|
return nil, fmt.Errorf("watching %s in %s: %w", pattern, bucket, err)
|
||||||
}
|
}
|
||||||
@@ -296,3 +299,6 @@ func (q *Session) KVWatchAwait(ctx context.Context, bucket, pattern string, wait
|
|||||||
}()
|
}()
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WatchPatterns is several key patterns as one pattern KVWatch takes.
|
||||||
|
func WatchPatterns(patterns ...string) string { return strings.Join(patterns, " ") }
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -94,6 +95,18 @@ type Finding struct {
|
|||||||
Writer string
|
Writer string
|
||||||
// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
|
// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
|
||||||
Toward []string
|
Toward []string
|
||||||
|
// Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer).
|
||||||
|
Peers []Peer
|
||||||
|
}
|
||||||
|
|
||||||
|
// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and
|
||||||
|
// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose
|
||||||
|
// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the
|
||||||
|
// fact only when a peer comes or goes.
|
||||||
|
type Peer struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Addresses []string `json:"addresses,omitempty"`
|
||||||
|
Seen bool `json:"seen"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Part is one part's state, as the statement says it.
|
// Part is one part's state, as the statement says it.
|
||||||
@@ -115,6 +128,8 @@ type Statement struct {
|
|||||||
Since time.Time `json:"since"`
|
Since time.Time `json:"since"`
|
||||||
At time.Time `json:"at"`
|
At time.Time `json:"at"`
|
||||||
Parts []Part `json:"parts"`
|
Parts []Part `json:"parts"`
|
||||||
|
// Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read.
|
||||||
|
Peers []Peer `json:"peers,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Machine is what a look reads, replaced in tests.
|
// Machine is what a look reads, replaced in tests.
|
||||||
@@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
|
|||||||
changed = true
|
changed = true
|
||||||
}
|
}
|
||||||
st.State, st.Since = worst, j.overall.since
|
st.State, st.Since = worst, j.overall.since
|
||||||
|
// The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable).
|
||||||
|
st.Peers = findings[PartTunnel].Peers
|
||||||
|
changed = changed || !samePeers(j.said.Peers, st.Peers)
|
||||||
j.said = st
|
j.said = st
|
||||||
return st, changed
|
return st, changed
|
||||||
}
|
}
|
||||||
@@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
hub := ""
|
hub := ""
|
||||||
|
allowed := map[string][]string{}
|
||||||
for _, line := range strings.Split(ips, "\n") {
|
for _, line := range strings.Split(ips, "\n") {
|
||||||
f := strings.Fields(line)
|
f := strings.Fields(line)
|
||||||
for _, prefix := range f[min(1, len(f)):] {
|
for _, prefix := range f[min(1, len(f)):] {
|
||||||
|
allowed[f[0]] = append(allowed[f[0]], prefix)
|
||||||
if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
|
if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
|
||||||
hub = f[0]
|
hub = f[0]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
peers := make([]Peer, 0, len(handshakes))
|
||||||
|
for key, at := range handshakes {
|
||||||
|
peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake})
|
||||||
|
}
|
||||||
|
sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key })
|
||||||
|
found := func(f Finding) Finding { f.Peers = peers; return f }
|
||||||
age := func(at int64) string {
|
age := func(at int64) string {
|
||||||
if at == 0 {
|
if at == 0 {
|
||||||
return "never"
|
return "never"
|
||||||
@@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
|
|||||||
if hub != "" {
|
if hub != "" {
|
||||||
at := handshakes[hub]
|
at := handshakes[hub]
|
||||||
if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
|
if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
|
||||||
return Finding{OK: true}
|
return found(Finding{OK: true})
|
||||||
}
|
}
|
||||||
return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
|
||||||
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}
|
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}})
|
||||||
}
|
}
|
||||||
if len(handshakes) == 0 {
|
if len(handshakes) == 0 {
|
||||||
return Finding{OK: true}
|
return found(Finding{OK: true})
|
||||||
}
|
}
|
||||||
var newest int64
|
var newest int64
|
||||||
for _, at := range handshakes {
|
for _, at := range handshakes {
|
||||||
newest = max(newest, at)
|
newest = max(newest, at)
|
||||||
}
|
}
|
||||||
if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
|
if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
|
||||||
return Finding{OK: true}
|
return found(Finding{OK: true})
|
||||||
}
|
}
|
||||||
return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
|
return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
|
||||||
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}
|
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))})
|
||||||
|
}
|
||||||
|
|
||||||
|
// samePeers says two looks saw the same peers, each seen or not alike.
|
||||||
|
func samePeers(a, b []Peer) bool {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range a {
|
||||||
|
if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *Judge) lookBus() Finding {
|
func (j *Judge) lookBus() Finding {
|
||||||
|
|||||||
@@ -302,9 +302,27 @@ func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) {
|
|||||||
f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n"
|
f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n"
|
||||||
f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n"
|
f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n"
|
||||||
j := f.judge(t)
|
j := f.judge(t)
|
||||||
if st := f.look(t, j); st.State != Healthy {
|
st := f.look(t, j)
|
||||||
|
if st.State != Healthy {
|
||||||
t.Fatalf("the hub with one fresh peer is %+v", st.Parts)
|
t.Fatalf("the hub with one fresh peer is %+v", st.Parts)
|
||||||
}
|
}
|
||||||
|
// Each peer and whether it was seen within five minutes, for the watchman's network-unreachable (issue 774); a
|
||||||
|
// peer seen again is a change, a handshake renewed alone is not.
|
||||||
|
want := []Peer{{Key: "A=", Addresses: []string{"10.10.0.2/32"}}, {Key: "B=", Addresses: []string{"10.10.0.3/32"}, Seen: true},
|
||||||
|
{Key: "C=", Addresses: []string{"10.10.0.4/32"}}}
|
||||||
|
if !samePeers(st.Peers, want) {
|
||||||
|
t.Fatalf("the peers are %+v", st.Peers)
|
||||||
|
}
|
||||||
|
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
|
||||||
|
f.now = f.now.Add(LookEvery)
|
||||||
|
if st, changed := j.Look(t.Context()); !changed || !st.Peers[0].Seen {
|
||||||
|
t.Fatalf("a peer seen again is no change: changed %v, %+v", changed, st.Peers)
|
||||||
|
}
|
||||||
|
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
|
||||||
|
f.now = f.now.Add(LookEvery)
|
||||||
|
if _, changed := j.Look(t.Context()); changed {
|
||||||
|
t.Fatal("handshakes renewed alone are a change")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) {
|
func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) {
|
||||||
|
|||||||
@@ -0,0 +1,239 @@
|
|||||||
|
package selfcompose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The data items of each module placed here, as this engine composed them** (novox/hq ADR 0577 clause 1). After a
|
||||||
|
// pass that composed and handed something new (or the first one of a run), the composer works out each placed module's
|
||||||
|
// data items from the same inputs it composed from (frombus.DataItems, which reads them through this composer's own
|
||||||
|
// view of the bus) and keeps each module's in the engine's state data, key `<node>.<module>`, with the digest of the
|
||||||
|
// module's composition: written only when the value differs, deleted when the module no longer declares any here or left
|
||||||
|
// the node. That node's disk-load watches its node's keys and measures what they name; it composes nothing (clause 2).
|
||||||
|
|
||||||
|
// DataStore is this node's keys of the engine's state data (link.EngineData).
|
||||||
|
type DataStore interface {
|
||||||
|
// Get is a key's value, nil when it has none.
|
||||||
|
Get(ctx context.Context, key string) ([]byte, error)
|
||||||
|
Put(ctx context.Context, key string, value []byte) error
|
||||||
|
Delete(ctx context.Context, key string) error
|
||||||
|
// Keys is every key matching pattern that holds a value.
|
||||||
|
Keys(ctx context.Context, pattern string) ([]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// itemsReader is the composer's view of the bus for working out data items: no own secret is asked of the vault, since
|
||||||
|
// working out where an item lies reads no secret (the items are rendered as a plan is, compose.Planning).
|
||||||
|
type itemsReader struct{ reader }
|
||||||
|
|
||||||
|
func (itemsReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dataItemsOf works out the data items of the scope's node and, by module, why each module held for a manifest the
|
||||||
|
// indexer refused is left out (frombus.DataItemsHeld, ADR 0578 §3); a test sets what it answers.
|
||||||
|
var dataItemsOf = frombus.DataItemsHeld
|
||||||
|
|
||||||
|
// publishData keeps the data items of this node's modules in the engine's state data, when a pass asked it (dataDue).
|
||||||
|
func (c *Composer) publishData(ctx context.Context) {
|
||||||
|
c.mu.Lock()
|
||||||
|
due := c.dataDue
|
||||||
|
c.dataDue = false
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !due || c.env.Data == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
scope := c.dataScope
|
||||||
|
c.mu.Unlock()
|
||||||
|
if scope.Node == "" {
|
||||||
|
scope.Node = c.env.Node
|
||||||
|
}
|
||||||
|
items, unplaced, held, err := dataItemsOf(ctx, itemsReader{reader{c: c}}, c.env.Mesh, scope)
|
||||||
|
if err != nil {
|
||||||
|
c.sayData("the data items of this node's modules cannot be worked out, and what the engine's state " +
|
||||||
|
delivery.EngineDataBucket + " holds stands; " + c.oweData() + ": " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// A module held for a refused manifest declares nothing this pass can read: it is named, its other modules' items
|
||||||
|
// are kept as ever, and its own key stands as it was until the manifest is accepted (ADR 0578 §3).
|
||||||
|
if len(held) > 0 {
|
||||||
|
var lines []string
|
||||||
|
for _, m := range heldModules(held) {
|
||||||
|
lines = append(lines, m+": "+held[m])
|
||||||
|
}
|
||||||
|
c.env.Say("data items of " + strings.Join(heldModules(held), ", ") + " not worked out, as the manifest is refused " +
|
||||||
|
"and the module held (its key stands as it was): " + strings.Join(lines, "; "))
|
||||||
|
}
|
||||||
|
if len(unplaced) > 0 {
|
||||||
|
c.env.Say("data items left out as not placeable: " + strings.Join(unplaced, "; "))
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
digests := c.digestsByModule()
|
||||||
|
c.mu.Unlock()
|
||||||
|
wrote, removed, err := c.writeData(ctx, items, digests, held)
|
||||||
|
if err != nil {
|
||||||
|
c.sayData("the data items of this node's modules are not all kept in " + delivery.EngineDataBucket +
|
||||||
|
"; " + c.oweData() + ": " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
c.dataTries = 0
|
||||||
|
c.mu.Unlock()
|
||||||
|
c.sayData("")
|
||||||
|
if wrote+removed > 0 {
|
||||||
|
c.env.Say(fmt.Sprintf("the data items of this node's modules: %d key(s) of %s written, %d deleted",
|
||||||
|
wrote, delivery.EngineDataBucket, removed))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// oweData keeps the data items due and has the composer's loop make a pass again after the growing bound (LaterWait), as
|
||||||
|
// a pass refused for a bus that did not answer is: the first write after a delivery may be refused while the state or
|
||||||
|
// its grant is not on the bus yet, and nothing else would try again until an unrelated composition changed. It answers
|
||||||
|
// the words that say when.
|
||||||
|
func (c *Composer) oweData() string {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.dataDue = true
|
||||||
|
c.wait = LaterWait(c.dataTries)
|
||||||
|
c.dataTries++
|
||||||
|
c.owed.Store(true)
|
||||||
|
return fmt.Sprintf("tried again at %s (try %d)", time.Now().Add(c.wait).Format("15:04:05"), c.dataTries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayData says a line about the data items once until it changes; "" clears it.
|
||||||
|
func (c *Composer) sayData(line string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
same := c.dataSaid == line
|
||||||
|
c.dataSaid = line
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !same && line != "" {
|
||||||
|
c.env.Say(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// digestsByModule is the digest of each module's composition handed last, and the node's own under "" for a module run
|
||||||
|
// inside it. Under c.mu.
|
||||||
|
func (c *Composer) digestsByModule() map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
for key, dig := range c.handed {
|
||||||
|
_, module, _, isNode, err := delivery.ParseComposedKey(key)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if isNode {
|
||||||
|
module = ""
|
||||||
|
}
|
||||||
|
out[module] = dig
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeData writes each module's key whose value differs from what the key holds, and deletes this node's keys of
|
||||||
|
// modules with no items here now. The keys are read once per run (dataKept), so a restart that finds the same items
|
||||||
|
// writes nothing.
|
||||||
|
func (c *Composer) writeData(ctx context.Context, items map[string][]catalogue.PlacedDataItem, digests map[string]string, held map[string]string) (wrote, removed int, err error) {
|
||||||
|
store := c.env.Data
|
||||||
|
if c.dataKept == nil {
|
||||||
|
pattern, err := delivery.EngineDataNodeKeys(c.env.Node)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, err
|
||||||
|
}
|
||||||
|
keys, err := store.Keys(ctx, pattern)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("this node's keys cannot be listed: %w", err)
|
||||||
|
}
|
||||||
|
kept := map[string][]byte{}
|
||||||
|
for _, k := range keys {
|
||||||
|
raw, err := store.Get(ctx, k)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("%s cannot be read: %w", k, err)
|
||||||
|
}
|
||||||
|
if raw != nil {
|
||||||
|
kept[k] = raw
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.dataKept = kept
|
||||||
|
}
|
||||||
|
want := map[string][]byte{}
|
||||||
|
modules := make([]string, 0, len(items))
|
||||||
|
for m := range items {
|
||||||
|
modules = append(modules, m)
|
||||||
|
}
|
||||||
|
sort.Strings(modules)
|
||||||
|
var failed []string
|
||||||
|
for _, module := range modules {
|
||||||
|
if len(items[module]) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, err := delivery.EngineDataKey(c.env.Node, module)
|
||||||
|
if err != nil {
|
||||||
|
failed = append(failed, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
digest, ok := digests[module]
|
||||||
|
if !ok {
|
||||||
|
digest = digests[""] // run inside the node's own composition
|
||||||
|
}
|
||||||
|
raw, err := delivery.EncodeEngineData(delivery.EngineData{Node: c.env.Node, Module: module, Digest: digest, Items: items[module]})
|
||||||
|
if err != nil {
|
||||||
|
failed = append(failed, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
want[key] = raw
|
||||||
|
}
|
||||||
|
keys := make([]string, 0, len(want))
|
||||||
|
for k := range want {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, key := range keys {
|
||||||
|
if bytes.Equal(c.dataKept[key], want[key]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := store.Put(ctx, key, want[key]); err != nil {
|
||||||
|
failed = append(failed, key+": "+err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.dataKept[key] = want[key]
|
||||||
|
wrote++
|
||||||
|
}
|
||||||
|
var gone []string
|
||||||
|
for key := range c.dataKept {
|
||||||
|
if _, still := want[key]; !still && !heldKey(c.env.Node, key, held) {
|
||||||
|
gone = append(gone, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(gone)
|
||||||
|
for _, key := range gone {
|
||||||
|
if err := store.Delete(ctx, key); err != nil {
|
||||||
|
failed = append(failed, key+": "+err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(c.dataKept, key)
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
if len(failed) > 0 {
|
||||||
|
return wrote, removed, fmt.Errorf("%s", strings.Join(failed, "; "))
|
||||||
|
}
|
||||||
|
return wrote, removed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldKey is whether key is the key of a module held for a refused manifest, which stands as it was.
|
||||||
|
func heldKey(node, key string, held map[string]string) bool {
|
||||||
|
for module := range held {
|
||||||
|
if k, err := delivery.EngineDataKey(node, module); err == nil && k == key {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
package selfcompose
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeData is the engine's state data as a map, counting the writes and deletes made.
|
||||||
|
type fakeData struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
kv map[string][]byte
|
||||||
|
puts []string
|
||||||
|
deletes []string
|
||||||
|
fail error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeData) Get(_ context.Context, key string) ([]byte, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.kv[key], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeData) Put(_ context.Context, key string, value []byte) error {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if f.fail != nil {
|
||||||
|
return f.fail
|
||||||
|
}
|
||||||
|
f.kv[key] = value
|
||||||
|
f.puts = append(f.puts, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeData) Delete(_ context.Context, key string) error {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
delete(f.kv, key)
|
||||||
|
f.deletes = append(f.deletes, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeData) Keys(_ context.Context, pattern string) ([]string, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
var out []string
|
||||||
|
for k := range f.kv {
|
||||||
|
if matches(pattern, k) {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0577 clause 1 (and its check in clause 5): after a pass that composed something new, the engine keeps each placed
|
||||||
|
// module's data items in its state data, key `<node>.<module>`, with its composition's digest: written when the module's
|
||||||
|
// items change, not written when they do not, and deleted when the module leaves the node; a key of another node is never
|
||||||
|
// touched, a pass that handed nothing new reads nothing, and items that could not be worked out or written are tried
|
||||||
|
// again with no new composition, after the composer's growing bound (the loop's owed pass).
|
||||||
|
func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) {
|
||||||
|
item := func(module, id, path string) catalogue.PlacedDataItem {
|
||||||
|
return catalogue.PlacedDataItem{Module: module, ID: id, Class: "valuable", Path: path, Measure: catalogue.MeasureWalk}
|
||||||
|
}
|
||||||
|
store := &fakeData{kv: map[string][]byte{
|
||||||
|
// Left by a module that went while the engine was down, and another node's key.
|
||||||
|
"n1.gone": []byte(`{"v":1}`), "n2.web": []byte(`{"v":1}`)}}
|
||||||
|
var items map[string][]catalogue.PlacedDataItem
|
||||||
|
var asked int
|
||||||
|
var itemsErr error
|
||||||
|
var heldNow map[string]string
|
||||||
|
setFor(t, &dataItemsOf, func(context.Context, frombus.Reader, frombus.Mesh, frombus.Scope) (map[string][]catalogue.PlacedDataItem, []string, map[string]string, error) {
|
||||||
|
asked++
|
||||||
|
return items, nil, heldNow, itemsErr
|
||||||
|
})
|
||||||
|
var said []string
|
||||||
|
env := Env{Node: "n1", Data: store, Say: func(s string) { said = append(said, s) }}
|
||||||
|
// The engine's grant lets it read its own node's keys, as the listing at a run's first pass does, and no other's.
|
||||||
|
if !grantAllows(t, "n1", delivery.EngineDataBucket, "n1.*") || grantAllows(t, "n1", delivery.EngineDataBucket, "n2.*") {
|
||||||
|
t.Error("the node's grant reads other keys of the engine's state data than its own")
|
||||||
|
}
|
||||||
|
c := New(env)
|
||||||
|
webKey, _ := delivery.ComposedKey("n1", "web", delivery.SourcePrefix2+strings.Repeat("a", 64))
|
||||||
|
nodeKey, _ := delivery.ComposedNodeKey("n1")
|
||||||
|
for _, step := range []struct {
|
||||||
|
name string
|
||||||
|
due bool
|
||||||
|
items map[string][]catalogue.PlacedDataItem
|
||||||
|
digests map[string]string // composed key to digest handed
|
||||||
|
fail error
|
||||||
|
refused error // what working out the items says
|
||||||
|
held map[string]string // modules held for a refused manifest, with why (ADR 0578 §3)
|
||||||
|
owed time.Duration // the bound the loop waits before trying again; 0 when nothing is owed
|
||||||
|
puts []string
|
||||||
|
deletes []string
|
||||||
|
asked int
|
||||||
|
// want is each key's items and digest after the step.
|
||||||
|
want map[string]delivery.EngineData
|
||||||
|
}{
|
||||||
|
{name: "the run's first pass writes each module's items and deletes what left", due: true,
|
||||||
|
items: map[string][]catalogue.PlacedDataItem{"web": {item("web", "db", "/srv/web/db")}, "oldie": {item("oldie", "x", "/srv/oldie")}, "bare": nil},
|
||||||
|
digests: map[string]string{webKey: "sha256:w1", nodeKey: "sha256:n1"},
|
||||||
|
puts: []string{"n1.oldie", "n1.web"}, deletes: []string{"n1.gone"}, asked: 1,
|
||||||
|
want: map[string]delivery.EngineData{
|
||||||
|
"n1.web": {Module: "web", Digest: "sha256:w1", Items: []catalogue.PlacedDataItem{item("web", "db", "/srv/web/db")}},
|
||||||
|
"n1.oldie": {Module: "oldie", Digest: "sha256:n1", Items: []catalogue.PlacedDataItem{item("oldie", "x", "/srv/oldie")}}}},
|
||||||
|
{name: "a pass that handed nothing new reads nothing", due: false, asked: 1},
|
||||||
|
{name: "a pass that changes no module's items writes nothing", due: true,
|
||||||
|
items: map[string][]catalogue.PlacedDataItem{"web": {item("web", "db", "/srv/web/db")}, "oldie": {item("oldie", "x", "/srv/oldie")}},
|
||||||
|
digests: map[string]string{webKey: "sha256:w1", nodeKey: "sha256:n1"}, asked: 2},
|
||||||
|
{name: "items that cannot be worked out are tried again after the bound", due: true, refused: errors.New("no answer"),
|
||||||
|
digests: map[string]string{webKey: "sha256:w2", nodeKey: "sha256:n1"}, asked: 3, owed: LaterWait(0)},
|
||||||
|
{name: "a write that fails is tried again with no new composition, after a longer bound", due: false, fail: errors.New("the bus said no"),
|
||||||
|
items: map[string][]catalogue.PlacedDataItem{"web": {item("web", "db", "/srv/web/db2")}, "oldie": {item("oldie", "x", "/srv/oldie")}},
|
||||||
|
digests: map[string]string{webKey: "sha256:w2", nodeKey: "sha256:n1"}, asked: 4, owed: LaterWait(1)},
|
||||||
|
{name: "a module whose composition changed its items is written again", due: false,
|
||||||
|
items: map[string][]catalogue.PlacedDataItem{"web": {item("web", "db", "/srv/web/db2")}, "oldie": {item("oldie", "x", "/srv/oldie")}},
|
||||||
|
digests: map[string]string{webKey: "sha256:w2", nodeKey: "sha256:n1"}, puts: []string{"n1.web"}, asked: 5,
|
||||||
|
want: map[string]delivery.EngineData{
|
||||||
|
"n1.web": {Module: "web", Digest: "sha256:w2", Items: []catalogue.PlacedDataItem{item("web", "db", "/srv/web/db2")}}}},
|
||||||
|
{name: "a module that left the node is deleted", due: true,
|
||||||
|
items: map[string][]catalogue.PlacedDataItem{"oldie": {item("oldie", "x", "/srv/oldie")}},
|
||||||
|
digests: map[string]string{nodeKey: "sha256:n1"}, deletes: []string{"n1.web"}, asked: 6},
|
||||||
|
// ADR 0578 §3: one refused manifest never ends the read; its module is named, its key stands, the rest is kept.
|
||||||
|
{name: "a module held for a refused manifest is named, its key stands and every other module's items are written", due: true,
|
||||||
|
items: map[string][]catalogue.PlacedDataItem{"web": {item("web", "db", "/srv/web/db")}},
|
||||||
|
held: map[string]string{"oldie": "the manifest of oldie is refused: a bar item is no icon"},
|
||||||
|
digests: map[string]string{webKey: "sha256:w3", nodeKey: "sha256:n1"}, puts: []string{"n1.web"}, asked: 7,
|
||||||
|
want: map[string]delivery.EngineData{
|
||||||
|
"n1.web": {Module: "web", Digest: "sha256:w3", Items: []catalogue.PlacedDataItem{item("web", "db", "/srv/web/db")}}}},
|
||||||
|
} {
|
||||||
|
t.Run(step.name, func(t *testing.T) {
|
||||||
|
store.mu.Lock()
|
||||||
|
store.puts, store.deletes, store.fail = nil, nil, step.fail
|
||||||
|
store.mu.Unlock()
|
||||||
|
items, itemsErr, heldNow, said = step.items, step.refused, step.held, nil
|
||||||
|
c.paid() // the pass before composed: nothing owed
|
||||||
|
c.mu.Lock()
|
||||||
|
c.handed = step.digests
|
||||||
|
c.dataDue = c.dataDue || step.due
|
||||||
|
c.mu.Unlock()
|
||||||
|
c.publishData(context.Background())
|
||||||
|
slices.Sort(store.puts)
|
||||||
|
if !slices.Equal(store.puts, step.puts) || !slices.Equal(store.deletes, step.deletes) || asked != step.asked {
|
||||||
|
t.Fatalf("wrote %v and deleted %v, items asked %d times; want %v, %v, %d", store.puts, store.deletes, asked,
|
||||||
|
step.puts, step.deletes, step.asked)
|
||||||
|
}
|
||||||
|
if wait, _ := c.waitNow(); c.owed.Load() != (step.owed > 0) || wait != step.owed {
|
||||||
|
t.Errorf("owed a pass: %v after %s; want after %s", c.owed.Load(), wait, step.owed)
|
||||||
|
}
|
||||||
|
for key, want := range step.want {
|
||||||
|
got, err := delivery.DecodeEngineData(store.kv[key])
|
||||||
|
if err != nil || got.Node != "n1" || got.Module != want.Module || got.Digest != want.Digest ||
|
||||||
|
!slices.Equal(got.Items, want.Items) {
|
||||||
|
t.Errorf("%s holds %+v (%v), want %+v", key, got, err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for module, why := range step.held {
|
||||||
|
if !slices.ContainsFunc(said, func(l string) bool { return strings.Contains(l, module) && strings.Contains(l, why) }) {
|
||||||
|
t.Errorf("the held %s is not named with why in %q", module, said)
|
||||||
|
}
|
||||||
|
if store.kv["n1."+module] == nil {
|
||||||
|
t.Errorf("the key of the held %s was deleted", module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if store.kv["n2.web"] == nil {
|
||||||
|
t.Error("another node's key was deleted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// The items are worked out from what the pass read: a done build is read over the bus once, not again for the data
|
||||||
|
// items after the pass (a module in a running build is read again, since it may land).
|
||||||
|
b := newFakeBus()
|
||||||
|
done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: delivery.ManifestDigest([]byte(lemurs))})
|
||||||
|
running, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: "building"})
|
||||||
|
fp := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||||
|
doneKey, _ := delivery.BuilderBuildKey("web", fp)
|
||||||
|
runningKey, _ := delivery.BuilderBuildKey("pg", fp)
|
||||||
|
b.kv[delivery.BuilderBuildsBucket+"/"+doneKey], b.kv[delivery.BuilderBuildsBucket+"/"+runningKey] = done, running
|
||||||
|
r := reader{c: New(b.env())}
|
||||||
|
for range 3 {
|
||||||
|
if got, err := r.Build(context.Background(), "web", fp); err != nil || got.State != delivery.BuildDone {
|
||||||
|
t.Fatalf("the done build: %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := r.Build(context.Background(), "pg", fp); err != nil || got.State != "building" {
|
||||||
|
t.Fatalf("the running build: %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reads := map[string]int{}
|
||||||
|
for _, read := range b.reads {
|
||||||
|
reads[read[1]]++
|
||||||
|
}
|
||||||
|
if reads[doneKey] != 1 || reads[runningKey] != 3 {
|
||||||
|
t.Errorf("the builds were read %v times over the bus; want the done one once, the running one each time", reads)
|
||||||
|
}
|
||||||
|
// A done key the builder claims again (a gone step, issue 1937) and lands done again is read again, not served kept.
|
||||||
|
again, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: delivery.ManifestDigest([]byte("{}"))})
|
||||||
|
b.mu.Lock()
|
||||||
|
b.kv[delivery.BuilderBuildsBucket+"/"+doneKey] = again
|
||||||
|
b.mu.Unlock()
|
||||||
|
r.c.takeBuildDone(agent.Update{Key: doneKey, Value: running})
|
||||||
|
if got, err := r.Build(context.Background(), "web", fp); err != nil || got.Manifest != delivery.ManifestDigest([]byte("{}")) {
|
||||||
|
t.Errorf("a done build written again is served as %+v (%v), not read again", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,11 +12,15 @@ package selfcompose
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"reflect"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -28,6 +32,7 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
@@ -75,6 +80,16 @@ type Env struct {
|
|||||||
// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
|
// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
|
||||||
// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
|
// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
|
||||||
Fetch func(ctx context.Context, url string) ([]byte, error)
|
Fetch func(ctx context.Context, url string) ([]byte, error)
|
||||||
|
// VaultAuthority is the vault's authority as this node read it from the vault's `signers` (ADR 0550 §2), composed
|
||||||
|
// beside the controller's as the mesh's authority (frombus.SignersReader); "" or nil composes the controller's alone.
|
||||||
|
VaultAuthority func() string
|
||||||
|
// Certify asks the vault's `sign` for this node's own certificate, keeps it and writes the node's record again (ADR
|
||||||
|
// 0550 §2): asked only when a module placed here asks for a certificate and the record holds none from an authority
|
||||||
|
// the node trusts. Nil asks nothing, and such a module stands in, as before.
|
||||||
|
Certify func(ctx context.Context) error
|
||||||
|
// Data is this node's keys of the engine's state data (novox/hq ADR 0577 clause 1), where each placed module's data
|
||||||
|
// items are kept after a pass that composed something new. Nil keeps none.
|
||||||
|
Data DataStore
|
||||||
}
|
}
|
||||||
|
|
||||||
// VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value.
|
// VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value.
|
||||||
@@ -87,18 +102,31 @@ type Composer struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
wants map[string]delivery.Want // by module, the node's own under agent.NodeModule
|
wants map[string]delivery.Want // by module, the node's own under agent.NodeModule
|
||||||
placements map[string]delivery.Placement
|
placements map[string]delivery.Placement
|
||||||
|
// placementKeys is the seat's state placement key by key in either form, each node's placement assembled from it
|
||||||
|
// (delivery.PlacementKeys; research 072 row 37): a write of the holder's under way is never composed half.
|
||||||
|
placementKeys *delivery.PlacementKeys
|
||||||
records map[string]delivery.NodeRecord
|
records map[string]delivery.NodeRecord
|
||||||
// inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module
|
// inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module
|
||||||
// at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126).
|
// at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126).
|
||||||
inOwn map[string]bool
|
inOwn map[string]bool
|
||||||
modules map[string]delivery.Module
|
modules map[string]delivery.Module
|
||||||
|
told map[string]bool // lines said once each (sayOnceEach)
|
||||||
values map[string]map[string]any // by module, this node's
|
values map[string]map[string]any // by module, this node's
|
||||||
manifests map[string][]byte // by digest, read once
|
manifests map[string][]byte // by digest, read once
|
||||||
archives map[string][]byte // templates archives by digest, read once
|
archives map[string][]byte // templates archives by digest, read once
|
||||||
|
// builds are the builder's done builds by key, read once: a done build at a fingerprint never changes, so a pass and
|
||||||
|
// the data items after it (publishData) do not read each placed module's build again over the bus (ADR 0577).
|
||||||
|
builds map[string]delivery.BuilderBuild
|
||||||
handed map[string]string // composed key to the digest handed
|
handed map[string]string // composed key to the digest handed
|
||||||
// sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again
|
// sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again
|
||||||
// is other bytes, so the one first had is kept while the generation stands, and the composition stays the same.
|
// is other bytes, so the one first had is kept while the generation stands, and the composition stays the same.
|
||||||
sealedOwn map[string]ownSealed
|
sealedOwn map[string]ownSealed
|
||||||
|
// ownHanded is, by module, whether the vault judged it handed to the provisioner in its last own answer (its
|
||||||
|
// `handed`), nil when that answer said nothing of it (a vault older than the field): the composition hands exactly
|
||||||
|
// those (frombus.HandedReader, issue 774).
|
||||||
|
ownHanded map[string]bool
|
||||||
|
// certified is the serving key a certificate was last asked for successfully, so it is asked once per key.
|
||||||
|
certified string
|
||||||
said string
|
said string
|
||||||
refusal string
|
refusal string
|
||||||
// held is what the composer's last pass held: each module refused alone, and the node's own ("node") when it could
|
// held is what the composer's last pass held: each module refused alone, and the node's own ("node") when it could
|
||||||
@@ -115,6 +143,80 @@ type Composer struct {
|
|||||||
missed atomic.Uint64
|
missed atomic.Uint64
|
||||||
// owed is set when a pass refused for a bus that did not answer: the composer composes again after a bound.
|
// owed is set when a pass refused for a bus that did not answer: the composer composes again after a bound.
|
||||||
owed atomic.Bool
|
owed atomic.Bool
|
||||||
|
// tries counts the passes owed in a row, and wait is the bound before the next (LaterWait): set by the pass that owes
|
||||||
|
// one, so the refusal it marks says when the node composes again (issue 774, 2026-10-31). Under mu.
|
||||||
|
tries int
|
||||||
|
wait time.Duration
|
||||||
|
// asked are the compose verb's asks (ComposeNow), each answered once the pass it asks is made.
|
||||||
|
asked chan chan Pass
|
||||||
|
// last is how the last pass went, under mu.
|
||||||
|
last Pass
|
||||||
|
// watching says the composer's loop runs (Watch), so an ask of a pass can be answered.
|
||||||
|
watching atomic.Bool
|
||||||
|
// dataDue says a pass composed something new (or is the run's first), so the data items are kept again
|
||||||
|
// (publishData); dataSaid is the last line said of them. Under mu. dataKept is what this node's keys of the state
|
||||||
|
// data hold, read once per run, nil before; the loop's alone.
|
||||||
|
dataDue bool
|
||||||
|
dataTries int // data runs failed in a row, for oweData's bound
|
||||||
|
dataSaid string
|
||||||
|
dataScope frombus.Scope
|
||||||
|
dataKept map[string][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pass is how one pass went, as the node-engine's verb compose answers it.
|
||||||
|
type Pass struct {
|
||||||
|
// At is when the pass ended.
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Refused is why the node composed none of its modules; empty when it composed.
|
||||||
|
Refused string `json:"refused,omitempty"`
|
||||||
|
// Held is each module refused alone, with why.
|
||||||
|
Held map[string]string `json:"held,omitempty"`
|
||||||
|
// Handed is how many compositions the pass handed the agent that differ from what it handed before.
|
||||||
|
Handed int `json:"handed"`
|
||||||
|
// Again is when the node composes again on its own, after a seat or a read that did not answer; zero when it does
|
||||||
|
// not need to.
|
||||||
|
Again time.Time `json:"again,omitempty"`
|
||||||
|
// Earlier says the answer is the last pass, made less than askedWithin ago, and no new one was made for the ask.
|
||||||
|
Earlier bool `json:"earlier,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// askedWithin is how recent a pass answers an ask of the verb compose without a new one: the first wait after a seat
|
||||||
|
// that did not answer (laterAgain's first).
|
||||||
|
var askedWithin = 5 * time.Second
|
||||||
|
|
||||||
|
// passSaysEvery is how often a pass that runs on says, in the journal, how long it has run and what it waits on.
|
||||||
|
var passSaysEvery = time.Minute
|
||||||
|
|
||||||
|
// owe marks a pass owed after a seat or a read that did not answer: the next wait of the growing bound (LaterWait), and
|
||||||
|
// the words that say when the node composes again, for the refusal it marks.
|
||||||
|
func (c *Composer) owe() string {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.wait = LaterWait(c.tries)
|
||||||
|
c.tries++
|
||||||
|
c.owed.Store(true)
|
||||||
|
return fmt.Sprintf("the node composes again at %s (try %d)", time.Now().Add(c.wait).Format("15:04:05"), c.tries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// paid clears what owe marked: the pass composed, or owes nothing.
|
||||||
|
func (c *Composer) paid() {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.tries, c.wait = 0, 0
|
||||||
|
c.owed.Store(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// againLine is what a retry pass says as it starts: the try its refusal named.
|
||||||
|
func (c *Composer) againLine() string {
|
||||||
|
_, try := c.waitNow()
|
||||||
|
return fmt.Sprintf("the node composes again now (try %d), after a seat or a read that did not answer", try)
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitNow is the bound before the pass owed, and its try.
|
||||||
|
func (c *Composer) waitNow() (time.Duration, int) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.wait, c.tries
|
||||||
}
|
}
|
||||||
|
|
||||||
// note counts err when it is the bus not answering, and returns it.
|
// note counts err when it is the bus not answering, and returns it.
|
||||||
@@ -136,9 +238,41 @@ func New(env Env) *Composer {
|
|||||||
if env.Say == nil {
|
if env.Say == nil {
|
||||||
env.Say = func(string) {}
|
env.Say = func(string) {}
|
||||||
}
|
}
|
||||||
return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{},
|
return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{}, placementKeys: delivery.NewPlacementKeys(),
|
||||||
records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{},
|
records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{},
|
||||||
manifests: map[string][]byte{}, archives: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}}
|
manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]delivery.BuilderBuild{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{},
|
||||||
|
asked: make(chan chan Pass, 8)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ComposeNow asks the composer for a pass now (the node-engine's verb compose, issue 774): the growing bound after a seat
|
||||||
|
// that did not answer starts again, and the answer is how the pass went. Refused in words when no pass is made before
|
||||||
|
// ctx ends: the composer not running, or its inputs not all read yet.
|
||||||
|
//
|
||||||
|
// A pass ended less than askedWithin ago is the answer, and no new one is made, so a caller asking in a loop never
|
||||||
|
// drives passes back to back against the vault; an ask never restarts the growing bound either (review round 1).
|
||||||
|
func (c *Composer) ComposeNow(ctx context.Context) (Pass, error) {
|
||||||
|
if !c.watching.Load() {
|
||||||
|
return Pass{}, errors.New("the node's composer runs no loop now: it composes nothing until its agent starts again")
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
last := c.last
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !last.At.IsZero() && time.Since(last.At) < askedWithin {
|
||||||
|
last.Earlier = true
|
||||||
|
return last, nil
|
||||||
|
}
|
||||||
|
reply := make(chan Pass, 1)
|
||||||
|
select {
|
||||||
|
case c.asked <- reply:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Pass{}, errors.New("the node's composer took no ask: it composes nothing now")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case p := <-reply:
|
||||||
|
return p, nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return Pass{}, fmt.Errorf("the node made no pass in time; it waits on %s", c.waitingOn())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ownSealed is one own secret's sealed value at a generation and the key it is sealed to.
|
// ownSealed is one own secret's sealed value at a generation and the key it is sealed to.
|
||||||
@@ -168,9 +302,38 @@ func (c *Composer) inputs() []input {
|
|||||||
{delivery.HaveBucket, node + ".>", nil, (*Composer).takeModuleHave},
|
{delivery.HaveBucket, node + ".>", nil, (*Composer).takeModuleHave},
|
||||||
{delivery.ModulesBucket, ">", (*Composer).takeModule, nil},
|
{delivery.ModulesBucket, ">", (*Composer).takeModule, nil},
|
||||||
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
||||||
|
// The builder's builds: a pass that held a module for a build not yet done is made again when it lands (issue
|
||||||
|
// 774, 2026-10-30 19:30: node-tools' build landed and ace, g14 and shanks stayed refused until another input moved).
|
||||||
|
{delivery.BuilderBuildsBucket, ">", nil, (*Composer).takeBuildDone},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// takeBuildDone says whether a build that landed is one this node composes from: done, of a module it wants at that
|
||||||
|
// build or whose trunk's build it is. Every other build, and a build not done, moves nothing.
|
||||||
|
func (c *Composer) takeBuildDone(u agent.Update) bool {
|
||||||
|
// Any write or delete of a build's key forgets the build kept as done: the builder claims a done key again as a new
|
||||||
|
// revision (a gone step, issue 1937), which lands done with other artifacts.
|
||||||
|
c.mu.Lock()
|
||||||
|
delete(c.builds, u.Key)
|
||||||
|
c.mu.Unlock()
|
||||||
|
if u.Deleted {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
module, fp, err := delivery.ParseBuilderBuildKey(u.Key)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
w, wanted := c.wants[module]
|
||||||
|
m := c.modules[module]
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !wanted || (w.FP != fp && m.FP2 != fp) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
b, err := delivery.DecodeBuilderBuild(u.Value)
|
||||||
|
return err == nil && b.State == delivery.BuildDone
|
||||||
|
}
|
||||||
|
|
||||||
// Watch starts the composer and answers the compositions it makes: each as a key of `composed-<node>` would be, then a
|
// Watch starts the composer and answers the compositions it makes: each as a key of `composed-<node>` would be, then a
|
||||||
// Caught update once every input was read and the node composed once, then every change. It ends when ctx ends or an
|
// Caught update once every input was read and the node composed once, then every change. It ends when ctx ends or an
|
||||||
// input's watch ends.
|
// input's watch ends.
|
||||||
@@ -226,15 +389,17 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
out := make(chan agent.Update, 64)
|
out := make(chan agent.Update, 64)
|
||||||
|
c.watching.Store(true)
|
||||||
go func() {
|
go func() {
|
||||||
defer close(out)
|
defer close(out)
|
||||||
|
defer c.watching.Store(false)
|
||||||
caught := make([]bool, len(ins))
|
caught := make([]bool, len(ins))
|
||||||
ready := false
|
ready := false
|
||||||
dirty := false
|
dirty := false
|
||||||
// A pass refused for a bus that did not answer is made again after a bound (laterAgain), not kept until another
|
// A pass refused for a bus that did not answer is made again after a bound (laterAgain), not kept until another
|
||||||
// input moves (issue 1126, 2026-10-28).
|
// input moves (issue 1126, 2026-10-28).
|
||||||
var again <-chan time.Time
|
var again <-chan time.Time
|
||||||
tries := 0
|
var replies []chan Pass
|
||||||
for {
|
for {
|
||||||
// Drain whatever came, then compose once for all of it.
|
// Drain whatever came, then compose once for all of it.
|
||||||
select {
|
select {
|
||||||
@@ -243,6 +408,13 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
|||||||
case <-again:
|
case <-again:
|
||||||
again = nil
|
again = nil
|
||||||
dirty = true
|
dirty = true
|
||||||
|
c.env.Say(c.againLine())
|
||||||
|
case reply := <-c.asked:
|
||||||
|
// Asked by hand (the verb compose): composed now; the growing bound is not restarted, so a pass that
|
||||||
|
// fails again waits longer, not less.
|
||||||
|
replies = append(replies, reply)
|
||||||
|
again = nil
|
||||||
|
dirty = true
|
||||||
case t := <-merged:
|
case t := <-merged:
|
||||||
if !t.ok {
|
if !t.ok {
|
||||||
c.env.Say(fmt.Sprintf("the watch of %s ended; the node composes no more until its agent starts again", ins[t.in].bucket))
|
c.env.Say(fmt.Sprintf("the watch of %s ended; the node composes no more until its agent starts again", ins[t.in].bucket))
|
||||||
@@ -256,8 +428,7 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
|||||||
case ins[t.in].changed != nil:
|
case ins[t.in].changed != nil:
|
||||||
dirty = ins[t.in].changed(c, t.u) || dirty
|
dirty = ins[t.in].changed(c, t.u) || dirty
|
||||||
default:
|
default:
|
||||||
ins[t.in].take(c, t.u)
|
dirty = c.takeChanged(ins[t.in], t.u) || dirty
|
||||||
dirty = true
|
|
||||||
}
|
}
|
||||||
if len(merged) > 0 {
|
if len(merged) > 0 {
|
||||||
continue
|
continue
|
||||||
@@ -281,14 +452,21 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
c.publishData(ctx)
|
||||||
if c.owed.Load() {
|
if c.owed.Load() {
|
||||||
wait := LaterWait(tries)
|
wait, _ := c.waitNow()
|
||||||
tries++
|
|
||||||
c.env.Say(fmt.Sprintf("the node composes again in %s: a read on the bus did not answer", wait))
|
c.env.Say(fmt.Sprintf("the node composes again in %s: a read on the bus did not answer", wait))
|
||||||
again = time.After(wait)
|
again = time.After(wait)
|
||||||
} else {
|
} else {
|
||||||
tries, again = 0, nil
|
again = nil
|
||||||
}
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
last := c.last
|
||||||
|
c.mu.Unlock()
|
||||||
|
for _, r := range replies {
|
||||||
|
r <- last
|
||||||
|
}
|
||||||
|
replies = nil
|
||||||
if !c.caughtSent() {
|
if !c.caughtSent() {
|
||||||
select {
|
select {
|
||||||
case out <- agent.Update{Caught: true}:
|
case out <- agent.Update{Caught: true}:
|
||||||
@@ -319,6 +497,67 @@ func (c *Composer) caughtSent() bool {
|
|||||||
return sent
|
return sent
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// takeChanged takes u through in.take and says whether the entry it names changed: a want, a placement, a module or a
|
||||||
|
// config value written again with the value it held changes nothing composed, so it makes no pass (issue 774,
|
||||||
|
// 2026-10-31: every put under want.<node>.> made a pass, and each pass asked the vault for the node's own secrets).
|
||||||
|
func (c *Composer) takeChanged(in input, u agent.Update) bool {
|
||||||
|
before, had, known := c.entry(in.bucket, u.Key)
|
||||||
|
in.take(c, u)
|
||||||
|
if !known {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
after, has, _ := c.entry(in.bucket, u.Key)
|
||||||
|
return had != has || !reflect.DeepEqual(before, after)
|
||||||
|
}
|
||||||
|
|
||||||
|
// entry is what the composer holds of one key of bucket, whether it holds it, and whether the bucket is one it knows.
|
||||||
|
func (c *Composer) entry(bucket, key string) (v any, has, known bool) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
switch bucket {
|
||||||
|
case delivery.AssignerWantsBucket:
|
||||||
|
// A want is composed by its fingerprint alone (fingerprints); its words, rule and newest change nothing composed.
|
||||||
|
module := agent.NodeModule
|
||||||
|
if _, err := delivery.ParseWantNodeKey(key); err != nil {
|
||||||
|
node, m, err := delivery.ParseWantKey(key)
|
||||||
|
if err != nil || node != c.env.Node {
|
||||||
|
return nil, false, true
|
||||||
|
}
|
||||||
|
module = m
|
||||||
|
}
|
||||||
|
w, ok := c.wants[module]
|
||||||
|
return w.FP, ok, true
|
||||||
|
case delivery.PlacementBucket:
|
||||||
|
// Any key of a node, its node key, a module key or its legacy key, stands for the node's whole placement: a
|
||||||
|
// module key written before its node key names it changes nothing composed.
|
||||||
|
node, _, _, err := delivery.ParsePlacementKey(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, true
|
||||||
|
}
|
||||||
|
p, ok := c.placements[node]
|
||||||
|
return p, ok, true
|
||||||
|
case delivery.ModulesBucket:
|
||||||
|
name, err := delivery.ParseModulesKey(key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, true
|
||||||
|
}
|
||||||
|
m, ok := c.modules[name]
|
||||||
|
return m, ok, true
|
||||||
|
case ResolvedBucket:
|
||||||
|
rest, ok := strings.CutPrefix(key, c.env.Node+".")
|
||||||
|
if !ok {
|
||||||
|
return nil, false, true
|
||||||
|
}
|
||||||
|
module, k, ok := strings.Cut(rest, ".")
|
||||||
|
if !ok {
|
||||||
|
return nil, false, true
|
||||||
|
}
|
||||||
|
val, ok := c.values[module][k]
|
||||||
|
return val, ok, true
|
||||||
|
}
|
||||||
|
return nil, false, false
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Composer) takeWant(u agent.Update) {
|
func (c *Composer) takeWant(u agent.Update) {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
@@ -343,18 +582,25 @@ func (c *Composer) takeWant(u agent.Update) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *Composer) takePlacement(u agent.Update) {
|
func (c *Composer) takePlacement(u agent.Update) {
|
||||||
node, err := delivery.ParsePlacementKey(u.Key)
|
if u.Caught {
|
||||||
if err != nil {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
if c.placementKeys == nil {
|
||||||
p, err := delivery.DecodePlacement(u.Value)
|
c.placementKeys = delivery.NewPlacementKeys()
|
||||||
if u.Deleted || err != nil || p.Node != node {
|
|
||||||
delete(c.placements, node)
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
node, err := c.placementKeys.Put(u.Key, u.Value, u.Revision, u.Deleted || len(u.Value) == 0)
|
||||||
|
if node != "" {
|
||||||
|
if p, _, ok := c.placementKeys.Of(node); ok {
|
||||||
c.placements[node] = p
|
c.placements[node] = p
|
||||||
|
} else {
|
||||||
|
delete(c.placements, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
if node != "" && err != nil && c.env.Say != nil {
|
||||||
|
c.env.Say("the seat's placement of " + node + ": " + err.Error())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Composer) takeRecord(u agent.Update) {
|
func (c *Composer) takeRecord(u agent.Update) {
|
||||||
@@ -393,7 +639,12 @@ func (c *Composer) takeModuleHave(u agent.Update) bool {
|
|||||||
// A have that names nothing it runs, failed (the node could not compose a module it never ran: NotComposed), is
|
// A have that names nothing it runs, failed (the node could not compose a module it never ran: NotComposed), is
|
||||||
// no module run inside the node's own composition: taken as one, it was composed from its manifest past its
|
// no module run inside the node's own composition: taken as one, it was composed from its manifest past its
|
||||||
// build (issue 1126, review round 1 of the builds).
|
// build (issue 1126, review round 1 of the builds).
|
||||||
runsNothing := h.Phase == delivery.PhaseFailed && h.FP == "" && h.Composed == ""
|
// Nor is one held by a backup the node agent's own stop interrupted that names nothing (a have written before
|
||||||
|
// mesh-host #189, novox's postgres 2026-10-31): taken as one, it was composed from its manifest and refused
|
||||||
|
// for its secret reader before the start backed it up again. Any other have naming nothing keeps the rule
|
||||||
|
// (waiting for a yes inside the node's own stays in).
|
||||||
|
interrupted := h.Phase == delivery.PhaseHeld && h.Detail == agent.BackupInterrupted
|
||||||
|
runsNothing := (h.Phase == delivery.PhaseFailed || interrupted) && h.FP == "" && h.Composed == ""
|
||||||
in = h.Phase != delivery.PhaseRemoved && !runsNothing && !strings.HasPrefix(h.FP, delivery.SourcePrefix2)
|
in = h.Phase != delivery.PhaseRemoved && !runsNothing && !strings.HasPrefix(h.FP, delivery.SourcePrefix2)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -411,6 +662,24 @@ func (c *Composer) takeModuleHave(u agent.Update) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held.
|
// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held.
|
||||||
|
// Machines is, from the placements this composer holds, each node by its address on the private network, and the hub's
|
||||||
|
// node: what names the machine a failing network part points at (delivery.NetworkPart.Targets, issue 774). Empty
|
||||||
|
// before any placement is read.
|
||||||
|
func (c *Composer) Machines() (byAddress map[string]string, hub string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
byAddress = map[string]string{}
|
||||||
|
for node, p := range c.placements {
|
||||||
|
if a, _, _ := strings.Cut(p.Decided.Address, "/"); a != "" {
|
||||||
|
byAddress[a] = node
|
||||||
|
}
|
||||||
|
if p.Decided.Hub {
|
||||||
|
hub = node
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return byAddress, hub
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Composer) inOwnOf(p delivery.Placement) []string {
|
func (c *Composer) inOwnOf(p delivery.Placement) []string {
|
||||||
var out []string
|
var out []string
|
||||||
for _, m := range p.Modules {
|
for _, m := range p.Modules {
|
||||||
@@ -464,6 +733,35 @@ func (c *Composer) takeValue(u agent.Update) {
|
|||||||
// compose composes the node once and answers the updates to hand: each composition whose digest differs from the one
|
// compose composes the node once and answers the updates to hand: each composition whose digest differs from the one
|
||||||
// handed, and a deletion for each handed one no want names any more.
|
// handed, and a deletion for each handed one no want names any more.
|
||||||
func (c *Composer) compose(ctx context.Context) []agent.Update {
|
func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||||
|
pass := Pass{}
|
||||||
|
defer func() {
|
||||||
|
pass.At = time.Now()
|
||||||
|
if c.owed.Load() {
|
||||||
|
wait, _ := c.waitNow()
|
||||||
|
pass.Again = pass.At.Add(wait)
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
c.last = pass
|
||||||
|
c.mu.Unlock()
|
||||||
|
}()
|
||||||
|
// A pass that runs on says so, and what it waits on, every passSaysEvery (issue 774, 2026-10-31: ace said nothing for
|
||||||
|
// five minutes while a pass waited after the vault restarted).
|
||||||
|
began := time.Now()
|
||||||
|
long, stopLong := context.WithCancel(ctx)
|
||||||
|
defer stopLong()
|
||||||
|
go func() {
|
||||||
|
t := time.NewTicker(passSaysEvery)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-long.Done():
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
c.env.Say(fmt.Sprintf("the node's composition has run for %s and waits on %s",
|
||||||
|
time.Since(began).Round(time.Second), c.waitingOn()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
fps := map[string]string{}
|
fps := map[string]string{}
|
||||||
kept := map[string]string{}
|
kept := map[string]string{}
|
||||||
@@ -488,7 +786,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
|||||||
// The catalogue no longer places it: the agent returns to the controller's compositions (endWhenUnplaced), and
|
// The catalogue no longer places it: the agent returns to the controller's compositions (endWhenUnplaced), and
|
||||||
// this is no refusal of what it runs (issue 1126, the seed's revert).
|
// this is no refusal of what it runs (issue 1126, the seed's revert).
|
||||||
c.sayOnce("this node's key in the seat's placement state is gone: it composes nothing of its own, and keeps running what it runs")
|
c.sayOnce("this node's key in the seat's placement state is gone: it composes nothing of its own, and keeps running what it runs")
|
||||||
c.owed.Store(false) // nothing of its own to compose, so no pass is owed and no timer armed again
|
c.paid() // nothing of its own to compose, so no pass is owed and no timer armed again
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// Composed from the build its want names (frombus.Scope.KeptAt, issue 1126).
|
// Composed from the build its want names (frombus.Scope.KeptAt, issue 1126).
|
||||||
@@ -501,7 +799,12 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
|||||||
if c.env.Running != nil {
|
if c.env.Running != nil {
|
||||||
running = c.env.Running()
|
running = c.env.Running()
|
||||||
}
|
}
|
||||||
|
// Its own certificate, asked of the vault when a module here asks for one and the record holds none it trusts.
|
||||||
|
c.certifyIfWanted(ctx, r, own)
|
||||||
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
|
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
|
||||||
|
c.mu.Lock()
|
||||||
|
c.dataScope = frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn} // the data items are worked out as composed
|
||||||
|
c.mu.Unlock()
|
||||||
missed := c.missed.Load()
|
missed := c.missed.Load()
|
||||||
compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
|
||||||
err = c.missedIn(missed, err)
|
err = c.missedIn(missed, err)
|
||||||
@@ -525,11 +828,20 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
why := fmt.Sprintf("%v", err)
|
||||||
|
// A bus that did not answer past every try is no answer: composed again after a bound (issue 1126, 2026-10-28),
|
||||||
|
// and the refusal says when (issue 774, 2026-10-31). A refusal is final until an input moves.
|
||||||
|
if BusFailed(err) {
|
||||||
|
why += "; " + c.owe()
|
||||||
|
} else {
|
||||||
|
c.paid()
|
||||||
|
}
|
||||||
|
pass.Refused = why
|
||||||
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
|
// Said before the agent marks the modules: the marks read each have first, which a paused bus delays.
|
||||||
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
|
c.sayOnce("the node cannot compose its modules, and keeps running what it runs: " + why)
|
||||||
c.refused(fmt.Sprintf("%v", err), nil)
|
done := c.waiting("the agent marking each wanted module's have")
|
||||||
// A bus that did not answer past every try is no answer: composed again after a bound (issue 1126, 2026-10-28).
|
c.refused(why, nil)
|
||||||
c.owed.Store(BusFailed(err))
|
done()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// The boxes it runs, slot by slot: read before c.mu is taken, since the agent's lock is taken to read what runs.
|
// The boxes it runs, slot by slot: read before c.mu is taken, since the agent's lock is taken to read what runs.
|
||||||
@@ -540,14 +852,17 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
|||||||
if c.missed.Load() > missed {
|
if c.missed.Load() > missed {
|
||||||
// A have of what runs the bus did not answer for: its boxes would be the vault's, sealed again, and the
|
// A have of what runs the bus did not answer for: its boxes would be the vault's, sealed again, and the
|
||||||
// composition another; nothing is handed, and the pass is made again after a bound (issue 1126, 2026-10-28).
|
// composition another; nothing is handed, and the pass is made again after a bound (issue 1126, 2026-10-28).
|
||||||
c.sayOnce("the node composes nothing new for now, and keeps running what it runs: the have of what it runs did not answer on the bus")
|
pass.Refused = "the have of what it runs did not answer on the bus; " + c.owe()
|
||||||
c.owed.Store(true)
|
c.sayOnce("the node composes nothing new for now, and keeps running what it runs: " + pass.Refused)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
c.owed.Store(false)
|
c.paid()
|
||||||
|
pass.Held = held
|
||||||
// A module refused alone is held: it keeps what it runs, its have says why, and every other module is applied (issue
|
// A module refused alone is held: it keeps what it runs, its have says why, and every other module is applied (issue
|
||||||
// 1126: one module's refusal froze the whole node).
|
// 1126: one module's refusal froze the whole node).
|
||||||
|
done := c.waiting("the agent marking each held module's have")
|
||||||
c.refused("", held)
|
c.refused("", held)
|
||||||
|
done()
|
||||||
if len(held) == 0 {
|
if len(held) == 0 {
|
||||||
c.sayOnce("")
|
c.sayOnce("")
|
||||||
} else {
|
} else {
|
||||||
@@ -602,6 +917,9 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
|
|||||||
out = append(out, agent.Update{Key: key, Deleted: true, Revision: c.rev})
|
out = append(out, agent.Update{Key: key, Deleted: true, Revision: c.rev})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
pass.Handed = len(out)
|
||||||
|
// The data items are kept again after a pass that handed something new, and after the run's first (ADR 0577 clause 1).
|
||||||
|
c.dataDue = c.dataDue || len(out) > 0 || c.dataKept == nil
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -735,17 +1053,146 @@ func (c *Composer) Policy(ctx context.Context, module string) string {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
parsed, err := catalogue.ParseManifestForInputs(raw)
|
parsed, held, err := c.taken(module, m, raw)
|
||||||
if err != nil {
|
if err != nil || held != "" {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
policy, _, _ := catalogue.DerivedUpgrade(parsed)
|
policy, _, _ := catalogue.DerivedUpgrade(parsed)
|
||||||
return policy
|
return policy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), as a composition's Data says
|
||||||
|
// it, and whether the manifest could be read: the agent's Data for what runs of a module from the kept declaration
|
||||||
|
// (agent.Env.HoldsData, issue 924).
|
||||||
|
func (c *Composer) HoldsData(ctx context.Context, module string) (data, known bool) {
|
||||||
|
c.mu.Lock()
|
||||||
|
m, ok := c.modules[module]
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !ok || m.Manifest == "" {
|
||||||
|
return false, false
|
||||||
|
}
|
||||||
|
raw, err := reader{c: c}.Manifest(ctx, m.Manifest)
|
||||||
|
if err != nil {
|
||||||
|
return false, false
|
||||||
|
}
|
||||||
|
parsed, held, err := c.taken(module, m, raw)
|
||||||
|
if err != nil || held != "" {
|
||||||
|
return false, false
|
||||||
|
}
|
||||||
|
return len(parsed.DataItems()) > 0, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// taken is a module's manifest as this engine reads it (novox/hq ADR 0578 §3): decoded leniently, judging nothing, with
|
||||||
|
// the indexer's verdict from the module's value (a missing one is accepted). held is why the module is held, "" when it is
|
||||||
|
// not: a manifest the indexer refused, or one not read past a field this engine does not know. A field set aside is said
|
||||||
|
// once, and an error is returned only for bytes that are no manifest, which the caller says in its own words.
|
||||||
|
func (c *Composer) taken(module string, m delivery.Module, raw []byte) (man catalogue.Manifest, held string, err error) {
|
||||||
|
if v := m.VerdictOf(module); v != nil {
|
||||||
|
if refused, _ := delivery.TakeManifestVerdict(v); refused {
|
||||||
|
return catalogue.Manifest{}, v.Why(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
man, err = catalogue.Decode(raw)
|
||||||
|
if err != nil {
|
||||||
|
c.sayOnceEach("the manifest of " + module + " does not decode, and the module is held: " + err.Error())
|
||||||
|
return man, "", err
|
||||||
|
}
|
||||||
|
if aside := man.SetAside(); aside != "" {
|
||||||
|
c.sayOnceEach("the manifest of " + module + " has a field this engine does not know, set aside: " + aside)
|
||||||
|
if man.ReadNoFurther() {
|
||||||
|
return man, aside, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return man, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayOnceEach says a line the first time only, apart from sayOnce's single current line.
|
||||||
|
func (c *Composer) sayOnceEach(line string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
if c.told == nil {
|
||||||
|
c.told = map[string]bool{}
|
||||||
|
}
|
||||||
|
again := c.told[line]
|
||||||
|
c.told[line] = true
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !again {
|
||||||
|
c.env.Say(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// reader is the composer's view of the bus as frombus reads it.
|
// reader is the composer's view of the bus as frombus reads it.
|
||||||
type reader struct{ c *Composer }
|
type reader struct{ c *Composer }
|
||||||
|
|
||||||
|
// VaultAuthority is the vault's authority the node follows (frombus.SignersReader).
|
||||||
|
func (r reader) VaultAuthority(context.Context) string {
|
||||||
|
if r.c.env.VaultAuthority == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return r.c.env.VaultAuthority()
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ frombus.SignersReader = reader{}
|
||||||
|
|
||||||
|
// certificateWanted says whether a module the placement p puts on this node asks for a certificate for the machine's
|
||||||
|
// internal name (its manifest's `certificate`), read from the indexer's manifests; a manifest that cannot be read is
|
||||||
|
// taken as asking none, and the composition says what it misses.
|
||||||
|
func (c *Composer) certificateWanted(ctx context.Context, r reader, p delivery.Placement) bool {
|
||||||
|
for _, m := range p.Modules {
|
||||||
|
c.mu.Lock()
|
||||||
|
digest := c.modules[m.Module].Manifest
|
||||||
|
c.mu.Unlock()
|
||||||
|
if digest == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := r.Manifest(ctx, digest)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
mod := c.modules[m.Module]
|
||||||
|
c.mu.Unlock()
|
||||||
|
parsed, held, err := c.taken(m.Module, mod, raw)
|
||||||
|
if err != nil || held != "" {
|
||||||
|
continue // held or not read: it asks none, and its refusal is named by the composition
|
||||||
|
}
|
||||||
|
if parsed.Certificate != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// certifyIfWanted asks Certify once per serving key when a module placed here asks for a certificate and this node's
|
||||||
|
// record holds none for its internal name and serving key from an authority it trusts (the one it holds, or the vault's).
|
||||||
|
func (c *Composer) certifyIfWanted(ctx context.Context, r reader, p delivery.Placement) {
|
||||||
|
if c.env.Certify == nil || !c.certificateWanted(ctx, r, p) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
rec, ok := c.records[c.env.Node]
|
||||||
|
asked := c.certified
|
||||||
|
c.mu.Unlock()
|
||||||
|
if !ok || rec.ServingKey == "" || asked == rec.ServingKey {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key, err := base64.StdEncoding.DecodeString(rec.ServingKey)
|
||||||
|
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
trusted := []string{rec.Authority, r.VaultAuthority(ctx)}
|
||||||
|
if rec.Certificate != "" && delivery.CheckNodeCertificate(rec.Certificate, trusted, overlay.InternalName(c.env.Node),
|
||||||
|
ed25519.PublicKey(key), time.Now()) == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := c.env.Certify(ctx); err != nil {
|
||||||
|
c.sayOnce(fmt.Sprintf("a module here asks for a certificate, and the vault's sign gave none: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
c.certified = rec.ServingKey
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||||
r.c.mu.Lock()
|
r.c.mu.Lock()
|
||||||
defer r.c.mu.Unlock()
|
defer r.c.mu.Unlock()
|
||||||
@@ -810,6 +1257,12 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
r.c.mu.Lock()
|
||||||
|
done, ok := r.c.builds[key]
|
||||||
|
r.c.mu.Unlock()
|
||||||
|
if ok {
|
||||||
|
return &done, nil
|
||||||
|
}
|
||||||
defer r.c.waiting("the build of " + module + " at " + fp + " (" + delivery.BuilderBuildsBucket + ")")()
|
defer r.c.waiting("the build of " + module + " at " + fp + " (" + delivery.BuilderBuildsBucket + ")")()
|
||||||
raw, err := r.c.get(ctx, delivery.BuilderBuildsBucket, key)
|
raw, err := r.c.get(ctx, delivery.BuilderBuildsBucket, key)
|
||||||
if err != nil || raw == nil {
|
if err != nil || raw == nil {
|
||||||
@@ -819,6 +1272,11 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the builder's %s: %w", key, err)
|
return nil, fmt.Errorf("the builder's %s: %w", key, err)
|
||||||
}
|
}
|
||||||
|
if b.State == delivery.BuildDone {
|
||||||
|
r.c.mu.Lock()
|
||||||
|
r.c.builds[key] = b
|
||||||
|
r.c.mu.Unlock()
|
||||||
|
}
|
||||||
return &b, nil
|
return &b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -913,6 +1371,7 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
|
|||||||
Generation int `json:"generation"`
|
Generation int `json:"generation"`
|
||||||
Sealed json.RawMessage `json:"sealed"`
|
Sealed json.RawMessage `json:"sealed"`
|
||||||
} `json:"own"`
|
} `json:"own"`
|
||||||
|
Handed map[string]bool `json:"handed"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(raw, &a); err != nil {
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
return nil, fmt.Errorf("the vault's own answer is not one: %w", err)
|
return nil, fmt.Errorf("the vault's own answer is not one: %w", err)
|
||||||
@@ -922,6 +1381,7 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
|
|||||||
}
|
}
|
||||||
r.c.mu.Lock()
|
r.c.mu.Lock()
|
||||||
defer r.c.mu.Unlock()
|
defer r.c.mu.Unlock()
|
||||||
|
r.c.ownHanded = a.Handed
|
||||||
out := make([]compose.OwnSecretRow, 0, len(a.Own))
|
out := make([]compose.OwnSecretRow, 0, len(a.Own))
|
||||||
for _, o := range a.Own {
|
for _, o := range a.Own {
|
||||||
// The box first had is kept while its generation stands, so sealing it again moves no composition; the box the
|
// The box first had is kept while its generation stands, so sealing it again moves no composition; the box the
|
||||||
@@ -938,6 +1398,19 @@ func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecre
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handed is, of this node's modules, which the vault judged handed to the provisioner when it last answered own
|
||||||
|
// (frombus.HandedReader, issue 774): the node hands exactly those, so the vault and the node judge a module one way.
|
||||||
|
func (r reader) Handed(_ context.Context, node string) map[string]bool {
|
||||||
|
if node != r.c.env.Node {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
r.c.mu.Lock()
|
||||||
|
defer r.c.mu.Unlock()
|
||||||
|
return maps.Clone(r.c.ownHanded)
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ frombus.HandedReader = reader{}
|
||||||
|
|
||||||
func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||||
if node != r.c.env.Node || r.c.env.Own == nil {
|
if node != r.c.env.Node || r.c.env.Own == nil {
|
||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
@@ -1016,13 +1489,18 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv
|
|||||||
// watch endWhenPlaced keeps). A node that switched never goes back on a failed read, however long it lasts.
|
// watch endWhenPlaced keeps). A node that switched never goes back on a failed read, however long it lasts.
|
||||||
func Placed(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
|
func Placed(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
|
||||||
wait func(try int) time.Duration, say func(string), switched bool) (bool, error) {
|
wait func(try int) time.Duration, say func(string), switched bool) (bool, error) {
|
||||||
key, err := delivery.PlacementKey(node)
|
key, err := delivery.PlacementNodeKey(node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
legacy, _ := delivery.LegacyPlacementKey(node)
|
||||||
said := ""
|
said := ""
|
||||||
for try := 1; ; try++ {
|
for try := 1; ; try++ {
|
||||||
|
// Its node key, or, before the seat's holder converted it, its legacy key (research 072 row 37).
|
||||||
raw, err := get(ctx, delivery.PlacementBucket, key)
|
raw, err := get(ctx, delivery.PlacementBucket, key)
|
||||||
|
if err == nil && len(raw) == 0 {
|
||||||
|
raw, err = get(ctx, delivery.PlacementBucket, legacy)
|
||||||
|
}
|
||||||
if errors.Is(err, nats.ErrBucketNotFound) {
|
if errors.Is(err, nats.ErrBucketNotFound) {
|
||||||
return false, nil // the seat has written no placement state yet: nothing places it, read and known
|
return false, nil // the seat has written no placement state yet: nothing places it, read and known
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,10 +3,13 @@ package selfcompose
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"reflect"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -19,6 +22,7 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/agent"
|
"github.com/novox/mesh-host/internal/agent"
|
||||||
@@ -160,7 +164,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
asks++
|
asks++
|
||||||
sealedAt[fmt.Sprintf(`{"ask":%d}`, asks)] = generation + 1
|
sealedAt[fmt.Sprintf(`{"ask":%d}`, asks)] = generation + 1
|
||||||
return json.RawMessage(fmt.Sprintf(`{"node":"n1","key":"sk","own":[{"module":"keeper","name":"password","origin":"made",`+
|
return json.RawMessage(fmt.Sprintf(`{"node":"n1","key":"sk","own":[{"module":"keeper","name":"password","origin":"made",`+
|
||||||
`"generation":%d,"sealed":{"ask":%d}}]}`, generation+1, asks)), nil
|
`"generation":%d,"sealed":{"ask":%d}}],"handed":{"keeper":false}}`, generation+1, asks)), nil
|
||||||
}
|
}
|
||||||
secretChanged := make(chan []byte, 1)
|
secretChanged := make(chan []byte, 1)
|
||||||
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
||||||
@@ -169,6 +173,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
return secretChanged, func() {}, nil
|
return secretChanged, func() {}, nil
|
||||||
}
|
}
|
||||||
|
var composed atomic.Int64 // passes made: each reads what runs once
|
||||||
|
env.Running = func() []delivery.Composed { composed.Add(1); return nil }
|
||||||
c := New(env)
|
c := New(env)
|
||||||
out, err := c.Watch(ctx, "n1.>")
|
out, err := c.Watch(ctx, "n1.>")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -196,7 +202,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
b.put(t, delivery.ModulesBucket, "oldie", omod)
|
b.put(t, delivery.ModulesBucket, "oldie", omod)
|
||||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", want)
|
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", want)
|
||||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.node", nodeWant)
|
b.put(t, delivery.AssignerWantsBucket, "want.n1.node", nodeWant)
|
||||||
for _, bucket := range []string{delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} {
|
for _, bucket := range []string{delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} {
|
||||||
b.send(bucket, agent.Update{Caught: true})
|
b.send(bucket, agent.Update{Caught: true})
|
||||||
}
|
}
|
||||||
got := map[string]bool{}
|
got := map[string]bool{}
|
||||||
@@ -234,6 +240,29 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
|
// Issue 774, 2026-10-31: the verb compose makes a pass now and says how it went; a composer that runs no watch
|
||||||
|
// makes none, and says so rather than waiting.
|
||||||
|
// The pass made at the start is less than askedWithin old for a moment: the ask waits that out first.
|
||||||
|
setFor(t, &askedWithin, 200*time.Millisecond)
|
||||||
|
time.Sleep(250 * time.Millisecond)
|
||||||
|
madeBefore := composed.Load()
|
||||||
|
cctx, ccancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
nowPass, err := c.ComposeNow(cctx)
|
||||||
|
ccancel()
|
||||||
|
if err != nil || nowPass.At.IsZero() || nowPass.Refused != "" || !nowPass.Again.IsZero() || composed.Load() == madeBefore {
|
||||||
|
t.Errorf("compose now: %+v (%v), %d pass(es) made; want one pass, composed", nowPass, err, composed.Load()-madeBefore)
|
||||||
|
}
|
||||||
|
// Asked again at once, the last pass is the answer: a caller in a loop drives no passes back to back (review round 1).
|
||||||
|
madeBefore = composed.Load()
|
||||||
|
if again, err := c.ComposeNow(ctx); err != nil || !again.Earlier || !again.At.Equal(nowPass.At) || composed.Load() != madeBefore {
|
||||||
|
t.Errorf("compose now asked again at once: %+v (%v), %d pass(es) made; want the last pass, none made", again, err, composed.Load()-madeBefore)
|
||||||
|
}
|
||||||
|
// A composer whose loop runs not refuses at once, not after the verb's whole bound.
|
||||||
|
idle, icancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
if _, err := New(env).ComposeNow(idle); err == nil || !strings.Contains(err.Error(), "runs no loop") || idle.Err() != nil {
|
||||||
|
t.Errorf("a composer that watches nothing answered compose now: %v", err)
|
||||||
|
}
|
||||||
|
icancel()
|
||||||
lemursKey, _ := delivery.ComposedKey("n1", "lemurs", fp)
|
lemursKey, _ := delivery.ComposedKey("n1", "lemurs", fp)
|
||||||
nodeKey, _ := delivery.ComposedNodeKey("n1")
|
nodeKey, _ := delivery.ComposedNodeKey("n1")
|
||||||
if !got[lemursKey] || !got[nodeKey] || len(got) != 2 {
|
if !got[lemursKey] || !got[nodeKey] || len(got) != 2 {
|
||||||
@@ -273,15 +302,27 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
return c2, &refusals
|
return c2, &refusals
|
||||||
}
|
}
|
||||||
|
setFor(t, &passSaysEvery, 5*time.Millisecond)
|
||||||
for name, row := range map[string]struct {
|
for name, row := range map[string]struct {
|
||||||
failures int // reads failing before the bus answers
|
failures int // reads failing before the bus answers
|
||||||
cancel bool
|
cancel bool
|
||||||
composes bool
|
composes bool
|
||||||
|
fail error // what each failing read says; JetStream not enabled when nil
|
||||||
|
slow time.Duration // each failing read takes this long
|
||||||
|
// again: the refusal says when the node composes again, and a pass is owed (issue 774, 2026-10-31: ace's
|
||||||
|
// route-proxy said only "the vault's own could not be asked ... context deadline exceeded").
|
||||||
|
again bool
|
||||||
}{
|
}{
|
||||||
"the bus paused for one read": {failures: 1, composes: true},
|
"the bus paused for one read": {failures: 1, composes: true},
|
||||||
"the bus paused past every try": {failures: 100},
|
"the bus paused past every try": {failures: 100, again: true},
|
||||||
"the run ended mid-pass": {failures: 100, cancel: true},
|
"the run ended mid-pass": {failures: 100, cancel: true},
|
||||||
|
"the vault did not answer in time": {failures: 100, again: true, slow: 30 * time.Millisecond,
|
||||||
|
fail: fmt.Errorf("the vault's own could not be asked: no answer from mesh.seat.mesh-vault.call.own.node~n1: %w", context.DeadlineExceeded)},
|
||||||
|
"a refusal": {failures: 100, fail: errors.New("nats says ${dir:mesh-state}, and nats declares no directory")},
|
||||||
} {
|
} {
|
||||||
|
b.mu.Lock()
|
||||||
|
saidBefore := len(b.said)
|
||||||
|
b.mu.Unlock()
|
||||||
failed := 0
|
failed := 0
|
||||||
pctx, pcancel := context.WithCancel(ctx)
|
pctx, pcancel := context.WithCancel(ctx)
|
||||||
c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) {
|
c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) {
|
||||||
@@ -291,6 +332,10 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
pcancel()
|
pcancel()
|
||||||
return nil, fmt.Errorf("the vault's own could not be asked: %w", context.Canceled)
|
return nil, fmt.Errorf("the vault's own could not be asked: %w", context.Canceled)
|
||||||
}
|
}
|
||||||
|
time.Sleep(row.slow)
|
||||||
|
if row.fail != nil {
|
||||||
|
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, row.fail)
|
||||||
|
}
|
||||||
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, nats.ErrJetStreamNotEnabled)
|
return nil, fmt.Errorf("the revisions of %s in %s: %w", key, bucket, nats.ErrJetStreamNotEnabled)
|
||||||
}
|
}
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
@@ -299,14 +344,39 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
})
|
})
|
||||||
handed := c2.compose(pctx)
|
handed := c2.compose(pctx)
|
||||||
pcancel()
|
pcancel()
|
||||||
|
tries := 1 + len(composeAgain) // a bus paused for a moment is tried again at once; a timeout or a refusal is not
|
||||||
|
if row.fail != nil {
|
||||||
|
tries = 1
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case row.composes && (len(handed) == 0 || len(*refusals) != 0):
|
case row.composes && (len(handed) == 0 || len(*refusals) != 0):
|
||||||
t.Errorf("%s: handed %d, refusals %q; want composed and nothing refused", name, len(handed), *refusals)
|
t.Errorf("%s: handed %d, refusals %q; want composed and nothing refused", name, len(handed), *refusals)
|
||||||
case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != 1+len(composeAgain)):
|
case !row.composes && !row.cancel && (len(*refusals) != 1 || failed != tries):
|
||||||
t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, 1+len(composeAgain))
|
t.Errorf("%s: refusals %q after %d reads; want one refusal after %d tries", name, *refusals, failed, tries)
|
||||||
case row.cancel && len(*refusals) != 0:
|
case row.cancel && len(*refusals) != 0:
|
||||||
t.Errorf("%s: a pass whose run ended refused %q", name, *refusals)
|
t.Errorf("%s: a pass whose run ended refused %q", name, *refusals)
|
||||||
}
|
}
|
||||||
|
if !row.composes && !row.cancel && len(*refusals) > 0 {
|
||||||
|
// The retry pass says the same try the refusal named (review round 1: it said try 2 for try 1).
|
||||||
|
says := strings.Contains((*refusals)[0], "; the node composes again at ") && strings.HasSuffix((*refusals)[0], "(try 1)") &&
|
||||||
|
strings.Contains(c2.againLine(), "(try 1)")
|
||||||
|
if says != row.again || c2.owed.Load() != row.again || (c2.last.Again.IsZero() == row.again) || c2.last.Refused != (*refusals)[0] {
|
||||||
|
t.Errorf("%s: refused %q, owes a pass %v, last pass %+v; want the time it composes again said and owed: %v",
|
||||||
|
name, (*refusals)[0], c2.owed.Load(), c2.last, row.again)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A pass that runs on says so, and what it waits on (issue 774, 2026-10-31: ace was silent for five minutes).
|
||||||
|
if row.slow > 0 {
|
||||||
|
b.mu.Lock()
|
||||||
|
long := false
|
||||||
|
for _, line := range b.said[saidBefore:] {
|
||||||
|
long = long || strings.Contains(line, "the node's composition has run for ") && strings.Contains(line, " and waits on ")
|
||||||
|
}
|
||||||
|
b.mu.Unlock()
|
||||||
|
if !long {
|
||||||
|
t.Errorf("%s: a pass that ran on said nothing of what it waits on", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !Transient(fmt.Errorf("the manifest of avahi: the revisions of x in mesh-delivery-indexer_manifests: nats: jetstream not enabled")) ||
|
if !Transient(fmt.Errorf("the manifest of avahi: the revisions of x in mesh-delivery-indexer_manifests: nats: jetstream not enabled")) ||
|
||||||
!Transient(fmt.Errorf("x: %w", nats.ErrNoResponders)) || Transient(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) ||
|
!Transient(fmt.Errorf("x: %w", nats.ErrNoResponders)) || Transient(fmt.Errorf("nats says ${dir:mesh-state}, and nats declares no directory")) ||
|
||||||
@@ -315,9 +385,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
// Issue 1126, the applying path (review of the seed): one module's own refusal holds that module alone. On novox,
|
// Issue 1126, the applying path (review of the seed): one module's own refusal holds that module alone. On novox,
|
||||||
// de-spiegel's unset route names (its manifest at the indexer's commit e25e1beb, captured) refused the whole node and
|
// de-spiegel's unset route names (its manifest at the indexer's commit e25e1beb, captured) refused the whole node and
|
||||||
// marked all its modules not composed. Now lemurs is handed; de-spiegel is held, and proxy with it, which receives its
|
// marked all its modules not composed. Now lemurs and proxy are handed; de-spiegel is held alone (issue 1960: a hold
|
||||||
// routes: what was handed of de-spiegel stays handed, so the node keeps running it, and the agent is told those two
|
// never spreads to proxy, which composes without its routes, nor to its smtp provider): what was handed of de-spiegel
|
||||||
// alone, de-spiegel in its config form's words.
|
// stays handed, so the node keeps running it, and the agent is told it alone, in its config form's words.
|
||||||
spiegel, err := os.ReadFile("testdata/de-spiegel-e25e1beb.module.json")
|
spiegel, err := os.ReadFile("testdata/de-spiegel-e25e1beb.module.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -353,15 +423,70 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
if u, ok := handed3[lemursKey3]; !ok || u.Deleted {
|
if u, ok := handed3[lemursKey3]; !ok || u.Deleted {
|
||||||
t.Errorf("lemurs, which composes, is not handed beside de-spiegel's refusal: %v (told %q)", handed3, *told)
|
t.Errorf("lemurs, which composes, is not handed beside de-spiegel's refusal: %v (told %q)", handed3, *told)
|
||||||
}
|
}
|
||||||
if u, ok := handed3[proxyKey]; ok {
|
// Issue 1960: a hold never spreads. proxy is handed, composed without held de-spiegel's routes, which do not compose.
|
||||||
t.Errorf("proxy is handed without held de-spiegel's routes: %+v", u)
|
if u, ok := handed3[proxyKey]; !ok || u.Deleted {
|
||||||
|
t.Errorf("proxy is not handed beside held de-spiegel: %v (told %q)", handed3, *told)
|
||||||
}
|
}
|
||||||
if u, ok := handed3[spiegelKey]; ok {
|
if u, ok := handed3[spiegelKey]; ok {
|
||||||
t.Errorf("held de-spiegel is handed %+v, want what was handed of it kept", u)
|
t.Errorf("held de-spiegel is handed %+v, want what was handed of it kept", u)
|
||||||
}
|
}
|
||||||
if len(*told) != 1 || !strings.HasPrefix((*told)[0], "held de-spiegel: ") || !strings.Contains((*told)[0], "${config:praktijk-name}") ||
|
if len(*told) != 1 || !strings.HasPrefix((*told)[0], "held de-spiegel: ") || !strings.Contains((*told)[0], "${config:praktijk-name}") ||
|
||||||
!strings.Contains((*told)[0], "held proxy: ") || strings.Contains((*told)[0], "held lemurs") {
|
!strings.Contains((*told)[0], "without what it contributes to route") || strings.Contains((*told)[0], "held proxy") ||
|
||||||
t.Errorf("the agent was told %q, want de-spiegel held in its config form's words, and proxy", *told)
|
strings.Contains((*told)[0], "held lemurs") {
|
||||||
|
t.Errorf("the agent was told %q, want de-spiegel held alone, in its config form's words, without its routes", *told)
|
||||||
|
}
|
||||||
|
// Issue 774, 2026-10-30 19:13: a manifest the mesh refuses (power's bar item at mesh-catalog ca8d6f33, captured, with
|
||||||
|
// attributes its bar places by none of) refused every node's composition and failed all its modules. Now power is
|
||||||
|
// held alone, naming why, and lemurs is handed.
|
||||||
|
power, err := os.ReadFile("testdata/power-ca8d6f33.module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c4, told4 := fresh(env.Get)
|
||||||
|
powerDig := delivery.ManifestDigest(power)
|
||||||
|
powerEnc, _ := delivery.EncodeManifest(delivery.Manifest{Module: "power", JSON: string(power)})
|
||||||
|
powerKey, _ := delivery.ManifestsKey(powerDig)
|
||||||
|
b.mu.Lock()
|
||||||
|
b.kv[delivery.ManifestsBucket+"/"+powerKey] = powerEnc
|
||||||
|
b.mu.Unlock()
|
||||||
|
powerFP := delivery.SourcePrefix2 + strings.Repeat("c", 64)
|
||||||
|
c4.modules["power"] = delivery.Module{FP2: powerFP, Manifest: powerDig, Repository: "novox/mesh-catalog", Commit: "ca8d6f33",
|
||||||
|
// The indexer's verdict on the captured manifest (ADR 0578): the engine judges nothing, it takes this.
|
||||||
|
Verdict: delivery.VerdictRefused, Problems: []string{`bar item "shows" is none of them`}}
|
||||||
|
c4.wants["power"] = delivery.Want{FP: powerFP, Self: true}
|
||||||
|
p4 := c4.placements["n1"]
|
||||||
|
p4.Modules = append(append([]delivery.PlacedModule{}, p4.Modules...), delivery.PlacedModule{Module: "power"})
|
||||||
|
c4.placements["n1"] = p4
|
||||||
|
handed4 := map[string]agent.Update{}
|
||||||
|
for _, u := range c4.compose(ctx) {
|
||||||
|
handed4[u.Key] = u
|
||||||
|
}
|
||||||
|
if u, ok := handed4[lemursKey3]; !ok || u.Deleted {
|
||||||
|
t.Errorf("lemurs is not handed beside power's refused manifest: %v (told %q)", handed4, *told4)
|
||||||
|
}
|
||||||
|
if len(*told4) != 1 || !strings.HasPrefix((*told4)[0], "held power: ") || !strings.Contains((*told4)[0], `"shows" is none of them`) {
|
||||||
|
t.Errorf("the agent was told %q, want power held alone, naming its item", *told4)
|
||||||
|
}
|
||||||
|
// Review of issue 774 (the kept read): mesh-wireguard, which the mesh provides and which never has a fingerprint, is
|
||||||
|
// folded in the node's own and composed from the manifest mesh-sdk provides (delivery.ProvidedManifest), not from
|
||||||
|
// what runs nor the kept declaration: with nothing running (the kept read retired, Running empty) the node's own
|
||||||
|
// still carries its resources, so retiring the read removes nothing of it.
|
||||||
|
c5, told5 := fresh(env.Get)
|
||||||
|
p5 := c5.placements["n1"]
|
||||||
|
p5.Modules = append(append([]delivery.PlacedModule{}, p5.Modules...), delivery.PlacedModule{Module: overlay.Name})
|
||||||
|
p5.Decided.Endpoint = "n1.example:51820"
|
||||||
|
c5.placements["n1"] = p5
|
||||||
|
c5.inOwn[overlay.Name] = true
|
||||||
|
nodeKey5, _ := delivery.ComposedNodeKey("n1")
|
||||||
|
var own5 string
|
||||||
|
for _, u := range c5.compose(ctx) {
|
||||||
|
if u.Key == nodeKey5 {
|
||||||
|
own5 = string(u.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(*told5) != 0 || !strings.Contains(own5, overlay.Name+".") {
|
||||||
|
t.Errorf("the node's own with %s folded and nothing running: told %q, carries %s: %.300s", overlay.Name, *told5,
|
||||||
|
overlay.Name, own5)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A read the bus did not answer, timed out or not, is tried again later; the run's own end and a refusal are not.
|
// A read the bus did not answer, timed out or not, is tried again later; the run's own end and a refusal are not.
|
||||||
@@ -395,6 +520,178 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1,"source":"node"}`))
|
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1,"source":"node"}`))
|
||||||
quiet(t, out)
|
quiet(t, out)
|
||||||
|
|
||||||
|
// Issue 774, 2026-10-30 19:30: node-tools' want moved to a build not yet done, and the node stayed refused after it
|
||||||
|
// landed until another input moved. A done build of a module the node wants, at its want's or the trunk's build,
|
||||||
|
// makes a pass; a build not done, of another build or of a module not wanted, makes none.
|
||||||
|
passes := func() int64 { return composed.Load() }
|
||||||
|
for _, row := range []struct {
|
||||||
|
name, module, fp, state string
|
||||||
|
pass bool
|
||||||
|
}{
|
||||||
|
{"lemurs's wanted build, not done", "lemurs", fp, "building", false},
|
||||||
|
{"lemurs's wanted build lands", "lemurs", fp, delivery.BuildDone, true},
|
||||||
|
{"another build of lemurs", "lemurs", delivery.SourcePrefix2 + strings.Repeat("e", 64), delivery.BuildDone, false},
|
||||||
|
{"a module the node does not want", "oldie", delivery.SourcePrefix2 + strings.Repeat("c", 64), delivery.BuildDone, false},
|
||||||
|
} {
|
||||||
|
before := passes()
|
||||||
|
key, _ := delivery.BuilderBuildKey(row.module, row.fp)
|
||||||
|
build := delivery.BuilderBuild{State: row.state}
|
||||||
|
if row.state == delivery.BuildDone {
|
||||||
|
build.Manifest = digest
|
||||||
|
}
|
||||||
|
raw, err := delivery.EncodeBuilderBuild(build)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.put(t, delivery.BuilderBuildsBucket, key, raw)
|
||||||
|
quiet(t, out)
|
||||||
|
if got := passes() > before; got != row.pass {
|
||||||
|
t.Errorf("%s: a pass made %v, want %v", row.name, got, row.pass)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Issue 774, 2026-10-31: every put under want.n1.> made a pass, and each pass asked the vault for the node's own
|
||||||
|
// secrets. A key written again with what it held, or a want whose words moved and not its build, makes none.
|
||||||
|
worded, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true, Words: "lemurs is held at aaaaaaaaaaaa"})
|
||||||
|
elsewhere, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("e", 64), Self: true})
|
||||||
|
// Research 072 row 37: the placement converted to one key per module and node moves nothing composed; a write in
|
||||||
|
// keys makes its pass at its node key, the commit, and not at a module key written before it nor at the mirror.
|
||||||
|
placeBase := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
||||||
|
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}}
|
||||||
|
placeKeys := delivery.NewPlacementKeys()
|
||||||
|
keyRows := func(after delivery.Placement, what string) []struct {
|
||||||
|
name, bucket, key string
|
||||||
|
value []byte
|
||||||
|
pass bool
|
||||||
|
} {
|
||||||
|
writes, err := placeKeys.Writes("n1", &after)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var rows []struct {
|
||||||
|
name, bucket, key string
|
||||||
|
value []byte
|
||||||
|
pass bool
|
||||||
|
}
|
||||||
|
for i, w := range writes {
|
||||||
|
_, _ = placeKeys.Put(w.Key, w.Value, uint64(i+1), w.Value == nil)
|
||||||
|
rows = append(rows, struct {
|
||||||
|
name, bucket, key string
|
||||||
|
value []byte
|
||||||
|
pass bool
|
||||||
|
}{what + ": " + w.Key, delivery.PlacementBucket, w.Key, w.Value, w.Commit && what != "converted"})
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
_, _ = placeKeys.Put("placement.n1", place, 1, false)
|
||||||
|
lemursMoved := placeBase
|
||||||
|
lemursMoved.Modules = []delivery.PlacedModule{{Module: "lemurs", Values: map[string]any{"greeting": "hoi"}}, {Module: "oldie"}}
|
||||||
|
var placementRows []struct {
|
||||||
|
name, bucket, key string
|
||||||
|
value []byte
|
||||||
|
pass bool
|
||||||
|
}
|
||||||
|
for _, step := range []struct {
|
||||||
|
what string
|
||||||
|
p delivery.Placement
|
||||||
|
}{{"converted", placeBase}, {"lemurs moved", lemursMoved}, {"lemurs back", placeBase}} {
|
||||||
|
placementRows = append(placementRows, keyRows(step.p, step.what)...)
|
||||||
|
}
|
||||||
|
for _, row := range append([]struct {
|
||||||
|
name, bucket, key string
|
||||||
|
value []byte
|
||||||
|
pass bool
|
||||||
|
}{
|
||||||
|
{"lemurs's want again", delivery.AssignerWantsBucket, "want.n1.lemurs", want, false},
|
||||||
|
{"the node's own want again", delivery.AssignerWantsBucket, "want.n1.node", nodeWant, false},
|
||||||
|
{"lemurs's want with other words", delivery.AssignerWantsBucket, "want.n1.lemurs", worded, false},
|
||||||
|
{"the placement again", delivery.PlacementBucket, "placement.n1", place, false},
|
||||||
|
{"lemurs's module again", delivery.ModulesBucket, "lemurs", mod, false},
|
||||||
|
{"lemurs's want at another build", delivery.AssignerWantsBucket, "want.n1.lemurs", elsewhere, true},
|
||||||
|
{"lemurs's want back at its build", delivery.AssignerWantsBucket, "want.n1.lemurs", want, true},
|
||||||
|
}, placementRows...) {
|
||||||
|
before := passes()
|
||||||
|
b.put(t, row.bucket, row.key, row.value)
|
||||||
|
if !row.pass {
|
||||||
|
quiet(t, out)
|
||||||
|
} else {
|
||||||
|
for done := false; !done; {
|
||||||
|
select {
|
||||||
|
case <-out:
|
||||||
|
case <-time.After(200 * time.Millisecond):
|
||||||
|
done = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := passes() > before; got != row.pass {
|
||||||
|
t.Errorf("%s: a pass made %v, want %v", row.name, got, row.pass)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And today's sequence whole: tool runs at its old build; its want and the trunk move to a build not yet done, and
|
||||||
|
// the node holds tool, keeping what runs, naming why; the build lands, and with no other change tool is handed at it.
|
||||||
|
{
|
||||||
|
tb := newFakeBus()
|
||||||
|
tool := `{"module":"tool","version":"1","capabilities":["systemd"],"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},` +
|
||||||
|
`"resources":[{"id":"service","type":"service","unit":"tool.service","state":"running","boot":"enabled"}]}`
|
||||||
|
tdigest := delivery.ManifestDigest([]byte(tool))
|
||||||
|
tman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "tool", JSON: tool})
|
||||||
|
tkey, _ := delivery.ManifestsKey(tdigest)
|
||||||
|
tb.kv[delivery.ManifestsBucket+"/"+tkey] = tman
|
||||||
|
old, next2 := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
|
||||||
|
tenv := tb.env()
|
||||||
|
var held []string
|
||||||
|
var hmu sync.Mutex
|
||||||
|
tenv.Refused = func(why string, h map[string]string) {
|
||||||
|
hmu.Lock()
|
||||||
|
defer hmu.Unlock()
|
||||||
|
held = append(held, why+fmt.Sprint(h))
|
||||||
|
}
|
||||||
|
tenv.Running = func() []delivery.Composed {
|
||||||
|
return []delivery.Composed{{Node: "n1", Module: "tool", FP: old, Resources: []json.RawMessage{
|
||||||
|
json.RawMessage(`{"id":"service","module":"tool","type":"service","unit":"tool.service","state":"running","boot":"enabled"}`)}}}
|
||||||
|
}
|
||||||
|
tc := New(tenv)
|
||||||
|
tout, err := tc.Watch(ctx, "n1.>")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
tplace, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
||||||
|
Modules: []delivery.PlacedModule{{Module: "tool"}}})
|
||||||
|
tmod, _ := delivery.EncodeModule(delivery.Module{FP2: next2, Manifest: tdigest, Repository: "novox/t", Commit: "c"})
|
||||||
|
twant, _ := delivery.EncodeWant(delivery.Want{FP: next2, Self: true})
|
||||||
|
building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: "building"})
|
||||||
|
bkey, _ := delivery.BuilderBuildKey("tool", next2)
|
||||||
|
tb.kv[delivery.BuilderBuildsBucket+"/"+bkey] = building
|
||||||
|
tb.put(t, delivery.PlacementBucket, "placement.n1", tplace)
|
||||||
|
tb.put(t, delivery.HaveBucket, "n1.node", have)
|
||||||
|
tb.put(t, delivery.ModulesBucket, "tool", tmod)
|
||||||
|
tb.put(t, delivery.AssignerWantsBucket, "want.n1.tool", twant)
|
||||||
|
for _, bucket := range []string{delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket,
|
||||||
|
ResolvedBucket, delivery.BuilderBuildsBucket} {
|
||||||
|
tb.send(bucket, agent.Update{Caught: true})
|
||||||
|
}
|
||||||
|
toolKey, _ := delivery.ComposedKey("n1", "tool", next2)
|
||||||
|
for u := next(t, tout); !u.Caught; u = next(t, tout) {
|
||||||
|
if u.Key == toolKey {
|
||||||
|
t.Errorf("tool is handed at a build not yet done")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hmu.Lock()
|
||||||
|
if len(held) == 0 || !strings.Contains(held[len(held)-1], "no done build yet") {
|
||||||
|
t.Errorf("the node was told %q, want tool held for its build not yet done", held)
|
||||||
|
}
|
||||||
|
hmu.Unlock()
|
||||||
|
done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: tdigest,
|
||||||
|
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://tool/files/blobs/sha256:" + strings.Repeat("9", 64),
|
||||||
|
Digest: "sha256:" + strings.Repeat("9", 64)}}})
|
||||||
|
tb.mu.Lock()
|
||||||
|
tb.kv[delivery.BuilderBuildsBucket+"/"+bkey] = done
|
||||||
|
tb.mu.Unlock()
|
||||||
|
tb.put(t, delivery.BuilderBuildsBucket, bkey, done)
|
||||||
|
if u := next(t, tout); u.Key != toolKey || u.Deleted {
|
||||||
|
t.Errorf("the build landed and the node handed %s, want tool at its new build", u.Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A want removed takes its composition back.
|
// A want removed takes its composition back.
|
||||||
removed, _ := delivery.EncodeWant(delivery.Want{Removed: true})
|
removed, _ := delivery.EncodeWant(delivery.Want{Removed: true})
|
||||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", removed)
|
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", removed)
|
||||||
@@ -412,7 +709,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
n := 0
|
n := 0
|
||||||
for _, s := range b.said {
|
for _, s := range b.said {
|
||||||
if strings.Contains(s, "lemurs is wanted at") {
|
if strings.Contains(s, "lemurs is wanted at "+gone) {
|
||||||
n++
|
n++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -439,7 +736,17 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
both, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
both, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
||||||
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}})
|
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}})
|
||||||
b.put(t, delivery.ModulesBucket, "keeper", kmod)
|
b.put(t, delivery.ModulesBucket, "keeper", kmod)
|
||||||
b.put(t, delivery.PlacementBucket, "placement.n1", both)
|
// n1 is held in keys per module and node since the rows above, which a legacy key no longer moves.
|
||||||
|
bothPlaced, _ := delivery.DecodePlacement(both)
|
||||||
|
bothWrites, err := placeKeys.Writes("n1", &bothPlaced)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, w := range bothWrites {
|
||||||
|
if w.Value != nil {
|
||||||
|
b.put(t, delivery.PlacementBucket, w.Key, w.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.keeper", want)
|
b.put(t, delivery.AssignerWantsBucket, "want.n1.keeper", want)
|
||||||
keeperKey, _ := delivery.ComposedKey("n1", "keeper", fp)
|
keeperKey, _ := delivery.ComposedKey("n1", "keeper", fp)
|
||||||
var handed string
|
var handed string
|
||||||
@@ -463,6 +770,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
if rotated.Key != keeperKey {
|
if rotated.Key != keeperKey {
|
||||||
t.Fatalf("a new generation handed %s, want keeper again", rotated.Key)
|
t.Fatalf("a new generation handed %s, want keeper again", rotated.Key)
|
||||||
}
|
}
|
||||||
|
// Issue 774: what the vault judged handed in its own answer is what the node's composition hands, for n1 alone.
|
||||||
|
b.mu.Lock()
|
||||||
|
asked := asks
|
||||||
|
b.mu.Unlock()
|
||||||
|
if h := (reader{c: c}).Handed(ctx, "n1"); asked == 0 || !reflect.DeepEqual(h, map[string]bool{"keeper": false}) {
|
||||||
|
t.Errorf("the vault's handed verdicts as the composition reads them, after %d asks: %v", asked, h)
|
||||||
|
}
|
||||||
|
if h := (reader{c: c}).Handed(ctx, "n2"); h != nil {
|
||||||
|
t.Errorf("another node's verdicts: %v", h)
|
||||||
|
}
|
||||||
|
|
||||||
// Review of L2 (round 2): rotated, then the node-engine restarts. The agent has applied nothing yet, so the box the
|
// Review of L2 (round 2): rotated, then the node-engine restarts. The agent has applied nothing yet, so the box the
|
||||||
// node runs comes from keeper's have (Held); the vault seals the value again, other bytes, and the box the have holds
|
// node runs comes from keeper's have (Held); the vault seals the value again, other bytes, and the box the have holds
|
||||||
@@ -729,7 +1046,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
// Review of L2b, round 1, finding 2: the candidates' state is not there when the node starts (a delivery before the
|
// Review of L2b, round 1, finding 2: the candidates' state is not there when the node starts (a delivery before the
|
||||||
// indexer made it): the shadow tries again, and answers once it appears.
|
// indexer made it): the shadow tries again, and answers once it appears.
|
||||||
sb.missing(delivery.PlacementCandidatesBucket, true)
|
sb.missing(delivery.PlacementCandidatesBucket, true)
|
||||||
go sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond })
|
kept := make(chan struct{})
|
||||||
|
go func() { sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond }); close(kept) }()
|
||||||
time.Sleep(30 * time.Millisecond)
|
time.Sleep(30 * time.Millisecond)
|
||||||
sb.missing(delivery.PlacementCandidatesBucket, false)
|
sb.missing(delivery.PlacementCandidatesBucket, false)
|
||||||
sb.made(7)
|
sb.made(7)
|
||||||
@@ -975,7 +1293,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
"switched: placed after two failures": {answers: []error{fmt.Errorf("nats: timeout"), fmt.Errorf("nats: timeout")}, value: []byte(`{}`), placed: true, tries: 3, switched: true},
|
"switched: placed after two failures": {answers: []error{fmt.Errorf("nats: timeout"), fmt.Errorf("nats: timeout")}, value: []byte(`{}`), placed: true, tries: 3, switched: true},
|
||||||
"never switched: a timeout is not placed, said once": {answers: []error{fmt.Errorf("nats: timeout")}, placed: false, tries: 1},
|
"never switched: a timeout is not placed, said once": {answers: []error{fmt.Errorf("nats: timeout")}, placed: false, tries: 1},
|
||||||
"no key": {placed: false, tries: 1},
|
"no key": {placed: false, tries: 2}, // its node key, then its legacy key (research 072 row 37)
|
||||||
"no placement state yet": {answers: []error{fmt.Errorf("the bucket %s: %w", delivery.PlacementBucket, nats.ErrBucketNotFound)}, placed: false, tries: 1, quiet: true},
|
"no placement state yet": {answers: []error{fmt.Errorf("the bucket %s: %w", delivery.PlacementBucket, nats.ErrBucketNotFound)}, placed: false, tries: 1, quiet: true},
|
||||||
"switched: failing until the link ends": {answers: []error{fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready"),
|
"switched: failing until the link ends": {answers: []error{fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready"),
|
||||||
fmt.Errorf("jetstream not ready")}, end: true, tries: 3, switched: true},
|
fmt.Errorf("jetstream not ready")}, end: true, tries: 3, switched: true},
|
||||||
@@ -1048,7 +1366,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "older", Placement: &trunk})
|
oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "older", Placement: &trunk})
|
||||||
oldKey, _ := delivery.PlacementCandidateKey("older", "n1")
|
oldKey, _ := delivery.PlacementCandidateKey("older", "n1")
|
||||||
nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2)
|
nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2)
|
||||||
for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} {
|
for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} {
|
||||||
nb.send(bucket, agent.Update{Caught: true})
|
nb.send(bucket, agent.Update{Caught: true})
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
@@ -1064,6 +1382,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
t.Errorf("a shadow not ready said %q", nsaid)
|
t.Errorf("a shadow not ready said %q", nsaid)
|
||||||
}
|
}
|
||||||
nmu.Unlock()
|
nmu.Unlock()
|
||||||
|
// The first shadow ends before slowAfter is set: it reads it while it answers (a race the race detector names).
|
||||||
|
scancel()
|
||||||
|
<-kept
|
||||||
setFor(t, &slowAfter, time.Nanosecond) // every answer is slow: said, and noted in words that do not move with the time
|
setFor(t, &slowAfter, time.Nanosecond) // every answer is slow: said, and noted in words that do not move with the time
|
||||||
nb.send(delivery.AssignerWantsBucket, agent.Update{Caught: true})
|
nb.send(delivery.AssignerWantsBucket, agent.Update{Caught: true})
|
||||||
select {
|
select {
|
||||||
@@ -1219,6 +1540,32 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
|
|
||||||
|
// L3 s7 (ADR 0550 §2): the node asks the vault for its own certificate only when a module placed here asks for one
|
||||||
|
// and its record holds none from an authority it trusts; once per serving key.
|
||||||
|
certBus := newFakeBus()
|
||||||
|
certAsks := 0
|
||||||
|
certEnv := certBus.env()
|
||||||
|
certEnv.Certify = func(context.Context) error { certAsks++; return nil }
|
||||||
|
certEnv.VaultAuthority = func() string { return "" }
|
||||||
|
cc := New(certEnv)
|
||||||
|
withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}`
|
||||||
|
cdigest := delivery.ManifestDigest([]byte(withCert))
|
||||||
|
cc.manifests[digest] = []byte(lemurs)
|
||||||
|
cc.manifests[cdigest] = []byte(withCert)
|
||||||
|
cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}}
|
||||||
|
cc.records["n1"] = delivery.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))}
|
||||||
|
onlyLemurs := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}}}
|
||||||
|
cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs)
|
||||||
|
if certAsks != 0 {
|
||||||
|
t.Errorf("no module asks for a certificate, and the vault was asked %d times", certAsks)
|
||||||
|
}
|
||||||
|
withWeb := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "web"}}}
|
||||||
|
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
|
||||||
|
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
|
||||||
|
if certAsks != 1 {
|
||||||
|
t.Errorf("a module asks for a certificate: the vault was asked %d times, want once", certAsks)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches
|
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches
|
||||||
@@ -1415,6 +1762,10 @@ func TestAHaveSaysWhetherItsModuleRunsInsideTheNodesOwn(t *testing.T) {
|
|||||||
{"running with no fingerprint (mesh-wireguard)", delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, true, false},
|
{"running with no fingerprint (mesh-wireguard)", delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, true, false},
|
||||||
{"held today", delivery.Have{FP: "build-0123abcd", Composed: "sha256:" + strings.Repeat("c", 64), Phase: delivery.PhaseHeld,
|
{"held today", delivery.Have{FP: "build-0123abcd", Composed: "sha256:" + strings.Repeat("c", 64), Phase: delivery.PhaseHeld,
|
||||||
Health: delivery.HealthWaits, Detail: "held: a required config value is unset"}, true, true},
|
Health: delivery.HealthWaits, Detail: "held: a required config value is unset"}, true, true},
|
||||||
|
// novox's postgres (2026-10-31), its have written before mesh-host #189: held by a backup the node agent's stop
|
||||||
|
// interrupted and naming nothing; counted in, it was composed from its manifest and refused for its secret reader.
|
||||||
|
{"held by an interrupted backup, naming nothing", delivery.Have{Phase: delivery.PhaseHeld, Health: delivery.HealthWaits,
|
||||||
|
Detail: agent.BackupInterrupted}, false, true},
|
||||||
} {
|
} {
|
||||||
t.Run(row.name, func(t *testing.T) {
|
t.Run(row.name, func(t *testing.T) {
|
||||||
c := New(Env{Node: "n1"})
|
c := New(Env{Node: "n1"})
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -231,7 +230,7 @@ func (s *Shadow) Run(ctx context.Context) error {
|
|||||||
case t.in == 6:
|
case t.in == 6:
|
||||||
changed = s.takeWant(t.u)
|
changed = s.takeWant(t.u)
|
||||||
default:
|
default:
|
||||||
changed = s.takeChanged(ins[t.in], t.u)
|
changed = c.takeChanged(ins[t.in], t.u)
|
||||||
}
|
}
|
||||||
if changed {
|
if changed {
|
||||||
if t.in != 0 {
|
if t.in != 0 {
|
||||||
@@ -264,53 +263,6 @@ func (s *Shadow) Run(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeChanged takes u through in.take and says whether the entry it names changed: a placement, a module or a config
|
|
||||||
// value written again with the value it held changes nothing composed.
|
|
||||||
func (s *Shadow) takeChanged(in input, u agent.Update) bool {
|
|
||||||
before, had, known := s.entry(in.bucket, u.Key)
|
|
||||||
in.take(s.c, u)
|
|
||||||
if !known {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
after, has, _ := s.entry(in.bucket, u.Key)
|
|
||||||
return had != has || !reflect.DeepEqual(before, after)
|
|
||||||
}
|
|
||||||
|
|
||||||
// entry is what the composer holds of one key of bucket, whether it holds it, and whether the bucket is one it knows.
|
|
||||||
func (s *Shadow) entry(bucket, key string) (v any, has, known bool) {
|
|
||||||
c := s.c
|
|
||||||
c.mu.Lock()
|
|
||||||
defer c.mu.Unlock()
|
|
||||||
switch bucket {
|
|
||||||
case delivery.PlacementBucket:
|
|
||||||
node, err := delivery.ParsePlacementKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, true
|
|
||||||
}
|
|
||||||
p, ok := c.placements[node]
|
|
||||||
return p, ok, true
|
|
||||||
case delivery.ModulesBucket:
|
|
||||||
name, err := delivery.ParseModulesKey(key)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, true
|
|
||||||
}
|
|
||||||
m, ok := c.modules[name]
|
|
||||||
return m, ok, true
|
|
||||||
case ResolvedBucket:
|
|
||||||
rest, ok := strings.CutPrefix(key, c.env.Node+".")
|
|
||||||
if !ok {
|
|
||||||
return nil, false, true
|
|
||||||
}
|
|
||||||
module, k, ok := strings.Cut(rest, ".")
|
|
||||||
if !ok {
|
|
||||||
return nil, false, true
|
|
||||||
}
|
|
||||||
val, ok := c.values[module][k]
|
|
||||||
return val, ok, true
|
|
||||||
}
|
|
||||||
return nil, false, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// revsOf is a proposal's keys' revisions, by node, as one word.
|
// revsOf is a proposal's keys' revisions, by node, as one word.
|
||||||
func revsOf(keys map[string]candidateKey) string {
|
func revsOf(keys map[string]candidateKey) string {
|
||||||
nodes := make([]string, 0, len(keys))
|
nodes := make([]string, 0, len(keys))
|
||||||
|
|||||||
@@ -0,0 +1,452 @@
|
|||||||
|
{
|
||||||
|
"module": "power",
|
||||||
|
"version": "1",
|
||||||
|
"moved": true,
|
||||||
|
"templates": "templates",
|
||||||
|
"capabilities": [
|
||||||
|
"package-manager",
|
||||||
|
"service-manager"
|
||||||
|
],
|
||||||
|
"claims": [
|
||||||
|
{
|
||||||
|
"name": "node-power",
|
||||||
|
"scope": "node"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"emits": [
|
||||||
|
"booted",
|
||||||
|
"sleeping",
|
||||||
|
"woke",
|
||||||
|
"shutting-down",
|
||||||
|
"on-mains",
|
||||||
|
"on-battery",
|
||||||
|
"battery-low"
|
||||||
|
],
|
||||||
|
"state": [
|
||||||
|
{
|
||||||
|
"name": "draw",
|
||||||
|
"ttl-seconds": 30,
|
||||||
|
"per-machine": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tools": [
|
||||||
|
"power_state",
|
||||||
|
"power_hooks",
|
||||||
|
"power_history",
|
||||||
|
"power_run",
|
||||||
|
"power_check"
|
||||||
|
],
|
||||||
|
"config": {
|
||||||
|
"handle-power-key": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "What logind does when the power key is pressed: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.",
|
||||||
|
"enum": [
|
||||||
|
"ignore",
|
||||||
|
"poweroff",
|
||||||
|
"reboot",
|
||||||
|
"halt",
|
||||||
|
"kexec",
|
||||||
|
"suspend",
|
||||||
|
"hibernate",
|
||||||
|
"hybrid-sleep",
|
||||||
|
"suspend-then-hibernate",
|
||||||
|
"lock",
|
||||||
|
"factory-reset",
|
||||||
|
"sleep"
|
||||||
|
],
|
||||||
|
"trusted": true,
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"handle-lid-switch": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "What logind does when the lid is closed on battery: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.",
|
||||||
|
"enum": [
|
||||||
|
"ignore",
|
||||||
|
"poweroff",
|
||||||
|
"reboot",
|
||||||
|
"halt",
|
||||||
|
"kexec",
|
||||||
|
"suspend",
|
||||||
|
"hibernate",
|
||||||
|
"hybrid-sleep",
|
||||||
|
"suspend-then-hibernate",
|
||||||
|
"lock",
|
||||||
|
"factory-reset",
|
||||||
|
"sleep"
|
||||||
|
],
|
||||||
|
"trusted": true,
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"handle-lid-switch-external-power": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "What logind does when the lid is closed on external power: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.",
|
||||||
|
"enum": [
|
||||||
|
"ignore",
|
||||||
|
"poweroff",
|
||||||
|
"reboot",
|
||||||
|
"halt",
|
||||||
|
"kexec",
|
||||||
|
"suspend",
|
||||||
|
"hibernate",
|
||||||
|
"hybrid-sleep",
|
||||||
|
"suspend-then-hibernate",
|
||||||
|
"lock",
|
||||||
|
"factory-reset",
|
||||||
|
"sleep"
|
||||||
|
],
|
||||||
|
"trusted": true,
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"handle-lid-switch-docked": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "What logind does when the lid is closed while docked or with another screen attached: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.",
|
||||||
|
"enum": [
|
||||||
|
"ignore",
|
||||||
|
"poweroff",
|
||||||
|
"reboot",
|
||||||
|
"halt",
|
||||||
|
"kexec",
|
||||||
|
"suspend",
|
||||||
|
"hibernate",
|
||||||
|
"hybrid-sleep",
|
||||||
|
"suspend-then-hibernate",
|
||||||
|
"lock",
|
||||||
|
"factory-reset",
|
||||||
|
"sleep"
|
||||||
|
],
|
||||||
|
"trusted": true,
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"id": "polkit",
|
||||||
|
"type": "package",
|
||||||
|
"package": "polkit"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "scripts",
|
||||||
|
"type": "archive",
|
||||||
|
"path": "/usr/local/lib/mesh-power",
|
||||||
|
"artifact": "scripts",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "config-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/mesh-power",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "moments-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/mesh-power/moments",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "moment-after-boot",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/mesh-power/moments/after-boot",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/mesh-power/moments/after-boot"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "moment-before-sleep",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/mesh-power/moments/before-sleep",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/mesh-power/moments/before-sleep"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "moment-after-wake",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/mesh-power/moments/after-wake",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/mesh-power/moments/after-wake"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "moment-before-shutdown",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/mesh-power/moments/before-shutdown",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/mesh-power/moments/before-shutdown"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "moment-on-mains",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/mesh-power/moments/on-mains",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/mesh-power/moments/on-mains"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "moment-on-battery",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/mesh-power/moments/on-battery",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/mesh-power/moments/on-battery"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "logind-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/logind.conf.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "logind-power",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/logind.conf.d/power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/logind.conf.d/power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "logind",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "systemd-logind.service",
|
||||||
|
"reload-on": [
|
||||||
|
"logind-power"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "after-boot-unit",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/mesh-power-after-boot.service",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/mesh-power-after-boot.service"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "before-shutdown-unit",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/mesh-power-before-shutdown.service",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/mesh-power-before-shutdown.service"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "before-sleep-unit",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/mesh-power-before-sleep.service",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/mesh-power-before-sleep.service"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "after-wake-unit",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/mesh-power-after-wake.service",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/mesh-power-after-wake.service"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "supply-unit",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/mesh-power-supply.service",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/mesh-power-supply.service"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sleep-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/sleep.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "sleep-wants",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/sleep.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/sleep.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "suspend-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/suspend.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "suspend-wants",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/suspend.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/suspend.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "hibernate-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/hibernate.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "hibernate-wants",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/hibernate.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/hibernate.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "hybrid-sleep-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/hybrid-sleep.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "hybrid-sleep-wants",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/hybrid-sleep.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/hybrid-sleep.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "suspend-then-hibernate-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/suspend-then-hibernate.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "suspend-then-hibernate-wants",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reboot-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/reboot.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "reboot-bound",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/reboot.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/reboot.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "poweroff-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/poweroff.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "poweroff-bound",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/poweroff.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/poweroff.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "halt-drop-ins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/etc/systemd/system/halt.target.d",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "halt-bound",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/halt.target.d/mesh-power.conf",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/systemd/system/halt.target.d/mesh-power.conf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "udev-rule",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/udev/rules.d/90-mesh-power.rules",
|
||||||
|
"mode": "0644",
|
||||||
|
"template": "templates/udev/rules.d/90-mesh-power.rules"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "after-boot",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "mesh-power-after-boot.service",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "before-shutdown",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "mesh-power-before-shutdown.service",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "draw-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/run/mesh-power",
|
||||||
|
"mode": "0755",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "draw",
|
||||||
|
"type": "process",
|
||||||
|
"name": "mesh-power-draw",
|
||||||
|
"artifact": "tools-go",
|
||||||
|
"run": [
|
||||||
|
"./power",
|
||||||
|
"draw"
|
||||||
|
],
|
||||||
|
"health": {
|
||||||
|
"kind": "unit"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"build": {
|
||||||
|
"artifacts": [
|
||||||
|
{
|
||||||
|
"name": "tools-go",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/power",
|
||||||
|
"binary": "power",
|
||||||
|
"loads": [
|
||||||
|
"power"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "scripts",
|
||||||
|
"kind": "archive",
|
||||||
|
"from": "files"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "templates",
|
||||||
|
"kind": "archive",
|
||||||
|
"from": "templates"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"collection": "contribution:node-bar:block",
|
||||||
|
"name": "battery",
|
||||||
|
"if-capability": "battery",
|
||||||
|
"attributes": {
|
||||||
|
"bar": "bottom",
|
||||||
|
"place": "status",
|
||||||
|
"order": "40"
|
||||||
|
},
|
||||||
|
"template": "templates/bar-battery"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collection": "contribution:node-bar:block",
|
||||||
|
"name": "power-draw",
|
||||||
|
"if-capability": "power-meter",
|
||||||
|
"attributes": {
|
||||||
|
"bar": "bottom",
|
||||||
|
"place": "status",
|
||||||
|
"order": "45",
|
||||||
|
"shows": "state",
|
||||||
|
"state": "power.draw"
|
||||||
|
},
|
||||||
|
"template": "templates/bar-power-draw"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -17,6 +17,14 @@
|
|||||||
"consumes": [
|
"consumes": [
|
||||||
"restic.backed-up"
|
"restic.backed-up"
|
||||||
],
|
],
|
||||||
|
"state": [
|
||||||
|
{
|
||||||
|
"name": "health"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "data"
|
||||||
|
}
|
||||||
|
],
|
||||||
"config": {
|
"config": {
|
||||||
"operator-signers": {
|
"operator-signers": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ func TestTheManifestDeclaresTheOperatorKeysFileFedByASetting(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := catalogue.ParseManifest(raw); err != nil {
|
if _, err := catalogue.Decode(raw); err != nil {
|
||||||
t.Fatalf("the manifest no longer parses: %v", err)
|
t.Fatalf("the manifest no longer parses: %v", err)
|
||||||
}
|
}
|
||||||
var m struct {
|
var m struct {
|
||||||
@@ -68,4 +68,19 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
|
|||||||
if len(m.Consumes) != 1 || m.Consumes[0] != "restic.backed-up" {
|
if len(m.Consumes) != 1 || m.Consumes[0] != "restic.backed-up" {
|
||||||
t.Fatalf("consumes %v", m.Consumes)
|
t.Fatalf("consumes %v", m.Consumes)
|
||||||
}
|
}
|
||||||
|
// 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account
|
||||||
|
// of its own is carried by every node's tool runner, which would be a second writer of the engine's own keys and
|
||||||
|
// has the grants part refuse the whole user list (2026-10-30). It comes back once the runtime never carries the engine.
|
||||||
|
parsed, err := catalogue.Decode(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// 774 (the alert table): the state health, one key per node, the node's own health report, kept until
|
||||||
|
// written again (no TTL: a node that stops writing keeps its last word, which the watchman reads with its time).
|
||||||
|
// ADR 0577 clause 1: and the state data, one key per module placed on the node that declares data items, kept until
|
||||||
|
// the module leaves the node; created by the grants part from this manifest as health is.
|
||||||
|
if len(parsed.State) != 2 || parsed.State[0].Name != "health" || parsed.State[0].TTLSeconds != 0 ||
|
||||||
|
parsed.State[1].Name != "data" || parsed.State[1].TTLSeconds != 0 {
|
||||||
|
t.Fatalf("the manifest declares the state %+v", parsed.State)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user