Compare commits
136
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a3c098a5c6 | ||
|
|
2fd165f5af | ||
|
|
db1151c773 | ||
|
|
8c5af3e4a3 | ||
|
|
ece607e215 | ||
|
|
86a05babed | ||
|
|
1ec73915b8 | ||
|
|
f81e6147ae | ||
|
|
0a4c35f731 | ||
|
|
4f9227a186 | ||
|
|
fb8ea0e683 | ||
|
|
a6c70803ab | ||
|
|
9bd1c84e0f | ||
|
|
1f3fb52470 | ||
|
|
bb6e472939 | ||
|
|
93f33e2305 | ||
|
|
abdf85aa4f | ||
|
|
91686fdb81 | ||
|
|
d2a868f99f | ||
|
|
a143678fd8 | ||
|
|
9914459830 | ||
|
|
b7159d71a6 | ||
|
|
e0c7f36ca9 | ||
|
|
9120f94aa4 | ||
|
|
1900b40192 | ||
|
|
6cbdb3712e | ||
|
|
2a0c428ef9 | ||
|
|
a5d1f7dde4 | ||
|
|
95dd64c774 | ||
|
|
81fa009bf4 | ||
|
|
05ff90e3c1 | ||
|
|
aaffc19232 | ||
|
|
479ac71050 | ||
|
|
4f6b389a37 | ||
|
|
45f5e27526 | ||
|
|
dd8ba97989 | ||
|
|
ab22efd8b1 | ||
|
|
54aee345fc | ||
|
|
8639b373d2 | ||
|
|
5fc69444f6 | ||
|
|
2fe9d678fc | ||
|
|
58b4f8b6bc | ||
|
|
bf667a602d | ||
|
|
9a8b3870ba | ||
|
|
24c23a1d06 | ||
|
|
a548a61e86 | ||
|
|
bc28606d45 | ||
|
|
7f0eb9c51d | ||
|
|
690aa662a8 | ||
|
|
8bd0b5db15 | ||
|
|
b4fd2cc210 | ||
|
|
1a021052fe | ||
|
|
cf738cdb90 | ||
|
|
7a2af23764 | ||
|
|
daaa1962bd | ||
|
|
df878ba1f7 | ||
|
|
1e01868d92 | ||
|
|
9d04ae7edd | ||
|
|
16df7d0b7a | ||
|
|
cd1d2ff447 | ||
|
|
faf6a23d91 | ||
|
|
2f4e81f3b0 | ||
|
|
4182f8fbfb | ||
|
|
c3a2ffde67 | ||
|
|
6502ba6ad6 | ||
|
|
f6c0520f26 | ||
|
|
c912bcab6e | ||
|
|
63c0b445d7 | ||
|
|
653ff61662 | ||
|
|
a7cd0a7807 | ||
|
|
7271cf151b | ||
|
|
82d0e125b2 | ||
|
|
a4c24a1979 | ||
|
|
cff69893a3 | ||
|
|
047237c911 | ||
|
|
449cda1b00 | ||
|
|
a6e53a4858 | ||
|
|
4993f1b1d7 | ||
|
|
199f6b48b1 | ||
|
|
4636261435 | ||
|
|
4e8fdfdf04 | ||
|
|
d00926e089 | ||
|
|
bb15aab50e | ||
|
|
6a4718b712 | ||
|
|
d5adf9b72b | ||
|
|
7365890298 | ||
|
|
0862f97972 | ||
|
|
45439fd742 | ||
|
|
0184e2a5b8 | ||
|
|
50c229f905 | ||
|
|
69b845c0f7 | ||
|
|
caafd9306e | ||
|
|
0d22776b20 | ||
|
|
1803d25c3a | ||
|
|
d56c576e9a | ||
|
|
fbc6d6330f | ||
|
|
fb0cdabe25 | ||
|
|
0474a2a3cb | ||
|
|
180a7e7b80 | ||
|
|
be94d2689e | ||
|
|
b225c7bb4c | ||
|
|
1d44288514 | ||
|
|
af2d2a2c14 | ||
|
|
59d77dd457 | ||
|
|
23d66187a8 | ||
|
|
4476e58080 | ||
|
|
58de6aa10a | ||
|
|
cdd3f3fd4f | ||
|
|
21d6583b5e | ||
|
|
bb46a6ad0f | ||
|
|
f8afaefba0 | ||
|
|
1e38c42646 | ||
|
|
1241ee7041 | ||
|
|
2dc384d641 | ||
|
|
e4dbb49d2f | ||
|
|
02d9d7b053 | ||
|
|
7fc0ea5478 | ||
|
|
8be1c2ab0d | ||
|
|
ff9386dccc | ||
|
|
bba9081ca1 | ||
|
|
65cd7b4787 | ||
|
|
a78239b2aa | ||
|
|
2ac0c4686a | ||
|
|
5233f7584d | ||
|
|
8858dba25e | ||
|
|
9254180cab | ||
|
|
94f9aa298c | ||
|
|
ee9fcaba20 | ||
|
|
43f2d6020e | ||
|
|
6250b1274b | ||
|
|
7f04d12047 | ||
|
|
e88ddab241 | ||
|
|
06f70e11ee | ||
|
|
61cec66ba8 | ||
|
|
5e2fadce15 | ||
|
|
882e75c954 |
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// authorityPath is where this engine keeps the mesh's own certificate as it last applied it (public): beside the
|
||||
// certificate issued for the machine's name, read into its record (novox/hq ADR 0555 §3).
|
||||
func authorityPath(state string) string { return filepath.Join(filepath.Dir(state), "authority.crt") }
|
||||
|
||||
// keepAuthority keeps the mesh's own certificate a composition carries (the resource a module's certificate.authority
|
||||
// asks for, catalogue.AuthorityID), so the record says the authority this machine holds. A composition carrying none
|
||||
// changes nothing.
|
||||
func keepAuthority(state string, c delivery.Composed) error {
|
||||
for _, raw := range c.Resources {
|
||||
var r struct {
|
||||
ID string `json:"id"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &r) != nil || r.Content == "" || !strings.HasSuffix(r.ID, "."+catalogue.AuthorityID()) {
|
||||
continue
|
||||
}
|
||||
path := authorityPath(state)
|
||||
if was, err := os.ReadFile(path); err == nil && string(was) == r.Content {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, []byte(r.Content), 0o644)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
||||
// record's keys read it; a composition carrying none changes nothing.
|
||||
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||
res := func(id, content string) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"id": id, "type": "file", "path": "/etc/x", "content": content})
|
||||
return raw
|
||||
}
|
||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("web.conf", "x")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(authorityPath(state)); !os.IsNotExist(err) {
|
||||
t.Fatalf("a composition with no authority kept one: %v", err)
|
||||
}
|
||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recordKeys(state)(); got.Authority != "CA-1" {
|
||||
t.Errorf("the record's keys read the authority %q", got.Authority)
|
||||
}
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func TestAModuleWithNothingLongRunningIsOkWithinALookOfItsApply(t *testing.T) {
|
||||
applying := fact
|
||||
applying.Phase = string(delivery.PhaseApplying)
|
||||
queue.NoteModule(ctx, applying)
|
||||
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil)
|
||||
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil, nil)
|
||||
// The facts look inside the apply, while the module is still applying.
|
||||
queue.SayFacts(ctx, link.Report{Health: &link.Health{Contract: link.LivenessContract, At: time.Now().UTC()}})
|
||||
if h, _ := bus.have("n1.xorg"); h.Health != delivery.HealthUnknown {
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
)
|
||||
|
||||
// heldBy is the containers an open maintenance window holds still, by runtime name (ADR 0189), at now.
|
||||
func heldBy(windows *apply.Windows) func(time.Time) map[string]bool {
|
||||
return func(now time.Time) map[string]bool {
|
||||
held := map[string]bool{}
|
||||
for _, w := range windows.Open(now) {
|
||||
for _, name := range w.Holds {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
return held
|
||||
}
|
||||
}
|
||||
|
||||
// inactiveIn is what the agent asks at its start of a module it believes runs (ADR 0568 §1): whether the machine shows a
|
||||
// long-running resource the composition declares running not active, read from the service manager and the container
|
||||
// runtime and from no record. A resource held by an open maintenance window, or waiting for a library, is not counted;
|
||||
// one that cannot be read is not counted either (unknown is not inactive), and neither is one the manager is restarting.
|
||||
func inactiveIn(rt liveness.Runtime, held func(time.Time) map[string]bool) func(context.Context, delivery.Composed) bool {
|
||||
return func(ctx context.Context, c delivery.Composed) bool {
|
||||
d, err := declarationOf(c)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
var holding map[string]bool
|
||||
if held != nil {
|
||||
holding = held(time.Now())
|
||||
}
|
||||
return anyInactive(ctx, rt, liveness.LongRunning(d, withWaiting(nil, apply.WaitingOnLibrary(d))), holding)
|
||||
}
|
||||
}
|
||||
|
||||
// anyInactive reads each group of resources once (the containers, each service manager's units) and says whether any is
|
||||
// not active.
|
||||
func anyInactive(ctx context.Context, rt liveness.Runtime, resources []liveness.Resource, held map[string]bool) bool {
|
||||
type group struct {
|
||||
kind, scope, user string
|
||||
targets []string
|
||||
}
|
||||
var groups []*group
|
||||
at := map[string]*group{}
|
||||
for _, r := range resources {
|
||||
if r.Kind == liveness.KindContainer && held[r.Target] {
|
||||
continue
|
||||
}
|
||||
g := &group{kind: r.Kind, scope: r.Scope, user: r.User}
|
||||
if r.Kind == liveness.KindContainer {
|
||||
g.scope, g.user = "", ""
|
||||
}
|
||||
key := g.kind + "/" + g.scope + "/" + g.user
|
||||
if r.Kind != liveness.KindContainer {
|
||||
key = "units/" + g.scope + "/" + g.user
|
||||
}
|
||||
if at[key] == nil {
|
||||
at[key] = g
|
||||
groups = append(groups, g)
|
||||
}
|
||||
at[key].targets = append(at[key].targets, r.Target)
|
||||
}
|
||||
for _, g := range groups {
|
||||
var got map[string]liveness.Observed
|
||||
var err error
|
||||
if g.kind == liveness.KindContainer {
|
||||
got, err = rt.Containers(ctx, g.targets)
|
||||
} else {
|
||||
got, err = rt.Units(ctx, g.scope, g.user, g.targets)
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, t := range g.targets {
|
||||
if o := got[t]; !o.Found || !(o.Running || o.Restarting) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
)
|
||||
|
||||
// 1957/AC2 and AC4 (unit; ADR 0568 §2, §4): the judge's look acts on a resource that goes down, once per edge. A repeated
|
||||
// statement acts nothing, the edge clears when the resource is healthy again, and a resource held, restarting, starting or
|
||||
// missing (the other path's) is no edge.
|
||||
func testOnlyAResourceNewlyDownIsAnEdge(t *testing.T) {
|
||||
state := func(id, s, reason string, missing bool) liveness.State {
|
||||
return liveness.State{Resource: liveness.Resource{Module: "dnsmasq", ID: id}, State: s, Reason: reason, Missing: missing}
|
||||
}
|
||||
down := state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, false)
|
||||
steps := []struct {
|
||||
name string
|
||||
st []liveness.State
|
||||
want []string
|
||||
}{
|
||||
{"down: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||
{"said again while down: none", []liveness.State{down}, nil},
|
||||
{"healthy again: clears", []liveness.State{state("dnsmasq.unit", liveness.Healthy, "", false)}, nil},
|
||||
{"down again: an edge", []liveness.State{down}, []string{"dnsmasq"}},
|
||||
{"held by a window: none", []liveness.State{state("dnsmasq.unit", liveness.Held, "", false)}, nil},
|
||||
{"restarting: none", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonRestarting, false)}, nil},
|
||||
{"starting: none", []liveness.State{state("dnsmasq.unit", liveness.Starting, "", false)}, nil},
|
||||
{"missing: the other path's", []liveness.State{state("dnsmasq.unit", liveness.Unhealthy, liveness.ReasonDown, true)}, nil},
|
||||
}
|
||||
var last map[string]bool
|
||||
for _, s := range steps {
|
||||
var got []string
|
||||
got, last = newlyInactive(last, liveness.Statement{Resources: s.st})
|
||||
if len(got) != len(s.want) || (len(got) == 1 && got[0] != s.want[0]) {
|
||||
t.Fatalf("%s: acted on %v, want %v", s.name, got, s.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fakeRuntime struct {
|
||||
units map[string]liveness.Observed
|
||||
containers map[string]liveness.Observed
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeRuntime) Containers(_ context.Context, _ []string) (map[string]liveness.Observed, error) {
|
||||
return f.containers, f.err
|
||||
}
|
||||
func (f fakeRuntime) Units(_ context.Context, _, _ string, _ []string) (map[string]liveness.Observed, error) {
|
||||
return f.units, f.err
|
||||
}
|
||||
|
||||
// 1957/AC1 (unit; ADR 0568 §1): the machine's facts say a declared-running resource is not active when its unit is stopped
|
||||
// or unknown to the manager; everything up, a restart in progress, a read that failed and a container held by a
|
||||
// maintenance window are not.
|
||||
func testTheMachineShowsAResourceNotActive(t *testing.T) {
|
||||
svc := []byte(`{"id":"dnsmasq.unit","type":"service","unit":"dnsmasq.service","state":"running","module":"dnsmasq"}`)
|
||||
ctr := []byte(`{"id":"dnsmasq.box","type":"container","name":"box","image":"x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","module":"dnsmasq"}`)
|
||||
up := liveness.Observed{Found: true, Running: true}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
rt fakeRuntime
|
||||
held map[string]bool
|
||||
want bool
|
||||
}{
|
||||
{"all up", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||
{"unit inactive", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||
{"unit unknown to the manager", fakeRuntime{containers: map[string]liveness.Observed{"box": up}}, nil, true},
|
||||
{"unit restarting", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": {Found: true, Restarting: true}}, containers: map[string]liveness.Observed{"box": up}}, nil, false},
|
||||
{"container not running", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, nil, true},
|
||||
{"container held by a window", fakeRuntime{units: map[string]liveness.Observed{"dnsmasq.service": up}, containers: map[string]liveness.Observed{"box": {Found: true}}}, map[string]bool{"box": true}, false},
|
||||
{"unread", fakeRuntime{err: errors.New("the service manager could not be read")}, nil, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
d, err := declarationOf(delivery.Composed{Node: "n1", Module: "dnsmasq", Resources: []json.RawMessage{svc, ctr}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := anyInactive(context.Background(), c.rt, liveness.LongRunning(d, nil), c.held); got != c.want {
|
||||
t.Fatalf("inactive = %v, want %v", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+68
-11
@@ -1046,15 +1046,7 @@ func runLink(ctx context.Context, opts options) error {
|
||||
say(err.Error())
|
||||
}
|
||||
windows := apply.WindowsIn(filepath.Dir(opts.state))
|
||||
j.HeldNow = func(now time.Time) map[string]bool {
|
||||
held := map[string]bool{}
|
||||
for _, w := range windows.Open(now) {
|
||||
for _, name := range w.Holds {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
return held
|
||||
}
|
||||
j.HeldNow = heldBy(windows)
|
||||
judging = j
|
||||
// And which units its service managers say failed, and whose each is (novox/hq issue 315).
|
||||
unitJudge = units.New(units.Exec{Run: apply.ExecRunner, System: builtFor})
|
||||
@@ -1115,6 +1107,8 @@ func runLink(ctx context.Context, opts options) error {
|
||||
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
||||
queue := &link.Session{
|
||||
Membership: membership,
|
||||
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
||||
Keys: recordKeys(opts.state),
|
||||
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
||||
HaveWritten: proof.haveWritten,
|
||||
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
||||
@@ -1171,6 +1165,10 @@ func runLink(ctx context.Context, opts options) error {
|
||||
if ag := current.get(); ag != nil {
|
||||
ag.Retry(modules...)
|
||||
}
|
||||
}, func(modules ...string) {
|
||||
if ag := current.get(); ag != nil {
|
||||
ag.Reconcile(modules...)
|
||||
}
|
||||
})
|
||||
}
|
||||
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
|
||||
@@ -1334,8 +1332,8 @@ const (
|
||||
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
|
||||
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
|
||||
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
|
||||
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply func(modules ...string)) {
|
||||
var missingBefore map[string]bool
|
||||
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, say link.Announce, reapply, reconcile func(modules ...string)) {
|
||||
var missingBefore, inactiveBefore map[string]bool
|
||||
ticker := time.NewTicker(liveness.LookEvery)
|
||||
defer ticker.Stop()
|
||||
var lastSaid time.Time
|
||||
@@ -1437,6 +1435,14 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
|
||||
say(fmt.Sprintf("applying %s again: a resource it declares is not on this machine", strings.Join(m, ", ")))
|
||||
reapply(m...)
|
||||
}
|
||||
// A resource its module declares running that is down but still known to the manager is started by the same
|
||||
// apply, once each time it goes down, never on a clock (ADR 0568 §2): the judge reads, the agent acts.
|
||||
var inactive []string
|
||||
inactive, inactiveBefore = newlyInactive(inactiveBefore, st)
|
||||
if len(inactive) > 0 && reconcile != nil {
|
||||
say(fmt.Sprintf("applying %s again: a resource it declares running is not active", strings.Join(inactive, ", ")))
|
||||
reconcile(inactive...)
|
||||
}
|
||||
}
|
||||
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||
unitSt), accountSt), found), accessSt)) {
|
||||
@@ -2131,3 +2137,54 @@ func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[st
|
||||
}
|
||||
return missingModules(fresh), now
|
||||
}
|
||||
|
||||
// newlyInactive are the modules of the resources a statement says down (not running after their grace, but still known to
|
||||
// their manager, and not restarting) that the last statement acted on did not, each once, sorted, and the resources down
|
||||
// now: a resource still down is not acted on again, and one that came back and went down again is (ADR 0568 §2).
|
||||
func newlyInactive(last map[string]bool, st liveness.Statement) ([]string, map[string]bool) {
|
||||
now := map[string]bool{}
|
||||
var fresh liveness.Statement
|
||||
for _, r := range st.Resources {
|
||||
if r.State != liveness.Unhealthy || r.Reason != liveness.ReasonDown || r.Missing {
|
||||
continue
|
||||
}
|
||||
now[r.ID] = true
|
||||
if !last[r.ID] {
|
||||
fresh.Resources = append(fresh.Resources, r)
|
||||
}
|
||||
}
|
||||
return missingModules(withMissing(fresh)), now
|
||||
}
|
||||
|
||||
// withMissing is the statement with each of its resources marked missing, so missingModules lists their modules.
|
||||
func withMissing(st liveness.Statement) liveness.Statement {
|
||||
for i := range st.Resources {
|
||||
st.Resources[i].Missing = true
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
|
||||
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
|
||||
// empty, never invented; the certificate is there only after the first declaration that carried it.
|
||||
func recordKeys(state string) func() link.RecordKeys {
|
||||
return func() link.RecordKeys {
|
||||
var k link.RecordKeys
|
||||
if id, err := identity.Load(identity.Path(state)); err == nil {
|
||||
k.TunnelKey = id.Overlay.Public
|
||||
}
|
||||
if sk, err := identity.LoadSealingKey(identity.SealingKeyPath(state)); err == nil {
|
||||
k.SealingKey = sk.Public
|
||||
}
|
||||
if sv, err := identity.LoadServingKey(identity.ServingKeyPath(state)); err == nil {
|
||||
k.ServingKey = sv.Public
|
||||
}
|
||||
if cert, err := os.ReadFile(identity.CertificatePath(state)); err == nil {
|
||||
k.Certificate = string(cert)
|
||||
}
|
||||
if authority, err := os.ReadFile(authorityPath(state)); err == nil {
|
||||
k.Authority = string(authority)
|
||||
}
|
||||
return k
|
||||
}
|
||||
}
|
||||
|
||||
+294
-6
@@ -13,11 +13,15 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
@@ -35,6 +39,20 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return w.updates(ctx, in), nil
|
||||
}
|
||||
|
||||
// WatchAwait is Watch for a state that its holder makes when it starts (the placement seat's): a bucket not on the bus
|
||||
// yet is waited for on the bus's word that it was created (link.Session.KVWatchAwait), and waiting is told why.
|
||||
func (w bucketWatch) WatchAwait(ctx context.Context, pattern string, waiting func(error)) (<-chan agent.Update, error) {
|
||||
in, err := w.session.KVWatchAwait(ctx, w.bucket, pattern, waiting)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return w.updates(ctx, in), nil
|
||||
}
|
||||
|
||||
func (w bucketWatch) updates(ctx context.Context, in <-chan link.KVUpdate) <-chan agent.Update {
|
||||
out := make(chan agent.Update, 64)
|
||||
go func() {
|
||||
defer close(out)
|
||||
@@ -46,25 +64,52 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
return out
|
||||
}
|
||||
|
||||
// sessionHaves writes the haves the agent says, through the session.
|
||||
type sessionHaves struct{ session *link.Session }
|
||||
|
||||
func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) {
|
||||
if module == agent.NodeModule {
|
||||
// The node's own key: the digest of its own composition it runs, and what that carries sealed.
|
||||
got, ok := h.session.ReadNodeHave(ctx)
|
||||
if !ok || got.Composed == "" {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.SealedAt}, true
|
||||
}
|
||||
got, ok := h.session.ReadHave(ctx, module)
|
||||
if !ok {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail}, true
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.SealedAt}, true
|
||||
}
|
||||
|
||||
// ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
|
||||
func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, bool, error) {
|
||||
if module == agent.NodeModule {
|
||||
// The node's own key, its read the bus did not answer told as such (issue 1126, 2026-10-28).
|
||||
got, ok, err := h.session.ReadNodeHaveErr(ctx)
|
||||
if !ok || err != nil || got.Composed == "" {
|
||||
return agent.Have{}, false, err
|
||||
}
|
||||
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.SealedAt}, true, nil
|
||||
}
|
||||
got, ok, err := h.session.ReadHaveErr(ctx, module)
|
||||
if !ok || err != nil {
|
||||
return agent.Have{}, false, err
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.SealedAt}, true, nil
|
||||
}
|
||||
|
||||
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
if x.Module == agent.NodeModule {
|
||||
// The node's own key holds its facts and the digest of the composition it runs.
|
||||
if x.Phase == "" || x.Phase == string(delivery.PhaseRunning) {
|
||||
h.session.NoteNode(ctx, x.Composed)
|
||||
h.session.NoteNode(ctx, x.Composed, x.Sealed)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -73,7 +118,8 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
phase = ""
|
||||
}
|
||||
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
|
||||
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail)})
|
||||
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
|
||||
Sealed: x.Sealed})
|
||||
}
|
||||
|
||||
func (h sessionHaves) Removed(ctx context.Context, module string) {
|
||||
@@ -401,6 +447,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
}
|
||||
var ag *agent.Agent
|
||||
kept := keptOnce(opts, signer)
|
||||
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
|
||||
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
|
||||
if byModule, _ := kept(); byModule != nil {
|
||||
for module, resources := range byModule {
|
||||
if err := keepAuthority(opts.state, delivery.Composed{Module: module, Resources: resources}); err != nil {
|
||||
say("the mesh's certificate the kept declaration carries could not be kept for this machine's record: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
|
||||
// node, and a node whose compositions carry no list writes nothing.
|
||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
|
||||
@@ -428,6 +483,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
|
||||
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
|
||||
Recorded: recordedIn(opts.state, say),
|
||||
Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
|
||||
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
||||
got, err := session.KVHistory(ctx, composedBucket, key)
|
||||
if err != nil {
|
||||
@@ -449,8 +505,22 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
Backoff: link.Backoff,
|
||||
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
|
||||
actx, done := applyContext(session.WithLink(ctx))
|
||||
res := applyComposition(actx, opts, c, others, sched, say)
|
||||
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
||||
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
||||
held, herr := users.Hold(c)
|
||||
var res agent.Result
|
||||
if herr != nil {
|
||||
res = failedWith(herr.Error())
|
||||
} else {
|
||||
res = applyComposition(actx, opts, held, others, sched, say)
|
||||
}
|
||||
done()
|
||||
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
|
||||
if !res.Failed {
|
||||
if err := keepAuthority(opts.state, c); err != nil {
|
||||
say("the mesh's certificate in " + c.Module + "'s composition could not be kept for this machine's record: " + err.Error())
|
||||
}
|
||||
}
|
||||
// An apply of the bus module may write an older list than the newest composition holds: looked at again.
|
||||
if module, _ := users.State(); module != "" && module == c.Module {
|
||||
users.Again()
|
||||
@@ -468,9 +538,85 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
return res
|
||||
},
|
||||
}
|
||||
// The node composes its own modules once the catalogue places it (novox/hq ADR 0547 §2, ADR 0549, issue 1126): its
|
||||
// placement key is read at the agent's start; until then it applies the controller's compositions as before.
|
||||
run, stopRun := context.WithCancel(linked)
|
||||
// What the node runs and how it opens a sealed value, for the composer to keep the boxes it runs; and the agent
|
||||
// told when the node cannot compose, so each wanted module's have says it.
|
||||
// The agent of this run alone: its composer tells it, never the agent of a run begun since (issue 1126: a pass in
|
||||
// flight at the switch marked the next agent's modules failed).
|
||||
own := &agentNow{}
|
||||
hooks := composerHooks{
|
||||
running: func() []delivery.Composed {
|
||||
if now := own.get(); now != nil {
|
||||
return now.Everything()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
unseal: sealOpener(opts.state),
|
||||
// And what each module's have says it runs sealed: after a restart the agent has applied nothing yet.
|
||||
// A have the bus did not answer for is said as such, never as one holding no seals (issue 1126, 2026-10-28).
|
||||
held: func(module string) (delivery.Seals, []string, error) {
|
||||
if module == agent.NodeModule {
|
||||
got, _, err := session.ReadNodeHaveErr(linked)
|
||||
return got.SealedAt, got.Sealed, err
|
||||
}
|
||||
got, ok, err := session.ReadHaveErr(linked, module)
|
||||
if !ok {
|
||||
return nil, nil, err
|
||||
}
|
||||
return got.SealedAt, got.Sealed, nil
|
||||
},
|
||||
refused: func(why string, held map[string]string) {
|
||||
if now := own.get(); now != nil && run.Err() == nil {
|
||||
now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126)
|
||||
}
|
||||
},
|
||||
}
|
||||
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
||||
// its own key, `<node>.node`, under candidates.
|
||||
shadowHooks := hooks
|
||||
shadowHooks.refused = nil
|
||||
shadowEnv := composerEnv(session, node, say, shadowHooks)
|
||||
// And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to
|
||||
// the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync).
|
||||
composes := &composesNow{}
|
||||
resync := ownResync(session, node, say, shadowEnv, composes)
|
||||
shadow := selfcompose.NewShadow(shadowEnv,
|
||||
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
session.NoteCandidates(ctx, delivery.WithValuesDiffer(answers))
|
||||
go resync.take(ctx, answers)
|
||||
})
|
||||
go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again
|
||||
composer, why := selfComposer(linked, session, node, say, hooks)
|
||||
if linked.Err() != nil {
|
||||
// The link ended before the node knew whose compositions to apply: it chose neither, and waits for the next.
|
||||
stopRun()
|
||||
cancel()
|
||||
continue
|
||||
}
|
||||
if composer != nil {
|
||||
composes.set(delivery.ComposesOwn)
|
||||
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
||||
session.NoteComposes(linked, delivery.ComposesOwn, why)
|
||||
// And once the catalogue no longer places it, the agent begins again on the controller's compositions: the
|
||||
// placement key deleted ends this run, the way back (issue 1126, the seed's revert).
|
||||
go endWhenUnplaced(run, session, node, stopRun, say)
|
||||
} else {
|
||||
composes.set(delivery.ComposesController)
|
||||
go resync.again(run)
|
||||
session.NoteComposes(linked, delivery.ComposesController, why)
|
||||
// And once the catalogue places it, the agent begins again composing its own: the placement key appearing ends
|
||||
// this run, so a node does not wait for a restart to follow the seed of its placement (ADR 0549 §4).
|
||||
go endWhenPlaced(run, session, node, stopRun, say, func(reason string) {
|
||||
session.NoteComposes(linked, delivery.ComposesController, why+"; "+reason)
|
||||
})
|
||||
}
|
||||
ag = agent.New(env)
|
||||
own.set(ag)
|
||||
current.set(ag)
|
||||
err = ag.Run(linked)
|
||||
err = ag.Run(run)
|
||||
stopRun()
|
||||
cancel()
|
||||
current.set(nil)
|
||||
if ctx.Err() != nil {
|
||||
@@ -748,3 +894,145 @@ const applyBound = 30 * time.Minute
|
||||
func applyContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), applyBound)
|
||||
}
|
||||
|
||||
// selfComposer is the node's own composer when the catalogue places the node (its key `placement.<node>` in the
|
||||
// indexer's state), and nil, said once, while it does not or the key cannot be read.
|
||||
// composerHooks are what the composer reads of the agent and the machine (selfcompose.Env Running, Unseal, Refused).
|
||||
type composerHooks struct {
|
||||
running func() []delivery.Composed
|
||||
unseal func(string) ([]byte, error)
|
||||
refused func(why string, held map[string]string)
|
||||
held func(module string) (delivery.Seals, []string, error)
|
||||
}
|
||||
|
||||
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
say("this node applies the controller's compositions: " + err.Error())
|
||||
return nil, err.Error()
|
||||
}
|
||||
env := composerEnv(session, node, say, hooks)
|
||||
unreadable := ""
|
||||
placed, err := selfcompose.Placed(ctx, env.Get, node, link.Backoff, func(m string) {
|
||||
if strings.Contains(m, "cannot be read") {
|
||||
unreadable = m
|
||||
}
|
||||
say(m)
|
||||
}, selfcompose.ComposedItsOwn(ctx, env.Get, node))
|
||||
if err != nil {
|
||||
return nil, "its placement could not be read before the link ended: " + err.Error()
|
||||
}
|
||||
if !placed {
|
||||
why := "the seat's placement state holds no key for it"
|
||||
if unreadable != "" {
|
||||
why = "its placement cannot be read, and it never composed its own: " + unreadable
|
||||
}
|
||||
say("this node applies the controller's compositions: " + why)
|
||||
return nil, why
|
||||
}
|
||||
why := "the seat's placement state holds its key (" + key + ")"
|
||||
say("this node composes its own modules: " + why)
|
||||
return selfcompose.New(env), why
|
||||
}
|
||||
|
||||
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
|
||||
// to placement candidates (selfcompose.Shadow) alike.
|
||||
func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env {
|
||||
return selfcompose.Env{Node: node,
|
||||
Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
if bucket == delivery.PlacementBucket || bucket == delivery.PlacementCandidatesBucket {
|
||||
// The seat's states are made when their holder starts: a node-engine that starts first waits for the
|
||||
// bus's word that the bucket was created, and the shadow and the composer say they wait.
|
||||
return bucketWatch{session, bucket}.WatchAwait(ctx, pattern, func(err error) {
|
||||
say("the seat's state " + bucket + " is not on the bus yet (" + err.Error() + "): it is watched when the bus says it was created")
|
||||
})
|
||||
}
|
||||
return bucketWatch{session, bucket}.Watch(ctx, pattern)
|
||||
},
|
||||
Get: func(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||
h, err := session.KVHistory(ctx, bucket, key)
|
||||
if err != nil || len(h) == 0 || h[len(h)-1].Deleted {
|
||||
return nil, err
|
||||
}
|
||||
return h[len(h)-1].Value, nil
|
||||
},
|
||||
Mesh: frombus.Mesh{InternalSuffix: overlay.Suffix()},
|
||||
Say: say,
|
||||
// Its modules' own secrets, sealed by the vault to this machine's key (ADR 0550 §5).
|
||||
Own: func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
return session.AskVault(ctx, "own", args)
|
||||
},
|
||||
// And the vault's word that a secret changed, on which it composes again (ADR 0547 §2).
|
||||
Hear: session.Hear,
|
||||
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
|
||||
Fetch: apply.Fetch,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
|
||||
}
|
||||
|
||||
// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch
|
||||
// caught up, or gone already), so the agent begins again and applies the controller's compositions, as a node the
|
||||
// catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be
|
||||
// made is said; the node then returns at its next start.
|
||||
func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, key)
|
||||
if err != nil {
|
||||
say("whether this node's key in the seat's placement state still exists is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start")
|
||||
return
|
||||
}
|
||||
placed := false
|
||||
for u := range updates {
|
||||
switch {
|
||||
case u.Caught && placed:
|
||||
continue
|
||||
case u.Caught || u.Deleted || len(u.Value) == 0:
|
||||
say("this node's key in the seat's placement state is gone (" + key + "): the node agent begins again, applying the controller's compositions")
|
||||
stop()
|
||||
return
|
||||
default:
|
||||
placed = true
|
||||
}
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state still exists is followed again at the node agent's next start")
|
||||
}
|
||||
}
|
||||
|
||||
// endWhenPlaced calls stop once this node's placement key appears in the seat's placement (a value after the watch
|
||||
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
|
||||
// the node then switches at its next start.
|
||||
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce, note func(string)) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// The seat's placement state is made when its holder starts; the delivery brings the grants part and the holder first,
|
||||
// so it is normally there. A node-engine that starts before it waits for the bus's word that the bucket was created
|
||||
// (no timer), says so once in the journal and in the node's own have, where the watchman reads it, and stays on the
|
||||
// controller's compositions until its key is written. A watch the account may not make is said the same way, and the
|
||||
// node stays until its next start.
|
||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, key, func(err error) {
|
||||
why := "this node's key in the seat's placement state is waited for, the state is not on the bus yet (" + err.Error() + "): it stays on the controller's compositions and is watched when the bus says the state was created"
|
||||
say(why)
|
||||
note(why)
|
||||
})
|
||||
if err != nil {
|
||||
why := "this node's key in the seat's placement state cannot be followed (" + err.Error() + "): it stays on the controller's compositions until its next start"
|
||||
say(why)
|
||||
note(why)
|
||||
return
|
||||
}
|
||||
for u := range updates {
|
||||
if !u.Caught && !u.Deleted && len(u.Value) > 0 {
|
||||
say("this node's key in the seat's placement state is written now (" + key + "): the node agent begins again, composing its own modules")
|
||||
stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
if ctx.Err() == nil {
|
||||
say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state is written is followed again at the node agent's next start")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,11 +14,14 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
|
||||
@@ -28,9 +31,14 @@ import (
|
||||
const itNode = "it1"
|
||||
|
||||
type itBed struct {
|
||||
t *testing.T
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
t *testing.T
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
// nodeConn and nodeJS are the node's own, under its real grant (buscompose.PermissionsFor, KindNode): the agent runs
|
||||
// on them, so a watch or a read the grant does not allow fails here as it fails on the bus (issue 1126, 2026-10-27
|
||||
// 09:21: a watch of every have key stopped every node agent). conn is the test's, unrestricted, for setting up.
|
||||
nodeConn *nats.Conn
|
||||
nodeJS nats.JetStreamContext
|
||||
wants nats.KeyValue
|
||||
composed nats.KeyValue
|
||||
haves nats.KeyValue
|
||||
@@ -40,12 +48,27 @@ type itBed struct {
|
||||
controll ed25519.PrivateKey
|
||||
operator ed25519.PrivateKey
|
||||
said []string
|
||||
// refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met.
|
||||
refusals []string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func aServer(t *testing.T) *nats.Conn {
|
||||
// aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what
|
||||
// buscompose.PermissionsFor gives a node (KindNode); it answers a connection of each.
|
||||
func aServer(t *testing.T, principal buscompose.Principal, refused func(error)) (admin, node *nats.Conn) {
|
||||
t.Helper()
|
||||
srv, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
perms, err := buscompose.PermissionsFor(principal)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
granted := &server.Permissions{Publish: &server.SubjectPermission{Allow: perms.Publish, Deny: perms.PublishDeny},
|
||||
Subscribe: &server.SubjectPermission{Allow: perms.Subscribe}}
|
||||
if perms.AllowResponses {
|
||||
granted.Response = &server.ResponsePermission{MaxMsgs: 1, Expires: buscompose.ResponseTTL}
|
||||
}
|
||||
nodeUser := buscompose.Principal{Kind: buscompose.KindNode, Node: itNode}.Username()
|
||||
srv, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Users: []*server.User{{Username: "test", Password: "test"}, {Username: nodeUser, Password: "node", Permissions: granted}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -54,15 +77,25 @@ func aServer(t *testing.T) *nats.Conn {
|
||||
t.Fatal("the embedded server did not start")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
conn, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
if admin, err = nats.Connect(srv.ClientURL(), nats.UserInfo("test", "test")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
return conn
|
||||
t.Cleanup(admin.Close)
|
||||
if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused(err) })); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(node.Close)
|
||||
return admin, node
|
||||
}
|
||||
|
||||
func newItBed(t *testing.T) *itBed {
|
||||
t.Helper()
|
||||
return newItBedAs(t, buscompose.Principal{Kind: buscompose.KindNode, Node: itNode})
|
||||
}
|
||||
|
||||
// newItBedAs is newItBed with the node's connection under the grant of principal, a node's (KindNode).
|
||||
func newItBedAs(t *testing.T, principal buscompose.Principal) *itBed {
|
||||
t.Helper()
|
||||
// Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node.
|
||||
bin := t.TempDir()
|
||||
@@ -74,11 +107,19 @@ func newItBed(t *testing.T) *itBed {
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
|
||||
b := &itBed{t: t, conn: aServer(t), dir: t.TempDir()}
|
||||
b := &itBed{t: t, dir: t.TempDir()}
|
||||
b.conn, b.nodeConn = aServer(t, principal, func(err error) {
|
||||
b.mu.Lock()
|
||||
b.refusals = append(b.refusals, err.Error())
|
||||
b.mu.Unlock()
|
||||
})
|
||||
var err error
|
||||
if b.js, err = b.conn.JetStream(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.nodeJS, err = b.nodeConn.JetStream(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composedBucket, _ := delivery.ComposedBucketFor(itNode)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
@@ -112,8 +153,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) {
|
||||
}
|
||||
}
|
||||
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
||||
Say: b.say}
|
||||
b.session.Attach(context.Background(), link.OverNATS{Conn: b.conn, JS: b.js, Node: itNode})
|
||||
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
||||
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
@@ -133,6 +174,8 @@ func (b *itBed) say(line string) {
|
||||
|
||||
func fpIt(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
|
||||
func fpDigest(c string) string { return "sha256:" + strings.Repeat(c, 64) }
|
||||
|
||||
// compose writes a module's signed composition of one file into the node's bucket and returns its digest.
|
||||
func (b *itBed) compose(module, fp, content string, data bool, signer ed25519.PrivateKey) string {
|
||||
b.t.Helper()
|
||||
@@ -197,6 +240,7 @@ func within(t *testing.T, d time.Duration, what string, ok func() bool) {
|
||||
// applied within one second and its have names the fingerprint and the digest; a composition whose digest no want names
|
||||
// is not applied.
|
||||
func TestNatsAWantAndItsCompositionAreAppliedWithinASecond(t *testing.T) {
|
||||
t.Run("1957: an engine start starts a declared-running unit", testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns)
|
||||
b := newItBed(t)
|
||||
stop := b.start(false)
|
||||
defer stop()
|
||||
@@ -311,3 +355,423 @@ func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) {
|
||||
return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2, the milestone's H8): the node agent started whole against a real bus, once the catalogue
|
||||
// places its node (the placement written after it started), composes a module from its want, the indexer's manifest, its record and its mesh-config value, and applies it,
|
||||
// with no key in the controller's compositions; a changed config value is composed again on its event and applied as a
|
||||
// revision of what runs. And a catalogue pull request's placement candidate equal to its placement is answered "same"
|
||||
// in the node's own key, and nothing is applied for it (ADR 0559 §4, subtask 10 of the milestone).
|
||||
//
|
||||
// And (issue 1126, ADR 0560) it reads, through its own session under the grant the grants part composes for it
|
||||
// (buscompose.Users, so it carries ServedReads), the build of a placed module that declares one and the one config value
|
||||
// a provider on another node serves from (a mail domain); under a grant without that served key the bus refuses the
|
||||
// read and the node composes nothing.
|
||||
func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
users, err := buscompose.Users(buscompose.Records{Nodes: []string{itNode, "it2"},
|
||||
Assigned: map[string][]buscompose.Declared{"it2": {{Module: "mailer", Served: []string{"domain"}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var principal buscompose.Principal
|
||||
for _, u := range users {
|
||||
if u.Kind == buscompose.KindNode && u.Node == itNode {
|
||||
principal = u
|
||||
}
|
||||
}
|
||||
if len(principal.ServedReads) != 1 {
|
||||
t.Fatalf("the grants part's principal of %s reads %v of the served keys", itNode, principal.ServedReads)
|
||||
}
|
||||
t.Run("without the served key the bus refuses the read", func(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
stop, _ := h8(t, b, false)
|
||||
defer stop()
|
||||
// 15 s, above link.historyWithin (10 s): the bus's refusal of a publish reaches only the connection's error
|
||||
// handler (nats.go fails no request on it), so the refused read ends at its own bound, then the node says it.
|
||||
within(t, 15*time.Second, "the bus did not refuse the served key's read, or the node did not say it", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return strings.Contains(strings.Join(b.refusals, "\n"), "Permissions Violation for Publish to \"$JS.API.CONSUMER.CREATE.KV_"+
|
||||
selfcompose.ResolvedBucket) && strings.Contains(strings.Join(b.refusals, "\n"), ".it2.mailer.domain") &&
|
||||
strings.Contains(strings.Join(b.said, "\n"), "what mailer serves on it2 (domain)")
|
||||
})
|
||||
if b.file("web") != "" {
|
||||
t.Errorf("composed and applied under a grant without the served key: %q", b.file("web"))
|
||||
}
|
||||
})
|
||||
b := newItBedAs(t, principal)
|
||||
stop, st := h8(t, b, true)
|
||||
defer stop()
|
||||
placement, resolved, candidates := st.placement, st.resolved, st.candidates
|
||||
hkey, place, fp := st.hkey, st.place, st.fp
|
||||
within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
|
||||
h8rest(t, b, resolved, candidates, hkey, place, fp)
|
||||
t.Run("one answer per candidate revision", func(t *testing.T) { h8once(t, b, resolved, candidates, place) })
|
||||
h8back(t, b, placement, resolved, candidates, hkey)
|
||||
}
|
||||
|
||||
// h8once is issue 1126, 2026-10-28 (shanks answered probe/1126-seed-7 at revision 84 every 13-17 s, its bus flooded):
|
||||
// a candidate revision is answered once. The node's own answer written, a module's have whose health words alone move
|
||||
// (each look rewrites them) and a config value written again with the value it held answer nothing again; the
|
||||
// candidate's next revision is answered once.
|
||||
func h8once(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, place []byte) {
|
||||
answering := func() int {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
n := 0
|
||||
for _, l := range b.said {
|
||||
if strings.HasPrefix(l, "answering placement candidate feat-x ") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
settled := answering()
|
||||
if settled == 0 {
|
||||
t.Fatal("feat-x was never answered")
|
||||
}
|
||||
hkey, _ := delivery.HaveKey(itNode, "web")
|
||||
for _, detail := range []string{"its tool web_health: no link to the bus is open", "its tool web_health: no answer within 10s"} {
|
||||
h := b.have("web")
|
||||
h.Health, h.Detail = delivery.HealthFailed, detail
|
||||
raw, err := delivery.EncodeHave(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.haves.Put(hkey, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
if got := answering(); got != settled {
|
||||
t.Errorf("feat-x answered %d more time(s) at the same revision, with nothing it is composed from changed", got-settled)
|
||||
}
|
||||
same, _ := delivery.DecodePlacement(place)
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "feat-x", Placement: &same})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("feat-x", itNode)
|
||||
if _, err := candidates.Put(ckey, craw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 10*time.Second, "the candidate's next revision was not answered", func() bool { return answering() > settled })
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
if got := answering(); got != settled+1 {
|
||||
t.Errorf("the candidate's next revision was answered %d time(s), want 1", got-settled)
|
||||
}
|
||||
}
|
||||
|
||||
// h8back is the way back (issue 1126, the seed's revert): a candidate deleting the node's placement is compared with
|
||||
// the controller's compositions at its wants (here none: refused, naming web), and the placement key deleted returns the agent to them without a
|
||||
// restart, said in the journal and the node's have, keeping what runs.
|
||||
func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValue, hkey string) {
|
||||
t.Helper()
|
||||
node := func() delivery.HaveNode {
|
||||
e, err := b.haves.Get(hkey)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}
|
||||
}
|
||||
n, _ := delivery.DecodeHaveNode(e.Value())
|
||||
return n
|
||||
}
|
||||
if n := node(); n.Composes != delivery.ComposesOwn {
|
||||
t.Errorf("a node composing its own says it composes %q", n.Composes)
|
||||
}
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "revert-x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("revert-x", itNode)
|
||||
if _, err := candidates.Put(ckey, craw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The controller holds no composition at web's want (the node composed its own): returning would leave web with
|
||||
// nothing to apply, and the answer says so rather than "same" (review of the shadow, round 1).
|
||||
within(t, 10*time.Second, "the node did not refuse returning to compositions the controller does not hold", func() bool {
|
||||
a := node().Candidates["revert-x"]
|
||||
return a.Proposal == "revert-x" && a.Returns == "" && a.Verdict == delivery.CandidateRefused &&
|
||||
strings.Contains(a.Refused, "web: the controller holds no composition")
|
||||
})
|
||||
ran := b.have("web").Composed
|
||||
pkey, _ := delivery.PlacementKey(itNode)
|
||||
// A pass in flight at the switch (issue 1126, 2026-10-28: the last pass of the own run, failing on its run's end,
|
||||
// marked the next agent's modules failed): a config value changed starts one, and the key goes at once.
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := placement.Delete(pkey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stopWatch := make(chan struct{})
|
||||
marked := make(chan Have1126, 1)
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-stopWatch:
|
||||
return
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
if h := b.have("web"); h.Phase == delivery.PhaseFailed {
|
||||
select {
|
||||
case marked <- Have1126{h.Phase, h.Detail}:
|
||||
default:
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
defer close(stopWatch)
|
||||
within(t, 15*time.Second, "the node did not return to the controller's compositions when its placement went", func() bool {
|
||||
b.mu.Lock()
|
||||
said := strings.Join(b.said, "\n")
|
||||
b.mu.Unlock()
|
||||
return strings.Contains(said, "key in the seat's placement state is gone") && node().Composes == delivery.ComposesController
|
||||
})
|
||||
time.Sleep(3 * time.Second) // what a pass in flight would mark lands within it
|
||||
select {
|
||||
case h := <-marked:
|
||||
t.Errorf("a pass in flight at the switch marked web failed: %+v", h)
|
||||
default:
|
||||
}
|
||||
if h := b.have("web"); h.Composed != ran || h.Phase == delivery.PhaseFailed || b.file("web") != "greeting=salut\n" {
|
||||
t.Errorf("returning to the controller's compositions moved what runs: %+v, %q", h, b.file("web"))
|
||||
}
|
||||
}
|
||||
|
||||
// Have1126 is what H8's way back watches of web's have.
|
||||
type Have1126 struct {
|
||||
Phase delivery.Phase
|
||||
Detail string
|
||||
}
|
||||
|
||||
// h8state is what h8 set up that the rest of H8 writes to.
|
||||
type h8state struct {
|
||||
placement, resolved, candidates nats.KeyValue
|
||||
hkey string
|
||||
place []byte
|
||||
fp string
|
||||
}
|
||||
|
||||
// h8 sets H8's bus up and starts the agent, then places the node. With checkBuild, the placed module's build key is
|
||||
// first one that does not decode, which the node must read through its session (and say), then a done one.
|
||||
func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
|
||||
t.Helper()
|
||||
kvOf := func(bucket string) nats.KeyValue {
|
||||
kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
placement, modules, manifests := kvOf(delivery.PlacementBucket), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket)
|
||||
resolved := kvOf(selfcompose.ResolvedBucket)
|
||||
builds := kvOf(delivery.BuilderBuildsBucket)
|
||||
candidates := kvOf(delivery.PlacementCandidatesBucket)
|
||||
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
|
||||
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(b.dir, "web.conf")
|
||||
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
|
||||
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}],` +
|
||||
`"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
|
||||
digest := delivery.ManifestDigest([]byte(man))
|
||||
mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man})
|
||||
mkey, _ := delivery.ManifestsKey(digest)
|
||||
fp := fpIt("a")
|
||||
modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)})
|
||||
place, _ := delivery.EncodePlacement(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}})
|
||||
pkey, _ := delivery.PlacementKey(itNode)
|
||||
hkey, _ := delivery.HaveNodeKey(itNode)
|
||||
// A provider on it2 serving from its own config value: a mail domain, which it1's composition settles.
|
||||
mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` +
|
||||
`"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}`
|
||||
mdigest := delivery.ManifestDigest([]byte(mailer))
|
||||
mlraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "mailer", JSON: mailer})
|
||||
mlkey, _ := delivery.ManifestsKey(mdigest)
|
||||
mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)})
|
||||
place2, _ := delivery.EncodePlacement(delivery.Placement{Node: "it2", Modules: []delivery.PlacedModule{{Module: "mailer"}}})
|
||||
rec2, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}})
|
||||
pkey2, _ := delivery.PlacementKey("it2")
|
||||
hkey2, _ := delivery.HaveNodeKey("it2")
|
||||
buildKey, _ := delivery.BuilderBuildKey("web", fp)
|
||||
done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest,
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}})
|
||||
firstBuild := done
|
||||
if checkBuild {
|
||||
firstBuild = []byte(`{"v":1,"state":"no such state"}`)
|
||||
}
|
||||
for _, w := range []struct {
|
||||
kv nats.KeyValue
|
||||
key string
|
||||
value []byte
|
||||
}{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec},
|
||||
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)},
|
||||
{manifests, mlkey, mlraw}, {modules, "mailer", mlmod}, {b.haves, hkey2, rec2}, {placement, pkey2, place2},
|
||||
{resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}, {builds, buildKey, firstBuild}} {
|
||||
if _, err := w.kv.Put(w.key, w.value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true})
|
||||
wkey, _ := delivery.WantKey(itNode, "web")
|
||||
if _, err := b.wants.Put(wkey, want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stop := b.start(false)
|
||||
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
|
||||
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if b.file("web") != "" {
|
||||
t.Fatal("a node the catalogue does not place composed its own")
|
||||
}
|
||||
if _, err := placement.Put(pkey, place); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if checkBuild {
|
||||
// The build of the placed module is read through the node's session: one that does not decode is said, and nothing
|
||||
// is applied; a done one, then a config event, and the node composes.
|
||||
within(t, 5*time.Second, "the node did not read web's build key through its session", func() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return strings.Contains(strings.Join(b.said, "\n"), "the builder's "+buildKey)
|
||||
})
|
||||
if b.file("web") != "" {
|
||||
t.Fatal("composed with a build key that does not decode")
|
||||
}
|
||||
if _, err := builds.Put(buildKey, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return stop, h8state{placement: placement, resolved: resolved, candidates: candidates, hkey: hkey, place: place, fp: fp}
|
||||
}
|
||||
|
||||
// h8rest is H8 once the node composes its own: the have, a config revision, and the candidate answered "same".
|
||||
func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey string, place []byte, fp string) {
|
||||
t.Helper()
|
||||
within(t, 2*time.Second, "the have does not name the build", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed != "" && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
if keys, _ := b.composed.Keys(); len(keys) != 0 {
|
||||
t.Errorf("the controller's compositions hold %v; the node composed its own", keys)
|
||||
}
|
||||
first := b.have("web").Composed
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 3*time.Second, "a changed config value was not composed and applied again", func() bool { return b.file("web") == "greeting=salut\n" })
|
||||
within(t, 2*time.Second, "the have does not name the revision", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed != first && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
|
||||
// The node says its facts (its record, as the test wrote it), then a candidate equal to its placement is written: the
|
||||
// node answers "same" at the candidate's head and revision in its own key, and applies nothing.
|
||||
b.session.SayFacts(context.Background(), link.Report{Profile: map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "service-manager", "present": true}}}})
|
||||
second := b.have("web").Composed
|
||||
same, _ := delivery.DecodePlacement(place)
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "feat-x", Placement: &same})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("feat-x", itNode)
|
||||
crev, err := candidates.Put(ckey, craw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var answer delivery.CandidateAnswer
|
||||
within(t, 10*time.Second, "the node did not answer its candidate", func() bool {
|
||||
e, err := b.haves.Get(hkey)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(e.Value())
|
||||
answer = n.Candidates["feat-x"]
|
||||
return err == nil && answer.Proposal == "feat-x" && n.Record != nil
|
||||
})
|
||||
if answer.Verdict != delivery.CandidateSame || answer.Revision != crev || len(answer.Same) != 1 || answer.Same[0] != "web" {
|
||||
t.Errorf("the answer to a candidate equal to the placement: %+v", answer)
|
||||
}
|
||||
if h := b.have("web"); h.Composed != second || b.file("web") != "greeting=salut\n" {
|
||||
t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web"))
|
||||
}
|
||||
}
|
||||
|
||||
// 1957/AC1 (integration; ADR 0568 §1): a node agent starting on a bus that holds a `running` have for a module at its want,
|
||||
// with a service manager (a systemctl on the PATH) that shows the module's unit inactive, starts the unit once and writes
|
||||
// the have running; the same start with the unit active applies nothing.
|
||||
func testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns(t *testing.T) {
|
||||
for _, active := range []bool{false, true} {
|
||||
b := newItBed(t)
|
||||
bin := t.TempDir()
|
||||
stateFile, calls := filepath.Join(bin, "state"), filepath.Join(bin, "calls")
|
||||
if active {
|
||||
os.WriteFile(stateFile, []byte("active"), 0o644)
|
||||
} else {
|
||||
os.WriteFile(stateFile, []byte("inactive"), 0o644)
|
||||
}
|
||||
script := "#!/bin/sh\necho \"$@\" >> " + calls + "\ncase \"$1\" in\n" +
|
||||
"show) printf 'LoadState=loaded\\nActiveState=%s\\nType=simple\\n' \"$(cat " + stateFile + ")\" ;;\n" +
|
||||
"start) printf active > " + stateFile + " ;;\nis-enabled) echo enabled ;;\nesac\nexit 0\n"
|
||||
if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
|
||||
// The engine's own record of the unit, as a previous run left it, and the have it wrote then.
|
||||
res, _ := json.Marshal(map[string]any{"id": "web.unit", "type": "service", "unit": "web-it.service", "state": "running", "module": "web"})
|
||||
var known store.State
|
||||
known.Record(store.Applied{ID: "web.unit", Type: "service", Target: "web-it.service"})
|
||||
if err := store.Save(b.opts.state, known); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fp := fpIt("a")
|
||||
c := delivery.Composed{Node: itNode, Module: "web", FP: fp, Resources: []json.RawMessage{res}}
|
||||
signed, _ := delivery.SignComposed(c, b.controll)
|
||||
raw, _ := delivery.EncodeComposed(signed)
|
||||
dig, _ := delivery.ComposedDigestOf(signed)
|
||||
key, _ := delivery.ComposedKey(itNode, "web", fp)
|
||||
if _, err := b.composed.Put(key, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.want("web", fp, dig, "")
|
||||
hkey, _ := delivery.HaveKey(itNode, "web")
|
||||
have, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.haves.Put(hkey, have); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stop := b.start(false)
|
||||
within(t, 5*time.Second, "the unit was not started and the have not running", func() bool {
|
||||
got, _ := os.ReadFile(stateFile)
|
||||
h := b.have("web")
|
||||
return string(got) == "active" && h.Phase == delivery.PhaseRunning && h.Composed == dig
|
||||
})
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
stop()
|
||||
starts := 0
|
||||
log, _ := os.ReadFile(calls)
|
||||
for _, l := range strings.Split(string(log), "\n") {
|
||||
if strings.HasPrefix(l, "start") {
|
||||
starts++
|
||||
}
|
||||
}
|
||||
if want := map[bool]int{false: 1, true: 0}[active]; starts != want {
|
||||
t.Fatalf("unit active=%v: started %d times, want %d", active, starts, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/ownresync"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
)
|
||||
|
||||
// composesNow is whose compositions this run of the agent applies, once decided ("" before).
|
||||
type composesNow struct {
|
||||
mu sync.Mutex
|
||||
now string
|
||||
}
|
||||
|
||||
func (c *composesNow) set(s string) { c.mu.Lock(); c.now = s; c.mu.Unlock() }
|
||||
func (c *composesNow) get() string { c.mu.Lock(); defer c.mu.Unlock(); return c.now }
|
||||
|
||||
// resyncStep is the step that hands the vault what runs of an own secret a shadow answer says another value of (issue
|
||||
// 1126, cause C4; internal/ownresync): one pass at a time, and the last answers taken again once the agent has decided
|
||||
// whose compositions it applies.
|
||||
type resyncStep struct {
|
||||
r *ownresync.Resync
|
||||
one sync.Mutex
|
||||
mu sync.Mutex
|
||||
last map[string]delivery.CandidateAnswer
|
||||
// tries and pending are the step taken again after a bound when a read or an ask the bus did not answer stopped it
|
||||
// (Outcome.Retry, issue 1126, 2026-10-28), one wait at a time.
|
||||
tries int
|
||||
pending bool
|
||||
}
|
||||
|
||||
func ownResync(session *link.Session, node string, say func(string), env selfcompose.Env, composes *composesNow) *resyncStep {
|
||||
return &resyncStep{r: ownresync.New(ownresync.Env{Node: node, Composes: composes.get, Running: env.Running,
|
||||
Get: env.Get, Unseal: env.Unseal, Seal: identity.Seal, Say: say,
|
||||
Ask: func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
return session.AskVault(ctx, verb, args)
|
||||
}})}
|
||||
}
|
||||
|
||||
// take keeps answers as the last and takes them.
|
||||
func (s *resyncStep) take(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
s.mu.Lock()
|
||||
s.last = answers
|
||||
s.mu.Unlock()
|
||||
s.again(ctx)
|
||||
}
|
||||
|
||||
// again takes the last answers, and once more after a bound (selfcompose.LaterWait) while the bus did not answer for one.
|
||||
func (s *resyncStep) again(ctx context.Context) {
|
||||
s.one.Lock()
|
||||
defer s.one.Unlock()
|
||||
s.mu.Lock()
|
||||
last := s.last
|
||||
s.mu.Unlock()
|
||||
if last == nil {
|
||||
return
|
||||
}
|
||||
retry := false
|
||||
for _, o := range s.r.Take(ctx, last) {
|
||||
retry = retry || o.Retry
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if !retry {
|
||||
s.tries = 0
|
||||
return
|
||||
}
|
||||
if s.pending || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
wait := selfcompose.LaterWait(s.tries)
|
||||
s.tries++
|
||||
s.pending = true
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(wait):
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.pending = false
|
||||
s.mu.Unlock()
|
||||
if ctx.Err() == nil {
|
||||
s.again(ctx)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -36,6 +36,7 @@ func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
|
||||
|
||||
// The modules applied again are those with a missing resource, each once; one down but there is not among them.
|
||||
func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
||||
t.Run("1957: the machine shows a resource not active", testTheMachineShowsAResourceNotActive)
|
||||
st := liveness.Statement{Resources: []liveness.State{
|
||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
|
||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
|
||||
@@ -51,6 +52,7 @@ func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
||||
// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One
|
||||
// that comes back and goes again is newly missing again (the review of issue 1945).
|
||||
func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) {
|
||||
t.Run("1957: a resource newly down is an edge", testOnlyAResourceNewlyDownIsAnEdge)
|
||||
missing := func(ids ...string) liveness.Statement {
|
||||
var st liveness.Statement
|
||||
for _, id := range ids {
|
||||
|
||||
@@ -7,9 +7,13 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
@@ -42,6 +46,35 @@ func userListFollower(opts options, session *link.Session, composedBucket string
|
||||
Report: func(ctx context.Context, module string, w userlist.Written) {
|
||||
session.NoteUserList(ctx, module, w)
|
||||
},
|
||||
// From take-over, the grants part's list (ADR 0547 clauses 7, 8): its one key watched, its object read by name and
|
||||
// opened with this machine's sealing key, the key's public half said in the have so the grants part seals to it.
|
||||
Grants: bucketWatch{session, delivery.GrantsUserListBucket},
|
||||
Fetch: func(ctx context.Context, name string) ([]byte, error) {
|
||||
return session.GrantsObject(ctx, name, grantsListLimit)
|
||||
},
|
||||
Open: func(b seal.Box) ([]byte, error) {
|
||||
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return key.OpenVaultBox(b)
|
||||
},
|
||||
Key: func() string {
|
||||
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
pem, err := key.PublicPEM()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return pem
|
||||
},
|
||||
// The grants part's key as last heard, kept beside the node's state, followed while it cannot be watched; its
|
||||
// watch tried again as a passing failure of the bus is (ADR 0547 clause 8).
|
||||
Kept: func() ([]byte, uint64) { return keptGrantsKey(opts) },
|
||||
Keep: func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) },
|
||||
Backoff: link.Backoff,
|
||||
// The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547
|
||||
// clause 9).
|
||||
VaultKey: vaultKey.get,
|
||||
@@ -49,6 +82,9 @@ func userListFollower(opts options, session *link.Session, composedBucket string
|
||||
})
|
||||
}
|
||||
|
||||
// grantsListLimit is the largest sealed list the bus node reads: the live list is about a hundred kilobytes.
|
||||
const grantsListLimit = 16 << 20
|
||||
|
||||
// userListFile is a user-list resource as the file it declares.
|
||||
func userListFile(raw []byte) (*declaration.File, error) {
|
||||
r, err := declaration.ParseResource(raw)
|
||||
@@ -125,3 +161,44 @@ func writeUserListAlone(ctx context.Context, opts options, raw []byte, body []by
|
||||
}
|
||||
return digest, writeErr == nil && out.Action != "unchanged", writeErr
|
||||
}
|
||||
|
||||
// grantsKeyPath is where the grants part's user-list key last heard is kept, beside the node's state.
|
||||
func grantsKeyPath(opts options) string {
|
||||
return filepath.Join(filepath.Dir(opts.state), "grants-user-list.json")
|
||||
}
|
||||
|
||||
type keptKey struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
|
||||
func keptGrantsKey(opts options) ([]byte, uint64) {
|
||||
raw, err := os.ReadFile(grantsKeyPath(opts))
|
||||
if err != nil {
|
||||
return nil, 0
|
||||
}
|
||||
var k keptKey
|
||||
if json.Unmarshal(raw, &k) != nil || len(k.Value) == 0 {
|
||||
return nil, 0
|
||||
}
|
||||
return k.Value, k.Revision
|
||||
}
|
||||
|
||||
func keepGrantsKey(opts options, value []byte, rev uint64, say link.Announce) {
|
||||
path := grantsKeyPath(opts)
|
||||
if value == nil {
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
announceOr(say)("the grants part's user-list key kept on this machine could not be taken away: " + err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(keptKey{Revision: rev, Value: value})
|
||||
tmp := path + ".new"
|
||||
err := os.WriteFile(tmp, raw, 0o600)
|
||||
if err == nil {
|
||||
err = os.Rename(tmp, path)
|
||||
}
|
||||
if err != nil {
|
||||
announceOr(say)("the grants part's user-list key could not be kept on this machine: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456 h1:He2o9vSKjgKbWOCOgOJY0WlaDh1n8RcGhvGY/dwXsBQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e h1:k/1NjjW5HuuuHW2ldyWTvjp+1u3fN0RO18Q57Iup6Dk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936 h1:HGBoWimZXYZp+Qn9VCstYDitOP84t9EUQYd1yM9j2eg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
+433
-16
@@ -23,6 +23,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -49,9 +51,17 @@ type Have struct {
|
||||
Module string
|
||||
FP string
|
||||
Composed string
|
||||
// Phase is "running", "failed", "held" or "removed"; empty reads as running.
|
||||
// Phase is "running", "failed", "held", "waiting-yes" or "removed"; empty reads as running.
|
||||
Phase string
|
||||
Detail string
|
||||
// Judgement and Waits are, in phase waiting-yes, the judgement of the move and the digest of the composition that
|
||||
// waits for a person's yes (ADR 0547 §3).
|
||||
Judgement string
|
||||
Waits string
|
||||
// Sealed is, on a node composing its own, each value the composition it names carries sealed to the node's key, by
|
||||
// slot (delivery.Have.SealedAt): kept in the have so the composer reuses them after a restart (ADR 0547 §3), each at
|
||||
// its own slot alone (issue 1126, C1).
|
||||
Sealed delivery.Seals
|
||||
}
|
||||
|
||||
// Haves is the node's have bucket as the agent uses it.
|
||||
@@ -108,6 +118,11 @@ type Env struct {
|
||||
// Recorded says the engine holds a record of every resource of the composition, so it was applied and not removed
|
||||
// since; nil says yes. A composition the kept declaration or a have says runs is applied again when it says no.
|
||||
Recorded func(ctx context.Context, c delivery.Composed) bool
|
||||
// Inactive says a resource the composition declares running is not active on the machine now (a unit stopped, a
|
||||
// container not running), read from the machine and not from any record (ADR 0568 §1). A have or a kept declaration
|
||||
// that says the module runs is believed at the agent's start only when this says no; nil says no, and an unreadable
|
||||
// machine says no (unknown is not inactive).
|
||||
Inactive func(ctx context.Context, c delivery.Composed) bool
|
||||
// Revisions are a composition key's revisions the bucket keeps, oldest first; nil reads none. The composition a want
|
||||
// names is taken from them when a newer one replaced it under the same key (ADR 0539 clause 1).
|
||||
Revisions func(ctx context.Context, key string) ([]Update, error)
|
||||
@@ -120,6 +135,14 @@ type Env struct {
|
||||
// transient failure only on RetryTransient or a change of either key.
|
||||
Backoff func(try int) time.Duration
|
||||
Say func(string)
|
||||
// SelfComposed says the compositions are the node's own (novox/hq ADR 0547 §2, issue 1126): Composed hands what the
|
||||
// node composed itself, unsigned, and a want names the build alone. The composition composed for the want's build is
|
||||
// the one applied, when it differs from what runs.
|
||||
SelfComposed bool
|
||||
// Policy is a module's upgrade policy as its manifest derives it (catalogue.DerivedUpgrade); with SelfComposed, a
|
||||
// move of a module of policy record that recreates a container or moves its data waits for a person's yes in its
|
||||
// want (ADR 0547 §3). Nil judges no move.
|
||||
Policy func(ctx context.Context, module string) string
|
||||
}
|
||||
|
||||
// transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has
|
||||
@@ -143,7 +166,14 @@ type Agent struct {
|
||||
// prior is a module the node ran before this agent started, so it has something to back up.
|
||||
prior map[string]bool
|
||||
said map[string]string
|
||||
retry chan []string
|
||||
// notComposed is the modules whose have says the node could not compose them: why, and the have before (NotComposed).
|
||||
notComposed map[string]notComposedMark
|
||||
// heldAlone is the modules the composer holds alone (NotComposedEach): what runs of each in the node's own
|
||||
// composition is held out of the node's own apply until a composition of its own is handed (heldOut).
|
||||
heldAlone map[string]bool
|
||||
retry chan []string
|
||||
// reconcile carries the judge's edges: a resource found not active (ADR 0568 §2).
|
||||
reconcile chan []string
|
||||
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
|
||||
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
|
||||
tries map[string]int
|
||||
@@ -189,8 +219,8 @@ func New(env Env) *Agent {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{},
|
||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{},
|
||||
retry: make(chan []string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
|
||||
retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
|
||||
}
|
||||
|
||||
@@ -222,6 +252,17 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
dirty[m] = true
|
||||
}
|
||||
a.mu.Unlock()
|
||||
case modules := <-a.reconcile:
|
||||
a.mu.Lock()
|
||||
for _, m := range modules {
|
||||
if w := a.wants[m]; !w.Present || w.Want.Removed || a.attempt[m].Held {
|
||||
continue
|
||||
}
|
||||
delete(a.running, m)
|
||||
delete(a.attempt, m)
|
||||
dirty[m] = true
|
||||
}
|
||||
a.mu.Unlock()
|
||||
case ask := <-a.again:
|
||||
// The wait of a transiently failed apply ran out: tried again from the keys it has, if nothing tried it since.
|
||||
a.mu.Lock()
|
||||
@@ -280,6 +321,16 @@ func (a *Agent) Retry(modules ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Reconcile asks for the modules to be applied again because a resource each declares running was found not active, once
|
||||
// per edge (the judge's look, ADR 0568 §2). It is Retry, but a module whose apply is held (a failed backup) is left
|
||||
// alone, as is one the node wants no more (ADR 0568 §4). It never blocks.
|
||||
func (a *Agent) Reconcile(modules ...string) {
|
||||
select {
|
||||
case a.reconcile <- modules:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// RetryTransient asks for every module whose last apply failed transiently to be applied again now: the engine says it
|
||||
// when it knows the bus answers again (after it wrote the bus's user list and the bus reloaded its accounts). It never
|
||||
// blocks.
|
||||
@@ -290,6 +341,208 @@ func (a *Agent) RetryTransient() {
|
||||
}
|
||||
}
|
||||
|
||||
// notComposedMark is why a module's have says the node could not compose it, and the have it said before.
|
||||
type notComposedMark struct {
|
||||
why string
|
||||
prior Have
|
||||
}
|
||||
|
||||
// NotComposedDetail opens the detail of a have NotComposed marks: the module keeps running what its build and digest
|
||||
// name, and only the node's composition of it failed.
|
||||
const NotComposedDetail = "the node cannot compose it, and keeps running what it ran: "
|
||||
|
||||
// composerMark says h is a mark NotComposed wrote: failed, its detail opening with NotComposedDetail.
|
||||
func composerMark(h Have) bool {
|
||||
return h.Phase == string(delivery.PhaseFailed) && strings.HasPrefix(h.Detail, NotComposedDetail)
|
||||
}
|
||||
|
||||
// HavesReadErr is what a Haves may answer besides Read: a have absent (false, nil) told from one that could not be read
|
||||
// (false, the error).
|
||||
type HavesReadErr interface {
|
||||
ReadErr(ctx context.Context, module string) (Have, bool, error)
|
||||
}
|
||||
|
||||
// readHave reads a module's have: found, absent, or unread. A read that fails is tried again after a short wait while
|
||||
// the bus does not answer (a reload pauses it for a few seconds, issue 1126), and is unread when it still fails. A
|
||||
// Haves that cannot tell an absent key from a failed read says absent.
|
||||
func (a *Agent) readHave(ctx context.Context, module string) (h Have, found, unread bool) {
|
||||
re, tells := a.env.Haves.(HavesReadErr)
|
||||
if !tells {
|
||||
h, found = a.env.Haves.Read(ctx, module)
|
||||
return h, found, false
|
||||
}
|
||||
for try := 0; ; try++ {
|
||||
h, found, err := re.ReadErr(ctx, module)
|
||||
if err == nil {
|
||||
return h, found, false
|
||||
}
|
||||
if try == 2 || ctx.Err() != nil {
|
||||
return Have{}, false, true
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return Have{}, false, true
|
||||
case <-time.After(time.Duration(1<<try) * readAgain):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readAgain is the first wait before a have is read again.
|
||||
var readAgain = time.Second
|
||||
|
||||
// NotComposed says the node composing its own could not compose its modules (ADR 0547 §2: a stand-in refused, an
|
||||
// input that cannot be read): each module with a want gets its have failed, naming why, while it keeps running what it
|
||||
// ran, so the failure is seen where every have is read and not only in the journal. The have keeps what runs: the
|
||||
// fingerprint, the digest and the sealed values the agent runs, or, before it ran anything since its start, those its
|
||||
// last have said (a refusal at the first compose after a start). Called with "" once the node composes again: each
|
||||
// module so marked has its have back as it was, unless an apply wrote it since.
|
||||
func (a *Agent) NotComposed(ctx context.Context, why string) { a.NotComposedEach(ctx, why, nil) }
|
||||
|
||||
// NotComposedEach is NotComposed for the node refused whole (why), or, with why empty, for the modules the node holds
|
||||
// alone (held, each with its own why: refused alone, the rest composed and applied, issue 1126): only those are marked,
|
||||
// each naming its own why, and every other module marked before has its have back. Both empty, every mark is cleared.
|
||||
func (a *Agent) NotComposedEach(ctx context.Context, why string, held map[string]string) {
|
||||
a.mu.Lock()
|
||||
if why == "" {
|
||||
a.heldAlone = make(map[string]bool, len(held))
|
||||
for m := range held {
|
||||
a.heldAlone[m] = true
|
||||
}
|
||||
}
|
||||
whyOf := map[string]string{} // the mark each module is to carry; a marked module not in it is cleared
|
||||
for module, w := range a.wants {
|
||||
if !w.Present || w.Want.Removed || module == NodeModule {
|
||||
continue
|
||||
}
|
||||
if why != "" {
|
||||
whyOf[module] = why
|
||||
} else if h, ok := held[module]; ok && h != "" {
|
||||
whyOf[module] = h
|
||||
}
|
||||
}
|
||||
var modules []string
|
||||
for module, w := range whyOf {
|
||||
if a.notComposed[module].why != w {
|
||||
modules = append(modules, module)
|
||||
}
|
||||
}
|
||||
for module := range a.notComposed {
|
||||
if _, still := whyOf[module]; !still && why == "" { // the node refused whole marks; it clears none
|
||||
modules = append(modules, module)
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
why := whyOf[module]
|
||||
if ctx.Err() != nil {
|
||||
return // the run ended: what it would mark or clear is the next run's to say (issue 1126)
|
||||
}
|
||||
a.mu.Lock()
|
||||
mark, marked := a.notComposed[module]
|
||||
r, ran := a.running[module]
|
||||
a.mu.Unlock()
|
||||
if why == "" {
|
||||
a.mu.Lock()
|
||||
delete(a.notComposed, module)
|
||||
a.mu.Unlock()
|
||||
back := mark.prior
|
||||
if ran {
|
||||
back = Have{Module: module, FP: r.FP, Composed: r.Composed, Phase: string(delivery.PhaseRunning)}
|
||||
if r.Phase != "" {
|
||||
back.Phase = string(r.Phase)
|
||||
}
|
||||
}
|
||||
if back.Composed == "" {
|
||||
continue // nothing said what ran: nothing to say back
|
||||
}
|
||||
a.writeHave(ctx, back)
|
||||
continue
|
||||
}
|
||||
if !marked {
|
||||
if ran {
|
||||
mark.prior = Have{Module: module, FP: r.FP, Composed: r.Composed, Phase: string(r.Phase)}
|
||||
if mark.prior.Phase == "" {
|
||||
mark.prior.Phase = string(delivery.PhaseRunning)
|
||||
}
|
||||
} else if h, found, unread := a.readHave(ctx, module); found {
|
||||
mark.prior = h // what its last have said runs: this agent has run nothing of it since its start
|
||||
} else if unread {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
// What its have names cannot be read (the bus paused, issue 1126): a mark written now would drop the build
|
||||
// and digest it names. Nothing is written; the next refusal marks it once the have reads.
|
||||
a.mu.Lock()
|
||||
a.sayOnce("unmarked:"+module, fmt.Sprintf("%s's have cannot be read, so it is not marked: the node cannot compose it: %s", module, why))
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
}
|
||||
// A have that already says another cause (an apply that failed, a backup held, an apply under way) keeps it: a
|
||||
// composer's mark over it would hide that cause, and baseline would take the mark for what runs (review of the
|
||||
// switch, round 1).
|
||||
if p := mark.prior.Phase; !marked && p != "" && p != string(delivery.PhaseRunning) && !composerMark(mark.prior) {
|
||||
a.mu.Lock()
|
||||
a.sayOnce("unmarked-cause:"+module, fmt.Sprintf("%s is not marked: its have says %s (%s), and the node cannot compose it: %s",
|
||||
module, p, mark.prior.Detail, why))
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
mark.why = why
|
||||
a.mu.Lock()
|
||||
a.notComposed[module] = mark
|
||||
a.mu.Unlock()
|
||||
a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed,
|
||||
Phase: string(delivery.PhaseFailed), Detail: NotComposedDetail + why})
|
||||
}
|
||||
}
|
||||
|
||||
// writeHave writes a module's have (the node's own key for NodeModule); on a node composing its own, with the sealed
|
||||
// values of the composition it names when the caller gave none (Have.Sealed): those of the composition the agent applied or holds at that digest, else
|
||||
// those the module's last have said at that digest.
|
||||
//
|
||||
// A writer that names no composition (one written from the agent's running entry before this agent ran anything of the
|
||||
// module since its start: held by a failed backup, waiting for a yes, not composed) keeps what the module's last have
|
||||
// said runs, its fingerprint, digest and sealed values, rather than a have naming nothing: what runs is not lost, and
|
||||
// the boxes the composer reuses after a restart stay (review of L2, rounds 2 and 3).
|
||||
func (a *Agent) writeHave(ctx context.Context, h Have) {
|
||||
if h.Composed == "" && h.Module != NodeModule {
|
||||
if last, ok := a.env.Haves.Read(ctx, h.Module); ok && last.Composed != "" {
|
||||
h.FP, h.Composed = last.FP, last.Composed
|
||||
if h.Sealed == nil {
|
||||
h.Sealed = last.Sealed
|
||||
}
|
||||
}
|
||||
}
|
||||
// The node's own composition too: its core boxes are sealed again by the vault at a restart, and kept in its key the
|
||||
// composer reuses them, so the restart applies nothing again (review of L2, round 3, finding 1).
|
||||
if a.env.SelfComposed && h.Sealed == nil && h.Composed != "" {
|
||||
h.Sealed = a.sealedOf(ctx, h.Module, h.FP, h.Composed)
|
||||
}
|
||||
a.env.Haves.Write(ctx, h)
|
||||
}
|
||||
|
||||
func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) delivery.Seals {
|
||||
a.mu.Lock()
|
||||
if c, ok := a.applied[module]; ok && c.Err == nil && c.Dig == composed {
|
||||
a.mu.Unlock()
|
||||
return delivery.SealsIn(c.C)
|
||||
}
|
||||
if c, ok := a.compositionFor(module, fp); ok && c.Err == nil && c.Dig == composed {
|
||||
a.mu.Unlock()
|
||||
return delivery.SealsIn(c.C)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if h, ok := a.env.Haves.Read(ctx, module); ok && h.Composed == composed {
|
||||
return h.Sealed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Running is the compositions the node runs now: each module's that the agent applied or found already running, and
|
||||
// the node's own. What the node judges its health by and says of itself is built from them.
|
||||
func (a *Agent) Running() []delivery.Composed {
|
||||
@@ -366,7 +619,14 @@ func (a *Agent) takeComposition(u Update) string {
|
||||
}
|
||||
now := compNow{Raw: u.Value, Rev: u.Revision}
|
||||
c, err := delivery.DecodeComposed(u.Value)
|
||||
if err == nil {
|
||||
if err == nil && a.env.SelfComposed {
|
||||
// The node composed it itself: nothing crossed the bus, so nothing is signed (ADR 0547 §2).
|
||||
now.C = c
|
||||
now.Dig, err = delivery.ComposedDigestOf(c)
|
||||
if err != nil {
|
||||
now.C = delivery.Composed{}
|
||||
}
|
||||
} else if err == nil {
|
||||
var trusted []ed25519.PublicKey
|
||||
if a.env.Trusted != nil {
|
||||
trusted = a.env.Trusted()
|
||||
@@ -600,6 +860,49 @@ func (a *Agent) heldOut(except string) []delivery.Composed {
|
||||
sort.Strings(names)
|
||||
for _, m := range names {
|
||||
out = append(out, waiting[m].C)
|
||||
held[m] = true
|
||||
}
|
||||
// A module the composer holds alone keeps what runs of it in the node's own composition, until a composition of its
|
||||
// own is handed: held out as one of its own, so the node's own apply never removes it (issue 1126, review of the
|
||||
// apply, round 1: de-spiegel leaving the fold with its names unset; issue 1945).
|
||||
if len(a.heldAlone) == 0 {
|
||||
return out
|
||||
}
|
||||
var own *delivery.Composed
|
||||
for _, c := range a.others("\x00") {
|
||||
if c.Module == "" {
|
||||
own = &c
|
||||
break
|
||||
}
|
||||
}
|
||||
alone := make([]string, 0, len(a.heldAlone))
|
||||
for m := range a.heldAlone {
|
||||
if !held[m] && m != except && own != nil {
|
||||
alone = append(alone, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(alone)
|
||||
for _, m := range alone {
|
||||
var rs []json.RawMessage
|
||||
for _, raw := range own.Resources {
|
||||
var head struct {
|
||||
ID string `json:"id"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
if json.Unmarshal(raw, &head) != nil {
|
||||
continue
|
||||
}
|
||||
owner := head.Module
|
||||
if owner == "" {
|
||||
owner, _ = declaration.ModuleGuess(head.ID)
|
||||
}
|
||||
if owner == m {
|
||||
rs = append(rs, raw)
|
||||
}
|
||||
}
|
||||
if len(rs) > 0 {
|
||||
out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -636,7 +939,16 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
|
||||
a.mu.Lock()
|
||||
comp, have := a.compositionFor(module, w.Want.FP)
|
||||
a.mu.Unlock()
|
||||
if (!have || comp.Err != nil || comp.Dig != w.Want.Composed) && w.Want.Composed != "" {
|
||||
if a.env.SelfComposed {
|
||||
if !have || comp.Err != nil {
|
||||
return // waits for the node's own composition of the want's build
|
||||
}
|
||||
// The composition the node composed for the want's build is the one it applies (ADR 0547 §2).
|
||||
w.Want.Composed = comp.Dig
|
||||
if a.waitsForYes(ctx, module, w, comp) {
|
||||
return
|
||||
}
|
||||
} else if (!have || comp.Err != nil || comp.Dig != w.Want.Composed) && w.Want.Composed != "" {
|
||||
// The composition the want names may be an older revision of the key, replaced by a newer one at the same
|
||||
// fingerprint (a want held there keeps its digest): taken from the revisions the bucket keeps (ADR 0539 clause 1).
|
||||
if old, found := a.revisionNamed(ctx, module, w.Want.FP, w.Want.Composed); found {
|
||||
@@ -726,18 +1038,40 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
||||
a.mu.Lock()
|
||||
a.prior[module] = true
|
||||
a.mu.Unlock()
|
||||
if h.Phase == string(delivery.PhaseHeld) {
|
||||
// Held for a failed backup, and the operator's answer is still to come: the hold is kept, nothing is
|
||||
// backed up again or applied, and the next revision of the want is the answer (ADR 0568 §4).
|
||||
a.mu.Lock()
|
||||
a.attempt[module] = attempt{FP: w.Want.FP, Composed: w.Want.Composed, WantRev: w.Rev, CompRev: comp.Rev, Held: true,
|
||||
AnsweredRev: w.Rev}
|
||||
if h.Composed != "" {
|
||||
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
if h.Composed != "" {
|
||||
// This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A
|
||||
// failed apply or a different composition is applied again.
|
||||
// A have a composer marked failed (NotComposed) names what still runs: taken as running, so a mark this
|
||||
// agent did not make clears once what it names is what the want names (issue 1126, the stale marks after
|
||||
// the seed's revert).
|
||||
marked := composerMark(h)
|
||||
if marked {
|
||||
h.Detail = ""
|
||||
}
|
||||
if h.FP == w.Want.FP && h.Composed == w.Want.Composed &&
|
||||
(h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) {
|
||||
(h.Phase == "" || h.Phase == string(delivery.PhaseRunning) || marked) {
|
||||
if a.inactiveAtStart(ctx, module, comp) {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
|
||||
a.mu.Unlock()
|
||||
// Said again, so the session holds the module and its key carries the health this engine judges,
|
||||
// not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25).
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
|
||||
Phase: string(delivery.PhaseRunning), Detail: h.Detail})
|
||||
a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
|
||||
Phase: string(delivery.PhaseRunning), Detail: h.Detail, Sealed: h.Sealed})
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -756,10 +1090,27 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
||||
if !ok || !SameResources(kept, comp.C.Resources) {
|
||||
return false
|
||||
}
|
||||
if a.inactiveAtStart(ctx, module, comp) {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning}
|
||||
a.mu.Unlock()
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
|
||||
a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
|
||||
return true
|
||||
}
|
||||
|
||||
// inactiveAtStart says the machine shows a resource the module declares running not active, so what a have or the kept
|
||||
// declaration says runs is not believed and the module is applied once, as for a changed want (ADR 0568 §1).
|
||||
func (a *Agent) inactiveAtStart(ctx context.Context, module string, comp compNow) bool {
|
||||
if a.env.Inactive == nil || !a.env.Inactive(ctx, comp.C) {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.prior[module] = true
|
||||
a.sayOnce("inactive:"+module+":"+comp.Dig, fmt.Sprintf("%s is applied again: a resource it declares running is not "+
|
||||
"active on this machine", module))
|
||||
a.mu.Unlock()
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -773,7 +1124,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
// A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on
|
||||
// without one (ADR 0539 §16).
|
||||
if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != delivery.BackupSkip {
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)})
|
||||
if got := a.env.Backup(ctx, module); !got.Done {
|
||||
a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, got.Words))
|
||||
a.mu.Lock()
|
||||
@@ -781,12 +1132,12 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
prev := a.running[module]
|
||||
a.mu.Unlock()
|
||||
// It keeps running what it has: the have says what it holds and why it waits.
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld),
|
||||
a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld),
|
||||
Detail: got.Words})
|
||||
return
|
||||
}
|
||||
}
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
|
||||
res := a.env.Apply(ctx, comp.C, others)
|
||||
a.mu.Lock()
|
||||
what := fmt.Sprintf("%s at %s", module, fp)
|
||||
@@ -816,7 +1167,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
phase = delivery.PhaseFailed // still retried, and said as failed: no longer a moment
|
||||
}
|
||||
a.env.Say(fmt.Sprintf("applying %s failed because the bus did not answer (try %d), retried %s: %s", what, try, when, res.Detail))
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(phase),
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(phase),
|
||||
Detail: fmt.Sprintf("retrying %s (try %d): %s", when, try, res.Detail)})
|
||||
return
|
||||
}
|
||||
@@ -825,17 +1176,18 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev}
|
||||
a.mu.Unlock()
|
||||
a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail))
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
|
||||
return
|
||||
}
|
||||
delete(a.attempt, module)
|
||||
delete(a.notComposed, module) // the apply says what it runs now
|
||||
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
|
||||
a.applied[module] = comp
|
||||
a.mu.Unlock()
|
||||
if a.env.Settled != nil {
|
||||
a.env.Settled(module)
|
||||
}
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail})
|
||||
}
|
||||
|
||||
func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
|
||||
@@ -925,3 +1277,68 @@ func short(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// waitsForYes judges a move the node composed for a module of policy record (ADR 0547 §3, ADR 0377 §3): one that
|
||||
// recreates a container or moves the module's data, or that cannot be judged, is not applied while the want names no yes
|
||||
// for this composition; the have says phase waiting-yes with the judgement, and the assigner asks a person. A composition
|
||||
// equal to what runs is no move. True when the module waits.
|
||||
func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp compNow) bool {
|
||||
if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
r, ran := a.running[module]
|
||||
if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved {
|
||||
a.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
from, known := a.runningComposition(module)
|
||||
prior := ran || a.prior[module]
|
||||
a.mu.Unlock()
|
||||
if !ran && !known {
|
||||
// Nothing of it run since this agent's start, and no kept declaration: its last have says what runs (review of L2,
|
||||
// round 3). The same composition is no move; another one is a move whose source cannot be read, so it waits for
|
||||
// a yes rather than being taken as the module's first composition.
|
||||
if h, ok := a.env.Haves.Read(ctx, module); ok && h.Composed != "" {
|
||||
if h.Composed == comp.Dig {
|
||||
return false
|
||||
}
|
||||
prior = true
|
||||
}
|
||||
}
|
||||
var j delivery.Judged
|
||||
switch {
|
||||
case known:
|
||||
j = delivery.JudgeMove(&from, comp.C)
|
||||
case prior:
|
||||
j = delivery.Judged{Judgement: delivery.JudgedCannot, Words: "what " + module + " runs on this node cannot be read"}
|
||||
default:
|
||||
j = delivery.JudgeMove(nil, comp.C)
|
||||
}
|
||||
switch j.Judgement {
|
||||
case delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.sayOnce("yes:"+module+":"+comp.Dig, fmt.Sprintf("%s at %s waits for a person's yes: %s", module, w.Want.FP, j.Words))
|
||||
prev := a.running[module]
|
||||
a.mu.Unlock()
|
||||
a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseWaitingYes),
|
||||
Detail: j.Words, Judgement: string(j.Judgement), Waits: comp.Dig})
|
||||
return true
|
||||
}
|
||||
|
||||
// runningComposition is what the module runs now: the composition last applied, else its resources in the declaration
|
||||
// the node kept from before the switch. Called with a.mu held.
|
||||
func (a *Agent) runningComposition(module string) (delivery.Composed, bool) {
|
||||
if c, ok := a.applied[module]; ok && c.Err == nil {
|
||||
return c.C, true
|
||||
}
|
||||
if a.env.Kept != nil {
|
||||
if rs, ok := a.env.Kept()[module]; ok {
|
||||
return delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}, true
|
||||
}
|
||||
}
|
||||
return delivery.Composed{}, false
|
||||
}
|
||||
|
||||
@@ -4,12 +4,15 @@ import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -71,6 +74,19 @@ type haves struct {
|
||||
stored map[string]Have
|
||||
log []Have
|
||||
removed []string
|
||||
// unreadable is each module whose have the bus cannot read now (ReadErr).
|
||||
unreadable map[string]bool
|
||||
}
|
||||
|
||||
func (h *haves) ReadErr(ctx context.Context, module string) (Have, bool, error) {
|
||||
h.mu.Lock()
|
||||
bad := h.unreadable[module]
|
||||
h.mu.Unlock()
|
||||
if bad {
|
||||
return Have{}, false, fmt.Errorf("nats: no responders available for request")
|
||||
}
|
||||
x, ok := h.Read(ctx, module)
|
||||
return x, ok, nil
|
||||
}
|
||||
|
||||
func (h *haves) Read(_ context.Context, module string) (Have, bool) {
|
||||
@@ -326,6 +342,44 @@ func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
|
||||
|
||||
// What the node-engine itself last applied is what runs after a restart, not the kept declaration.
|
||||
func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
|
||||
t.Run("1957: an engine start reads the machine's facts", testAnEngineStartReconcilesAModuleFromTheMachinesFacts)
|
||||
t.Run("says the have again", testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines)
|
||||
// An apply that failed is not what runs, though its have names the want's build and digest: the agent starting on
|
||||
// it applies again (review of the switch, round 1).
|
||||
t.Run("applies again after an apply failure", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
|
||||
wk, wraw := wantFor(t, "avahi", fp, dig, "")
|
||||
b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
|
||||
Detail: "apply: unit failed to start"}}
|
||||
b.comps.before(ck, craw)
|
||||
b.wants.before(wk, wraw)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "a failed apply was not applied again", func() bool { return len(b.appliedList()) == 1 })
|
||||
})
|
||||
// Issue 1126, the stale marks after the seed's revert: a have a composer marked failed names what still runs; the
|
||||
// agent starting on it applies nothing and says it running, so a mark it did not make clears.
|
||||
t.Run("clears a composer's mark", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "avahi", fp, false, file("avahi", "x", "1"))
|
||||
wk, wraw := wantFor(t, "avahi", fp, dig, "")
|
||||
b.haves.stored = map[string]Have{"avahi": {Module: "avahi", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed),
|
||||
Detail: NotComposedDetail + "frombus: the credentials of ace: context canceled"}}
|
||||
b.comps.before(ck, craw)
|
||||
b.wants.before(wk, wraw)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "the mark was not cleared", func() bool {
|
||||
h := b.haves.last("avahi")
|
||||
return h.Phase == string(delivery.PhaseRunning) && h.FP == fp && h.Composed == dig && h.Detail == ""
|
||||
})
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a module whose mark names what the want names was applied: %v", got)
|
||||
}
|
||||
})
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
@@ -345,7 +399,7 @@ func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
|
||||
// it and writes the health this engine judges. Not said, its key kept the health an older engine wrote: xorg's have on
|
||||
// g14 and shanks said unknown from before the engine said a module with nothing to judge ok, and the assigner held every
|
||||
// module requiring x11-display back for it (2026-10-25).
|
||||
func TestStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing.T) {
|
||||
func testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "xorg", fp, false, file("xorg", "x", "1"))
|
||||
@@ -378,6 +432,7 @@ func TestStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing
|
||||
// A module whose kept resources differ from the composition is applied, and a failed apply is retried only by a change
|
||||
// of either key.
|
||||
func TestAFailedApplyIsRetriedOnlyByAChangeOfEitherKey(t *testing.T) {
|
||||
t.Run("1957: the judge's edge reconciles, and a failed re-apply is failed", testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone)
|
||||
// A real failure is not retried by the clock, however short the backoff: only by a change of either key.
|
||||
t.Run("a real failure", func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
@@ -545,6 +600,7 @@ func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T)
|
||||
|
||||
// R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed.
|
||||
func TestARemovedWantRemovesThatModuleOnly(t *testing.T) {
|
||||
t.Run("its composition holds nothing back", testARemovedModulesCompositionHoldsNothingBack)
|
||||
b := newBed(t)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
@@ -598,6 +654,7 @@ func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) {
|
||||
// R6-A AC9: a composition signed by either trusted key is applied; one signed by another key, one changed after it was
|
||||
// signed and one unsigned are refused and nothing of them is applied (ADR 0539 §3).
|
||||
func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) {
|
||||
t.Run("with no trusted key", testWithNoTrustedKeyNothingIsApplied)
|
||||
b := newBed(t)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
@@ -648,7 +705,7 @@ func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) {
|
||||
}
|
||||
|
||||
// With no key trusted, nothing is applied: a node that does not know its signers fails closed.
|
||||
func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) {
|
||||
func testWithNoTrustedKeyNothingIsApplied(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.env.Trusted = nil
|
||||
stop := b.run()
|
||||
@@ -783,11 +840,42 @@ func TestTheNodesOwnApplyHoldsOutAModuleThatMovedToItsOwnComposition(t *testing.
|
||||
if !strings.Contains(strings.Join(names, ","), "systemd-resolved") {
|
||||
t.Fatalf("the others of the node's own apply were %v: systemd-resolved's resources would be removed", names)
|
||||
}
|
||||
|
||||
// Issue 1126, review of the apply, round 1: a module the composer holds alone (de-spiegel leaving the fold with its
|
||||
// names unset) has no composition of its own in the bucket: what runs of it in the node's own is held out of the
|
||||
// node's own apply until one is handed.
|
||||
b2 := newBed(t)
|
||||
spiegel := file("de-spiegel", "server-env", "A=1")
|
||||
var bare map[string]any
|
||||
_ = json.Unmarshal(spiegel, &bare)
|
||||
delete(bare, "module")
|
||||
spiegel, _ = json.Marshal(bare)
|
||||
b2.env.Kept = func() map[string][]json.RawMessage { return map[string][]json.RawMessage{NodeModule: {spiegel}} }
|
||||
nk2, nraw2, ndig2 := composition(t, NodeModule, "", false)
|
||||
nwraw2, _ := delivery.EncodeNodeWant(delivery.NodeWant{Composed: ndig2})
|
||||
b2.comps.before(nk2, nraw2)
|
||||
b2.wants.before(nwk, nwraw2)
|
||||
ctx2, cancel2 := context.WithCancel(context.Background())
|
||||
b2.ag = New(b2.env)
|
||||
b2.ag.NotComposedEach(ctx2, "", map[string]string{"de-spiegel": "de-spiegel says ${config:praktijk-name}, and nothing sets it"})
|
||||
done2 := make(chan struct{})
|
||||
go func() { defer close(done2); _ = b2.ag.Run(ctx2) }()
|
||||
defer func() { cancel2(); <-done2 }()
|
||||
eventually(t, "the node's own not applied beside a held module", func() bool { return len(b2.appliedList()) == 1 })
|
||||
b2.mu.Lock()
|
||||
defer b2.mu.Unlock()
|
||||
heldOut := false
|
||||
for _, c := range b2.others[0] {
|
||||
heldOut = heldOut || (c.Module == "de-spiegel" && len(c.Resources) == 1)
|
||||
}
|
||||
if !heldOut {
|
||||
t.Errorf("the others of the node's own apply were %+v: held de-spiegel's resources would be removed", b2.others[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A module the assigner removed holds nothing back, its compositions still in the bucket or not: its resources are its
|
||||
// removal's to take (issue 1945).
|
||||
func TestARemovedModulesCompositionHoldsNothingBack(t *testing.T) {
|
||||
func testARemovedModulesCompositionHoldsNothingBack(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("b")
|
||||
rk, rraw, _ := composition(t, "gone", fp, false, file("gone", "x", "1"))
|
||||
@@ -952,3 +1040,317 @@ func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) {
|
||||
t.Fatalf("what runs from the kept declaration: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2, §3): with the node's own compositions, unsigned, and wants naming the build alone, the
|
||||
// composition composed for the want's build is applied, and only when it differs from what runs; for a module of
|
||||
// policy record a move that recreates a container waits in phase waiting-yes with its judgement until the want names
|
||||
// the yes for that composition, and is then applied.
|
||||
func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.env.SelfComposed, b.env.Trusted = true, nil
|
||||
b.env.Policy = func(_ context.Context, m string) string {
|
||||
if m == "db" || m == "rec" {
|
||||
return catalogue.PolicyRecord
|
||||
}
|
||||
return catalogue.PolicyRoll
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
unsigned := func(module, fp string, rs ...json.RawMessage) (string, []byte, string) {
|
||||
c := delivery.Composed{Node: node, Module: module, FP: fp, Resources: rs}
|
||||
key, _ := delivery.ComposedKey(node, module, fp)
|
||||
raw, err := delivery.EncodeComposed(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dig, _ := delivery.ComposedDigestOf(c)
|
||||
return key, raw, dig
|
||||
}
|
||||
want := func(module, fp, yes string) {
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Yes: yes, Self: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.wants.put(key, raw)
|
||||
}
|
||||
|
||||
fp := fpOf("a")
|
||||
pw, _ := json.Marshal(map[string]any{"id": "web.pw", "type": "file", "path": "/tmp/r6/web/pw", "sealed": "1", "module": "web"})
|
||||
ck, craw, dig := unsigned("web", fp, file("web", "x", "1"), pw)
|
||||
b.comps.put(ck, craw)
|
||||
want("web", fp, "")
|
||||
eventually(t, "the node's own composition was not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||
eventually(t, "the have does not name the composed digest", func() bool { return b.haves.last("web").Composed == dig })
|
||||
// The same composition handed again is no move.
|
||||
b.comps.put(ck, craw)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
// The node's own composition carries its sealed values in its key, as a module's have does, so a restart reuses its
|
||||
// core boxes and applies nothing again (review of L2, round 3, finding 1).
|
||||
box, _ := json.Marshal(map[string]any{"id": "core-box", "type": "file", "path": "/tmp/r6/core-box", "sealed": "core"})
|
||||
own := delivery.Composed{Node: node, Resources: []json.RawMessage{box}}
|
||||
nk, _ := delivery.ComposedNodeKey(node)
|
||||
nraw, _ := delivery.EncodeComposed(own)
|
||||
ndig, _ := delivery.ComposedDigestOf(own)
|
||||
b.comps.put(nk, nraw)
|
||||
nwk, _ := delivery.WantNodeKey(node)
|
||||
nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true})
|
||||
b.wants.put(nwk, nwraw)
|
||||
eventually(t, "the node's own key does not carry its sealed values", func() bool {
|
||||
h := b.haves.last(NodeModule)
|
||||
return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed["core-box#sealed"] == "core"
|
||||
})
|
||||
// The node cannot compose (a stand-in refused): each wanted module's have says so, failed, naming why, and still
|
||||
// names what runs; composed again, the have is what it runs. web was applied by this agent; idle was not since the
|
||||
// agent's start (the switch and every delivery of the node-engine restart it), so what runs is what its have said.
|
||||
idleFP, idleDig := fpOf("c"), "sha256:"+strings.Repeat("9", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "idle", FP: idleFP, Composed: idleDig, Phase: string(delivery.PhaseRunning),
|
||||
Sealed: delivery.Seals{"idle.key#sealed": `{"box":"idle"}`}})
|
||||
want("idle", idleFP, "")
|
||||
eventually(t, "idle's want was not taken", func() bool {
|
||||
b.ag.mu.Lock()
|
||||
defer b.ag.mu.Unlock()
|
||||
return b.ag.wants["idle"].Present
|
||||
})
|
||||
if h := b.haves.last("web"); len(h.Sealed) != 1 || h.Sealed["web.pw#sealed"] != "1" {
|
||||
t.Errorf("web's running have does not carry its sealed value: %+v", h)
|
||||
}
|
||||
b.ag.NotComposed(context.Background(), "a stand-in for web's password")
|
||||
for _, row := range []struct {
|
||||
module, fp, composed, slot, sealed string
|
||||
}{{"web", fp, dig, "web.pw#sealed", "1"}, {"idle", idleFP, idleDig, "idle.key#sealed", `{"box":"idle"}`}} {
|
||||
h := b.haves.last(row.module)
|
||||
if h.Phase != string(delivery.PhaseFailed) || h.FP != row.fp || h.Composed != row.composed ||
|
||||
len(h.Sealed) != 1 || h.Sealed[row.slot] != row.sealed || !strings.Contains(h.Detail, "web's password") {
|
||||
t.Errorf("%s's have while the node cannot compose: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
// A wanted module whose have cannot be read (the bus paused, issue 1126) is not marked: a mark would drop the build
|
||||
// and digest its have names.
|
||||
readAgain = time.Millisecond
|
||||
b.haves.mu.Lock()
|
||||
b.haves.unreadable = map[string]bool{"unread": true}
|
||||
b.haves.mu.Unlock()
|
||||
want("unread", fpOf("d"), "")
|
||||
want("fresh", fpOf("e"), "") // never ran, its have absent: marked, naming nothing it runs
|
||||
// A have that already says another cause keeps it (review of the switch, round 1): an apply that failed.
|
||||
brokenFP, brokenDig := fpOf("f"), "sha256:"+strings.Repeat("8", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "broken", FP: brokenFP, Composed: brokenDig, Phase: string(delivery.PhaseFailed),
|
||||
Detail: "apply: unit failed to start"})
|
||||
want("broken", brokenFP, "")
|
||||
eventually(t, "unread's want was not taken", func() bool {
|
||||
b.ag.mu.Lock()
|
||||
defer b.ag.mu.Unlock()
|
||||
return b.ag.wants["unread"].Present
|
||||
})
|
||||
b.ag.NotComposed(context.Background(), "a stand-in for web's password, again")
|
||||
if h := b.haves.last("unread"); h.Module != "" {
|
||||
t.Errorf("a module whose have could not be read was marked: %+v", h)
|
||||
}
|
||||
if h := b.haves.last("fresh"); h.Phase != string(delivery.PhaseFailed) || h.FP != "" {
|
||||
t.Errorf("a module that never ran, its have absent, is not marked: %+v", h)
|
||||
}
|
||||
if h := b.haves.last("broken"); h.Detail != "apply: unit failed to start" || h.Composed != brokenDig {
|
||||
t.Errorf("an apply failure was overwritten by a composer's mark: %+v", h)
|
||||
}
|
||||
// A run that ended marks nothing: what it would say is the next run's (review of the switch, round 1).
|
||||
ended, end := context.WithCancel(context.Background())
|
||||
end()
|
||||
before := len(b.haves.log)
|
||||
b.ag.NotComposed(ended, "a refusal from a run that ended")
|
||||
if len(b.haves.log) != before {
|
||||
t.Errorf("a run that ended wrote %d haves", len(b.haves.log)-before)
|
||||
}
|
||||
b.ag.NotComposed(context.Background(), "")
|
||||
for _, row := range []struct{ module, composed string }{{"web", dig}, {"idle", idleDig}} {
|
||||
if h := b.haves.last(row.module); h.Phase != string(delivery.PhaseRunning) || h.Composed != row.composed || len(h.Sealed) != 1 {
|
||||
t.Errorf("%s's have once the node composes again: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
// Issue 1126, the applying path: a module the node holds alone (its own refusal: de-spiegel's unset route names on
|
||||
// novox) is marked alone, naming its own words; web, which composes, keeps running, unmarked. Composed again, idle's
|
||||
// have is what it runs.
|
||||
b.ag.NotComposedEach(context.Background(), "", map[string]string{"idle": "idle says ${config:name}, and nothing sets it"})
|
||||
if h := b.haves.last("idle"); h.Phase != string(delivery.PhaseFailed) || h.Composed != idleDig || !strings.Contains(h.Detail, "${config:name}") {
|
||||
t.Errorf("idle's have while it alone is held: %+v", h)
|
||||
}
|
||||
if h := b.haves.last("web"); h.Phase != string(delivery.PhaseRunning) || h.Composed != dig {
|
||||
t.Errorf("web, which composes, is marked beside idle: %+v", h)
|
||||
}
|
||||
b.ag.NotComposedEach(context.Background(), "", nil)
|
||||
if h := b.haves.last("idle"); h.Phase != string(delivery.PhaseRunning) || h.Composed != idleDig {
|
||||
t.Errorf("idle's have once it composes again: %+v", h)
|
||||
}
|
||||
// A revision at the same build (a config value changed) is applied.
|
||||
_, craw2, dig2 := unsigned("web", fp, file("web", "x", "2"))
|
||||
b.comps.put(ck, craw2)
|
||||
eventually(t, "the revision was not applied", func() bool { return len(b.appliedList()) == 3 && b.haves.last("web").Composed == dig2 })
|
||||
|
||||
// A record module: its first composition is no move; a new image recreates its container and waits for a yes.
|
||||
container := func(image string) json.RawMessage {
|
||||
module := strings.SplitN(image, "@", 2)[0]
|
||||
raw, _ := json.Marshal(map[string]any{"id": module + ".server", "type": "container", "module": module, "name": module, "image": image})
|
||||
return raw
|
||||
}
|
||||
dk, draw, _ := unsigned("db", fp, container("db@sha256:"+strings.Repeat("1", 64)))
|
||||
b.comps.put(dk, draw)
|
||||
want("db", fp, "")
|
||||
eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 4 })
|
||||
fpB := fpOf("b")
|
||||
dk2, draw2, ddig2 := unsigned("db", fpB, container("db@sha256:"+strings.Repeat("2", 64)))
|
||||
// rec is a record module this agent has run nothing of since its start (the switch and every delivery of the
|
||||
// node-engine restart it): its have alone says what runs. Its move waits for a yes as db's does, and its have keeps
|
||||
// what runs (fingerprint, digest, sealed values) rather than naming nothing (review of L2, round 3).
|
||||
recFP, recDig := fpOf("d"), "sha256:"+strings.Repeat("8", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "rec", FP: recFP, Composed: recDig, Phase: string(delivery.PhaseRunning),
|
||||
Sealed: delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}})
|
||||
rk, rraw, rdig := unsigned("rec", fpB, container("rec@sha256:"+strings.Repeat("3", 64)))
|
||||
b.comps.put(dk2, draw2)
|
||||
b.comps.put(rk, rraw)
|
||||
want("db", fpB, "")
|
||||
want("rec", fpB, "")
|
||||
applied := len(b.appliedList())
|
||||
for _, row := range []struct {
|
||||
module, fp, composed, waits string
|
||||
sealed delivery.Seals
|
||||
}{{"db", fp, "", ddig2, nil}, {"rec", recFP, recDig, rdig, delivery.Seals{"rec.key#sealed": `{"box":"rec"}`}}} {
|
||||
eventually(t, row.module+"'s move did not wait for a yes", func() bool {
|
||||
h := b.haves.last(row.module)
|
||||
return h.Phase == string(delivery.PhaseWaitingYes) && h.Waits == row.waits
|
||||
})
|
||||
h := b.haves.last(row.module)
|
||||
if h.FP != row.fp || (row.composed != "" && h.Composed != row.composed) || h.Composed == "" ||
|
||||
(row.sealed != nil && !maps.Equal(h.Sealed, row.sealed)) {
|
||||
t.Errorf("%s's have waiting for a yes does not keep what runs: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
if got := b.appliedList(); len(got) != applied {
|
||||
t.Fatalf("a move waiting for a yes was applied: %v", got)
|
||||
}
|
||||
want("db", fpB, ddig2)
|
||||
want("rec", fpB, rdig)
|
||||
eventually(t, "the yes did not apply the move", func() bool {
|
||||
h, r := b.haves.last("db"), b.haves.last("rec")
|
||||
return len(b.appliedList()) == applied+2 && h.Phase == string(delivery.PhaseRunning) && h.FP == fpB &&
|
||||
r.Phase == string(delivery.PhaseRunning) && r.FP == fpB && r.Composed == rdig
|
||||
})
|
||||
}
|
||||
|
||||
// 1957/AC1 and AC4 (unit; ADR 0568 §1, §4): an engine start takes a module from what the machine shows, not only from the
|
||||
// have or the kept declaration it kept. Every resource up applies nothing; one declared running and not active applies the
|
||||
// module once, from a have or from the kept declaration; a module with no want is not looked at.
|
||||
func testAnEngineStartReconcilesAModuleFromTheMachinesFacts(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
have bool // the engine's own have says running at the want
|
||||
held bool // else its have says held, from a failed backup
|
||||
kept bool // else the kept declaration says it runs
|
||||
inactive bool
|
||||
unwanted bool
|
||||
applied int
|
||||
asked bool // the machine was looked at
|
||||
}{
|
||||
{"all up: nothing is applied", true, false, false, false, false, 0, true},
|
||||
{"a unit not active: applied once", true, false, false, true, false, 1, true},
|
||||
{"kept declaration, a unit not active: applied once", false, false, true, true, false, 1, true},
|
||||
{"kept declaration, all up: nothing is applied", false, false, true, false, false, 0, true},
|
||||
{"a held module applies nothing at start", false, true, false, true, false, 0, false},
|
||||
{"no want: nothing is looked at or applied", true, false, false, true, true, 0, false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
rs := []json.RawMessage{file("web", "x", "1")}
|
||||
ck, craw, dig := composition(t, "web", fp, false, rs...)
|
||||
wk, wraw := wantFor(t, "web", fp, dig, "")
|
||||
b.comps.before(ck, craw)
|
||||
if !c.unwanted {
|
||||
b.wants.before(wk, wraw)
|
||||
}
|
||||
if c.have {
|
||||
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: "running"}}
|
||||
}
|
||||
if c.held {
|
||||
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64),
|
||||
Phase: string(delivery.PhaseHeld), Detail: "restic: the repository is locked"}}
|
||||
}
|
||||
if c.kept {
|
||||
b.kept = map[string][]json.RawMessage{"web": rs}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
asked := false
|
||||
b.env.Inactive = func(context.Context, delivery.Composed) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
asked = true
|
||||
return c.inactive
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
if !c.unwanted && !c.held {
|
||||
eventually(t, "the module was not settled", func() bool { return b.haves.last("web").Composed == dig })
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != c.applied {
|
||||
t.Fatalf("applied %v, want %d", got, c.applied)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if asked != c.asked {
|
||||
t.Fatalf("the machine was asked: %v, want %v", asked, c.asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1957/AC2, AC3 and AC4 (unit; ADR 0568 §2, §3, §4): the judge's edge, Reconcile, applies a running module again; one held
|
||||
// for a failed backup and one with no want are left alone; and an apply that fails writes the have `failed` with the
|
||||
// apply's own words.
|
||||
func testReconcileAppliesARunningModuleAgainAndLeavesTheHeldAndTheUnwantedAlone(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ckW, crawW, digW := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
wkW, wrawW := wantFor(t, "web", fp, digW, "")
|
||||
ckD, crawD, digD := composition(t, "db", fp, true, file("db", "x", "1"))
|
||||
wkD, wrawD := wantFor(t, "db", fp, digD, "")
|
||||
b.comps.before(ckW, crawW)
|
||||
b.wants.before(wkW, wrawW)
|
||||
b.comps.before(ckD, crawD)
|
||||
b.wants.before(wkD, wrawD)
|
||||
b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: digW, Phase: "running"},
|
||||
"db": {Module: "db", FP: fpOf("0"), Composed: "sha256:" + strings.Repeat("0", 64), Phase: "running"}}
|
||||
b.backup = func(string) Backup { return Backup{Words: "restic: the repository is locked"} }
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
eventually(t, "web was not settled", func() bool { return b.haves.last("web").Composed == digW })
|
||||
eventually(t, "db was not held", func() bool { return b.haves.last("db").Phase == string(delivery.PhaseHeld) })
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("applied at the start: %v", got)
|
||||
}
|
||||
|
||||
b.ag.Reconcile("db", "ghost")
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 0 {
|
||||
t.Fatalf("a held module and an unwanted one were applied: %v", got)
|
||||
}
|
||||
if h := b.haves.last("db"); h.Phase != string(delivery.PhaseHeld) {
|
||||
t.Fatalf("the held module's have changed: %+v", h)
|
||||
}
|
||||
|
||||
b.ag.Reconcile("web")
|
||||
eventually(t, "web was not applied again", func() bool { return len(b.appliedList()) == 1 })
|
||||
eventually(t, "web is not running again", func() bool { return b.haves.last("web").Phase == string(delivery.PhaseRunning) })
|
||||
|
||||
b.mu.Lock()
|
||||
b.failWith = "systemctl start web.service: exit status 1"
|
||||
b.mu.Unlock()
|
||||
b.ag.Reconcile("web")
|
||||
eventually(t, "the failed re-apply did not write the have failed with its words", func() bool {
|
||||
h := b.haves.last("web")
|
||||
return h.Phase == string(delivery.PhaseFailed) && h.Detail == "systemctl start web.service: exit status 1"
|
||||
})
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if got := b.appliedList(); len(got) != 2 {
|
||||
t.Fatalf("a failed apply was tried again with no edge: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
+72
-2
@@ -1223,6 +1223,19 @@ func applyScoped(
|
||||
}
|
||||
}
|
||||
|
||||
// A file written or removed in a unit directory is read by the service manager only after a reload, whether or not
|
||||
// a service reflects it: a drop-in on reboot.target is reflected by no service, and without the reload the next
|
||||
// reboot still runs the target as it was loaded (issue 1126). Once per apply.
|
||||
if at := unitFileMoved(report.Outcomes); at != "" {
|
||||
if u, ok := sys.(unitReloader); ok {
|
||||
if err := u.ReloadUnits(ctx, run); err != nil {
|
||||
failures = append(failures, &Error{Resource: at, Err: fmt.Errorf("reloading the service manager's units: %w", err)})
|
||||
} else {
|
||||
log(fmt.Sprintf(" reloaded the service manager's units: %s changed", at))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder.
|
||||
if origin == store.OriginDeclared && scope.isNode() {
|
||||
// And an identity, then a library group, no longer declared goes, once what ran as it went above (ADR 0328
|
||||
@@ -1997,11 +2010,68 @@ type serviceReloader interface {
|
||||
ReloadService(ctx context.Context, run system.Runner, unit string) error
|
||||
}
|
||||
|
||||
// UnitDirs are the system service manager's unit directories: a file the apply writes or removes in one of them, or in a
|
||||
// drop-in directory below one, is read by the manager only after a reload (issue 1126). Replaceable in tests.
|
||||
var UnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/usr/local/lib/systemd/system"}
|
||||
|
||||
// unitFileMoved is the first file this apply wrote or removed in a unit directory or a drop-in directory below one,
|
||||
// or "" when it moved none.
|
||||
func unitFileMoved(outcomes []Outcome) string {
|
||||
for _, o := range outcomes {
|
||||
// Only what changed the file on disk, as MovesRoot reads it: skipped, held, kept, forgotten and waiting come
|
||||
// back on every apply and change nothing.
|
||||
if declaration.Type(o.Type) != declaration.TypeFile {
|
||||
continue
|
||||
}
|
||||
switch o.Action {
|
||||
case "created", "updated", "corrected", "removed":
|
||||
default:
|
||||
continue
|
||||
}
|
||||
at := filepath.Clean(o.Target)
|
||||
for _, dir := range UnitDirs {
|
||||
if rel, err := filepath.Rel(dir, at); err == nil && rel != "." && !strings.HasPrefix(rel, "..") {
|
||||
return at
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unitReloader is a service manager that caches unit files and must be told to read them again.
|
||||
type unitReloader interface {
|
||||
ReloadUnits(ctx context.Context, run system.Runner) error
|
||||
}
|
||||
|
||||
// restReader is a service manager that can say whether a one-shot has nothing running now
|
||||
// (novox/hq issue 1126).
|
||||
type restReader interface {
|
||||
ServiceAtRest(ctx context.Context, run system.Runner, unit string) (bool, error)
|
||||
}
|
||||
|
||||
// declaredState is a unit's state as a declaration of want reads it. **A one-shot at rest is
|
||||
// stopped to a declaration that it be stopped** (novox/hq issue 1126): a one-shot that does not
|
||||
// linger reads "running" once it ran and worked — that is what a declaration that it run asks — but
|
||||
// it has no process between its runs, so it is also as stopped as it can be, and stopping it changes
|
||||
// nothing. Read only as "running", a boot-time cleanup declared stopped and enabled failed every
|
||||
// apply after the boot that ran it: "was asked to be stopped and is running".
|
||||
func declaredState(ctx context.Context, sys system.System, run Runner, unit, want string) (string, error) {
|
||||
state, err := sys.ServiceState(ctx, run, unit)
|
||||
if err != nil || want != "stopped" || state != "running" {
|
||||
return state, err
|
||||
}
|
||||
if r, ok := sys.(restReader); ok {
|
||||
rest, err := r.ServiceAtRest(ctx, run, unit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rest {
|
||||
return "stopped", nil
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// serviceSettle is how long the host waits before looking at a unit a second time. A test sets it
|
||||
// to nothing; on a machine it is the window in which a daemon that refuses its configuration dies.
|
||||
var serviceSettle = 2 * time.Second
|
||||
@@ -2154,7 +2224,7 @@ func applyServiceIn(ctx context.Context, sys system.System, r *declaration.Servi
|
||||
// own stopping from anybody else's (novox/hq ADR 0269).
|
||||
out.declared = &store.FoundUnit{Unit: r.Unit, State: r.State, Boot: r.Boot}
|
||||
|
||||
before, err := sys.ServiceState(ctx, run, r.Unit)
|
||||
before, err := declaredState(ctx, sys, run, r.Unit, r.State)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -2165,7 +2235,7 @@ func applyServiceIn(ctx context.Context, sys system.System, r *declaration.Servi
|
||||
|
||||
// Read back. A service manager accepting a command says the transaction was accepted,
|
||||
// not that the unit is running — one that starts and immediately dies satisfies it.
|
||||
after, err := sys.ServiceState(ctx, run, r.Unit)
|
||||
after, err := declaredState(ctx, sys, run, r.Unit, r.State)
|
||||
if err == nil && r.State == "running" {
|
||||
after, err = stayedRunning(ctx, sys, run, r.Unit)
|
||||
}
|
||||
|
||||
@@ -350,6 +350,42 @@ func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A boot-time one-shot a package ships, declared stopped and enabled at boot, is satisfied while
|
||||
// nothing of it runs, whatever its last run did (novox/hq issue 1126). The kernel module's
|
||||
// linux-modules-cleanup.service failed on ace with "was asked to be stopped and is running": an
|
||||
// inactive one-shot with ExecMainStatus=0 reads "running", and stopping it changes nothing.
|
||||
func TestABootOneShotDeclaredStoppedIsSatisfiedAtRest(t *testing.T) {
|
||||
for _, c := range []struct{ name, status string }{
|
||||
{"exited 0: never ran, ran at boot, or unloaded", "0"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var asked []string
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
args = oldShape(name, args)
|
||||
switch args[0] {
|
||||
case "show":
|
||||
return "LoadState=loaded\nActiveState=inactive\nType=oneshot\nRemainAfterExit=no\n" +
|
||||
"ExecMainStatus=" + c.status + "\n", nil
|
||||
case "is-enabled":
|
||||
return "enabled\n", nil
|
||||
}
|
||||
asked = append(asked, strings.Join(args, " "))
|
||||
return "", nil
|
||||
}
|
||||
d := parse(t, `{"declaration":1,"resources":[
|
||||
{"id":"cleanup","type":"service","unit":"linux-modules-cleanup.service","state":"stopped","boot":"enabled"}
|
||||
]}`)
|
||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("a one-shot at rest declared stopped failed: %v", err)
|
||||
}
|
||||
if len(asked) > 0 {
|
||||
t.Errorf("the service manager was asked %q for a unit already as declared", asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestADroppedServiceIsGivenBackTheStateItWasFoundIn(t *testing.T) {
|
||||
// novox/hq ADR 0118: undeclaring removes what the mesh made, gives back what it changed, and
|
||||
// leaves what was the machine's. A service resource never installs a unit — so undeclaring it
|
||||
@@ -1225,6 +1261,52 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
|
||||
if reloaded < 0 || reloaded > stop {
|
||||
t.Errorf("the service was restarted without the unit files being read again first; commands were %v", commands)
|
||||
}
|
||||
|
||||
// A drop-in in a unit directory that no service reflects — the power module's bound on reboot.target
|
||||
// (issue 1126) — is read by the service manager only after a reload: one per apply, and no restart.
|
||||
// Unchanged, it reloads nothing.
|
||||
units := filepath.Join(dir, "system")
|
||||
saved := UnitDirs
|
||||
UnitDirs = []string{units}
|
||||
t.Cleanup(func() { UnitDirs = saved })
|
||||
dropIn := filepath.Join(units, "reboot.target.d", "mesh-power.conf")
|
||||
withDropIn := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
|
||||
{"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"},
|
||||
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]},
|
||||
{"id":"units","type":"directory","path":%q,"mode":"0755"},
|
||||
{"id":"drop-ins","type":"directory","path":%q,"mode":"0755"},
|
||||
{"id":"bound","type":"file","path":%q,"content":"[Unit]\nJobTimeoutSec=10min\n","mode":"0644"},
|
||||
{"id":"bound-too","type":"file","path":%q,"content":"[Unit]\nJobTimeoutSec=10min\n","mode":"0644"}
|
||||
]}`, path, units, filepath.Dir(dropIn), dropIn, filepath.Join(filepath.Dir(dropIn), "other.conf")))
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
reloads int
|
||||
}{{"a drop-in written", 1}, {"the same drop-in again", 0}} {
|
||||
commands = nil
|
||||
if _, state, err = Apply(context.Background(), archHost(t), withDropIn, state,
|
||||
store.OriginCarried, run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloads := 0
|
||||
for _, c := range commands {
|
||||
if strings.Contains(c, "daemon-reload") {
|
||||
reloads++
|
||||
}
|
||||
if strings.Contains(c, "stop thing.service") {
|
||||
t.Errorf("%s: a drop-in no service reflects restarted one: %v", row.name, commands)
|
||||
}
|
||||
}
|
||||
if reloads != row.reloads {
|
||||
t.Errorf("%s: %d reloads, want %d; commands were %v", row.name, reloads, row.reloads, commands)
|
||||
}
|
||||
}
|
||||
// A drop-in whose module is skipped behind its step, or held as found, comes back so on every apply: it moved
|
||||
// nothing, and reloads nothing.
|
||||
for _, action := range []string{"skipped", "held"} {
|
||||
if at := unitFileMoved([]Outcome{{ID: "bound", Type: "file", Target: dropIn, Action: action}}); at != "" {
|
||||
t.Errorf("a drop-in %s reloads the service manager (%s)", action, at)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -290,3 +290,7 @@ func unpack(body []byte, into string) (int, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch is an archive at source as the host fetches every archive it unpacks, at most maxArchive bytes: what the node's
|
||||
// own composer reads a moved module's templates archive with (issue 1126). Its digest is the caller's to check.
|
||||
func Fetch(ctx context.Context, source string) ([]byte, error) { return fetch(ctx, source) }
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -94,6 +95,14 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
||||
if r.Schedule != "" {
|
||||
unit = r.Name + ".timer"
|
||||
}
|
||||
// The yielding slice is checked even then: a slice file removed by hand would leave the build unbound.
|
||||
if wrote, err := placeYieldSlice(r); err != nil {
|
||||
return out, err
|
||||
} else if wrote {
|
||||
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil {
|
||||
_ = active
|
||||
return out, nil
|
||||
@@ -121,7 +130,12 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
||||
}
|
||||
// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
|
||||
// previous version would be loaded by a runtime that walks a directory.
|
||||
if err := os.RemoveAll(at); err != nil {
|
||||
// **Retired, never deleted under its process** (novox/hq issue 734): the old process runs until
|
||||
// the restart below, from this directory and in it. Deleted here, every command it started
|
||||
// before the restart failed with "getwd: no such file or directory" — a builder's toolchain probe
|
||||
// refused three checks on 2026-10-26. Renamed aside, its process keeps a directory that exists,
|
||||
// and the build is deleted once nothing runs from it, as a judged build's is.
|
||||
if err := witness.Retire(daemonRoot, r.Name, at, time.Now()); err != nil {
|
||||
return out, err
|
||||
}
|
||||
} else {
|
||||
@@ -189,6 +203,9 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
||||
if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if _, err := placeYieldSlice(r); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -277,6 +294,7 @@ func unitFor(r *declaration.Process) string {
|
||||
if r.User != "" {
|
||||
fmt.Fprintf(&b, "User=%s\n", r.User)
|
||||
}
|
||||
writeYields(&b, r.Yields)
|
||||
fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
|
||||
if r.Schedule != "" || r.RunOnce {
|
||||
// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
|
||||
@@ -293,6 +311,78 @@ func unitFor(r *declaration.Process) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// machineCPUs is how many processors this machine has, for a share of them in a quota. A variable only so a test can
|
||||
// fix it.
|
||||
var machineCPUs = runtime.NumCPU
|
||||
|
||||
// writeYields writes how a process gives way to everything else on its machine (novox/hq ADR 0553, issue 1806): a
|
||||
// build seat's holder, whose builds must never slow the operator's desktop.
|
||||
//
|
||||
// **Into the node's one yielding slice**, where the CPU weight and the quota are (yieldSlice): every yielding process
|
||||
// of the machine shares them, so three build modules take one share together, never one each. **Per process**, what a
|
||||
// slice cannot say: Nice=19 for every process it starts, and the lowest best-effort IO level. The IO level binds only
|
||||
// where the disk's scheduler honours it (bfq); under `none`, the usual one for NVMe, it does nothing. Not the idle IO
|
||||
// class: a disk that is never idle would starve a build for good.
|
||||
//
|
||||
// The share is written into the unit as a comment, so a changed share changes the unit, the daemon's identity, and the
|
||||
// slice is written again.
|
||||
func writeYields(b *strings.Builder, y *declaration.Yields) {
|
||||
if y == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(b, "# Yields: %s, %d%% of the machine shared in %s\n", y.Priority, y.CPUShare, declaration.YieldSlice)
|
||||
fmt.Fprintf(b, "Slice=%s\n", declaration.YieldSlice)
|
||||
if y.Priority == declaration.YieldIdle {
|
||||
b.WriteString("Nice=19\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n")
|
||||
}
|
||||
}
|
||||
|
||||
// yieldSlice is the node's one yielding slice (declaration.YieldSlice) for a process that yields: idle CPU weight and
|
||||
// IO weight 1 for an idle priority, and CPUQuota for its share of the processors this machine has (systemd counts a
|
||||
// quota in percent of one processor, so half of 24 is 1200%), at least one processor. At the top of the hierarchy, so
|
||||
// its weight and quota bind against the operator's user slice and the system's.
|
||||
//
|
||||
// The IO weight binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under `none` it does
|
||||
// nothing, and the CPU bound is what holds.
|
||||
//
|
||||
// Every yielding process of a machine carries the node's one share (the composer's, ADR 0553 §3), so whichever writes
|
||||
// it writes the same file. Left in place when nothing yields any more: an empty slice binds nothing.
|
||||
func yieldSlice(y *declaration.Yields) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a yielding\n")
|
||||
b.WriteString("# process is declared again (novox/hq ADR 0553).\n")
|
||||
b.WriteString("[Unit]\nDescription=The mesh's yielding processes: builds, at idle priority under one share\n\n")
|
||||
b.WriteString("[Slice]\n")
|
||||
if y.Priority == declaration.YieldIdle {
|
||||
b.WriteString("CPUWeight=idle\nIOWeight=1\n")
|
||||
}
|
||||
if y.CPUShare > 0 {
|
||||
quota := y.CPUShare * machineCPUs()
|
||||
if quota < 100 {
|
||||
quota = 100
|
||||
}
|
||||
fmt.Fprintf(&b, "CPUQuota=%d%%\n", quota)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// placeYieldSlice writes the yielding slice for a process that yields, when what is there differs, and says whether it
|
||||
// wrote: the caller reloads the service manager before it starts anything.
|
||||
func placeYieldSlice(r *declaration.Process) (bool, error) {
|
||||
if r.Yields == nil {
|
||||
return false, nil
|
||||
}
|
||||
path := filepath.Join(unitDir, declaration.YieldSlice)
|
||||
want := yieldSlice(r.Yields)
|
||||
if have, err := os.ReadFile(path); err == nil && string(have) == want {
|
||||
return false, nil
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
|
||||
return false, fmt.Errorf("the yielding slice %s: %w", path, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// timerFor is the cadence a scheduled process runs on.
|
||||
//
|
||||
// **The mesh's cron expression, handed to the machine's own timer.** The host already parses and
|
||||
|
||||
@@ -21,10 +21,16 @@ func aProcess() *declaration.Process {
|
||||
}
|
||||
}
|
||||
|
||||
// The unit the mesh writes says what it runs, where, and that it comes back.
|
||||
// The unit the mesh writes says what it runs, where, and that it comes back — and that it is the
|
||||
// mesh's: **generated whole and saying so**, because an edit that survives until the next declaration
|
||||
// and then vanishes is worse than one that is refused. The long-running mode stays up and is not a step.
|
||||
func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
||||
unit := unitFor(aProcess())
|
||||
if !strings.HasPrefix(unit, "#") {
|
||||
t.Fatalf("the unit does not open saying it is generated:\n%s", unit)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"Do not edit",
|
||||
"ExecStart=node index.js",
|
||||
"WorkingDirectory=/var/lib/mesh/daemons/greeter",
|
||||
"Restart=always",
|
||||
@@ -34,14 +40,8 @@ func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
||||
t.Fatalf("the unit does not say %q:\n%s", want, unit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Generated whole and saying so.** Every managed file on a machine carries this, because an edit
|
||||
// that survives until the next declaration and then vanishes is worse than one that is refused.
|
||||
func TestTheUnitSaysItIsTheMeshs(t *testing.T) {
|
||||
unit := unitFor(aProcess())
|
||||
if !strings.HasPrefix(unit, "#") || !strings.Contains(unit, "Do not edit") {
|
||||
t.Fatalf("the unit does not say it is generated:\n%s", unit)
|
||||
if strings.Contains(unit, "Type=oneshot") {
|
||||
t.Fatalf("a process that should stay up is declared a step:\n%s", unit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,60 @@ func TestAProcessRunsAsWhoItSaid(t *testing.T) {
|
||||
if strings.Contains(unitFor(aProcess()), "User=") {
|
||||
t.Fatalf("a process that named no user had one written for it:\n%s", unitFor(aProcess()))
|
||||
}
|
||||
|
||||
// A process that yields (ADR 0553) runs in the node's one yielding slice at nice 19; the slice holds the idle weight
|
||||
// and the quota for its share of the machine's processors; one that does not yield has none of it.
|
||||
defer func(n func() int) { machineCPUs = n }(machineCPUs)
|
||||
for _, c := range []struct {
|
||||
cpus int
|
||||
y *declaration.Yields
|
||||
unit, slice, not []string
|
||||
}{
|
||||
{24, &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50},
|
||||
[]string{"Slice=meshyield.slice\n", "Nice=19\n", "IOSchedulingPriority=7\n", "50%"},
|
||||
[]string{"CPUWeight=idle\n", "IOWeight=1\n", "CPUQuota=1200%\n"}, []string{"CPUQuota=", "CPUWeight="}},
|
||||
{1, &declaration.Yields{CPUShare: 25}, []string{"Slice=meshyield.slice\n"}, []string{"CPUQuota=100%\n"},
|
||||
[]string{"Nice="}},
|
||||
{24, nil, nil, nil, []string{"Slice=", "Nice=", "CPUQuota="}},
|
||||
} {
|
||||
machineCPUs = func() int { return c.cpus }
|
||||
p := aProcess()
|
||||
p.Yields = c.y
|
||||
unit := unitFor(p)
|
||||
for _, w := range c.unit {
|
||||
if !strings.Contains(unit, w) {
|
||||
t.Errorf("yields %+v: the unit does not say %q:\n%s", c.y, w, unit)
|
||||
}
|
||||
}
|
||||
for _, n := range c.not {
|
||||
if strings.Contains(unit, n) {
|
||||
t.Errorf("yields %+v: the unit says %q:\n%s", c.y, n, unit)
|
||||
}
|
||||
}
|
||||
if c.y == nil {
|
||||
continue
|
||||
}
|
||||
slice := yieldSlice(c.y)
|
||||
for _, w := range c.slice {
|
||||
if !strings.Contains(slice, w) {
|
||||
t.Errorf("yields %+v on %d processors: the slice does not say %q:\n%s", c.y, c.cpus, w, slice)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The slice is placed beside the units, once, and again only when it differs.
|
||||
was := unitDir
|
||||
unitDir = t.TempDir()
|
||||
defer func() { unitDir = was }()
|
||||
p := aProcess()
|
||||
p.Yields = &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50}
|
||||
for i, want := range []bool{true, false} {
|
||||
if wrote, err := placeYieldSlice(p); err != nil || wrote != want {
|
||||
t.Errorf("placing %d: wrote %v (%v), want %v", i, wrote, err, want)
|
||||
}
|
||||
}
|
||||
if b, err := os.ReadFile(filepath.Join(unitDir, "meshyield.slice")); err != nil || !strings.Contains(string(b), "[Slice]") {
|
||||
t.Errorf("the slice file: %q %v", b, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Three modes, one kind.** A scheduled process is a timer plus a unit that finishes, not a unit
|
||||
@@ -124,17 +178,6 @@ func TestACronBecomesATimersCalendar(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// And the long-running mode is unchanged by any of it: it stays up and comes back.
|
||||
func TestAProcessThatStaysUpIsStillRestartedWhenItExits(t *testing.T) {
|
||||
unit := unitFor(aProcess())
|
||||
if !strings.Contains(unit, "Restart=always") {
|
||||
t.Fatalf("a process that should stay up is not restarted when it exits:\n%s", unit)
|
||||
}
|
||||
if strings.Contains(unit, "Type=oneshot") {
|
||||
t.Fatalf("a process that should stay up is declared a step:\n%s", unit)
|
||||
}
|
||||
}
|
||||
|
||||
// **A unit file reinterprets a value in three ways nothing else does**, and a module's environment
|
||||
// routinely contains all three — a generated password is arbitrary bytes.
|
||||
func TestAnEnvironmentValueMeansWhatTheDeclarationSaid(t *testing.T) {
|
||||
|
||||
@@ -3,7 +3,9 @@ package apply
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -103,3 +105,79 @@ func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
|
||||
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// **A process nobody judges keeps the directory it works in until its restart** (novox/hq issue 734).
|
||||
// On 2026-10-26 three checks on one builder were refused in the second a new build of it arrived: the
|
||||
// applier deleted the running build's directory before the restart, and the `go version` the old
|
||||
// process started in it failed with "getwd: no such file or directory". Here a process works in the
|
||||
// build's directory without running from it, as that `go version` did: the apply of a new build leaves
|
||||
// its directory in place, and the retired build goes once nothing works in it.
|
||||
func TestAProcessNobodyJudgesKeepsItsDirectoryUntilItsRestart(t *testing.T) {
|
||||
if _, err := os.Stat("/proc/self/cwd"); err != nil {
|
||||
t.Skip("which process works in a build is read from /proc")
|
||||
}
|
||||
units, bundles := t.TempDir(), t.TempDir()
|
||||
wasUnits, wasBundles := unitDir, daemonRoot
|
||||
unitDir, daemonRoot = units, bundles
|
||||
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
||||
|
||||
oldBody, oldDigest := anArchive(t, map[string]string{"mesh-build-go": "old\n"})
|
||||
newBody, newDigest := anArchive(t, map[string]string{"mesh-build-go": "new\n"})
|
||||
builder := func(body []byte, digest string) string {
|
||||
return `{"id":"mesh-build-go.builder","type":"process","name":"mesh-build-go","source":"` + serving(t, body) +
|
||||
`","digest":"` + digest + `","run":["./mesh-build-go"]}`
|
||||
}
|
||||
_, state, err := Apply(context.Background(), archHost(t), declare(t, builder(oldBody, oldDigest)), store.State{},
|
||||
store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sleep, err := exec.LookPath("sleep")
|
||||
if err != nil {
|
||||
t.Skip("no sleep to run")
|
||||
}
|
||||
working := exec.Command(sleep, "60")
|
||||
working.Dir = filepath.Join(bundles, "mesh-build-go")
|
||||
if err := working.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = working.Process.Kill(); _ = working.Wait() })
|
||||
cwd := filepath.Join("/proc", strconv.Itoa(working.Process.Pid), "cwd")
|
||||
for deadline := time.Now().Add(5 * time.Second); ; {
|
||||
if at, err := os.Readlink(cwd); err == nil && at == working.Dir {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("PID %d does not work in %s within 5s", working.Process.Pid, working.Dir)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
if _, _, err := Apply(context.Background(), archHost(t), declare(t, builder(newBody, newDigest)), state,
|
||||
store.OriginDeclared, run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-build-go", "mesh-build-go")); string(got) != "new\n" {
|
||||
t.Fatalf("the new build is not where the unit runs it from: %q", got)
|
||||
}
|
||||
at, err := os.Readlink(cwd)
|
||||
if err != nil || strings.HasSuffix(at, " (deleted)") {
|
||||
t.Fatalf("the directory a running process works in was deleted before its restart: %q (%v)", at, err)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(at, "mesh-build-go")); string(got) != "old\n" {
|
||||
t.Fatalf("the process's directory no longer holds its own build: %q", got)
|
||||
}
|
||||
if len(witness.Trials(bundles)) != 0 {
|
||||
t.Fatal("a process nobody judges went on trial")
|
||||
}
|
||||
|
||||
// Its process gone, the next look deletes the retired build and keeps nothing about the process.
|
||||
_ = working.Process.Kill()
|
||||
_ = working.Wait()
|
||||
witness.SweepAll(bundles)
|
||||
if _, err := os.Stat(witness.Dir(bundles, "mesh-build-go")); !os.IsNotExist(err) {
|
||||
t.Fatalf("a retired build nothing works in was kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+50
-2
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
@@ -92,7 +93,32 @@ func (b OverNATS) FileObject(ctx context.Context, name string, limit int64) ([]b
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stream := declaration.FileStreamPrefix + bucket
|
||||
return objectByName(ctx, js, bucket, declaration.FileStreamPrefix+bucket, name, limit)
|
||||
}
|
||||
|
||||
// GrantsObject reads one object of the grants part's bucket of sealed user lists by its name (novox/hq ADR 0547 clause 7),
|
||||
// with the same subjects and nothing more as FileObject: the bus node is granted exactly those (mesh-sdk
|
||||
// buscompose.GrantsUserListFollows).
|
||||
func (b OverNATS) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) {
|
||||
js := b.JS
|
||||
wait := 10 * time.Second
|
||||
if at, ok := ctx.Deadline(); ok {
|
||||
if wait = time.Until(at); wait <= 0 {
|
||||
return nil, context.DeadlineExceeded
|
||||
}
|
||||
}
|
||||
if b.Conn != nil {
|
||||
bounded, err := b.Conn.JetStream(nats.MaxWait(wait))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js = bounded
|
||||
}
|
||||
return objectByName(ctx, js, delivery.GrantsUserListObjects, delivery.GrantsUserListObjectStream, name, limit)
|
||||
}
|
||||
|
||||
// objectByName is one object of bucket (its stream stream) by its name, no larger than limit, verified against its digest.
|
||||
func objectByName(ctx context.Context, js nats.JetStreamContext, bucket, stream, name string, limit int64) ([]byte, error) {
|
||||
meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
|
||||
nats.DirectGet(), nats.Context(ctx))
|
||||
if err != nil {
|
||||
@@ -109,7 +135,7 @@ func (b OverNATS) FileObject(ctx context.Context, name string, limit int64) ([]b
|
||||
return nil, fmt.Errorf("the object %s was deleted from the bucket %s", name, bucket)
|
||||
}
|
||||
if info.Size > uint64(limit) {
|
||||
return nil, fmt.Errorf("the object is larger than the %d bytes the declaration names", limit)
|
||||
return nil, fmt.Errorf("the object is larger than the %d bytes allowed", limit)
|
||||
}
|
||||
if info.Size == 0 {
|
||||
return []byte{}, nil
|
||||
@@ -142,6 +168,28 @@ func (l *natsLink) FileObject(ctx context.Context, name string, limit int64) ([]
|
||||
return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.FileObject(ctx, name, limit)
|
||||
}
|
||||
|
||||
func (l *natsLink) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) {
|
||||
return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.GrantsObject(ctx, name, limit)
|
||||
}
|
||||
|
||||
// GrantsObjects is a link that can read an object of the grants part's bucket of sealed user lists.
|
||||
type GrantsObjects interface {
|
||||
GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error)
|
||||
}
|
||||
|
||||
// GrantsObject reads an object of the grants part's bucket by its name on the link open now.
|
||||
func (q *Session) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
g, ok := bus.(GrantsObjects)
|
||||
if bus == nil || !ok {
|
||||
return nil, ErrNoLink
|
||||
}
|
||||
return g.GrantsObject(ctx, name, limit)
|
||||
}
|
||||
|
||||
var (
|
||||
_ FilesBus = OverNATS{}
|
||||
_ FilesBus = (*natsLink)(nil)
|
||||
|
||||
+281
-28
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -37,12 +38,23 @@ type haveState struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]ModuleFact
|
||||
health map[string]delivery.Health
|
||||
// words is, by module, what the last look found of the resource that made its health failed or waits: the have says
|
||||
// it, so a failed health is never without words (issue 1496: "the node gave no words" hid a directory used as found).
|
||||
words map[string]string
|
||||
// looked says a look's health was taken: a running module it judges no resource of then has nothing that could be
|
||||
// down, and is ok (issue 1945).
|
||||
looked bool
|
||||
node *delivery.HaveNode
|
||||
// nodeComposed is the digest of the node's own composition it runs (ADR 0539 §9), kept beside the node's facts.
|
||||
nodeComposed string
|
||||
// nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3,
|
||||
// issue 1126): kept in its key so a restart reuses them and applies nothing again.
|
||||
nodeSealed delivery.Seals
|
||||
// nodeCandidates is the node's answer to each placement candidate proposal (novox/hq ADR 0559 §4), written into its key.
|
||||
nodeCandidates map[string]delivery.CandidateAnswer
|
||||
// nodeComposes and nodeComposesWhy are whose compositions the node applies and why (HaveNode.Composes), written into
|
||||
// its key.
|
||||
nodeComposes, nodeComposesWhy string
|
||||
// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
|
||||
// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
|
||||
// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
|
||||
@@ -51,6 +63,8 @@ type haveState struct {
|
||||
// userList is what the bus node's engine last wrote of the bus's user list, by the module whose have says it (novox/hq
|
||||
// ADR 0547 §8); kept apart from the module's fact, which every apply replaces.
|
||||
userList map[string]delivery.UserListWritten
|
||||
// userListNoKey is each module whose user list was said to have no have key to ride on, so it is said once.
|
||||
userListNoKey map[string]bool
|
||||
// unjudged is each module whose composition moved (an apply began, or the fact names another build) and that no look
|
||||
// has judged since it stopped moving: its health is not yet judged, never the last look's of what ran before.
|
||||
unjudged map[string]bool
|
||||
@@ -106,10 +120,11 @@ func (s *haveState) owed() bool {
|
||||
// takeLook keeps a look's health: of a module still in motion nothing, since the look judged what is being replaced;
|
||||
// of every other module what the look found, which is then judged. The caller holds the lock.
|
||||
func (s *haveState) takeLook(h *Health) {
|
||||
next := moduleHealths(h)
|
||||
next, words := moduleHealths(h)
|
||||
for module, f := range s.modules {
|
||||
if inMotion(f.Phase) {
|
||||
delete(next, module)
|
||||
delete(words, module)
|
||||
if s.unjudged == nil {
|
||||
s.unjudged = map[string]bool{}
|
||||
}
|
||||
@@ -118,19 +133,32 @@ func (s *haveState) takeLook(h *Health) {
|
||||
}
|
||||
delete(s.unjudged, module)
|
||||
}
|
||||
s.health, s.looked = next, true
|
||||
s.health, s.words, s.looked = next, words, true
|
||||
}
|
||||
|
||||
// wordsOf is what a have says of its health when the apply said nothing: the words of the resource that made it failed
|
||||
// or waits, while it is so. The caller holds the lock.
|
||||
func (s *haveState) wordsOf(module string, health delivery.Health) string {
|
||||
if health != delivery.HealthFailed && health != delivery.HealthWaits {
|
||||
return ""
|
||||
}
|
||||
return s.words[module]
|
||||
}
|
||||
|
||||
// moduleHealths is the health of each module a look names, from the states of its resources: failed when any is
|
||||
// unhealthy, waits when any waits (waiting, starting, held), ok when all are healthy, unknown otherwise.
|
||||
func moduleHealths(h *Health) map[string]delivery.Health {
|
||||
out := map[string]delivery.Health{}
|
||||
// unhealthy, waits when any waits (waiting, starting, held), ok when all are healthy, unknown otherwise; and, for a
|
||||
// module failed or waiting, the words of the first resource that made it so. A directory used as found is not the
|
||||
// module's health: it says nothing of how the module runs, waits for the operator's hand-over and keeps its own
|
||||
// condition, raised from the same statement (issue 1496: on ace mesh-build-node's cache directory, there before the
|
||||
// mesh, made its have failed while its check answered healthy, and the assigner held its version on every node).
|
||||
func moduleHealths(h *Health) (map[string]delivery.Health, map[string]string) {
|
||||
out, words := map[string]delivery.Health{}, map[string]string{}
|
||||
if h == nil {
|
||||
return out
|
||||
return out, words
|
||||
}
|
||||
rank := map[string]int{}
|
||||
for _, r := range h.Resources {
|
||||
if r.Module == "" {
|
||||
if r.Module == "" || usedAsFound(r) {
|
||||
continue
|
||||
}
|
||||
var this int
|
||||
@@ -146,12 +174,32 @@ func moduleHealths(h *Health) map[string]delivery.Health {
|
||||
}
|
||||
if was, seen := rank[r.Module]; !seen || this > was {
|
||||
rank[r.Module] = this
|
||||
if this >= 2 {
|
||||
words[r.Module] = resourceWords(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
for module, r := range rank {
|
||||
out[module] = [...]delivery.Health{delivery.HealthOK, delivery.HealthUnknown, delivery.HealthWaits, delivery.HealthFailed}[r]
|
||||
if r < 2 {
|
||||
delete(words, module)
|
||||
}
|
||||
}
|
||||
return out
|
||||
return out, words
|
||||
}
|
||||
|
||||
// usedAsFound says a resource of a statement is a directory used as found (issue 339).
|
||||
func usedAsFound(r ResourceHealth) bool {
|
||||
return r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound)
|
||||
}
|
||||
|
||||
// resourceWords is one resource's state in the words a have carries: its id, its state and why.
|
||||
func resourceWords(r ResourceHealth) string {
|
||||
w := r.Resource + " " + r.State
|
||||
if why := strings.TrimSpace(r.Reason); why != "" {
|
||||
w += ": " + why
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// nodeFacts is the node's own facts of a report, each as the JSON the report carries it in.
|
||||
@@ -187,8 +235,7 @@ func (q *Session) noteApply(r Report) bool {
|
||||
if r.Health != nil {
|
||||
s.takeLook(r.Health)
|
||||
}
|
||||
n := nodeFacts(r)
|
||||
s.node = &n
|
||||
s.node = q.nodeFactsWithRecord(r, s.node)
|
||||
return true
|
||||
}
|
||||
s.mu.Lock()
|
||||
@@ -240,24 +287,45 @@ func (q *Session) noteApply(r Report) bool {
|
||||
if s.owed() {
|
||||
q.oweHealth()
|
||||
}
|
||||
n := nodeFacts(r)
|
||||
s.node = &n
|
||||
s.node = q.nodeFactsWithRecord(r, s.node)
|
||||
return true
|
||||
}
|
||||
|
||||
// nodeFactsWithRecord is the node's own facts of a report with its record (ADR 0555 §3): a report that says nothing of
|
||||
// the machine leaves the record last made standing, and one without a health statement keeps the contract last stated.
|
||||
func (q *Session) nodeFactsWithRecord(r Report, before *delivery.HaveNode) *delivery.HaveNode {
|
||||
n := nodeFacts(r)
|
||||
if q.Keys != nil {
|
||||
n.Record = recordOf(q.Membership.Node, r, q.Keys())
|
||||
}
|
||||
if before != nil && before.Record != nil {
|
||||
switch {
|
||||
case n.Record == nil:
|
||||
n.Record = before.Record
|
||||
case n.Record.Contract == 0:
|
||||
n.Record.Contract = before.Record.Contract
|
||||
}
|
||||
}
|
||||
return &n
|
||||
}
|
||||
|
||||
// noteHealth takes a look's health, and says whether any module's changed.
|
||||
func (q *Session) noteHealth(h Health) bool {
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
before := map[string]delivery.Health{}
|
||||
said := func(module string, f ModuleFact) string {
|
||||
health := s.healthOf(module, f)
|
||||
return string(health) + "\x00" + s.wordsOf(module, health)
|
||||
}
|
||||
before := map[string]string{}
|
||||
for module, f := range s.modules {
|
||||
before[module] = s.healthOf(module, f)
|
||||
before[module] = said(module, f)
|
||||
}
|
||||
s.takeLook(&h)
|
||||
changed := false
|
||||
for module, f := range s.modules {
|
||||
if s.healthOf(module, f) != before[module] {
|
||||
if said(module, f) != before[module] {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
@@ -283,7 +351,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.modules == nil && s.node == nil && len(s.removed) == 0 {
|
||||
if s.modules == nil && s.node == nil && len(s.removed) == 0 && len(s.userList) == 0 {
|
||||
return
|
||||
}
|
||||
node := q.Membership.Node
|
||||
@@ -306,13 +374,20 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
f := s.modules[module]
|
||||
h := delivery.Have{V: delivery.Version, FP: f.FP, Composed: f.Composed, Build: f.Build,
|
||||
Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown, Detail: f.Detail}
|
||||
if delivery.Phase(f.Phase) == delivery.PhaseWaitingYes {
|
||||
h.Judgement, h.Waits = delivery.Judgement(f.Judgement), f.Waits
|
||||
}
|
||||
switch {
|
||||
case f.Phase != "":
|
||||
h.Phase = delivery.Phase(f.Phase)
|
||||
case f.Failed:
|
||||
h.Phase = delivery.PhaseFailed
|
||||
}
|
||||
h.SealedAt = f.Sealed
|
||||
h.Health = s.healthOf(module, f)
|
||||
if h.Detail == "" {
|
||||
h.Detail = s.wordsOf(module, h.Health)
|
||||
}
|
||||
if w, ok := s.userList[module]; ok {
|
||||
h.UserList = &w
|
||||
}
|
||||
@@ -340,12 +415,18 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
if !q.writeRemoved(write, bucket, node) {
|
||||
return
|
||||
}
|
||||
if !q.writeUserListsAlone(write, bucket, node) {
|
||||
return
|
||||
}
|
||||
if s.node != nil {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err == nil {
|
||||
var raw []byte
|
||||
facts := *s.node
|
||||
facts.Composed = s.nodeComposed
|
||||
facts.SealedAt = s.nodeSealed
|
||||
facts.Candidates = s.nodeCandidates
|
||||
facts.Composes, facts.ComposesWhy = s.nodeComposes, s.nodeComposesWhy
|
||||
if raw, err = delivery.EncodeHaveNode(facts); err == nil {
|
||||
want := facts
|
||||
if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
|
||||
@@ -455,6 +536,78 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b
|
||||
return true
|
||||
}
|
||||
|
||||
// writeUserListsAlone writes the user list of each module this node-engine holds no fact of (no apply here said its have)
|
||||
// into that module's key as the bucket holds it, keeping every other field the key says (novox/hq ADR 0547 §8). The have
|
||||
// of a module is one value with parts from more than one source: the apply's (fingerprint, digest, build, phase, health)
|
||||
// and the user-list follower's. Written only on the pass over the modules this engine applied, the follower's part was
|
||||
// lost whenever the bus module's apply was not this engine's to say (2026-10-26 on the bus node: the bus module had no
|
||||
// want and had left the declaration, so its key kept an older engine's value and the vault read no hashes in it). A key
|
||||
// that does not exist is not created: a have says a phase, and nothing here knows it; that is said once. False when the
|
||||
// pass should stop. The caller holds the lock of the haves.
|
||||
func (q *Session) writeUserListsAlone(ctx context.Context, b HaveBucket, node string) bool {
|
||||
s := &q.haves
|
||||
alone := make([]string, 0, len(s.userList))
|
||||
for module := range s.userList {
|
||||
_, held := s.modules[module]
|
||||
_, gone := s.removed[module]
|
||||
if !held && !gone {
|
||||
alone = append(alone, module)
|
||||
}
|
||||
}
|
||||
sort.Strings(alone)
|
||||
for _, module := range alone {
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
q.sayHave(fmt.Sprintf("module %q has no key in the have bucket: %v", module, err))
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
q.sayHave(fmt.Sprintf("the time for writing the have bucket ran out before %s; the next apply or look writes it", key))
|
||||
return false
|
||||
}
|
||||
current, revision, err := b.Get(ctx, key)
|
||||
switch {
|
||||
case errors.Is(err, ErrHaveAbsent):
|
||||
if !s.userListNoKey[module] {
|
||||
if s.userListNoKey == nil {
|
||||
s.userListNoKey = map[string]bool{}
|
||||
}
|
||||
s.userListNoKey[module] = true
|
||||
q.sayHave(fmt.Sprintf("the bus's user list written for %s is in no have: %s does not exist and no apply on "+
|
||||
"this node says %s's phase, so it is written once one does", module, key, module))
|
||||
}
|
||||
continue
|
||||
case err != nil:
|
||||
q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err))
|
||||
q.haveFailed(err)
|
||||
return false
|
||||
}
|
||||
delete(s.userListNoKey, module)
|
||||
h, err := delivery.DecodeHave(current)
|
||||
if err != nil {
|
||||
q.sayHave(fmt.Sprintf("the bus's user list is not written in %s: its value cannot be read: %v", key, err))
|
||||
continue
|
||||
}
|
||||
w := s.userList[module]
|
||||
next := h
|
||||
next.UserList = &w
|
||||
if delivery.SameHave(h, next) {
|
||||
continue
|
||||
}
|
||||
raw, err := delivery.EncodeHave(next)
|
||||
if err != nil {
|
||||
q.sayHave(fmt.Sprintf("cannot encode %s: %v", key, err))
|
||||
continue
|
||||
}
|
||||
if err := b.Set(ctx, key, raw, revision); err != nil {
|
||||
q.sayHave(fmt.Sprintf("could not write %s in the have bucket; the next apply or look writes it again: %v", key, err))
|
||||
q.haveFailed(err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// putHave reads a key and writes it unless same says it already holds the value. False when the pass should stop: the
|
||||
// bucket failed, the error is said once, and the rest wait for the next pass.
|
||||
func (q *Session) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool {
|
||||
@@ -561,12 +714,14 @@ func (q *Session) NoteRemoved(ctx context.Context, module, build string) {
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// NoteNode takes the digest of the node's own composition it runs, and writes the node's key where it differs.
|
||||
func (q *Session) NoteNode(ctx context.Context, composed string) {
|
||||
// NoteNode takes the digest of the node's own composition it runs and the values it carries sealed to the node's key
|
||||
// (nil for a composition the node did not make itself), and writes the node's key where it differs.
|
||||
func (q *Session) NoteNode(ctx context.Context, composed string, sealed delivery.Seals) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
s.nodeComposed = composed
|
||||
s.nodeSealed = sealed
|
||||
if s.node == nil {
|
||||
n := delivery.HaveNode{V: delivery.Version}
|
||||
s.node = &n
|
||||
@@ -578,36 +733,134 @@ func (q *Session) NoteNode(ctx context.Context, composed string) {
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot
|
||||
// be read answers as none: the agent then compares with the declaration it kept, and applies only what differs.
|
||||
func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) {
|
||||
// NoteCandidates takes the node's answers to the placement candidates, by proposal (its shadow composer's, novox/hq ADR
|
||||
// 0559 §4), and writes the node's key where it differs; a proposal left out is no longer answered.
|
||||
func (q *Session) NoteCandidates(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
s.nodeCandidates = answers
|
||||
if len(answers) == 0 {
|
||||
s.nodeCandidates = nil
|
||||
}
|
||||
// Before the node's facts are first said, its key is not written for an answer alone: it would carry no record. The
|
||||
// answers ride with the first facts.
|
||||
known := s.node != nil
|
||||
s.mu.Unlock()
|
||||
if !known {
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn or
|
||||
// delivery.ComposesController) and why: written at each start of its agent, so a node returning to the controller's
|
||||
// compositions says so where every have is read (issue 1126).
|
||||
func (q *Session) NoteComposes(ctx context.Context, composes, why string) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
changed := s.nodeComposes != composes || s.nodeComposesWhy != why
|
||||
s.nodeComposes, s.nodeComposesWhy = composes, why
|
||||
known := s.node != nil
|
||||
s.mu.Unlock()
|
||||
if !known || !changed {
|
||||
return // said with the node's first facts
|
||||
}
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition
|
||||
// it last ran is and carries sealed, read after a restart (ADR 0547 §3).
|
||||
func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) {
|
||||
n, ok, _ := q.ReadNodeHaveErr(ctx)
|
||||
return n, ok
|
||||
}
|
||||
|
||||
// ReadNodeHaveErr is ReadNodeHave telling the key absent (false, nil) from one that could not be read (false, the
|
||||
// error): a bus that did not answer is never kept as a node that ran nothing (issue 1126, 2026-10-28).
|
||||
func (q *Session) ReadNodeHaveErr(ctx context.Context) (delivery.HaveNode, bool, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
hb, ok := bus.(HaveBus)
|
||||
if !ok || bus == nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.HaveNode{}, false, ErrNotLinkedHaves
|
||||
}
|
||||
key, err := delivery.HaveKey(q.Membership.Node, module)
|
||||
key, err := delivery.HaveNodeKey(q.Membership.Node)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.HaveNode{}, false, err
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, q.timeout())
|
||||
defer cancel()
|
||||
bucket, err := hb.HaveBind(read)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.HaveNode{}, false, err
|
||||
}
|
||||
value, _, err := bucket.Get(read, key)
|
||||
if errors.Is(err, ErrHaveAbsent) {
|
||||
return delivery.HaveNode{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.HaveNode{}, false, err
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false, nil // a key no engine can read names nothing it runs
|
||||
}
|
||||
return n, true, nil
|
||||
}
|
||||
|
||||
// ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot
|
||||
// be read answers as none: the agent then compares with the declaration it kept, and applies only what differs.
|
||||
func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) {
|
||||
h, ok, _ := q.ReadHaveErr(ctx, module)
|
||||
return h, ok
|
||||
}
|
||||
|
||||
// ErrNotLinkedHaves is a have read with no link to the bus open that can bind the have bucket.
|
||||
var ErrNotLinkedHaves = errors.New("no link to the bus that binds the have bucket is open")
|
||||
|
||||
// ReadHaveErr is ReadHave telling a key absent (false, nil) from one that could not be read (false, the error): a bus
|
||||
// that does not answer for a moment is not a module that never ran (issue 1126).
|
||||
func (q *Session) ReadHaveErr(ctx context.Context, module string) (delivery.Have, bool, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
hb, ok := bus.(HaveBus)
|
||||
if !ok || bus == nil {
|
||||
return delivery.Have{}, false, ErrNotLinkedHaves
|
||||
}
|
||||
key, err := delivery.HaveKey(q.Membership.Node, module)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false, err
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, q.timeout())
|
||||
defer cancel()
|
||||
bucket, err := hb.HaveBind(read)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false, err
|
||||
}
|
||||
value, _, err := bucket.Get(read, key)
|
||||
if errors.Is(err, ErrHaveAbsent) {
|
||||
return delivery.Have{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return delivery.Have{}, false, err
|
||||
}
|
||||
h, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
return delivery.Have{}, false
|
||||
return delivery.Have{}, false, nil // a have no engine can read names nothing it runs
|
||||
}
|
||||
return h, true
|
||||
return h, true, nil
|
||||
}
|
||||
|
||||
// nothingJudged is the health of a module a look judged no resource of: ok when it was looked at and the module applied
|
||||
|
||||
+134
-9
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
@@ -198,25 +199,35 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
|
||||
l := newHaveLink()
|
||||
q, _ := aHaveQueue(l)
|
||||
rep := Report{Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"},
|
||||
{Module: "c", Build: "b"}, {Module: "d", Build: "b"}}}
|
||||
{Module: "c", Build: "b"}, {Module: "d", Build: "b"}, {Module: "e", Build: "b"}}}
|
||||
q.SayFacts(context.Background(), rep)
|
||||
for _, m := range []string{"a", "b", "c", "d"} {
|
||||
for _, m := range []string{"a", "b", "c", "d", "e"} {
|
||||
if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown {
|
||||
t.Errorf("%s with no look is %s, want unknown", m, h)
|
||||
}
|
||||
}
|
||||
h := healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy, "c": StateStarting})
|
||||
h.Resources[slicesIndex(h.Resources, "b.server")].Reason = ReasonRestarting
|
||||
// 1496: a directory used as found is not the module's health (e runs healthy beside one); a failed or waiting module's
|
||||
// have says the words of the resource that made it so.
|
||||
h.Resources = append(h.Resources, ResourceHealth{Module: "a", Resource: "a.two", State: StateWaiting},
|
||||
ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown})
|
||||
ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown},
|
||||
ResourceHealth{Module: "e", Resource: "e.server", State: StateHealthy},
|
||||
ResourceHealth{Module: "e", Resource: "e.cache", Kind: KindDirectory, State: StateUnhealthy, Reason: ReasonUsedAsFound + " owned by 1:1"})
|
||||
if !q.SayHealth(context.Background(), *h) {
|
||||
t.Fatal("not said")
|
||||
}
|
||||
for m, want := range map[string]delivery.Health{"a": delivery.HealthWaits, "b": delivery.HealthFailed,
|
||||
"c": delivery.HealthWaits, "d": delivery.HealthUnknown} {
|
||||
"c": delivery.HealthWaits, "d": delivery.HealthUnknown, "e": delivery.HealthOK} {
|
||||
if got := l.have(t, "n1."+m).Health; got != want {
|
||||
t.Errorf("%s is %s, want %s", m, got, want)
|
||||
}
|
||||
}
|
||||
for m, want := range map[string]string{"a": "a.two waiting", "b": "b.server unhealthy: restarting", "d": "", "e": ""} {
|
||||
if got := l.have(t, "n1."+m).Detail; got != want {
|
||||
t.Errorf("%s says %q, want %q", m, got, want)
|
||||
}
|
||||
}
|
||||
before := l.setCount()
|
||||
h2 := *h
|
||||
q.SayHealth(context.Background(), h2)
|
||||
@@ -225,8 +236,8 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
|
||||
}
|
||||
h2.Resources = []ResourceHealth{{Module: "a", Resource: "a.server", State: StateHealthy}}
|
||||
q.SayHealth(context.Background(), h2)
|
||||
if got := l.have(t, "n1.a").Health; got != delivery.HealthOK {
|
||||
t.Errorf("a after a healthy look is %s", got)
|
||||
if got := l.have(t, "n1.a"); got.Health != delivery.HealthOK || got.Detail != "" {
|
||||
t.Errorf("a after a healthy look is %s, saying %q", got.Health, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,6 +270,43 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||
if !delivery.SameHaveNode(got, want) {
|
||||
t.Errorf("the node key holds %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
|
||||
// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
|
||||
// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
|
||||
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"}
|
||||
q.Keys = func() RecordKeys { return keys }
|
||||
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
|
||||
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
|
||||
"machine": map[string]any{"networks": []any{map[string]any{"link": "eth0", "network": "192.0.2.0/24"}}}}
|
||||
rep.Outward = []string{"eth0"}
|
||||
rep.Health = &Health{Contract: NodeKeyContract}
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if got, err = delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || got.Record == nil {
|
||||
t.Fatalf("the node key holds no record: %+v, %v", got, err)
|
||||
}
|
||||
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
|
||||
Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
||||
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
|
||||
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
|
||||
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
|
||||
}
|
||||
at := l.kv["n1.node"].rev
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if l.kv["n1.node"].rev != at {
|
||||
t.Errorf("a report changing nothing wrote the node key again")
|
||||
}
|
||||
keys.Certificate = "cert-2"
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if got, _ = delivery.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || got.Record.Certificate != "cert-2" {
|
||||
t.Errorf("a new certificate: revision %d (was %d), record %+v", l.kv["n1.node"].rev, at, got.Record)
|
||||
}
|
||||
bare := rep
|
||||
bare.Health = nil
|
||||
q.SayFacts(context.Background(), bare)
|
||||
if got, _ = delivery.DecodeHaveNode(l.kv["n1.node"].value); got.Record.Contract != NodeKeyContract {
|
||||
t.Errorf("a report without a health statement dropped the contract: %+v", got.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again.
|
||||
@@ -534,9 +582,22 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
|
||||
if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseRemoved {
|
||||
t.Fatalf("removed: %+v", got)
|
||||
}
|
||||
q.NoteNode(context.Background(), dig)
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig {
|
||||
t.Fatalf("the node key: %+v %v", n, err)
|
||||
q.NoteNode(context.Background(), dig, delivery.Seals{"core-box#sealed": "core"})
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.SealedAt) != 1 || n.SealedAt["core-box#sealed"] != "core" {
|
||||
t.Fatalf("the node key, with its own composition's sealed values by slot: %+v %v", n, err)
|
||||
}
|
||||
if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.SealedAt) != 1 {
|
||||
t.Fatalf("the node key read back: %+v %v", n, ok)
|
||||
}
|
||||
// The node's answers to placement candidates ride in the same key (ADR 0559 §4), and go when no group is answered.
|
||||
answers := map[string]delivery.CandidateAnswer{"p-1": {Proposal: "p-1", Revision: 1, Verdict: delivery.CandidateSame, Same: []string{"web"}}}
|
||||
q.NoteCandidates(context.Background(), answers)
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Candidates["p-1"].Proposal != "p-1" || n.Composed != dig {
|
||||
t.Fatalf("the node key with its answers: %+v %v", n, err)
|
||||
}
|
||||
q.NoteCandidates(context.Background(), nil)
|
||||
if n, _ := delivery.DecodeHaveNode(l.kv["n1.node"].value); n.Candidates != nil {
|
||||
t.Fatalf("an answer left after its group went: %+v", n.Candidates)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -660,3 +721,67 @@ func TestADeclarationReportMovesAHealthAndARemovalForgetsIt(t *testing.T) {
|
||||
t.Fatal("a removed module is still marked unjudged")
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0547 §8, 2026-10-26 on the bus node: the bus's user list of a module this engine says no have of (no want, and no
|
||||
// longer in the declaration) still reaches the module's key, merged into what the key says, every other field kept; a
|
||||
// key that does not exist is not made up and is said once; a writer that drops the list again is written over.
|
||||
func TestAUserListReachesTheHaveOfAModuleNoApplyHereSays(t *testing.T) {
|
||||
l := newHaveLink()
|
||||
older := delivery.Have{V: delivery.Version, Build: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
l.put(t, "n1.nats", older)
|
||||
q, said := aHaveQueue(l)
|
||||
dig := "sha256:" + strings.Repeat("c", 64)
|
||||
w := delivery.UserListWritten{Source: "n1.nats", Revision: 7, Wanted: dig, Digest: dig, Users: 58,
|
||||
At: time.Date(2026, 10, 26, 12, 26, 51, 0, time.UTC)}
|
||||
q.NoteUserList(context.Background(), "nats", w)
|
||||
got := l.have(t, "n1.nats")
|
||||
if got.UserList == nil || got.UserList.Users != 58 || got.UserList.Digest != dig {
|
||||
t.Fatalf("the user list is not in the have: %+v", got)
|
||||
}
|
||||
if got.Build != "build-1" || got.Phase != delivery.PhaseRunning || got.Health != delivery.HealthOK {
|
||||
t.Fatalf("the merge lost the key's own fields: %+v", got)
|
||||
}
|
||||
rev := l.kv["n1.nats"].rev
|
||||
q.NoteUserList(context.Background(), "nats", w)
|
||||
if l.kv["n1.nats"].rev != rev {
|
||||
t.Errorf("the same list was written again, to revision %d", l.kv["n1.nats"].rev)
|
||||
}
|
||||
|
||||
// Another writer replaces the value without the list: the next pass merges it back.
|
||||
l.put(t, "n1.nats", older)
|
||||
q.SayFacts(context.Background(), Report{})
|
||||
if got := l.have(t, "n1.nats"); got.UserList == nil || got.Build != "build-1" {
|
||||
t.Fatalf("a value written without the list kept it out: %+v", got)
|
||||
}
|
||||
|
||||
// No key at all: nothing is created, said once; the module's first have carries the list.
|
||||
q.NoteUserList(context.Background(), "other", w)
|
||||
q.NoteUserList(context.Background(), "other", w)
|
||||
if _, ok := l.kv["n1.other"]; ok {
|
||||
t.Fatalf("a have was made up for a module no apply says")
|
||||
}
|
||||
n := 0
|
||||
for _, s := range *said {
|
||||
if strings.Contains(s, "is in no have") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("the missing key was said %d times: %q", n, *said)
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("a", 64)
|
||||
q.NoteModule(context.Background(), ModuleFact{Module: "other", FP: fp, Composed: dig})
|
||||
if got := l.have(t, "n1.other"); got.UserList == nil || got.FP != fp {
|
||||
t.Fatalf("the first have of the module lost the list: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// slicesIndex is the index of the resource with id in rs, or -1.
|
||||
func slicesIndex(rs []ResourceHealth, id string) int {
|
||||
for i, r := range rs {
|
||||
if r.Resource == id {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,7 +15,10 @@ import (
|
||||
// The revisions a bucket keeps of a key, read over a real bus: oldest first, a delete and a purge said as deleted, and
|
||||
// none for a key that never was (issue 1945: the node agent takes the composition its want names from them).
|
||||
func TestNatsAKeysRevisionsAreReadBack(t *testing.T) {
|
||||
srv, err := natsserver.NewServer(&natsserver.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
// A second account reads as a node's does: refused the one bucket "denied-it" (the seat's state before its grant arrives).
|
||||
srv, err := natsserver.NewServer(&natsserver.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Users: []*natsserver.User{{Username: "admin", Password: "a"}, {Username: "node", Password: "n", Permissions: &natsserver.Permissions{
|
||||
Publish: &natsserver.SubjectPermission{Allow: []string{">"}, Deny: []string{"$JS.API.*.*.KV_denied-it", "$JS.API.*.*.*.KV_denied-it", "$JS.API.*.*.*.KV_denied-it.>", "$KV.denied-it.>"}}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -23,7 +27,7 @@ func TestNatsAKeysRevisionsAreReadBack(t *testing.T) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
conn, err := nats.Connect(srv.ClientURL())
|
||||
conn, err := nats.Connect(srv.ClientURL(), nats.UserInfo("admin", "a"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -65,4 +69,58 @@ func TestNatsAKeysRevisionsAreReadBack(t *testing.T) {
|
||||
if got, err := bus.KVHistory(ctx, "composed-it", "it.never.x"); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a key that never was: %+v (%v)", got, err)
|
||||
}
|
||||
|
||||
// A bucket that is not there, and one the account may not read, are errors at once, never a wait: the node agent
|
||||
// follows its placement key by one watch at its start (endWhenPlaced) and, refused, stays on the controller's compositions and says why.
|
||||
if _, err := bus.KVWatch(ctx, "never-it", ">"); err == nil {
|
||||
t.Error("a watch of a bucket that is not there gave no error")
|
||||
}
|
||||
if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "denied-it", History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodeConn, err := nats.Connect(srv.ClientURL(), nats.UserInfo("node", "n"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nodeConn.Close)
|
||||
started := time.Now()
|
||||
wctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := (OverNATS{Conn: nodeConn}).KVWatch(wctx, "denied-it", ">"); err == nil || wctx.Err() != nil {
|
||||
t.Errorf("a watch the account may not make: error %v after %s, context %v", err, time.Since(started), wctx.Err())
|
||||
} else {
|
||||
t.Logf("a refused watch is an error after %s: %v", time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
|
||||
// A bucket made after the watch was asked is then watched, on the bus's word that it was created and with no timer;
|
||||
// a bucket the account may not read is no waiting but the error (endWhenPlaced, the shadow's candidates).
|
||||
q := &Session{Membership: Membership{Node: "n1"}}
|
||||
q.Attach(ctx, OverNATS{Conn: conn, JS: js})
|
||||
var waited atomic.Int32
|
||||
actx, acancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer acancel()
|
||||
ch, err := q.KVWatchAwait(actx, "later-it", ">", func(error) { waited.Add(1) })
|
||||
if err != nil || waited.Load() != 1 {
|
||||
t.Fatalf("a bucket not there yet: watch %v, waiting told %d time(s)", err, waited.Load())
|
||||
}
|
||||
select {
|
||||
case u := <-ch:
|
||||
t.Fatalf("an update before the bucket exists: %+v", u)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
later, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "later-it", History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := later.PutString("placement.n1", "v"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for seen := false; !seen; {
|
||||
select {
|
||||
case u := <-ch:
|
||||
seen = u.Key == "placement.n1" && string(u.Value) == "v"
|
||||
case <-actx.Done():
|
||||
t.Fatal("the bucket made after the watch was never watched")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
@@ -743,4 +744,9 @@ type ModuleFact struct {
|
||||
Phase string
|
||||
Failed bool
|
||||
Detail string
|
||||
// Judgement and Waits are, in phase waiting-yes, the judgement of the move and the composition waiting (ADR 0547 §3).
|
||||
Judgement string
|
||||
Waits string
|
||||
// Sealed is the values the composition it names carries sealed to this node's key, by slot (delivery.Have.SealedAt).
|
||||
Sealed delivery.Seals
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The node's record (novox/hq ADR 0555 §3, issue 1126): what the machine observes of itself, written in its own have key
|
||||
// `<node>.node` (delivery.HaveNode.Record) with its facts, so every node and the parts that compose read it from the bus
|
||||
// and hear its change as the key's. Each value is the one the controller took from this node's enrolment and reports
|
||||
// until now (its profile's present capabilities and networks, its outward links, its health statement's contract, and
|
||||
// the public halves of the keys it enrolled with), read here from the same sources, never from the controller.
|
||||
|
||||
// RecordKeys are the public halves the node holds and the certificate the mesh issued for its internal name, as read
|
||||
// from its state directory at the time of a pass: the certificate arrives after enrolment, a rekey changes the tunnel key.
|
||||
type RecordKeys struct {
|
||||
TunnelKey string
|
||||
SealingKey string
|
||||
ServingKey string
|
||||
Certificate string
|
||||
// Authority is the mesh's own certificate as this machine last applied it (public).
|
||||
Authority string
|
||||
}
|
||||
|
||||
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
|
||||
// says nothing of the machine, such as a rekey), so the record last written stands.
|
||||
func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
|
||||
if r.Profile == nil {
|
||||
return nil
|
||||
}
|
||||
var p struct {
|
||||
Capabilities []struct {
|
||||
Name string `json:"name"`
|
||||
Present bool `json:"present"`
|
||||
} `json:"capabilities"`
|
||||
Machine *struct {
|
||||
Networks []catalogue.AttachedNetwork `json:"networks"`
|
||||
} `json:"machine"`
|
||||
}
|
||||
raw, err := json.Marshal(r.Profile)
|
||||
if err != nil || json.Unmarshal(raw, &p) != nil {
|
||||
return nil
|
||||
}
|
||||
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
|
||||
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority}
|
||||
for _, c := range p.Capabilities {
|
||||
if c.Present && c.Name != "" {
|
||||
rec.Capabilities = append(rec.Capabilities, c.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(rec.Capabilities)
|
||||
if p.Machine != nil {
|
||||
rec.NetworksSaid = true
|
||||
rec.Networks = p.Machine.Networks
|
||||
}
|
||||
if len(r.Outward) > 0 {
|
||||
rec.OutwardLinks = append([]string(nil), r.Outward...)
|
||||
}
|
||||
if r.Health != nil {
|
||||
rec.Contract = r.Health.Contract
|
||||
}
|
||||
return rec
|
||||
}
|
||||
@@ -47,8 +47,11 @@ type Session struct {
|
||||
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
|
||||
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
|
||||
MoveSecret MoveSecretFunc
|
||||
Say Announce
|
||||
Timeout time.Duration
|
||||
// Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes
|
||||
// no record.
|
||||
Keys func() RecordKeys
|
||||
Say Announce
|
||||
Timeout time.Duration
|
||||
|
||||
once sync.Once
|
||||
|
||||
|
||||
@@ -239,3 +239,60 @@ func (q *Session) Hear(subject string) (<-chan []byte, func(), error) {
|
||||
}
|
||||
return h.Hear(subject)
|
||||
}
|
||||
|
||||
// CreatedAdvisory is the subject the bus says a bucket's stream was created on (the server's own advisory; the same
|
||||
// subject mesh-sdk's process.AwaitBucket binds on, and the grants part gives a principal for each bucket it may read).
|
||||
func CreatedAdvisory(bucket string) string {
|
||||
return "$JS.EVENT.ADVISORY.STREAM.CREATED.KV_" + bucket
|
||||
}
|
||||
|
||||
// KVWatchAwait is KVWatch for a bucket that may not be on the bus yet (the placement seat's states, made when their
|
||||
// holder starts): the channel is returned at once, and while the bucket is missing it yields nothing and waiting is told
|
||||
// why; when the bus says the bucket's stream was created the watch is made and its updates follow. No timer: the
|
||||
// advisory is heard before the first try, so a creation between the try and the wait is not missed. Any other error of
|
||||
// the first try is the caller's, as for KVWatch: a bucket the account may not read is not waited for.
|
||||
func (q *Session) KVWatchAwait(ctx context.Context, bucket, pattern string, waiting func(error)) (<-chan KVUpdate, error) {
|
||||
created, stop, err := q.Hear(CreatedAdvisory(bucket))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
in, err := q.KVWatch(ctx, bucket, pattern)
|
||||
if err == nil {
|
||||
stop()
|
||||
return in, nil
|
||||
}
|
||||
if !errors.Is(err, nats.ErrBucketNotFound) {
|
||||
stop()
|
||||
return nil, err
|
||||
}
|
||||
waiting(err)
|
||||
out := make(chan KVUpdate, 64)
|
||||
go func() {
|
||||
defer close(out)
|
||||
defer stop()
|
||||
for in == nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case _, ok := <-created:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w, err := q.KVWatch(ctx, bucket, pattern)
|
||||
if err != nil {
|
||||
waiting(err)
|
||||
continue
|
||||
}
|
||||
in = w
|
||||
}
|
||||
}
|
||||
for u := range in {
|
||||
select {
|
||||
case out <- u:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
// Package ownresync hands the vault the value of this machine's own secret that runs, where the vault holds another
|
||||
// (novox/hq issue 1126, cause C4).
|
||||
//
|
||||
// The vault's `move-in` never replaces a value it holds, so its first import of an own secret stays fixed; where the
|
||||
// controller later made that secret anew (a rotation at start, a re-issue), the new value runs and the vault keeps the
|
||||
// old one. The node's shadow composer sees it: composing from the vault's value, it opens both seals with this
|
||||
// machine's key and its answer's shown body says delivery.MaskedAnotherValue. After each shadow answer this step asks the
|
||||
// vault's `move-in` with replace_running: true and what runs, for each own secret so masked, once per running value, and
|
||||
// says what it asked and what the vault answered. It is move-in's, the verb this node may ask already, so no seat and no
|
||||
// bus grant changes for it.
|
||||
//
|
||||
// **Only while this node applies the controller's compositions**, and only for a module whose composition that runs is
|
||||
// the one its want names: then the controller's value is what runs, and what runs is the truth the vault lacks. Once the
|
||||
// node composes its own, the vault's value is the source and nothing is handed. **The switch** is the vault's
|
||||
// (`own_take_switch`, a person's act): while it is off the vault refuses every ask, and this step says so once per value.
|
||||
// The vault also refuses any value it changed in the last 15 minutes, so a re-issue the node has not applied yet is
|
||||
// never put back.
|
||||
package ownresync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
)
|
||||
|
||||
// Verb is the vault's verb this step asks, with ReplaceRunning true in both steps.
|
||||
const Verb = "move-in"
|
||||
|
||||
// ReplaceRunning is move-in's argument that makes it take the running value over the one the vault holds.
|
||||
const ReplaceRunning = "replace_running"
|
||||
|
||||
// Env is what the step reads and asks.
|
||||
type Env struct {
|
||||
Node string
|
||||
// Composes is whose compositions the node applies now: delivery.ComposesController, delivery.ComposesOwn, or ""
|
||||
// while its agent has not decided. The step hands nothing but for ComposesController.
|
||||
Composes func() string
|
||||
// Running is what the node runs now (agent.Everything).
|
||||
Running func() []delivery.Composed
|
||||
// Get reads one key's latest value; nil and no error when there is none.
|
||||
Get func(ctx context.Context, bucket, key string) ([]byte, error)
|
||||
// Unseal opens a value sealed to this machine; Seal seals a value to a raw X25519 key in base64 (identity.Seal).
|
||||
Unseal func(sealed string) ([]byte, error)
|
||||
Seal func(to string, value []byte) (string, error)
|
||||
// Ask asks one of the vault's verbs in this engine's own name (link.Session.AskVault).
|
||||
Ask func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error)
|
||||
Say func(string)
|
||||
}
|
||||
|
||||
// Outcome is what the step did for one own secret: asked or not, and what was said.
|
||||
type Outcome struct {
|
||||
Module string
|
||||
Name string
|
||||
Asked bool
|
||||
Said string
|
||||
// Retry says a read or an ask the bus did not answer stopped it: no answer, kept as none, and the step is to be
|
||||
// taken again after a bound (issue 1126, 2026-10-28: "want.shanks.brother could not be read ... no responders").
|
||||
Retry bool
|
||||
}
|
||||
|
||||
// Resync is the step of one node agent's run.
|
||||
type Resync struct {
|
||||
env Env
|
||||
|
||||
mu sync.Mutex
|
||||
// done is, by module/name, the running value's fingerprint the vault was asked with or why it was not: each is
|
||||
// asked, and each reason said, once.
|
||||
done map[string]string
|
||||
}
|
||||
|
||||
// New is the step for one node.
|
||||
func New(env Env) *Resync {
|
||||
if env.Say == nil {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Resync{env: env, done: map[string]string{}}
|
||||
}
|
||||
|
||||
// Take reads the node's shadow answers and hands the vault what runs of each own secret they say another value of.
|
||||
func (r *Resync) Take(ctx context.Context, answers map[string]delivery.CandidateAnswer) []Outcome {
|
||||
seen := map[delivery.OwnValueDiffers]bool{}
|
||||
var drift []delivery.OwnValueDiffers
|
||||
proposals := make([]string, 0, len(answers))
|
||||
for g := range answers {
|
||||
proposals = append(proposals, g)
|
||||
}
|
||||
sort.Strings(proposals)
|
||||
for _, g := range proposals {
|
||||
for _, o := range delivery.OwnValuesDiffer(answers[g]) {
|
||||
if !seen[o] {
|
||||
seen[o] = true
|
||||
drift = append(drift, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(drift) == 0 {
|
||||
return nil
|
||||
}
|
||||
composes := ""
|
||||
if r.env.Composes != nil {
|
||||
composes = r.env.Composes()
|
||||
}
|
||||
var out []Outcome
|
||||
for _, o := range drift {
|
||||
res := r.one(ctx, composes, o)
|
||||
if res.Said != "" || res.Retry {
|
||||
out = append(out, res)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// one is the step for one own secret: Said is empty when it was done before for the same value and reason.
|
||||
func (r *Resync) one(ctx context.Context, composes string, o delivery.OwnValueDiffers) Outcome {
|
||||
res := Outcome{Module: o.Module, Name: o.Name}
|
||||
key := o.Module + "/" + o.Name
|
||||
if composes == "" {
|
||||
return res // the agent has not decided whose compositions it applies: taken again once it has
|
||||
}
|
||||
if composes != delivery.ComposesController {
|
||||
return r.said(key, "not:composes", res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
||||
"nothing is handed: this node composes its own (%q), so the vault's value is the source", key, composes))
|
||||
}
|
||||
sealed, why, failed := r.running(ctx, o)
|
||||
if failed != nil {
|
||||
// The bus did not answer: no reason to keep, and taken again (Retry); said once per reason.
|
||||
res.Retry = true
|
||||
return r.said(key, "retry:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
||||
"nothing is handed yet: %s; asked again", key, why))
|
||||
}
|
||||
if why != "" {
|
||||
return r.said(key, "not:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
||||
"nothing is handed: %s", key, why))
|
||||
}
|
||||
sum := sha256.Sum256([]byte(sealed))
|
||||
fp := hex.EncodeToString(sum[:])
|
||||
r.mu.Lock()
|
||||
asked := r.done[key] == fp
|
||||
r.mu.Unlock()
|
||||
if asked {
|
||||
return res
|
||||
}
|
||||
value, err := r.env.Unseal(sealed)
|
||||
if err != nil {
|
||||
return r.said(key, "not:unseal:"+fp, res, fmt.Sprintf("%s: what runs does not open with this machine's key, so "+
|
||||
"nothing is handed: %v", key, err))
|
||||
}
|
||||
defer wipe(value)
|
||||
said, err := r.hand(ctx, o, value)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return res // the run ended: asked again in the next
|
||||
}
|
||||
if selfcompose.BusFailed(err) {
|
||||
// The vault did not answer on the bus: no word of the vault's, so not kept as asked (Retry).
|
||||
res.Retry = true
|
||||
return r.said(key, "retry:ask", res, fmt.Sprintf("%s: the vault's %s with replace_running did not answer, "+
|
||||
"and is asked again: %v", key, Verb, err))
|
||||
}
|
||||
said = fmt.Sprintf("%s: the vault's %s with replace_running refused: %v", key, Verb, err)
|
||||
}
|
||||
// Once per running value, whatever the vault answered: a refusal is the vault's word, said once, and a new value
|
||||
// that runs asks again.
|
||||
r.mu.Lock()
|
||||
r.done[key] = fp
|
||||
r.mu.Unlock()
|
||||
res.Asked, res.Said = true, said
|
||||
r.env.Say(said)
|
||||
return res
|
||||
}
|
||||
|
||||
// said says why once per reason, and keeps the reason as done.
|
||||
func (r *Resync) said(key, reason string, res Outcome, why string) Outcome {
|
||||
r.mu.Lock()
|
||||
again := r.done[key] == reason
|
||||
r.done[key] = reason
|
||||
r.mu.Unlock()
|
||||
if again {
|
||||
return res
|
||||
}
|
||||
res.Said = why
|
||||
r.env.Say(why)
|
||||
return res
|
||||
}
|
||||
|
||||
// running is the sealed value of o in the composition that runs, when that composition is the one the module's want
|
||||
// names; or why it is not handed, with the error of a read the bus did not answer, which is no answer.
|
||||
func (r *Resync) running(ctx context.Context, o delivery.OwnValueDiffers) (sealed, why string, failed error) {
|
||||
var comp *delivery.Composed
|
||||
if r.env.Running != nil {
|
||||
for _, c := range r.env.Running() {
|
||||
if c.Module == o.Module {
|
||||
c := c
|
||||
comp = &c
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if comp == nil {
|
||||
return "", o.Module + " runs no composition here", nil
|
||||
}
|
||||
wantKey, err := delivery.WantKey(r.env.Node, o.Module)
|
||||
if err != nil {
|
||||
return "", err.Error(), nil
|
||||
}
|
||||
raw, err := r.env.Get(ctx, delivery.AssignerWantsBucket, wantKey)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its want %s could not be read (%v)", wantKey, err), err
|
||||
}
|
||||
if raw == nil {
|
||||
return "", fmt.Sprintf("it has no want %s", wantKey), nil
|
||||
}
|
||||
want, err := delivery.DecodeWant(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its want %s is not one: %v", wantKey, err), nil
|
||||
}
|
||||
if want.Self || want.Removed || want.Composed == "" {
|
||||
return "", fmt.Sprintf("its want %s names no composition of the controller's", wantKey), nil
|
||||
}
|
||||
// What runs is the composition the want names, as the module's have says: applied and running.
|
||||
haveKey, err := delivery.HaveKey(r.env.Node, o.Module)
|
||||
if err != nil {
|
||||
return "", err.Error(), nil
|
||||
}
|
||||
raw, err = r.env.Get(ctx, delivery.HaveBucket, haveKey)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its have %s could not be read (%v)", haveKey, err), err
|
||||
}
|
||||
if raw == nil {
|
||||
return "", fmt.Sprintf("it has no have %s", haveKey), nil
|
||||
}
|
||||
have, err := delivery.DecodeHave(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Sprintf("its have %s is not one: %v", haveKey, err), nil
|
||||
}
|
||||
if have.Composed != want.Composed || have.Phase != delivery.PhaseRunning {
|
||||
return "", fmt.Sprintf("what runs (%s, %s) is not yet the composition its want names (%s)", have.Composed,
|
||||
have.Phase, want.Composed), nil
|
||||
}
|
||||
// On a node the resource's id carries its module's name (`only-office.needs-jwt`); delivery.OwnNeedName reads it.
|
||||
id := o.Module + "." + catalogue.NeedID(o.Name)
|
||||
for _, res := range comp.Resources {
|
||||
var body struct {
|
||||
ID string `json:"id"`
|
||||
Sealed any `json:"sealed"`
|
||||
}
|
||||
if json.Unmarshal(res, &body) != nil {
|
||||
continue
|
||||
}
|
||||
if name, own := delivery.OwnNeedName(o.Module, body.ID); !own || name != o.Name {
|
||||
continue
|
||||
}
|
||||
if s, ok := body.Sealed.(string); ok && s != "" {
|
||||
return s, "", nil
|
||||
}
|
||||
return "", fmt.Sprintf("its resource %s carries no value sealed to this machine", id), nil
|
||||
}
|
||||
return "", fmt.Sprintf("what runs holds no resource %s", id), nil
|
||||
}
|
||||
|
||||
// hand asks the vault's two steps with value, and checks what the vault holds after by its fingerprint.
|
||||
func (r *Resync) hand(ctx context.Context, o delivery.OwnValueDiffers, value []byte) (string, error) {
|
||||
raw, err := r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "module": o.Module, "name": o.Name,
|
||||
"why": "this node's shadow answer says the vault holds another value than runs here"})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var opened struct {
|
||||
Call string `json:"call"`
|
||||
SealTo string `json:"seal_to"`
|
||||
Secret string `json:"secret"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
|
||||
return "", fmt.Errorf("the vault answered no call and no key to seal to")
|
||||
}
|
||||
sealed, err := r.env.Seal(opened.SealTo, value)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
|
||||
}
|
||||
raw, err = r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "call": opened.Call, "sealed": sealed})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var kept struct {
|
||||
Secret string `json:"secret"`
|
||||
From int `json:"from_generation"`
|
||||
To int `json:"to_generation"`
|
||||
Unchanged bool `json:"unchanged"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &kept); err != nil || kept.Secret == "" {
|
||||
return "", fmt.Errorf("the vault answered %s with no secret", opened.Secret)
|
||||
}
|
||||
sum := sha256.Sum256(value)
|
||||
if kept.Fingerprint != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
return "", fmt.Errorf("the vault says it holds %s at generation %d, but not the value handed in", kept.Secret, kept.To)
|
||||
}
|
||||
if kept.Unchanged {
|
||||
return fmt.Sprintf("handed the vault what runs of %s: it held that value already (generation %d)", kept.Secret, kept.To), nil
|
||||
}
|
||||
return fmt.Sprintf("handed the vault what runs of %s: generation %d replaced by %d, the value that runs", kept.Secret,
|
||||
kept.From, kept.To), nil
|
||||
}
|
||||
|
||||
func wipe(b []byte) {
|
||||
for i := range b {
|
||||
b[i] = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package ownresync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Issue 1126, cause C4: after a shadow answer saying the vault holds another value of an own secret than runs, the node
|
||||
// hands the vault what runs, in the verb's two steps, once per running value; and only while it applies the
|
||||
// controller's compositions and runs the one its want names.
|
||||
func TestTheNodeHandsTheVaultWhatRunsOfADriftedOwnSecret(t *testing.T) {
|
||||
const running = "the-value-that-runs"
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
fp := "src2:" + strings.Repeat("b", 64)
|
||||
sum := sha256.Sum256([]byte(running))
|
||||
answers := map[string]delivery.CandidateAnswer{"g": {Proposal: "p-1", Revision: 1, Verdict: delivery.CandidateDiffers,
|
||||
Differs: []delivery.ModuleDiffers{{Module: "only-office", Why: "the resource differs", Bodies: []delivery.ResourceBodies{
|
||||
{ID: "only-office.needs-jwt", Composed: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`, Running: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`},
|
||||
{ID: "only-office.needs-salt", Composed: `{"sealed":"<masked: another seal of the same value>"}`}}}}}}
|
||||
for name, row := range map[string]struct {
|
||||
composes string
|
||||
runs string // the have's composition digest
|
||||
refuse string // the vault's refusal
|
||||
asks int // the vault's verb asked, both passes
|
||||
says string
|
||||
// Issue 1126, 2026-10-28: a read or an ask the bus did not answer is no answer: taken again (Retry), never kept.
|
||||
wantRead string // the want's read: "" answers it, "fails" the bus does not answer, "absent" no key
|
||||
retry bool
|
||||
}{
|
||||
"applies the controller's, runs its want": {delivery.ComposesController, digest, "", 2, "generation 1 replaced by 2", "", false},
|
||||
"composes its own": {delivery.ComposesOwn, digest, "", 0, "composes its own", "", false},
|
||||
"not decided yet": {"", digest, "", 0, "", "", false},
|
||||
"runs not yet what its want names": {delivery.ComposesController, "sha256:" + strings.Repeat("c", 64), "", 0, "not yet the composition", "", false},
|
||||
"the vault refuses": {delivery.ComposesController, digest, "switched off", 1, "switched off", "", false},
|
||||
"the vault does not answer": {delivery.ComposesController, digest, "nats: no responders available for request", 2, "asked again", "", true},
|
||||
"the want's read does not answer": {delivery.ComposesController, digest, "", 0, "asked again", "fails", true},
|
||||
"it has no want": {delivery.ComposesController, digest, "", 0, "it has no want", "absent", false},
|
||||
} {
|
||||
var asked []map[string]any
|
||||
var said []string
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest})
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: row.runs, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
r := New(Env{Node: "novox", Composes: func() string { return row.composes },
|
||||
Running: func() []delivery.Composed {
|
||||
return []delivery.Composed{{Node: "novox", Module: "only-office", FP: fp, Resources: []json.RawMessage{
|
||||
// As novox runs it (its shadow answer, 2026-10-28): the id carries the module's name.
|
||||
json.RawMessage(`{"id":"only-office.needs-jwt","module":"only-office","type":"file",` +
|
||||
`"path":"/var/lib/only-office/jwt.secret","sealed":"box-of-the-running-value"}`)}}}
|
||||
},
|
||||
Get: func(_ context.Context, bucket, key string) ([]byte, error) {
|
||||
switch {
|
||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "fails":
|
||||
return nil, errors.New("the bucket mesh-delivery-assigner_wants: nats: no responders available for request")
|
||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "absent":
|
||||
return nil, nil
|
||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office":
|
||||
return want, nil
|
||||
case bucket == delivery.HaveBucket && key == "novox.only-office":
|
||||
return have, nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
Unseal: func(sealed string) ([]byte, error) {
|
||||
if sealed != "box-of-the-running-value" {
|
||||
return nil, errors.New("not this machine's")
|
||||
}
|
||||
return []byte(running), nil
|
||||
},
|
||||
Seal: func(to string, value []byte) (string, error) { return to + "(" + string(value) + ")", nil },
|
||||
Ask: func(_ context.Context, verb string, args map[string]any) (json.RawMessage, error) {
|
||||
asked = append(asked, args)
|
||||
if verb != "move-in" || args["replace_running"] != true {
|
||||
t.Errorf("%s: asked %s with %v, not move-in with replace_running", name, verb, args)
|
||||
}
|
||||
if row.refuse != "" {
|
||||
return nil, errors.New(row.refuse)
|
||||
}
|
||||
if args["call"] == nil {
|
||||
if args["module"] != "only-office" || args["name"] != "jwt" || args["node"] != nil {
|
||||
t.Errorf("%s: the first step names %v", name, args)
|
||||
}
|
||||
return json.RawMessage(`{"call":"c1","seal_to":"key","secret":"own/novox/only-office/jwt"}`), nil
|
||||
}
|
||||
if args["sealed"] != "key("+running+")" {
|
||||
t.Errorf("%s: handed %v, not what runs sealed to the call's key", name, args["sealed"])
|
||||
}
|
||||
return json.RawMessage(`{"secret":"own/novox/only-office/jwt","from_generation":1,"to_generation":2,` +
|
||||
`"fingerprint":"sha256:` + hex.EncodeToString(sum[:]) + `"}`), nil
|
||||
},
|
||||
Say: func(s string) { said = append(said, s) }})
|
||||
first := r.Take(context.Background(), answers)
|
||||
again := r.Take(context.Background(), answers)
|
||||
if len(asked) != row.asks {
|
||||
t.Errorf("%s: the vault asked %d times, want %d: %v", name, len(asked), row.asks, asked)
|
||||
}
|
||||
if retry := len(first) == 1 && first[0].Retry; retry != row.retry {
|
||||
t.Errorf("%s: taken again %v, want %v: %+v", name, retry, row.retry, first)
|
||||
}
|
||||
if row.retry && (len(again) != 1 || !again[0].Retry || again[0].Said != "") || !row.retry && len(again) != 0 {
|
||||
t.Errorf("%s: the same answer did or said something again: %+v", name, again)
|
||||
}
|
||||
all := strings.Join(said, "\n")
|
||||
if row.says == "" && (len(first) != 0 || all != "") || row.says != "" && (len(first) != 1 || !strings.Contains(all, row.says)) {
|
||||
t.Errorf("%s: did %+v and said %q, want %q", name, first, all, row.says)
|
||||
}
|
||||
if strings.Contains(all, running) {
|
||||
t.Errorf("%s: the value is in what the node said", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,839 @@
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
// Shadow is the node's answer to placement candidates (novox/hq ADR 0559 §4, issue 1126): for every proposal with a key in
|
||||
// the placement seat's state `candidates`, it composes this node in shadow from the trunk's placement with the proposal's
|
||||
// keys laid over it and its own inputs (the sealed values and boxes it holds among them), compares the result with what
|
||||
// the node runs, and hands the answers to be written into the node's own have key, `<node>.node`, under `candidates`.
|
||||
// It applies nothing and hands no composition: what runs is still composed from `placement` alone, by the Composer.
|
||||
//
|
||||
// **Event-driven, like the Composer.** It watches every candidate key, the trunk's placement, every node's record, the
|
||||
// indexer's modules, this node's config values, and this node's module haves (what it runs moved); a burst of changes
|
||||
// answers once. It runs whether or not the node is placed yet: a node that applies the controller's compositions answers
|
||||
// against what it runs from them. An answer is handed only when it changes, and a proposal's answer is dropped when its
|
||||
// keys go.
|
||||
type Shadow struct {
|
||||
c *Composer
|
||||
answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)
|
||||
|
||||
mu sync.Mutex
|
||||
candidates map[string]map[string]candidateKey // by proposal, by node
|
||||
answers map[string]delivery.CandidateAnswer
|
||||
// wants are the node's wants by module (agent.NodeModule for its own): what a proposal deleting its placement returns
|
||||
// it to, the controller's compositions at them.
|
||||
wants map[string]delivery.Want
|
||||
// ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy
|
||||
// from the declaration kept before the switch (issue 1126, probe #532 on shanks).
|
||||
ran map[string]string
|
||||
// heldNow is, by module, how its have says the node keeps it today as it runs: "held", "failed", or "runs a build of
|
||||
// the old form" (a `build` and no `fp`, which no registered build of the trunk names); "" when it is composed as any
|
||||
// other. A candidate that cannot compose it, or holds it only behind one of these, keeps it as today
|
||||
// (frombus.Scope.HeldToday, issue 1126).
|
||||
heldNow map[string]string
|
||||
// answered is, by proposal, the candidate revisions and the input changes (moved) each answer was composed at: a proposal
|
||||
// is answered again only when either moved (issue 1126, 2026-10-28: shanks answered probe/1126-seed-7 at revision 84
|
||||
// every 13-17 s, woken by its own modules' haves whose health words changed at every look).
|
||||
answered map[string]answeredAt
|
||||
// moved counts, by input (inputWords), the changes of each of the node's own inputs, and retry names the proposals
|
||||
// answered again after a read the bus did not answer: what woke each answer is said with it (issue 1126,
|
||||
// 2026-10-28, seed-9's probe: a safety check could not tell a loop from inputs moving).
|
||||
moved [len(inputWords)]uint64
|
||||
retry map[string]bool
|
||||
// owed says an answer met a read the bus did not answer: it is no answer to keep, and its proposal is answered again
|
||||
// after a bound (laterAgain), whether or not an input moves (issue 1126, 2026-10-28: ace kept "the vault's own: not
|
||||
// yet ... no responders" in its answer to seed-7).
|
||||
owed bool
|
||||
}
|
||||
|
||||
// answeredAt is what one proposal's answer was composed at: its keys' revisions, by node, and the changes of each input.
|
||||
type answeredAt struct {
|
||||
revs string
|
||||
moved [len(inputWords)]uint64
|
||||
}
|
||||
|
||||
// inputWords name the shadow's inputs, in Run's order, as an answer says which of them woke it.
|
||||
var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves", "wants"}
|
||||
|
||||
// wokenBy says what moved between the answer a proposal stands at (had, when done) and the one about to be composed (at):
|
||||
// its first answer, a retry, its candidate's revisions, or the inputs that changed, by their words.
|
||||
func wokenBy(had answeredAt, done, retry bool, at answeredAt) string {
|
||||
switch {
|
||||
case retry:
|
||||
return "a retry after a read the bus did not answer"
|
||||
case !done:
|
||||
return "its first answer since the shadow began"
|
||||
}
|
||||
var words []string
|
||||
if had.revs != at.revs {
|
||||
words = append(words, inputWords[0])
|
||||
}
|
||||
for i := 1; i < len(inputWords); i++ {
|
||||
if had.moved[i] != at.moved[i] {
|
||||
words = append(words, inputWords[i])
|
||||
}
|
||||
}
|
||||
if len(words) == 0 {
|
||||
return "nothing that moved"
|
||||
}
|
||||
return strings.Join(words, ", ")
|
||||
}
|
||||
|
||||
// candidateKey is one candidate key as last read: its value and revision.
|
||||
type candidateKey struct {
|
||||
c delivery.PlacementCandidate
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
|
||||
func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
|
||||
return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
|
||||
answers: map[string]delivery.CandidateAnswer{}, wants: map[string]delivery.Want{}, ran: map[string]string{}}
|
||||
}
|
||||
|
||||
// Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch
|
||||
// yet (the candidates' state not made, or the node's grant of it not loaded at a delivery) or a watch that ended is
|
||||
// tried again rather than leaving the node answering nothing for the rest of the link (review of L2b, round 1, finding
|
||||
// 2). Each distinct reason is said once.
|
||||
func (s *Shadow) Keep(ctx context.Context, backoff func(try int) time.Duration) {
|
||||
said := ""
|
||||
for try := 1; ctx.Err() == nil; try++ {
|
||||
began := time.Now()
|
||||
err := s.Run(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if time.Since(began) > time.Minute {
|
||||
try = 1 // it ran: a later stop begins the waits again
|
||||
}
|
||||
why := "it stopped"
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != said {
|
||||
s.c.env.Say("this node answers no placement candidate for now, and tries again: " + why)
|
||||
said = why
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(backoff(try)):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run answers until ctx ends or an input's watch ends.
|
||||
func (s *Shadow) Run(ctx context.Context) error {
|
||||
// Each run watches afresh and starts from nothing read: what changed while it was down (a candidate key deleted)
|
||||
// is then not missed, and the watches of a run before are not left open (review of L2b, round 2).
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
s.mu.Lock()
|
||||
s.candidates = map[string]map[string]candidateKey{}
|
||||
s.wants = map[string]delivery.Want{}
|
||||
s.ran = map[string]string{}
|
||||
s.answered = map[string]answeredAt{}
|
||||
s.moved = [len(inputWords)]uint64{}
|
||||
s.retry = map[string]bool{}
|
||||
s.c = New(s.c.env)
|
||||
s.mu.Unlock()
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
// In inputWords' order.
|
||||
ins := []input{
|
||||
{delivery.PlacementCandidatesBucket, ">", nil, nil},
|
||||
{delivery.PlacementBucket, ">", (*Composer).takePlacement, nil},
|
||||
{delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil, nil},
|
||||
{delivery.ModulesBucket, ">", (*Composer).takeModule, nil},
|
||||
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
||||
// What this node runs moved: an apply ends in its module's have.
|
||||
{delivery.HaveBucket, node + ".>", nil, nil},
|
||||
// Its wants: a proposal deleting its placement returns it to the controller's compositions at them.
|
||||
{delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil},
|
||||
}
|
||||
type tagged struct {
|
||||
in int
|
||||
u agent.Update
|
||||
ok bool
|
||||
}
|
||||
merged := make(chan tagged, 64)
|
||||
for i, in := range ins {
|
||||
ch, err := c.env.Watch(ctx, in.bucket, in.pattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the node answers placement candidates from %s, which cannot be watched: %w", in.bucket, err)
|
||||
}
|
||||
go func(i int, ch <-chan agent.Update) {
|
||||
for u := range ch {
|
||||
select {
|
||||
case merged <- tagged{in: i, u: u, ok: true}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case merged <- tagged{in: i}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}(i, ch)
|
||||
}
|
||||
caught := make([]bool, len(ins))
|
||||
ready, dirty := false, false
|
||||
// Not ready after a minute is said, naming each input not read yet, in the journal and in an answer to each proposal it
|
||||
// knows: a shadow that waits is never silent (issue 1126: ace, g14 and shanks answered nothing to seed #531).
|
||||
waiting := time.NewTimer(notReadyAfter)
|
||||
defer waiting.Stop()
|
||||
var again <-chan time.Time
|
||||
tries := 0
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-again:
|
||||
again = nil
|
||||
dirty = true
|
||||
case <-waiting.C:
|
||||
if !ready {
|
||||
s.notReady(ctx, ins, caught)
|
||||
}
|
||||
continue
|
||||
case t := <-merged:
|
||||
if !t.ok {
|
||||
return fmt.Errorf("the watch of %s ended; the node answers no placement candidate until its agent starts again", ins[t.in].bucket)
|
||||
}
|
||||
// Only a change of what an answer is composed from answers again: a key written again with the value it held,
|
||||
// this node's own key (its answers), or a module's have whose health or words moved but not what it runs, is
|
||||
// none (issue 1126, 2026-10-28: each look rewrote shanks' unhealthy modules' haves, and each woke a full answer).
|
||||
changed := false
|
||||
switch {
|
||||
case t.u.Caught:
|
||||
caught[t.in] = true
|
||||
case t.in == 0:
|
||||
changed = s.takeCandidate(t.u)
|
||||
case t.in == 2:
|
||||
changed = s.takeRecord(t.u)
|
||||
case t.in == 5:
|
||||
folded := c.takeModuleHave(t.u)
|
||||
changed = s.takeRan(t.u) || folded
|
||||
case t.in == 6:
|
||||
changed = s.takeWant(t.u)
|
||||
default:
|
||||
changed = s.takeChanged(ins[t.in], t.u)
|
||||
}
|
||||
if changed {
|
||||
if t.in != 0 {
|
||||
s.moved[t.in]++ // the candidates' own changes are their revisions
|
||||
}
|
||||
dirty = true
|
||||
}
|
||||
if len(merged) > 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !ready {
|
||||
if ready = all(caught); !ready {
|
||||
continue
|
||||
}
|
||||
dirty = true
|
||||
}
|
||||
if dirty {
|
||||
dirty = false
|
||||
s.answerAll(ctx)
|
||||
if s.owed {
|
||||
wait := LaterWait(tries)
|
||||
tries++
|
||||
c.env.Say(fmt.Sprintf("this node answers again in %s the placement candidates whose answer met a bus that did not answer", wait))
|
||||
again = time.After(wait)
|
||||
} else {
|
||||
tries, again = 0, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// takeChanged takes u through in.take and says whether the entry it names changed: a placement, a module or a config
|
||||
// value written again with the value it held changes nothing composed.
|
||||
func (s *Shadow) takeChanged(in input, u agent.Update) bool {
|
||||
before, had, known := s.entry(in.bucket, u.Key)
|
||||
in.take(s.c, u)
|
||||
if !known {
|
||||
return true
|
||||
}
|
||||
after, has, _ := s.entry(in.bucket, u.Key)
|
||||
return had != has || !reflect.DeepEqual(before, after)
|
||||
}
|
||||
|
||||
// entry is what the composer holds of one key of bucket, whether it holds it, and whether the bucket is one it knows.
|
||||
func (s *Shadow) entry(bucket, key string) (v any, has, known bool) {
|
||||
c := s.c
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
switch bucket {
|
||||
case delivery.PlacementBucket:
|
||||
node, err := delivery.ParsePlacementKey(key)
|
||||
if err != nil {
|
||||
return nil, false, true
|
||||
}
|
||||
p, ok := c.placements[node]
|
||||
return p, ok, true
|
||||
case delivery.ModulesBucket:
|
||||
name, err := delivery.ParseModulesKey(key)
|
||||
if err != nil {
|
||||
return nil, false, true
|
||||
}
|
||||
m, ok := c.modules[name]
|
||||
return m, ok, true
|
||||
case ResolvedBucket:
|
||||
rest, ok := strings.CutPrefix(key, c.env.Node+".")
|
||||
if !ok {
|
||||
return nil, false, true
|
||||
}
|
||||
module, k, ok := strings.Cut(rest, ".")
|
||||
if !ok {
|
||||
return nil, false, true
|
||||
}
|
||||
val, ok := c.values[module][k]
|
||||
return val, ok, true
|
||||
}
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
// revsOf is a proposal's keys' revisions, by node, as one word.
|
||||
func revsOf(keys map[string]candidateKey) string {
|
||||
nodes := make([]string, 0, len(keys))
|
||||
for n := range keys {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var b strings.Builder
|
||||
for _, n := range nodes {
|
||||
fmt.Fprintf(&b, "%s=%d;", n, keys[n].rev)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// takeCandidate takes one candidate key, and says whether it changed anything.
|
||||
func (s *Shadow) takeCandidate(u agent.Update) bool {
|
||||
proposal, node, err := delivery.ParsePlacementCandidateKey(u.Key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u.Deleted {
|
||||
if _, ok := s.candidates[proposal][node]; !ok {
|
||||
return false
|
||||
}
|
||||
delete(s.candidates[proposal], node)
|
||||
if len(s.candidates[proposal]) == 0 {
|
||||
delete(s.candidates, proposal)
|
||||
}
|
||||
return true
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(u.Value)
|
||||
if err != nil || c.Proposal != proposal || (c.Placement != nil && c.Placement.Node != node) {
|
||||
// Not one: read as gone.
|
||||
if _, ok := s.candidates[proposal][node]; ok {
|
||||
delete(s.candidates[proposal], node)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if s.candidates[proposal] == nil {
|
||||
s.candidates[proposal] = map[string]candidateKey{}
|
||||
}
|
||||
s.candidates[proposal][node] = candidateKey{c: c, rev: u.Revision}
|
||||
return true
|
||||
}
|
||||
|
||||
// takeRecord takes a node's record, and says whether it changed: the node's own key changes with each answer written,
|
||||
// which moves nothing composed.
|
||||
func (s *Shadow) takeRecord(u agent.Update) bool {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(u.Key)
|
||||
if err != nil || !isNode {
|
||||
return false
|
||||
}
|
||||
c := s.c
|
||||
c.mu.Lock()
|
||||
before, had := c.records[node]
|
||||
c.mu.Unlock()
|
||||
c.takeRecord(u)
|
||||
c.mu.Lock()
|
||||
after, has := c.records[node]
|
||||
c.mu.Unlock()
|
||||
return had != has || (has && !delivery.SameNodeRecord(before, after))
|
||||
}
|
||||
|
||||
// How long the shadow waits before it says it is not ready, and how long a proposal's answer runs before it is said slow.
|
||||
var notReadyAfter, slowAfter = time.Minute, time.Minute
|
||||
|
||||
// notReady says which inputs the shadow has not read yet, and answers each proposal it knows refused for it.
|
||||
func (s *Shadow) notReady(ctx context.Context, ins []input, caught []bool) {
|
||||
var waits []string
|
||||
for i, in := range ins {
|
||||
if !caught[i] {
|
||||
waits = append(waits, in.bucket+" ("+in.pattern+")")
|
||||
}
|
||||
}
|
||||
why := fmt.Sprintf("the node has not read %s after %s; it answers once it has", strings.Join(waits, ", "), notReadyAfter)
|
||||
s.c.env.Say("this node answers no placement candidate yet: " + why)
|
||||
s.mu.Lock()
|
||||
next := map[string]delivery.CandidateAnswer{}
|
||||
for proposal, keys := range s.candidates {
|
||||
own, ok := keys[s.c.env.Node]
|
||||
if !ok {
|
||||
continue // no key of this node's: nothing asks it to answer
|
||||
}
|
||||
next[proposal] = delivery.CandidateAnswer{Proposal: proposal, Revision: own.rev, Refused: why, Verdict: delivery.CandidateRefused}
|
||||
}
|
||||
s.answers = next
|
||||
s.mu.Unlock()
|
||||
if len(next) > 0 && s.answer != nil {
|
||||
s.answer(ctx, next)
|
||||
}
|
||||
}
|
||||
|
||||
// answerAll answers every proposal, the newest candidate first, and hands the answers each time one changes (issue 1126,
|
||||
// T3: ace answered a closed pull request's group for 22 minutes before the newer one, all handed at the end). Newest
|
||||
// is the highest revision of the proposal's keys: a withdrawn proposal's candidate is the older and waits; the seat
|
||||
// drops a closed one's keys, which ends its answer here (a node does not read the forge). Each answer says when it
|
||||
// starts, every progressEvery while it runs what it waits on, and when it ends (T2: the node said nothing for 22
|
||||
// minutes).
|
||||
func (s *Shadow) answerAll(ctx context.Context) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
s.mu.Lock()
|
||||
type listed struct {
|
||||
proposal string
|
||||
rev uint64
|
||||
at answeredAt
|
||||
}
|
||||
var proposals []listed
|
||||
for g, keys := range s.candidates {
|
||||
l := listed{proposal: g, at: answeredAt{revs: revsOf(keys), moved: s.moved}}
|
||||
for _, k := range keys {
|
||||
l.rev = max(l.rev, k.rev)
|
||||
}
|
||||
proposals = append(proposals, l)
|
||||
}
|
||||
// A proposal whose keys' revisions and the node's inputs are those its answer stands at is not answered again.
|
||||
for g := range s.answered {
|
||||
if _, ok := s.candidates[g]; !ok {
|
||||
delete(s.answered, g)
|
||||
}
|
||||
}
|
||||
// Each proposal's last answer stands until it is answered again; a proposal whose keys went is dropped.
|
||||
next := map[string]delivery.CandidateAnswer{}
|
||||
for _, l := range proposals {
|
||||
if a, ok := s.answers[l.proposal]; ok {
|
||||
next[l.proposal] = a
|
||||
}
|
||||
}
|
||||
s.owed = false
|
||||
s.mu.Unlock()
|
||||
sort.Slice(proposals, func(i, j int) bool {
|
||||
if proposals[i].rev != proposals[j].rev {
|
||||
return proposals[i].rev > proposals[j].rev
|
||||
}
|
||||
return proposals[i].proposal < proposals[j].proposal
|
||||
})
|
||||
hand := func() {
|
||||
s.mu.Lock()
|
||||
same := len(next) == len(s.answers)
|
||||
for g, a := range next {
|
||||
if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) {
|
||||
same = false
|
||||
}
|
||||
}
|
||||
out := make(map[string]delivery.CandidateAnswer, len(next))
|
||||
for g, a := range next {
|
||||
out[g] = a
|
||||
}
|
||||
if !same {
|
||||
s.answers = out
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !same && s.answer != nil {
|
||||
given := make(map[string]delivery.CandidateAnswer, len(out))
|
||||
for g, a := range out {
|
||||
given[g] = a
|
||||
}
|
||||
s.answer(ctx, given)
|
||||
c.env.Say(fmt.Sprintf("%s answered %d placement candidate proposal(s)", node, len(given)))
|
||||
}
|
||||
}
|
||||
for _, l := range proposals {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
proposal := l.proposal
|
||||
s.mu.Lock()
|
||||
at, done := s.answered[proposal]
|
||||
retry := s.retry[proposal]
|
||||
s.mu.Unlock()
|
||||
if done && at == l.at {
|
||||
continue
|
||||
}
|
||||
began := time.Now()
|
||||
c.env.Say(fmt.Sprintf("answering placement candidate %s (revision %d; woken by %s)", proposal, l.rev, wokenBy(at, done, retry, l.at)))
|
||||
stop := make(chan struct{})
|
||||
go func() {
|
||||
tick := time.NewTicker(progressEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-tick.C:
|
||||
c.env.Say(fmt.Sprintf("the answer to %s has run %s and waits on %s", proposal,
|
||||
time.Since(began).Round(time.Second), c.waitingOn()))
|
||||
}
|
||||
}
|
||||
}()
|
||||
missed := c.missed.Load()
|
||||
a, ok := s.answerProposal(ctx, proposal)
|
||||
close(stop)
|
||||
provisional := c.missed.Load() > missed && ctx.Err() == nil
|
||||
took := time.Since(began)
|
||||
if took > slowAfter {
|
||||
c.env.Say(fmt.Sprintf("the answer to %s took %s", proposal, took.Round(time.Second)))
|
||||
if ok {
|
||||
// Words that do not move with the time it took, so an answer the same as before is not written again.
|
||||
a.Note = strings.TrimPrefix(a.Note+"; ", "; ") + "this answer took over " + slowAfter.String() + " to compose"
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
a = delivery.FitAnswer(a, delivery.MaxAnswerBytes)
|
||||
next[proposal] = a
|
||||
c.env.Say(fmt.Sprintf("answered placement candidate %s in %s: %s", proposal, took.Round(time.Millisecond), a.Verdict))
|
||||
} else {
|
||||
delete(next, proposal)
|
||||
c.env.Say(fmt.Sprintf("placement candidate %s does not place this node: nothing to answer", proposal))
|
||||
}
|
||||
s.mu.Lock()
|
||||
if provisional {
|
||||
// What the bus did not answer is no answer: handed as it stands, and composed again after a bound.
|
||||
s.owed = true
|
||||
s.retry[proposal] = true
|
||||
delete(s.answered, proposal)
|
||||
c.env.Say(fmt.Sprintf("the answer to %s met a read on the bus that did not answer: it is answered again", proposal))
|
||||
} else {
|
||||
s.answered[proposal] = l.at
|
||||
delete(s.retry, proposal)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
hand()
|
||||
}
|
||||
hand() // a proposal whose keys went, when no proposal is left to answer
|
||||
}
|
||||
|
||||
// How often a running answer says what it waits on.
|
||||
var progressEvery = time.Minute
|
||||
|
||||
// answerProposal is the node's answer to one proposal, and false when the node has nothing to answer: neither the trunk nor
|
||||
// the proposal places it.
|
||||
func (s *Shadow) answerProposal(ctx context.Context, proposal string) (delivery.CandidateAnswer, bool) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
defer c.waiting("composing")()
|
||||
s.mu.Lock()
|
||||
keys := s.candidates[proposal]
|
||||
given := make(map[string]*delivery.Placement, len(keys))
|
||||
for n, k := range keys {
|
||||
if k.c.Reached && k.c.Placement == nil {
|
||||
continue // reached, with no key of its own: nothing laid over it
|
||||
}
|
||||
given[n] = k.c.Placement
|
||||
}
|
||||
// The revision answered is the node's own candidate key's; a node with none is not asked.
|
||||
own, mine := keys[node]
|
||||
rev := own.rev
|
||||
s.mu.Unlock()
|
||||
|
||||
c.mu.Lock()
|
||||
modules := make(map[string]delivery.Module, len(c.modules))
|
||||
for k, v := range c.modules {
|
||||
modules[k] = v
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if !mine {
|
||||
return delivery.CandidateAnswer{}, false
|
||||
}
|
||||
|
||||
var running []delivery.Composed
|
||||
if c.env.Running != nil {
|
||||
running = c.env.Running()
|
||||
}
|
||||
if p, ok := given[node]; ok && p == nil && !own.c.Reached {
|
||||
// The proposal deletes this node's placement key: the node then returns to the controller's compositions, as one the
|
||||
// catalogue does not place (endWhenUnplaced). Those at its wants are compared with what runs (issue 1126, the
|
||||
// seed's revert, whose check read "would stop running node").
|
||||
s.mu.Lock()
|
||||
wants := make(map[string]delivery.Want, len(s.wants))
|
||||
for m, w := range s.wants {
|
||||
wants[m] = w
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return returning(ctx, c.get, node, wants, running, proposal, rev), true
|
||||
}
|
||||
|
||||
placement := given[node]
|
||||
if placement == nil {
|
||||
c.mu.Lock()
|
||||
p := c.placements[node]
|
||||
c.mu.Unlock()
|
||||
placement = &p
|
||||
}
|
||||
// What it runs inside its own composition, of what the candidate places (the seed's trunk places nothing).
|
||||
c.mu.Lock()
|
||||
inOwn := c.inOwnOf(*placement)
|
||||
c.mu.Unlock()
|
||||
fps := shadowBuilds(running, *placement, modules, inOwn)
|
||||
kept := map[string]string{}
|
||||
for module, fp := range fps {
|
||||
if mod, ok := modules[module]; ok && mod.FP2 == fp {
|
||||
continue
|
||||
}
|
||||
kept[module] = fp // composed from that build (frombus.Scope.KeptAt, issue 1126)
|
||||
}
|
||||
// The boxes it runs, by what each module's have and its own key say after a restart (Held), are reused for the
|
||||
// modules it runs: a value sealed again is no difference.
|
||||
held := map[string]delivery.Want{agent.NodeModule: {}}
|
||||
for module := range fps {
|
||||
held[module] = delivery.Want{FP: fps[module]}
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.wants = held
|
||||
c.mu.Unlock()
|
||||
a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, KeptAt: kept, Placements: given, InOwn: inOwn,
|
||||
SameValue: c.sameValue, HeldToday: s.heldToday()}, fps, running, proposal, rev)
|
||||
s.mu.Lock()
|
||||
ran := make(map[string]string, len(s.ran))
|
||||
for m, d := range s.ran {
|
||||
ran[m] = d
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return asHavesSay(a, ran), true
|
||||
}
|
||||
|
||||
// heldToday is what the node holds today: each module whose have's phase is held, and what its own composer holds, the
|
||||
// node's own composition ("node") among them when it composes none of its own.
|
||||
func (s *Shadow) heldToday() map[string]string {
|
||||
out := map[string]string{}
|
||||
s.mu.Lock()
|
||||
for m, how := range s.heldNow {
|
||||
if how != "" {
|
||||
out[m] = how
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for _, m := range s.c.heldNow() {
|
||||
if _, said := out[m]; !said {
|
||||
out[m] = "held"
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// takeRan keeps the digest one of this node's module haves says runs, and says whether it changed: a have whose health
|
||||
// or words alone moved runs the same.
|
||||
func (s *Shadow) takeRan(u agent.Update) bool {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(u.Key)
|
||||
if err != nil || isNode || node != s.c.env.Node {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
before, had := s.ran[module]
|
||||
h, err := delivery.DecodeHave(u.Value)
|
||||
if s.heldNow == nil {
|
||||
s.heldNow = map[string]string{}
|
||||
}
|
||||
wasHeld := s.heldNow[module]
|
||||
s.heldNow[module] = ""
|
||||
if !u.Deleted && err == nil {
|
||||
switch {
|
||||
case h.Phase == delivery.PhaseHeld:
|
||||
s.heldNow[module] = "held"
|
||||
case h.Phase == delivery.PhaseFailed:
|
||||
s.heldNow[module] = "failed"
|
||||
case h.Phase != delivery.PhaseRemoved && h.FP == "" && h.Build != "":
|
||||
// A build of the old form: its have names a `build` and no `fp`, which no registered build of the trunk
|
||||
// names, so the controller keeps it as it runs (issue 1126: novox's de-spiegel at build-…); a seed keeps it so.
|
||||
s.heldNow[module] = "runs a build of the old form"
|
||||
}
|
||||
}
|
||||
holdMoved := wasHeld != s.heldNow[module] // a hold's start or end changes the answer too
|
||||
if u.Deleted || err != nil || h.Composed == "" || h.Phase == delivery.PhaseRemoved {
|
||||
delete(s.ran, module)
|
||||
return had || holdMoved
|
||||
}
|
||||
s.ran[module] = h.Composed
|
||||
return !had || before != h.Composed || holdMoved
|
||||
}
|
||||
|
||||
// asHavesSay is a with each module that differs only from the copy the agent keeps of the declaration before the switch
|
||||
// counted the same where the composed digest is the one its have says runs: what runs is the composition at that
|
||||
// digest, never the kept copy (issue 1126, probe #532: shanks' xterm, zsh, zsh-autosuggestions and
|
||||
// zsh-syntax-highlighting).
|
||||
func asHavesSay(a delivery.CandidateAnswer, ran map[string]string) delivery.CandidateAnswer {
|
||||
if a.Refused != "" || len(a.Differs) == 0 {
|
||||
return a
|
||||
}
|
||||
kept := a.Differs[:0:0]
|
||||
for _, d := range a.Differs {
|
||||
if d.Composed != "" && ran[d.Module] == d.Composed {
|
||||
a.Same = append(a.Same, d.Module)
|
||||
continue
|
||||
}
|
||||
kept = append(kept, d)
|
||||
}
|
||||
a.Differs = kept
|
||||
sort.Strings(a.Same)
|
||||
a.Verdict = delivery.VerdictOf(a)
|
||||
return a
|
||||
}
|
||||
|
||||
// shadowBuilds is the build each module placed is composed at in shadow: the one it runs, and for one the candidate
|
||||
// adds, the trunk's. A module it runs inside its own composition (inOwn, from its haves: mesh-wireguard, one at a build
|
||||
// of the old form) is given none, so it is folded there again, as the node's own composer folds it, rather than
|
||||
// composed at the trunk's build as if the candidate added it (issue 1126: nats and photos have a trunk build that no
|
||||
// want names).
|
||||
func shadowBuilds(running []delivery.Composed, placement delivery.Placement, modules map[string]delivery.Module, inOwn []string) map[string]string {
|
||||
fps := map[string]string{}
|
||||
for _, r := range running {
|
||||
if r.Module != "" && r.FP != "" {
|
||||
fps[r.Module] = r.FP
|
||||
}
|
||||
}
|
||||
folded := map[string]bool{}
|
||||
for _, m := range inOwn {
|
||||
folded[m] = true
|
||||
}
|
||||
for _, m := range placement.Modules {
|
||||
if _, runs := fps[m.Module]; !runs && !folded[m.Module] {
|
||||
if mod, ok := modules[m.Module]; ok && mod.FP2 != "" {
|
||||
fps[m.Module] = mod.FP2
|
||||
}
|
||||
}
|
||||
}
|
||||
return fps
|
||||
}
|
||||
|
||||
func sameAnswer(a, b delivery.CandidateAnswer) bool {
|
||||
x, errA := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}})
|
||||
y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}})
|
||||
return errA == nil && errB == nil && string(x) == string(y)
|
||||
}
|
||||
|
||||
// takeWant takes one of the node's wants, and says whether it changed.
|
||||
func (s *Shadow) takeWant(u agent.Update) bool {
|
||||
module := ""
|
||||
var w delivery.Want
|
||||
if _, err := delivery.ParseWantNodeKey(u.Key); err == nil {
|
||||
module = agent.NodeModule
|
||||
} else if node, m, err := delivery.ParseWantKey(u.Key); err == nil && node == s.c.env.Node {
|
||||
module = m
|
||||
if !u.Deleted {
|
||||
if w, err = delivery.DecodeWant(u.Value); err != nil || w.Removed || w.FP == "" {
|
||||
u.Deleted = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
old, had := s.wants[module]
|
||||
if u.Deleted {
|
||||
delete(s.wants, module)
|
||||
return had
|
||||
}
|
||||
s.wants[module] = w
|
||||
return !had || old.FP != w.FP
|
||||
}
|
||||
|
||||
// returning is the node's answer to a proposal deleting its placement: it returns to the controller's compositions, each
|
||||
// read at the fingerprint its want names in the node's composition bucket and compared with what runs
|
||||
// (delivery.CompareComposed). A want the controller holds no composition for, or one that cannot be read, refuses the
|
||||
// answer, naming each: returning then would leave that module with nothing to apply.
|
||||
func returning(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string,
|
||||
wants map[string]delivery.Want, running []delivery.Composed, proposal string, rev uint64) delivery.CandidateAnswer {
|
||||
a := delivery.CandidateAnswer{Proposal: proposal, Revision: rev}
|
||||
bucket, err := delivery.ComposedBucketFor(node)
|
||||
if err != nil {
|
||||
a.Refused, a.Verdict = err.Error(), delivery.CandidateRefused
|
||||
return a
|
||||
}
|
||||
modules := make([]string, 0, len(wants))
|
||||
for m := range wants {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
comps := map[string]delivery.Composed{}
|
||||
var missing []string
|
||||
for _, m := range modules {
|
||||
var key string
|
||||
if m == agent.NodeModule {
|
||||
key, err = delivery.ComposedNodeKey(node)
|
||||
} else {
|
||||
key, err = delivery.ComposedKey(node, m, wants[m].FP)
|
||||
}
|
||||
if err != nil {
|
||||
missing = append(missing, m+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
raw, err := get(ctx, bucket, key)
|
||||
switch {
|
||||
case err != nil:
|
||||
missing = append(missing, fmt.Sprintf("%s: the controller's composition %s cannot be read: %v", m, key, err))
|
||||
continue
|
||||
case raw == nil:
|
||||
missing = append(missing, fmt.Sprintf("%s: the controller holds no composition %s, which its want names", m, key))
|
||||
continue
|
||||
}
|
||||
comp, err := delivery.DecodeComposed(raw)
|
||||
if err != nil {
|
||||
missing = append(missing, fmt.Sprintf("%s: the controller's composition %s: %v", m, key, err))
|
||||
continue
|
||||
}
|
||||
comps[key] = comp
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
a.Refused = "returning to the controller's compositions, " + strings.Join(missing, "; ")
|
||||
a.Verdict = delivery.CandidateRefused
|
||||
return a
|
||||
}
|
||||
cmp := delivery.CompareComposed(comps, running)
|
||||
a.Returns = delivery.ReturnsToController
|
||||
a.Same, a.Differs, a.Added, a.Removed = cmp.Same, cmp.Differs, cmp.Added, cmp.Removed
|
||||
a.Verdict = delivery.VerdictOf(a)
|
||||
return a
|
||||
}
|
||||
|
||||
// sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not
|
||||
// open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work).
|
||||
func (c *Composer) sameValue(a, b string) (same, known bool) {
|
||||
if c.env.Unseal == nil {
|
||||
return false, false
|
||||
}
|
||||
x, errA := c.env.Unseal(a)
|
||||
y, errB := c.env.Unseal(b)
|
||||
if errA != nil || errB != nil {
|
||||
return false, false
|
||||
}
|
||||
return bytes.Equal(x, y), true
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"module": "de-spiegel",
|
||||
"version": "1",
|
||||
"slug": "spiegel",
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"requires": [
|
||||
"route",
|
||||
"smtp"
|
||||
],
|
||||
"contributes": {
|
||||
"route": {
|
||||
"site": {
|
||||
"label": "de-spiegel",
|
||||
"port": 35621
|
||||
},
|
||||
"praktijk": {
|
||||
"name": "${config:praktijk-name}",
|
||||
"port": 35621
|
||||
},
|
||||
"praktijk-www": {
|
||||
"name": "${config:praktijk-www-name}",
|
||||
"redirect": "https://${config:praktijk-name}"
|
||||
},
|
||||
"psychologie": {
|
||||
"name": "${config:psychologie-name}",
|
||||
"port": 35621
|
||||
},
|
||||
"psychologie-www": {
|
||||
"name": "${config:psychologie-www-name}",
|
||||
"redirect": "https://${config:psychologie-name}"
|
||||
}
|
||||
},
|
||||
"smtp": {
|
||||
"account": "de-spiegel"
|
||||
}
|
||||
},
|
||||
"binds": {
|
||||
"route": "/var/lib/de-spiegel/route.json",
|
||||
"smtp": "/var/lib/de-spiegel/smtp.json"
|
||||
},
|
||||
"config": {
|
||||
"praktijk-name": {
|
||||
"type": "string",
|
||||
"description": "The public name the practice's site is served on; the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
},
|
||||
"praktijk-www-name": {
|
||||
"type": "string",
|
||||
"description": "The public name that redirects to the practice's site (its www form); the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
},
|
||||
"psychologie-name": {
|
||||
"type": "string",
|
||||
"description": "The public name the psychology practice's site is served on; the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
},
|
||||
"psychologie-www-name": {
|
||||
"type": "string",
|
||||
"description": "The public name that redirects to the psychology practice's site (its www form); the operator's, with no default (novox/hq ADR 0112).",
|
||||
"trusted": true,
|
||||
"required": true
|
||||
}
|
||||
},
|
||||
"listens": [
|
||||
{
|
||||
"port": 35621,
|
||||
"protocol": "tcp",
|
||||
"from": "mesh",
|
||||
"why": "the de-spiegel site and its /contact endpoint over http; the practice's three public names are route grants, and the proxy reaches every one of them here"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/de-spiegel",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "server-env",
|
||||
"type": "file",
|
||||
"path": "/var/lib/de-spiegel/server.env",
|
||||
"mode": "0600",
|
||||
"content": "SMTP_HOST=${bound:smtp:at}\nSMTP_PORT=${bound:smtp:port}\nSMTP_AUTH_USER=de-spiegel@${bound:smtp:domain}\nSMTP_AUTH_PASS=${secret:smtp}\n"
|
||||
},
|
||||
{
|
||||
"id": "net",
|
||||
"type": "network",
|
||||
"name": "de-spiegel"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
"name": "de-spiegel",
|
||||
"artifact": "site",
|
||||
"network": "de-spiegel",
|
||||
"env-file": [
|
||||
"/var/lib/de-spiegel/server.env"
|
||||
],
|
||||
"ports": [
|
||||
"35621"
|
||||
],
|
||||
"secrets-in-environment": "the application's own code reads SMTP_AUTH_USER/PASS from the environment (server/index.js); converting is this repository's change"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"on": [
|
||||
{
|
||||
"arg": "NODE_BASE",
|
||||
"image": "node@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9"
|
||||
}
|
||||
],
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "site",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile"
|
||||
}
|
||||
]
|
||||
},
|
||||
"secrets": {
|
||||
"smtp": "/var/lib/de-spiegel/smtp.secret"
|
||||
}
|
||||
}
|
||||
+73
-36
@@ -187,38 +187,11 @@ func describeAge(when, now time.Time) string {
|
||||
// so LoadState is what is read — and it is the thing an interface spanning systemd and OpenRC
|
||||
// would have had to drop.
|
||||
func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string, error) {
|
||||
out, asked := run(ctx, "systemctl", "show",
|
||||
"--property=LoadState", "--property=ActiveState", "--property=Type",
|
||||
"--property=RemainAfterExit", "--property=ExecMainStatus", "--", unit)
|
||||
|
||||
var load, active, kind, remains, exited string
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
key, value, found := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case "LoadState":
|
||||
load = value
|
||||
case "ActiveState":
|
||||
active = value
|
||||
case "Type":
|
||||
kind = value
|
||||
case "RemainAfterExit":
|
||||
remains = value
|
||||
case "ExecMainStatus":
|
||||
exited = value
|
||||
}
|
||||
u, err := showUnit(ctx, run, unit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch load {
|
||||
case "":
|
||||
// With why, where it said why: an account's manager that could not be reached says so
|
||||
// here, and nowhere else (novox/hq ADR 0177).
|
||||
if asked != nil {
|
||||
return "", fmt.Errorf("the service manager said nothing about %s: %w", unit, asked)
|
||||
}
|
||||
return "", fmt.Errorf("the service manager said nothing about %s", unit)
|
||||
switch u.load {
|
||||
case "not-found":
|
||||
return "", fmt.Errorf(
|
||||
"%s does not exist on this machine. A declaration naming a unit that is not "+
|
||||
@@ -227,7 +200,7 @@ func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string,
|
||||
case "masked":
|
||||
return "", fmt.Errorf("%s is masked, so its state cannot be declared", unit)
|
||||
case "error", "bad-setting":
|
||||
return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, load)
|
||||
return "", fmt.Errorf("%s is installed but its unit file cannot be loaded (%s)", unit, u.load)
|
||||
}
|
||||
|
||||
// **A one-shot that finished is not stopped.** A unit whose whole job is to apply something
|
||||
@@ -243,24 +216,88 @@ func (arch) ServiceState(ctx context.Context, run Runner, unit string) (string,
|
||||
// wrong with.
|
||||
//
|
||||
// So for that shape, what "running" means is "it ran, and it worked".
|
||||
if kind == "oneshot" && remains != "yes" && active == "inactive" {
|
||||
if exited == "0" || exited == "" {
|
||||
//
|
||||
// It cannot be told from one that never ran: systemd says ExecMainStatus=0 and a start timestamp
|
||||
// of 0 for a one-shot that never ran, and also for one that ran and was unloaded because nothing
|
||||
// referenced it, then loaded again. So a one-shot declared running that must be seen to have run
|
||||
// lingers (RemainAfterExit) or is enabled at boot; one declared stopped is judged at rest by the
|
||||
// apply (ServiceAtRest, novox/hq issue 1126).
|
||||
if u.oneShotAtRest() {
|
||||
if u.exited == "0" || u.exited == "" {
|
||||
return "running", nil
|
||||
}
|
||||
return "stopped", nil
|
||||
}
|
||||
|
||||
switch active {
|
||||
switch u.active {
|
||||
case "active", "activating", "reloading":
|
||||
return "running", nil
|
||||
case "inactive", "failed", "deactivating":
|
||||
return "stopped", nil
|
||||
default:
|
||||
return "", fmt.Errorf(
|
||||
"the service manager reports %s as %q, which is neither running nor stopped", unit, active)
|
||||
"the service manager reports %s as %q, which is neither running nor stopped", unit, u.active)
|
||||
}
|
||||
}
|
||||
|
||||
// ServiceAtRest is whether unit is a one-shot that does not linger and has nothing running now
|
||||
// (novox/hq issue 1126). Such a unit has no state in which a process is up between its runs, so a
|
||||
// declaration that it be stopped is satisfied whatever its last run did: there is nothing to stop,
|
||||
// and asking the manager to stop it changes nothing. A boot-time cleanup a package ships is declared
|
||||
// this way — enabled at boot, never started by an apply.
|
||||
func (arch) ServiceAtRest(ctx context.Context, run Runner, unit string) (bool, error) {
|
||||
u, err := showUnit(ctx, run, unit)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return u.oneShotAtRest(), nil
|
||||
}
|
||||
|
||||
// shownUnit is what `systemctl show` says of a unit, as far as its state is read.
|
||||
type shownUnit struct {
|
||||
load, active, kind, remains, exited string
|
||||
}
|
||||
|
||||
// oneShotAtRest is a one-shot that does not linger and is not running now.
|
||||
func (u shownUnit) oneShotAtRest() bool {
|
||||
return u.kind == "oneshot" && u.remains != "yes" && u.active == "inactive"
|
||||
}
|
||||
|
||||
// showUnit reads a unit's state properties. A manager that said nothing is an error, with why
|
||||
// where it said why: an account's manager that could not be reached says so here, and nowhere
|
||||
// else (novox/hq ADR 0177).
|
||||
func showUnit(ctx context.Context, run Runner, unit string) (shownUnit, error) {
|
||||
out, asked := run(ctx, "systemctl", "show",
|
||||
"--property=LoadState", "--property=ActiveState", "--property=Type",
|
||||
"--property=RemainAfterExit", "--property=ExecMainStatus", "--", unit)
|
||||
var u shownUnit
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
key, value, found := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case "LoadState":
|
||||
u.load = value
|
||||
case "ActiveState":
|
||||
u.active = value
|
||||
case "Type":
|
||||
u.kind = value
|
||||
case "RemainAfterExit":
|
||||
u.remains = value
|
||||
case "ExecMainStatus":
|
||||
u.exited = value
|
||||
}
|
||||
}
|
||||
if u.load == "" {
|
||||
if asked != nil {
|
||||
return u, fmt.Errorf("the service manager said nothing about %s: %w", unit, asked)
|
||||
}
|
||||
return u, fmt.Errorf("the service manager said nothing about %s", unit)
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (arch) SetServiceState(ctx context.Context, run Runner, unit, state string) error {
|
||||
verb := "start"
|
||||
if state == "stopped" {
|
||||
|
||||
@@ -5,47 +5,51 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A one-shot unit that did its work and exited is satisfied, not stopped.
|
||||
// How a one-shot unit's state is read: what it did, not whether a process is up.
|
||||
//
|
||||
// **This made the firewall permanently unsatisfiable.** Its unit loads a rule set and exits, so it
|
||||
// is inactive the instant it succeeds — the host started it, it worked, the host read back
|
||||
// "stopped" and reported the machine as not doing what it was told. On every apply, for ever, with
|
||||
// the rules correctly in place the whole time.
|
||||
func TestAOneShotThatFinishedIsRunning(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\nActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=0\n", nil
|
||||
}
|
||||
state, err := arch{}.ServiceState(context.Background(), run, "nftables.service")
|
||||
if err != nil {
|
||||
t.Fatalf("a one-shot that succeeded was an error: %v", err)
|
||||
}
|
||||
if state != "running" {
|
||||
t.Fatalf("a one-shot that did its work reads as %q, so it can never be satisfied", state)
|
||||
}
|
||||
}
|
||||
|
||||
// And one that failed is still stopped, or the host would report success for work that did not
|
||||
// happen — which is the opposite mistake and the worse one.
|
||||
func TestAOneShotThatFailedIsStopped(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\nActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=1\n", nil
|
||||
}
|
||||
state, err := arch{}.ServiceState(context.Background(), run, "nftables.service")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if state != "stopped" {
|
||||
t.Fatalf("a one-shot that failed reads as %q, so a broken firewall reports success", state)
|
||||
}
|
||||
}
|
||||
|
||||
// A unit that lingers deliberately is unaffected: it says so, and its active state is the answer.
|
||||
func TestAOneShotThatRemainsIsJudgedByItsActiveState(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\nActiveState=active\nType=oneshot\nRemainAfterExit=yes\nExecMainStatus=0\n", nil
|
||||
}
|
||||
state, _ := arch{}.ServiceState(context.Background(), run, "thing.service")
|
||||
if state != "running" {
|
||||
t.Fatalf("a lingering one-shot reads as %q", state)
|
||||
// **A one-shot that did its work and exited is satisfied, not stopped.** Its unit loads a rule set
|
||||
// and exits, so it is inactive the instant it succeeds; reading that as stopped made the firewall
|
||||
// permanently unsatisfiable, on every apply, with the rules correctly in place the whole time.
|
||||
//
|
||||
// **One that failed is still stopped**, or the host would report success for work that did not
|
||||
// happen — the opposite mistake and the worse one.
|
||||
//
|
||||
// **At rest** (novox/hq issue 1126, the kernel's boot-time cleanup on ace) is whether a one-shot that does not linger has nothing running now: a declaration
|
||||
// that it be stopped is then satisfied, whatever it last did, because there is nothing to stop.
|
||||
func TestAOneShotIsReadByWhatItDid(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, show, want string
|
||||
atRest bool
|
||||
}{
|
||||
{"finished", "ActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=0\n",
|
||||
"running", true},
|
||||
{"failed", "ActiveState=inactive\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=1\n",
|
||||
"stopped", true},
|
||||
{"running now", "ActiveState=activating\nType=oneshot\nRemainAfterExit=no\nExecMainStatus=0\n",
|
||||
"running", false},
|
||||
{"lingers", "ActiveState=active\nType=oneshot\nRemainAfterExit=yes\nExecMainStatus=0\n",
|
||||
"running", false},
|
||||
{"a daemon, stopped", "ActiveState=inactive\nType=simple\nRemainAfterExit=no\nExecMainStatus=0\n",
|
||||
"stopped", false},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
run := func(_ context.Context, _ string, _ ...string) (string, error) {
|
||||
return "LoadState=loaded\n" + c.show, nil
|
||||
}
|
||||
state, err := arch{}.ServiceState(context.Background(), run, "thing.service")
|
||||
if err != nil {
|
||||
t.Fatalf("read as an error: %v", err)
|
||||
}
|
||||
if state != c.want {
|
||||
t.Errorf("reads as %q, want %q", state, c.want)
|
||||
}
|
||||
rest, err := arch{}.ServiceAtRest(context.Background(), run, "thing.service")
|
||||
if err != nil {
|
||||
t.Fatalf("at rest read as an error: %v", err)
|
||||
}
|
||||
if rest != c.atRest {
|
||||
t.Errorf("at rest is %v, want %v", rest, c.atRest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,13 @@
|
||||
// (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to
|
||||
// that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only
|
||||
// when the list on the machine or the vault's key changes.
|
||||
//
|
||||
// **From take-over it follows the grants part's list instead** (ADR 0547 clauses 7 and 8, issue 1945). On the node whose
|
||||
// compositions carry the list, it also watches the grants part's one key `user-list.sealed`. While that key says
|
||||
// FollowGrants, the list is the object the key names, read by its name, opened with this node's sealing key and checked
|
||||
// against the key's digest, and written to the same file (mode 0600) only when the file's digest differs; otherwise the
|
||||
// controller's compositions are the source, as before. Never both. The have says which it follows and this node's key,
|
||||
// which the grants part seals to.
|
||||
package userlist
|
||||
|
||||
import (
|
||||
@@ -36,6 +43,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
@@ -64,6 +72,19 @@ type Env struct {
|
||||
Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error)
|
||||
// Report says what was written, in the module's have.
|
||||
Report func(ctx context.Context, module string, w Written)
|
||||
// Grants watches the grants part's state (its key delivery.GrantsUserListKey); nil follows the controller's alone.
|
||||
Grants agent.Watcher
|
||||
// Fetch reads an object of the grants part's bucket by its name; Open opens a box sealed to this node's key; Key is
|
||||
// this node's sealing key's public half (SPKI PEM), said in the have so the grants part seals to it ("" unknown).
|
||||
Fetch func(ctx context.Context, name string) ([]byte, error)
|
||||
Open func(seal.Box) ([]byte, error)
|
||||
Key func() string
|
||||
// Kept is the grants part's key as last heard on this machine and its revision (nil when none), and Keep keeps it:
|
||||
// what the follower follows while the key cannot be watched (ADR 0547 clause 8). Backoff is the wait before the
|
||||
// try-th watch again.
|
||||
Kept func() ([]byte, uint64)
|
||||
Keep func(value []byte, rev uint64)
|
||||
Backoff func(try int) time.Duration
|
||||
// VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the
|
||||
// key the loaded list's hashes are handed sealed to. Nil hands nothing.
|
||||
VaultKey func() string
|
||||
@@ -91,6 +112,13 @@ type Follower struct {
|
||||
state Written
|
||||
mod string
|
||||
said map[string]bool
|
||||
// grants is the grants part's key as last heard (nil when none), and following the source last taken: FollowGrants
|
||||
// or FollowController.
|
||||
grants *delivery.GrantsUserList
|
||||
grantsRev uint64
|
||||
following string
|
||||
// unwatched is why the grants part's key is not watched now; empty while it is, or where it need not be.
|
||||
unwatched string
|
||||
}
|
||||
|
||||
// New is a follower for a node.
|
||||
@@ -101,7 +129,18 @@ func New(env Env) *Follower {
|
||||
if env.Say == nil {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}}
|
||||
if env.Backoff == nil {
|
||||
env.Backoff = func(int) time.Duration { return time.Minute }
|
||||
}
|
||||
f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}}
|
||||
if env.Grants != nil && env.Kept != nil {
|
||||
if raw, rev := env.Kept(); raw != nil {
|
||||
if g, err := delivery.DecodeGrantsUserList(raw); err == nil {
|
||||
f.grants, f.grantsRev = &g, rev
|
||||
}
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module,
|
||||
@@ -128,6 +167,33 @@ func (f *Follower) Run(ctx context.Context) error {
|
||||
return fmt.Errorf("the user list cannot be followed: %w", err)
|
||||
}
|
||||
caught := false
|
||||
// The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk
|
||||
// buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link)
|
||||
// never writes the controller's list after take-over before it heard the switch; a watch refused or ended is tried
|
||||
// again on the backoff, and meanwhile the follow last heard (kept on the machine) holds.
|
||||
var grants <-chan agent.Update
|
||||
var retry <-chan time.Time
|
||||
opened, grantsCaught, tries := false, false, 0
|
||||
failed := func(why string) {
|
||||
grants, tries = nil, tries+1
|
||||
retry = time.After(f.env.Backoff(tries))
|
||||
f.mu.Lock()
|
||||
f.unwatched = why
|
||||
f.mu.Unlock()
|
||||
f.sayOnce("the grants part's user-list key is not watched, tried again: " + why)
|
||||
}
|
||||
open := func() {
|
||||
if grants != nil || retry != nil || f.env.Grants == nil || !f.busNode() {
|
||||
return
|
||||
}
|
||||
opened = true
|
||||
ch, err := f.env.Grants.Watch(ctx, delivery.GrantsUserListKey)
|
||||
if err != nil {
|
||||
failed(err.Error())
|
||||
return
|
||||
}
|
||||
grants = ch
|
||||
}
|
||||
for {
|
||||
again := false
|
||||
select {
|
||||
@@ -135,6 +201,8 @@ func (f *Follower) Run(ctx context.Context) error {
|
||||
return nil
|
||||
case <-f.kick:
|
||||
again = true
|
||||
case <-retry:
|
||||
retry = nil
|
||||
case u, ok := <-updates:
|
||||
if !ok {
|
||||
return fmt.Errorf("the watch of the compositions ended")
|
||||
@@ -144,13 +212,99 @@ func (f *Follower) Run(ctx context.Context) error {
|
||||
} else {
|
||||
f.take(u)
|
||||
}
|
||||
case u, ok := <-grants:
|
||||
if !ok {
|
||||
failed("its watch ended")
|
||||
break
|
||||
}
|
||||
if u.Caught {
|
||||
grantsCaught, tries = true, 0
|
||||
f.mu.Lock()
|
||||
f.unwatched = ""
|
||||
f.mu.Unlock()
|
||||
break
|
||||
}
|
||||
f.takeGrants(u)
|
||||
}
|
||||
if caught {
|
||||
f.settle(ctx, again)
|
||||
if !caught {
|
||||
continue
|
||||
}
|
||||
open()
|
||||
if opened && !grantsCaught && grants != nil {
|
||||
continue // the switch is not heard yet: nothing is written before it is
|
||||
}
|
||||
f.mu.Lock()
|
||||
if f.unwatched != f.state.Unwatched {
|
||||
again = true // the have says whether the key is watched
|
||||
}
|
||||
f.mu.Unlock()
|
||||
f.settle(ctx, again)
|
||||
}
|
||||
}
|
||||
|
||||
// busNode says some composition of this node carries the bus's user list.
|
||||
func (f *Follower) busNode() bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.cur.key != "" || len(f.srcs) > 0
|
||||
}
|
||||
|
||||
// takeGrants keeps the grants part's key as heard; a value it cannot read is said and leaves the controller's list
|
||||
// followed (no list is followed that the node cannot check).
|
||||
func (f *Follower) takeGrants(u agent.Update) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.grantsRev = u.Revision
|
||||
if u.Deleted {
|
||||
f.grants = nil
|
||||
f.keep(nil, 0)
|
||||
return
|
||||
}
|
||||
g, err := delivery.DecodeGrantsUserList(u.Value)
|
||||
if err != nil {
|
||||
f.grants = nil
|
||||
f.keep(nil, 0)
|
||||
f.sayOnceLocked("the grants part's user-list key cannot be read, so the controller's list is followed: " + err.Error())
|
||||
return
|
||||
}
|
||||
f.grants = &g
|
||||
f.keep(u.Value, u.Revision)
|
||||
}
|
||||
|
||||
// keep keeps the key last heard on the machine (Env.Keep), so a follower that cannot watch it follows what it said.
|
||||
func (f *Follower) keep(value []byte, rev uint64) {
|
||||
if f.env.Keep != nil {
|
||||
f.env.Keep(value, rev)
|
||||
}
|
||||
}
|
||||
|
||||
// Following is which list the follower takes now: the grants part's while its key says so, the controller's otherwise.
|
||||
func (f *Follower) Following() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.followingLocked()
|
||||
}
|
||||
|
||||
func (f *Follower) followingLocked() string {
|
||||
if f.grants != nil && f.grants.Follow == delivery.FollowGrants {
|
||||
return delivery.FollowGrants
|
||||
}
|
||||
return delivery.FollowController
|
||||
}
|
||||
|
||||
func (f *Follower) unwatchedNow() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.unwatched
|
||||
}
|
||||
|
||||
func (f *Follower) key() string {
|
||||
if f.env.Key == nil {
|
||||
return ""
|
||||
}
|
||||
return f.env.Key()
|
||||
}
|
||||
|
||||
// take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted.
|
||||
func (f *Follower) take(u agent.Update) {
|
||||
node, module, _, isNode, err := delivery.ParseComposedKey(u.Key)
|
||||
@@ -200,6 +354,17 @@ func (f *Follower) take(u agent.Update) {
|
||||
// A source never goes back: a key deleted, or a newer one refused, leaves the list written as it is, since an older
|
||||
// list would take away accounts a newer one gave.
|
||||
func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
f.mu.Lock()
|
||||
follow := f.followingLocked()
|
||||
if follow != f.following {
|
||||
// The switch turned: the other source is written, though neither changed.
|
||||
again = true
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if follow == delivery.FollowGrants {
|
||||
f.settleGrants(ctx, again)
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
newest := f.cur
|
||||
for _, s := range f.srcs {
|
||||
@@ -215,7 +380,8 @@ func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
at := fmt.Sprintf("%s@%d", newest.key, newest.rev)
|
||||
f.mu.Unlock()
|
||||
|
||||
w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC()}
|
||||
w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC(),
|
||||
Follows: delivery.FollowController, Key: f.key(), Unwatched: f.unwatchedNow()}
|
||||
var changed bool
|
||||
body, err := f.env.Resolve(ctx, newest.resource)
|
||||
if err == nil {
|
||||
@@ -248,6 +414,7 @@ func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
f.mu.Lock()
|
||||
f.mod, f.state = newest.module, w
|
||||
f.written = err == nil
|
||||
f.following = delivery.FollowController
|
||||
f.mu.Unlock()
|
||||
if err != nil {
|
||||
f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err))
|
||||
@@ -256,12 +423,149 @@ func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
at, w.Users, short(w.Digest)))
|
||||
}
|
||||
// Reported when what it says changed: a look again that found the file as it was says nothing new.
|
||||
if f.env.Report != nil && (changed || prev.Source != w.Source || prev.Revision != w.Revision ||
|
||||
prev.Digest != w.Digest || prev.Error != w.Error || handedKey(prev.Handed) != handedKey(w.Handed)) {
|
||||
if f.env.Report != nil && (changed || reportable(prev, w)) {
|
||||
f.env.Report(ctx, newest.module, w)
|
||||
}
|
||||
}
|
||||
|
||||
// reportable says a have's user list says something new: a look again that found the file as it was says nothing.
|
||||
func reportable(prev, w Written) bool {
|
||||
return prev.Source != w.Source || prev.Revision != w.Revision || prev.Digest != w.Digest || prev.Error != w.Error ||
|
||||
prev.Follows != w.Follows || prev.Key != w.Key || handedKey(prev.Handed) != handedKey(w.Handed) ||
|
||||
prev.Unwatched != w.Unwatched
|
||||
}
|
||||
|
||||
// settleGrants writes the grants part's list, the object its key names, into the file the controller's composition of
|
||||
// the bus module declares, made 0600 (ADR 0547 clause 7): when the key moved, when the switch turned, or when asked.
|
||||
func (f *Follower) settleGrants(ctx context.Context, again bool) {
|
||||
f.mu.Lock()
|
||||
g, rev := *f.grants, f.grantsRev
|
||||
file := f.cur
|
||||
for _, s := range f.srcs {
|
||||
if s.rev > file.rev {
|
||||
file = s
|
||||
}
|
||||
}
|
||||
if !again && f.following == delivery.FollowGrants && f.state.Revision == rev && f.written {
|
||||
f.mu.Unlock()
|
||||
return
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
w := Written{Source: delivery.GrantsUserListSource, Revision: rev, Wanted: g.Digest, At: f.env.Now().UTC(),
|
||||
Follows: delivery.FollowGrants, Key: f.key(), Unwatched: f.unwatchedNow()}
|
||||
var err error
|
||||
var changed bool
|
||||
resource, rerr := mode0600(file.resource)
|
||||
switch {
|
||||
case file.key == "" || rerr != nil:
|
||||
err = fmt.Errorf("no composition of this node declares the bus's user-list file, so the grants part's list has nowhere to be written")
|
||||
case g.Digest == "":
|
||||
err = fmt.Errorf("the grants part's key names no list yet; the list on the machine is kept")
|
||||
case f.env.Fetch == nil || f.env.Open == nil:
|
||||
err = fmt.Errorf("this engine cannot read or open the grants part's list")
|
||||
case g.SealedTo != "" && w.Key != "" && g.SealedTo != delivery.SealingKeyID(w.Key):
|
||||
err = fmt.Errorf("the grants part's list is sealed to another key than this node's (%s); the list on the machine is kept until it seals to this one", g.SealedTo)
|
||||
}
|
||||
var body []byte
|
||||
if err == nil {
|
||||
var object []byte
|
||||
if object, err = f.env.Fetch(ctx, g.Object); err == nil {
|
||||
body, err = delivery.OpenUserList(object, g.Digest, f.env.Open)
|
||||
}
|
||||
if err != nil {
|
||||
err = fmt.Errorf("the grants part's list %s could not be had: %w", short(g.Digest), err)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
w.Users = Users(body)
|
||||
if err = f.guard(resource, body); err != nil {
|
||||
if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil {
|
||||
w.Digest = Digest(disk)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
w.Digest, changed, err = f.env.Write(ctx, file.module, resource, body)
|
||||
} else if w.Digest == "" && file.key != "" && f.env.OnDisk != nil {
|
||||
if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil {
|
||||
w.Digest = Digest(disk)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
w.Error = err.Error()
|
||||
}
|
||||
f.mu.Lock()
|
||||
prev := f.state
|
||||
// The grants part's list is not handed to the vault: the handing is of what the controller's list loaded (clause 9).
|
||||
w.Handed = prev.Handed
|
||||
f.mod, f.state = file.module, w
|
||||
f.written = err == nil
|
||||
f.following = delivery.FollowGrants
|
||||
f.mu.Unlock()
|
||||
at := fmt.Sprintf("%s@%d", delivery.GrantsUserListSource, rev)
|
||||
if err != nil {
|
||||
f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err))
|
||||
} else if changed {
|
||||
f.env.Say(fmt.Sprintf("the bus's user list from the grants part (%s, revision %d) is written alone (%d users, %s); the bus reloads it itself",
|
||||
at, g.Revision, w.Users, short(w.Digest)))
|
||||
}
|
||||
if f.env.Report != nil && file.module != "" && (changed || reportable(prev, w)) {
|
||||
f.env.Report(ctx, file.module, w)
|
||||
}
|
||||
}
|
||||
|
||||
// Hold is c as the apply may write it (C3 of the switch's review): while the grants part's list is followed, the bus
|
||||
// module's composition from the controller carries the controller's list, so its user-list resource holds the list on the
|
||||
// machine instead (made 0600), or is left out when the machine holds none. A list on the machine that cannot be read
|
||||
// is an error: left out, the apply would remove the file as an orphan and the bus would refuse every connection at its
|
||||
// next reload, so the bus module's apply waits instead. Any other composition, or the controller's list followed, is c
|
||||
// unchanged.
|
||||
func (f *Follower) Hold(c delivery.Composed) (delivery.Composed, error) {
|
||||
if f.Following() != delivery.FollowGrants {
|
||||
return c, nil
|
||||
}
|
||||
out := c
|
||||
out.Resources = nil
|
||||
for _, raw := range c.Resources {
|
||||
if r, _, _, ok := userList([]json.RawMessage{raw}); ok {
|
||||
var held json.RawMessage
|
||||
body, err := f.env.OnDisk(r)
|
||||
if err != nil {
|
||||
return c, fmt.Errorf("the user list on the machine could not be read, so the apply waits and the file stays: %w", err)
|
||||
}
|
||||
if body != nil {
|
||||
var m map[string]any
|
||||
if json.Unmarshal(r, &m) == nil {
|
||||
delete(m, "digest")
|
||||
delete(m, "size")
|
||||
m["content"], m["mode"] = string(body), "0600"
|
||||
held, _ = json.Marshal(m)
|
||||
}
|
||||
}
|
||||
if held != nil {
|
||||
out.Resources = append(out.Resources, held)
|
||||
}
|
||||
continue
|
||||
}
|
||||
out.Resources = append(out.Resources, raw)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mode0600 is a user-list resource made mode 0600, whatever its composition said (ADR 0547 clause 7).
|
||||
func mode0600(raw json.RawMessage) (json.RawMessage, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no resource")
|
||||
}
|
||||
var r map[string]any
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r["mode"] = "0600"
|
||||
return json.Marshal(r)
|
||||
}
|
||||
|
||||
// hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a
|
||||
// new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash.
|
||||
func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded {
|
||||
|
||||
@@ -33,18 +33,29 @@ func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 6
|
||||
// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once
|
||||
// the watch began, so a put after it is never handed before the keys there before it.
|
||||
type feed struct {
|
||||
ch chan agent.Update
|
||||
rev uint64
|
||||
first []agent.Update
|
||||
ch chan agent.Update
|
||||
rev uint64
|
||||
first []agent.Update
|
||||
// watching is closed once the watch caught up, so a later put is not read before the keys there before it.
|
||||
watching chan struct{}
|
||||
once sync.Once
|
||||
// err refuses the watch, as the bus refuses a watch it grants nothing of.
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
for _, u := range f.first {
|
||||
f.ch <- u
|
||||
}
|
||||
f.ch <- agent.Update{Caught: true}
|
||||
close(f.watching)
|
||||
f.once.Do(func() {
|
||||
if f.watching != nil {
|
||||
close(f.watching)
|
||||
}
|
||||
})
|
||||
return f.ch, nil
|
||||
}
|
||||
|
||||
@@ -114,6 +125,15 @@ func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p
|
||||
return k, raw
|
||||
}
|
||||
|
||||
// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to
|
||||
// follow, and the list sealed to the node's key; one sealed to another key, or one that does not open to the key's digest.
|
||||
type gput struct {
|
||||
follow, content string
|
||||
otherKey, badDigest bool
|
||||
// after puts it once the grants part's list was written, as a hand-back comes after a take-over.
|
||||
after bool
|
||||
}
|
||||
|
||||
type put struct {
|
||||
module, fp, content string
|
||||
stranger, deleted bool
|
||||
@@ -135,6 +155,14 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
wantUsers int
|
||||
wantErr string
|
||||
wantRefused bool
|
||||
grants []gput
|
||||
// grantsRefused refuses the watch of the grants part's key; kept is the key last heard, kept on the machine;
|
||||
// wantUnwatched says the have then says why it is not watched.
|
||||
grantsRefused bool
|
||||
kept *gput
|
||||
wantUnwatched bool
|
||||
// wantSource and wantFollows are what the last have says it took the list from and follows.
|
||||
wantSource, wantFollows string
|
||||
// vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the
|
||||
// machine (ADR 0547 clause 9).
|
||||
vault bool
|
||||
@@ -180,6 +208,30 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3},
|
||||
{name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true,
|
||||
wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"},
|
||||
// ADR 0547 clauses 7, 8: never both lists; the grants part's from take-over, the controller's otherwise.
|
||||
{name: "from take-over the grants part's list is written into the same file",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
|
||||
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
||||
{name: "a hand-back writes the newest controller composition once",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
||||
grants: []gput{{follow: delivery.FollowGrants, content: three}, {follow: delivery.FollowController, content: three, after: true}},
|
||||
wantWrites: []string{three, two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
||||
{name: "a refused watch with no key kept follows the controller's list and says it is not watched",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
||||
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
||||
{name: "a refused watch keeps the follow last heard, never the controller's list",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
||||
kept: &gput{follow: delivery.FollowGrants, content: three},
|
||||
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
||||
{name: "in shadow the grants part's list is not taken",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
|
||||
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
||||
{name: "a grants list that does not open to its digest is refused and the file kept",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, badDigest: true}},
|
||||
onDisk: two, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
||||
{name: "a grants list sealed to another key is refused and the file kept",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, otherKey: true}},
|
||||
onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})}
|
||||
@@ -188,7 +240,66 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
var writes []string
|
||||
var reports []Written
|
||||
reported := make(chan struct{}, 64)
|
||||
fl := New(Env{Node: node, Composed: f,
|
||||
gf := &feed{ch: make(chan agent.Update, 64)}
|
||||
if c.grantsRefused {
|
||||
gf.err = errors.New("Permissions Violation for Publish to $JS.API.STREAM.INFO.KV_mesh-grants_user-list")
|
||||
}
|
||||
var grantsAfter []agent.Update
|
||||
var keptRaw []byte
|
||||
sealed := map[string][]byte{}
|
||||
nodeKey, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gs := c.grants
|
||||
if c.kept != nil {
|
||||
gs = append(append([]gput(nil), gs...), *c.kept)
|
||||
}
|
||||
for i, g := range gs {
|
||||
to := nodeKey.PublicKey
|
||||
if g.otherKey {
|
||||
other, _ := seal.GenerateKeyPair()
|
||||
to = other.PublicKey
|
||||
}
|
||||
digest := delivery.UserListDigest([]byte(g.content))
|
||||
if g.badDigest {
|
||||
digest = delivery.UserListDigest([]byte("another list"))
|
||||
}
|
||||
object, err := delivery.SealUserList([]byte(g.content), to)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sealed[delivery.UserListObjectName(digest)] = object
|
||||
raw, err := delivery.EncodeGrantsUserList(delivery.GrantsUserList{Follow: g.follow, Digest: digest,
|
||||
Object: delivery.UserListObjectName(digest), SealedTo: delivery.SealingKeyID(to), Node: node, Revision: uint64(i + 1)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u := agent.Update{Key: delivery.GrantsUserListKey, Value: raw, Revision: uint64(i + 1)}
|
||||
switch {
|
||||
case c.kept != nil && i == len(gs)-1:
|
||||
keptRaw = raw
|
||||
case g.after:
|
||||
grantsAfter = append(grantsAfter, u)
|
||||
default:
|
||||
gf.first = append(gf.first, u)
|
||||
}
|
||||
}
|
||||
fl := New(Env{Node: node, Composed: f, Grants: gf,
|
||||
Kept: func() ([]byte, uint64) { return keptRaw, 9 },
|
||||
Keep: func([]byte, uint64) {},
|
||||
Backoff: func(int) time.Duration { return time.Hour },
|
||||
Fetch: func(_ context.Context, name string) ([]byte, error) {
|
||||
if o, ok := sealed[name]; ok {
|
||||
return o, nil
|
||||
}
|
||||
return nil, errors.New("no object " + name)
|
||||
},
|
||||
Open: func(b seal.Box) ([]byte, error) {
|
||||
out, err := seal.Open(b, nodeKey.PrivateKey)
|
||||
return []byte(out), err
|
||||
},
|
||||
Key: func() string { return nodeKey.PublicKey },
|
||||
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} },
|
||||
Resolve: func(_ context.Context, r json.RawMessage) ([]byte, error) {
|
||||
var res struct{ Content, Digest string }
|
||||
@@ -254,9 +365,30 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- fl.Run(ctx) }()
|
||||
<-f.watching
|
||||
// A later put after the first list is written: the first write waits for the grants part's key to catch up.
|
||||
if len(c.after) > 0 {
|
||||
for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) {
|
||||
mu.Lock()
|
||||
n := len(writes)
|
||||
mu.Unlock()
|
||||
if n > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range c.after {
|
||||
f.ch <- update(p)
|
||||
}
|
||||
if len(grantsAfter) > 0 {
|
||||
for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
||||
if _, st := fl.State(); st.Source == delivery.GrantsUserListSource || time.Now().After(until) {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, u := range grantsAfter {
|
||||
gf.ch <- u
|
||||
}
|
||||
}
|
||||
if c.again {
|
||||
// After the first write is reported, so the look again is not folded into it.
|
||||
select {
|
||||
@@ -286,6 +418,14 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
t.Fatal("a refused list was not reported")
|
||||
}
|
||||
}
|
||||
if c.wantSource != "" {
|
||||
for until := time.Now().Add(5 * time.Second); ; {
|
||||
if _, st := fl.State(); st.Source == c.wantSource && (c.wantErr == "" || st.Error != "") || time.Now().After(until) {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond) // nothing more is written
|
||||
cancel()
|
||||
<-done
|
||||
@@ -295,6 +435,38 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
t.Fatalf("written %q, want %q", writes, c.wantWrites)
|
||||
}
|
||||
module, st := fl.State()
|
||||
if c.wantUnwatched != (st.Unwatched != "") {
|
||||
t.Fatalf("the have says unwatched %q", st.Unwatched)
|
||||
}
|
||||
// While the grants part's list is followed, an apply of the controller's composition writes not its list.
|
||||
if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" {
|
||||
cmp, err := delivery.DecodeComposed(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := fl.Hold(cmp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, _, hasList := userList(held.Resources)
|
||||
if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 {
|
||||
t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList)
|
||||
}
|
||||
// A list on the machine that cannot be read stops the apply; it is never left out to be removed.
|
||||
if c.wantFollows == delivery.FollowGrants {
|
||||
readErr := New(Env{Node: node, Composed: f, OnDisk: func(json.RawMessage) ([]byte, error) {
|
||||
return nil, errors.New("permission denied")
|
||||
}})
|
||||
readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants}
|
||||
if _, err := readErr.Hold(cmp); err == nil || !strings.Contains(err.Error(), "could not be read") {
|
||||
t.Fatalf("an unreadable list on the machine was held as %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.wantSource != "" && (st.Source != c.wantSource || st.Key != nodeKey.PublicKey ||
|
||||
(c.wantFollows != "" && st.Follows != c.wantFollows)) {
|
||||
t.Fatalf("the have says source %q follows %q key %t; want %q %q", st.Source, st.Follows, st.Key == nodeKey.PublicKey, c.wantSource, c.wantFollows)
|
||||
}
|
||||
switch {
|
||||
case c.wantHanded == 0 && st.Handed != nil:
|
||||
t.Fatalf("handed with no vault key or no hash: %+v", st.Handed)
|
||||
|
||||
@@ -49,6 +49,13 @@ func keep(root, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Retire takes a process's build out of the way before another is unpacked where it was: renamed under
|
||||
// retired/, and deleted at once if nothing runs from it, else at a later sweep. A process nobody judges
|
||||
// is replaced through it as a judged one is (novox/hq issue 734): deleting the running build before the
|
||||
// restart deleted the working directory of the process still running from it, and every command it
|
||||
// started until the restart failed with "getwd: no such file or directory".
|
||||
func Retire(root, name, dir string, now time.Time) error { return retire(root, name, dir, now) }
|
||||
|
||||
// retire takes a build that has no reader any more out of the way: renamed under retired/, and deleted
|
||||
// at once if nothing runs from it, else at a later sweep. A path that is not there is nothing to do.
|
||||
func retire(root, name, dir string, now time.Time) error {
|
||||
@@ -98,8 +105,11 @@ func Sweep(root, name string) ([]string, error) {
|
||||
return draining, nil
|
||||
}
|
||||
|
||||
// RunningFrom is every process whose executable is a file under dir — renamed there, or deleted from
|
||||
// there since it started. A variable so a test can name the machine's processes.
|
||||
// RunningFrom is every process whose executable is a file under dir, or whose working directory is in
|
||||
// it — renamed there, or deleted from there since it started. A working directory counts as much as an
|
||||
// image: a process started by the one that runs from a build (a `go version` a builder asks) works in
|
||||
// the build's directory and runs nothing from it (novox/hq issue 734). A variable so a test can name the
|
||||
// machine's processes.
|
||||
var RunningFrom = func(dir string) []int {
|
||||
return runningFromProc("/proc", dir)
|
||||
}
|
||||
@@ -116,14 +126,17 @@ func runningFromProc(proc, dir string) []int {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
exe, err := os.Readlink(filepath.Join(proc, e.Name(), "exe"))
|
||||
if err != nil {
|
||||
// Gone between the listing and the read, a kernel thread, or another user's process this
|
||||
// engine may not read — it runs as root, so in practice the first two.
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSuffix(exe, " (deleted)"), dir) {
|
||||
pids = append(pids, pid)
|
||||
for _, link := range []string{"exe", "cwd"} {
|
||||
at, err := os.Readlink(filepath.Join(proc, e.Name(), link))
|
||||
if err != nil {
|
||||
// Gone between the listing and the read, a kernel thread, or another user's process this
|
||||
// engine may not read — it runs as root, so in practice the first two.
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSuffix(at, " (deleted)")+string(filepath.Separator), dir) {
|
||||
pids = append(pids, pid)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Ints(pids)
|
||||
@@ -140,14 +153,30 @@ func joinPIDs(pids []int) string {
|
||||
|
||||
// SweepAll is Sweep for every process the engine keeps anything about: what the witness does on every
|
||||
// look, so a build retired while its process drained goes once the process has.
|
||||
//
|
||||
// Every process the engine keeps a directory for, not only those it judges: a process nobody judges keeps
|
||||
// nothing but the builds it retired, and those drain the same way (novox/hq issue 734).
|
||||
func SweepAll(root string) []string {
|
||||
entries, err := os.ReadDir(filepath.Join(root, ".witness"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var draining []string
|
||||
for _, s := range all(root) {
|
||||
kept, err := Sweep(root, s.Process)
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
kept, err := Sweep(root, name)
|
||||
if err != nil {
|
||||
draining = append(draining, fmt.Sprintf("%s: %v", s.Process, err))
|
||||
draining = append(draining, fmt.Sprintf("%s: %v", name, err))
|
||||
}
|
||||
draining = append(draining, kept...)
|
||||
// A process nobody judges whose last retired build has drained leaves nothing to keep.
|
||||
if s, err := Load(root, name); err == nil && s.Process == "" && len(kept) == 0 {
|
||||
_ = os.Remove(retiredDir(root, name))
|
||||
_ = os.Remove(Dir(root, name))
|
||||
}
|
||||
}
|
||||
return draining
|
||||
}
|
||||
|
||||
@@ -126,8 +126,26 @@ func Save(root string, s State) error {
|
||||
return os.Rename(tmp.Name(), statePath(root, s.Process))
|
||||
}
|
||||
|
||||
// Forget is a witnessed process no longer declared: everything kept about it goes with it.
|
||||
func Forget(root, name string) error { return os.RemoveAll(Dir(root, name)) }
|
||||
// Forget is a process no longer witnessed, or no longer declared: everything kept about it goes with it,
|
||||
// but for a build a process still runs from. That one is retired and deleted once nothing runs from it
|
||||
// (draining.go): a process whose witness was taken away, or which never had one, is still running from
|
||||
// whatever was retired for it until its unit restarts (novox/hq issue 734).
|
||||
func Forget(root, name string) error {
|
||||
dir := Dir(root, name)
|
||||
if _, err := os.Lstat(dir); os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err := retire(root, name, previousDir(root, name), time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(statePath(root, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if draining, err := Sweep(root, name); err != nil || len(draining) > 0 {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(dir)
|
||||
}
|
||||
|
||||
// Placing is what the applier is to do with a declared build of a witnessed process.
|
||||
type Placing struct {
|
||||
|
||||
@@ -22,6 +22,14 @@
|
||||
"type": "string",
|
||||
"description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.",
|
||||
"default": ""
|
||||
},
|
||||
"build-cpu-share": {
|
||||
"type": "number",
|
||||
"description": "The most of this node's processors its builds may take together, in percent (1 to 100): the node-engine holds every build module's runtime of the node in one slice at idle priority under this quota, and each build step sizes its threads from it (novox/hq ADR 0553). Set per node through the node's settings layer; the composer reads it for the build seats' runtimes, the engine's own code does not.",
|
||||
"minimum": 1,
|
||||
"maximum": 100,
|
||||
"default": 50,
|
||||
"runtime": true
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
|
||||
Reference in New Issue
Block a user