novox/mesh-host
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

A service can be declared to reflect a file

Browse Source 
Because a running service does not re-read its configuration. Replace the file,
find the service running, do nothing -- and the machine keeps behaving as it
did while every check passes, because the file is right and the service is up.

That is not hypothetical. It is how a third node joining a mesh left the first
two carrying a private network that no longer existed, with every part of it
reporting success.

Declared state rather than a command: the declaration says the running service
must reflect these files, and the host works out that it does not. A command to
restart would be an action, and the link may not carry one -- the host refused
precisely that when I tried it, correctly, which is how this shape was arrived
at rather than the other.

Scoped to one apply. A change from an earlier one has already been reflected,
and restarting for it every time would make a steady machine bounce its
services for ever.

Also: the node generates its overlay key at enrolment and reports the public
half, and the store waits three minutes rather than one for the database --
sixty seconds is not enough for a cold machine running initdb, and it failed
that way three times, which is the worst kind of flake because a second run
always fixed it.
This commit is contained in:
jschoubben
2026-08-29 18:04:16 +02:00
parent 732905a6a6
commit 1bc97ed50d
6 changed files with 223 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apply/apply.go
+51 -4
View File
@@ -112,8 +112,13 @@ func Apply(
log(fmt.Sprintf(" %s %s (%s)", action, orphan.ID, orphan.Target))
}
// What moved in this apply, so a service that must reflect a file can be told the file
// moved. Only within one apply: a change from an earlier one has already been reflected, and
// restarting for it every time would make a steady machine restart its services for ever.
changed := map[string]bool{}
for _, resource := range d.Resources {
outcome, err := applyOne(ctx, sys, resource, run)
outcome, err := applyOne(ctx, sys, resource, run, changed)
if err != nil {
return report, known, &Error{Resource: resource.Identity(), Err: err, Done: report}
}
@@ -126,20 +131,22 @@ func Apply(
})
report.Outcomes = append(report.Outcomes, outcome)
if outcome.Action != "unchanged" {
changed[resource.Identity()] = true
log(fmt.Sprintf(" %s %s (%s)", outcome.Action, outcome.ID, outcome.Target))
}
}
return report, known, nil
}
func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner) (Outcome, error) {
func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner,
changed map[string]bool) (Outcome, error) {
switch res := r.(type) {
case *declaration.Directory:
return applyDirectory(res)
case *declaration.File:
return applyFile(res)
case *declaration.Service:
return applyService(ctx, sys, res, run)
return applyService(ctx, sys, res, run, changed)
case *declaration.Package:
return applyPackage(ctx, sys, res, run)
case *declaration.Container:
@@ -318,7 +325,25 @@ func writeAtomically(path string, content []byte, mode os.FileMode) error {
return os.Rename(tmp.Name(), path)
}
func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner) (Outcome, error) {
// reflects reports whether anything this service must mirror changed in this apply.
func reflects(r *declaration.Service, changed map[string]bool) bool {
return len(reflected(r, changed)) > 0
}
// reflected is which of them changed, so the outcome can say why the service was restarted. A
// restart with no reason given is indistinguishable from a service that keeps falling over.
func reflected(r *declaration.Service, changed map[string]bool) []string {
var which []string
for _, id := range r.RestartOn {
if changed[id] {
which = append(which, id)
}
}
return which
}
func applyService(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
changed map[string]bool) (Outcome, error) {
out := begin(r)
var changes []string
@@ -365,6 +390,28 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
return out, fmt.Errorf("%s was asked to be %s and is %s", r.Unit, r.State, after)
}
changes = append(changes, before+" to "+after)
} else if r.State == "running" && reflects(r, changed) {
// The service is already in the state it was asked for, and something it must reflect
// changed in this same apply. A running service does not re-read its configuration, so
// leaving it alone here is how a machine ends up correct on disk and wrong in fact —
// with every check passing.
if err := sys.SetServiceState(ctx, run, r.Unit, "stopped"); err != nil {
return out, fmt.Errorf("restarting %s: stopping it: %w", r.Unit, err)
}
if err := sys.SetServiceState(ctx, run, r.Unit, "running"); err != nil {
return out, fmt.Errorf("restarting %s: starting it again: %w", r.Unit, err)
}
// Read back, for the same reason as above: a unit that starts and immediately dies
// satisfies a service manager and nothing else.
after, err := sys.ServiceState(ctx, run, r.Unit)
if err != nil {
return out, err
}
if after != "running" {
return out, fmt.Errorf(
"%s was restarted to pick up a change and is %s", r.Unit, after)
}
changes = append(changes, "restarted for "+strings.Join(reflected(r, changed), ", "))
}
if len(changes) == 0 {
internal/apply/apply_test.go
+116
View File
@@ -3,6 +3,7 @@ package apply
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -943,3 +944,118 @@ func archHost(t *testing.T) system.System {
}
return s
}
func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) {
// A running service does not re-read its configuration. Replace the file, find the service
// already running, do nothing — and the machine keeps behaving as it did while every check
// passes, because the file is right and the service is up.
//
// That is how a third node joining a mesh left the first two carrying a network that no
// longer existed. Found in the lab; this is the shape of the fix.
dir := t.TempDir()
path := filepath.Join(dir, "thing.conf")
d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"first\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, path))
var commands []string
run := recordingServices(&commands)
if _, state, err := Apply(context.Background(), archHost(t), d, store.State{},
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
} else {
// Second apply with the same content: nothing moved, so nothing restarts. A machine that
// restarted its services on every reconcile would never be steady.
commands = nil
if _, _, err := Apply(context.Background(), archHost(t), d, state,
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
}
for _, c := range commands {
if strings.Contains(c, "stop") {
t.Errorf("an unchanged declaration restarted the service: %s", c)
}
}
// Now the file changes. The service is already running and must still be restarted.
changedDecl := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"second\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, path))
commands = nil
if _, _, err := Apply(context.Background(), archHost(t), changedDecl, state,
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
}
var stopped, started bool
for _, c := range commands {
if strings.Contains(c, "stop thing.service") {
stopped = true
}
if strings.Contains(c, "start thing.service") {
started = true
}
}
if !stopped || !started {
t.Errorf("the file changed and the service was not restarted; commands were %v", commands)
}
}
}
func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) {
// The list is what it reflects, not everything in the declaration. A service restarted by any
// change anywhere would make every apply a fleet-wide bounce.
dir := t.TempDir()
conf := filepath.Join(dir, "thing.conf")
other := filepath.Join(dir, "unrelated")
first := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
{"id":"other","type":"file","path":%q,"content":"one\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, conf, other))
var commands []string
run := recordingServices(&commands)
_, state, err := Apply(context.Background(), archHost(t), first, store.State{},
store.OriginCarried, run, nil)
if err != nil {
t.Fatal(err)
}
second := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"conf","type":"file","path":%q,"content":"same\n","mode":"0644"},
{"id":"other","type":"file","path":%q,"content":"two\n","mode":"0644"},
{"id":"svc","type":"service","unit":"thing.service","state":"running","restart-on":["conf"]}
]}`, conf, other))
commands = nil
if _, _, err := Apply(context.Background(), archHost(t), second, state,
store.OriginCarried, run, nil); err != nil {
t.Fatal(err)
}
for _, c := range commands {
if strings.Contains(c, "stop") {
t.Errorf("a change to a file the service does not name restarted it: %s", c)
}
}
}
// recordingServices answers the way a machine with a running unit would, and remembers what it
// was asked to do — which is what a restart has to be proved by, since "running" looks the same
// before and after one.
func recordingServices(commands *[]string) Runner {
return func(_ context.Context, name string, args ...string) (string, error) {
line := name + " " + strings.Join(args, " ")
*commands = append(*commands, line)
switch {
case strings.Contains(line, "is-enabled"):
return "enabled", nil
case strings.Contains(line, "show") && strings.Contains(line, "ActiveState"):
return "LoadState=loaded\nActiveState=active\nSubState=running", nil
}
return "", nil
}
}
internal/declaration/declaration.go
+14
View File
@@ -113,6 +113,20 @@ type Service struct {
// Without this the host could start a unit and not make it survive a reboot, which is a
// declaration that reports success and stops being true at the next power cut.
Boot string `json:"boot,omitempty"`
// RestartOn names resources whose change means this service must be restarted.
//
// Because a running service does not re-read its configuration. Replace the file, find the
// service already running, do nothing, and the machine keeps behaving the way it did before —
// while every check passes, because the file is right and the service is up. That is not
// hypothetical: it is how a third node joining a mesh left the first two carrying a network
// that no longer existed, and every part of it reported success.
//
// This is declared state rather than a command. The declaration says the running service must
// reflect these files; the host works out that it does not and acts. A *command* to restart
// would be an action, and the link may not carry one (novox/hq ADR 0005) — so this is not a
// way around that rule, it is the shape the rule leaves.
RestartOn []string `json:"restart-on,omitempty"`
}
func (s *Service) Identity() string { return s.ID }
internal/link/enrol.go
+16 -6
View File
@@ -23,10 +23,19 @@ func QueueFor(node string) string { return "node." + node }
// EnrolRequest is what this node says when joining.
type EnrolRequest struct {
Node string `json:"node"`
Secret string `json:"secret"`
PublicKey []byte `json:"public_key"`
Profile map[string]any `json:"profile,omitempty"`
Node string `json:"node"`
Secret string `json:"secret"`
PublicKey []byte `json:"public_key"`
// OverlayKey is the public half of this node's key on the private network — a different key
// from PublicKey above, generated at the same moment and for a different purpose.
//
// Sent with enrolment because the overlay is the first declaration a node receives, and the
// mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled
// and unreachable for a round trip, which is the state everything else here works to avoid.
OverlayKey string `json:"overlay_key,omitempty"`
Profile map[string]any `json:"profile,omitempty"`
}
// EnrolReply is what the mesh says back.
@@ -56,7 +65,7 @@ var ErrRefused = errors.New("the mesh refused this enrolment")
// says once it is in, and the secret travels again because the control plane must not have to ask
// the broker who connected.
func Enrol(ctx context.Context, address, pin, node, secret string, public []byte,
profile map[string]any, timeout time.Duration) (EnrolReply, error) {
overlayKey string, profile map[string]any, timeout time.Duration) (EnrolReply, error) {
config, err := PinnedConfig(pin)
if err != nil {
@@ -101,7 +110,8 @@ func Enrol(ctx context.Context, address, pin, node, secret string, public []byte
return EnrolReply{}, err
}
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, Profile: profile}
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
OverlayKey: overlayKey, Profile: profile}
body, err := json.Marshal(request)
if err != nil {
return EnrolReply{}, err