novox/mesh-host
6
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

Judge the machine's own networking beside what its modules run (hq ADR 0241)

Browse Source 
A VPN client rewrote the laptop's resolver file and every mesh name failed
while each module read healthy: nothing asked the machine. The engine now
looks every 30 s at the resolver file the uplink holder declared (naming
the program that rewrote it), the names through each listed resolver
(NXDOMAIN for a mesh name's IPv6 address is a finding, issue 262), the
tunnel's handshake with the hub, the bus and the default route; a part is
unhealthy on its second failing look, and the statement carries it.
This commit is contained in:
jschoubben
2026-10-07 18:43:39 +02:00
parent 56e2ebec4b
commit 429ea42357
8 changed files with 1476 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/main.go
+105 -4
View File
@@ -17,6 +17,7 @@ import (
"flag"
"fmt"
"io"
"net"
"os"
"os/signal"
"path/filepath"
@@ -35,6 +36,7 @@ import (
"github.com/novox/mesh-host/internal/inventory"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/liveness"
"github.com/novox/mesh-host/internal/network"
"github.com/novox/mesh-host/internal/outward"
"github.com/novox/mesh-host/internal/profile"
"github.com/novox/mesh-host/internal/reachable"
@@ -1168,6 +1170,10 @@ func runLink(ctx context.Context, opts options) error {
// spaced to the budget (ADR 0240 Phase B); a tool is asked of this machine's node tools over the
// link open at the time.
judging.Probes = &liveness.Probes{AskTool: queue.AskTool}
// And the machine's own networking (novox/hq ADR 0241): the resolver file the uplink holder
// declared, the names through it, the tunnel, the bus and the route — said in the same statement.
netJudge.set(network.New(network.Machine{Run: apply.ExecRunner, Linked: queue.Linked},
hostOf(mine.Membership.Broker)))
go judging.Probe(aside)
go judgeWhatRuns(aside, judging, queue, say)
}
@@ -1361,6 +1367,74 @@ const ReconcileEvery = 5 * time.Minute
// reports no health, and in a test.
var judging *liveness.Judge
// netJudge is the serving host's judge of its machine's networking (novox/hq ADR 0241); empty in a
// one-shot command and in a test, which say nothing of the network.
var netJudge networkJudge
type networkJudge struct {
mu sync.Mutex
m *network.Judge
// The resolver file the last apply declared, kept for a judge made after it.
content, owner string
declared bool
}
func (n *networkJudge) get() *network.Judge {
n.mu.Lock()
defer n.mu.Unlock()
return n.m
}
func (n *networkJudge) set(m *network.Judge) {
n.mu.Lock()
defer n.mu.Unlock()
n.m = m
m.Declare(n.content, n.owner, n.declared)
}
// declare is the resolver file an apply just applied, for the judge now and any made later.
func (n *networkJudge) declare(content, owner string, ok bool) {
n.mu.Lock()
defer n.mu.Unlock()
n.content, n.owner, n.declared = content, owner, ok
if n.m != nil {
n.m.Declare(content, owner, ok)
}
}
// hostOf is the bus's name without its port: the mesh name the machine needs most. Empty when the bus
// is reached by address, and then no mesh name is asked.
func hostOf(broker string) string {
host := broker
if h, _, err := net.SplitHostPort(broker); err == nil {
host = h
}
if net.ParseIP(host) != nil {
return ""
}
return host
}
// declaredResolvConf is the resolver file a declaration has a module write whole — the uplink holder's
// (ADR 0223) — and that module.
func declaredResolvConf(d *declaration.Declaration) (string, string, bool) {
if d == nil {
return "", "", false
}
for _, r := range d.Resources {
f, ok := r.(*declaration.File)
if !ok || f.Path != network.ResolvConf || f.CreateOnce || f.Into != "" {
continue
}
module, ok := liveness.ModuleOf(f.ID)
if !ok {
return "", "", false
}
return f.Content, module, true
}
return "", "", false
}
// How often a statement of health is said between reports: again every minute while anything is not
// healthy (to-be 48 §4), so a lost event is not a lost fault; and every five minutes anyway, so a
// controller that restarted knows a healthy machine's state without waiting for its next apply.
@@ -1387,6 +1461,19 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
}
st, changed := j.Look(ctx)
owed = owed || changed
var netSt *network.Statement
if n := netJudge.get(); n != nil {
ns, netChanged := n.Look(ctx)
netSt = &ns
owed = owed || netChanged
if netChanged {
for _, p := range ns.Parts {
if p.State == network.Unhealthy {
say(fmt.Sprintf("this machine's network is unhealthy: %s: %s (%s)", p.Part, p.Reason, p.Said))
}
}
}
}
// Never more looks than the budget (ADR 0240): said when the engine has to space its own out.
if sp := j.Spacing(); sp != spaced {
if sp > 1 {
@@ -1396,7 +1483,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
spaced = sp
}
since := time.Since(lastSaid)
if !owed && !(!st.Healthy() && since >= sayUnhealthyAgain) && since < sayAnyway {
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy)
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
continue
}
if changed {
@@ -1407,14 +1495,14 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
}
}
}
if queue.SayHealth(ctx, *healthAsReported(st)) {
if queue.SayHealth(ctx, *healthAsReported(st, netSt)) {
lastSaid, owed = time.Now(), false
}
}
}
// healthAsReported is a statement as the report and the event carry it.
func healthAsReported(st liveness.Statement) *link.Health {
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
// ReadinessContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase
// B), which is what tells the controller it may be sent the field.
h := &link.Health{Contract: link.ReadinessContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
@@ -1423,6 +1511,13 @@ func healthAsReported(st liveness.Statement) *link.Health {
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
Restarts: r.Restarts, Check: r.CheckOf(), Needs: r.NeedsOf()})
}
if ns != nil && ns.State != "" {
h.Network = &link.NetworkHealth{State: ns.State, Since: ns.Since.UTC(), Parts: []link.NetworkPart{}}
for _, p := range ns.Parts {
h.Network.Parts = append(h.Network.Parts, link.NetworkPart{Part: p.Part, State: p.State, Reason: p.Reason,
Said: p.Said, Writer: p.Writer, Owner: p.Owner, Toward: p.Toward, Since: p.Since.UTC(), Streak: p.Streak})
}
}
return h
}
@@ -1657,7 +1752,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
if j := judging; j != nil {
j.Set(liveness.LongRunning(declared, held))
st, _ := j.Look(ctx)
report.Health = healthAsReported(st)
netJudge.declare(declaredResolvConf(declared))
var netSt *network.Statement
if n := netJudge.get(); n != nil {
ns := n.Last()
netSt = &ns
}
report.Health = healthAsReported(st, netSt)
}
// Which of this machine's links face outside, for the filter the mesh writes around them
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
cmd/mesh-host/network_test.go
+54
View File
@@ -0,0 +1,54 @@
package main
import (
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/liveness"
"github.com/novox/mesh-host/internal/network"
)
// The machine's networking in the engine's statement (novox/hq ADR 0241): the file judged is the one the
// uplink holder declares whole, the mesh name asked is the bus's, and the statement carries the parts.
func TestTheResolverFileJudgedIsTheOneAModuleDeclaresWhole(t *testing.T) {
d := &declaration.Declaration{Resources: []declaration.Resource{
&declaration.File{ID: "hosts.file", Type: "file", Path: "/etc/hosts", Content: "x"},
&declaration.File{ID: "networkmanager.resolv", Type: "file", Path: network.ResolvConf, Content: "nameserver 10.10.0.1\n"},
}}
content, owner, ok := declaredResolvConf(d)
if !ok || owner != "networkmanager" || content != "nameserver 10.10.0.1\n" {
t.Fatalf("got %q %q %v", content, owner, ok)
}
d.Resources[1].(*declaration.File).CreateOnce = true
if _, _, ok := declaredResolvConf(d); ok {
t.Fatal("a seed nobody holds the machine to was judged as the declared file")
}
if _, _, ok := declaredResolvConf(nil); ok {
t.Fatal("no declaration declared a file")
}
}
func TestTheMeshNameAskedIsTheBuses(t *testing.T) {
for broker, want := range map[string]string{"anchor.internal:4222": "anchor.internal", "192.0.2.10:5671": "",
"anchor.internal": "anchor.internal", "[2001:db8::1]:4222": ""} {
if got := hostOf(broker); got != want {
t.Errorf("%s: got %q, want %q", broker, got, want)
}
}
}
func TestTheStatementCarriesTheNetwork(t *testing.T) {
at := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC)
h := healthAsReported(liveness.Statement{At: at}, &network.Statement{State: network.Unhealthy, Since: at,
Parts: []network.Part{{Part: network.PartResolvConf, State: network.Unhealthy, Writer: "FortiClient",
Owner: "networkmanager", Reason: "the resolver file was rewritten by another program", Since: at}}})
if h.Network == nil || h.Network.State != network.Unhealthy || len(h.Network.Parts) != 1 ||
h.Network.Parts[0].Writer != "FortiClient" || h.Network.Parts[0].Owner != "networkmanager" {
t.Fatalf("the statement says %+v", h.Network)
}
if h := healthAsReported(liveness.Statement{At: at}, nil); h.Network != nil {
t.Fatal("an engine with no network judge said a network")
}
}