Name the machines a failing network part points at in the node's health report, so the watchman holds it where the controller does (issue 774)
delivery.NetworkPart.Targets (mesh-sdk 6eae076) is filled where the report is made, by the controller's own rule: - the hub's node for "hub"; - each resolver's node by its address on the private network; - none when the part points at nothing, at this machine, or at an address no other mesh machine has. The names come from the placements the node's own composer holds (Decided.Address, Decided.Hub). A spoke's tunnel names only the hub, so the tunnel cannot name a resolver on another spoke. A node that does not compose its own names no target. Test: rows in TestAHealthStatementIsSaidOnTheLinkOpenNow (hub, two resolvers, a VPN's resolver, its own, a healthy part).
This commit is contained in:
@@ -1488,7 +1488,11 @@ func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health,
|
|||||||
peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen})
|
peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers))
|
var machines link.Machines
|
||||||
|
if c := machinesNow.Load(); c != nil {
|
||||||
|
machines.ByAddress, machines.Hub = c.Machines()
|
||||||
|
}
|
||||||
|
_, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines))
|
||||||
switch {
|
switch {
|
||||||
case err != nil && err.Error() != *unwritten:
|
case err != nil && err.Error() != *unwritten:
|
||||||
*unwritten = err.Error()
|
*unwritten = err.Error()
|
||||||
|
|||||||
@@ -626,6 +626,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if composer != nil {
|
if composer != nil {
|
||||||
|
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
|
||||||
composes.set(delivery.ComposesOwn)
|
composes.set(delivery.ComposesOwn)
|
||||||
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
||||||
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
||||||
@@ -768,6 +769,10 @@ func busyModules() []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// machinesNow is the composer whose placements name the mesh's machines by address, for the health report's targets
|
||||||
|
// (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target.
|
||||||
|
var machinesNow atomic.Pointer[selfcompose.Composer]
|
||||||
|
|
||||||
// keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of
|
// keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of
|
||||||
// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9).
|
// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9).
|
||||||
var keptRetired atomic.Bool
|
var keptRetired atomic.Bool
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 h1:eQIsSB3Z72MNjirW3TbKbvFgkhwjCAgOAqWd0U4ldCE=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 h1:4qfwx2bv2vosFM6tEbBZUeaQ/pfRauXZuNI3whul30g=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package link
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -69,12 +70,32 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
|
|||||||
{Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}}}
|
{Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}}}
|
||||||
}
|
}
|
||||||
peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}}
|
peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}}
|
||||||
r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers)
|
r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers, Machines{})
|
||||||
if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" ||
|
if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" ||
|
||||||
len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" ||
|
len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" ||
|
||||||
len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" {
|
len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" {
|
||||||
t.Fatalf("the report is %+v", r)
|
t.Fatalf("the report is %+v", r)
|
||||||
}
|
}
|
||||||
|
// The machines an unhealthy part points at, as the controller holds it there; none for the machine's own.
|
||||||
|
machines := Machines{ByAddress: map[string]string{"10.10.0.1": "novox", "10.10.0.4": "ace", "10.10.0.9": "anchor"}, Hub: "novox"}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
toward []string
|
||||||
|
state string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"the tunnel to the hub", []string{"hub"}, StateUnhealthy, []string{"novox"}},
|
||||||
|
{"two mesh resolvers", []string{"10.10.0.1", "10.10.0.4"}, StateUnhealthy, []string{"novox", "ace"}},
|
||||||
|
{"a resolver a VPN client wrote in is the machine's own", []string{"10.10.0.1", "192.0.2.53"}, StateUnhealthy, nil},
|
||||||
|
{"its own resolver is its own", []string{"10.10.0.9"}, StateUnhealthy, nil},
|
||||||
|
{"a healthy part points at nothing", []string{"hub"}, StateHealthy, nil},
|
||||||
|
} {
|
||||||
|
h := Health{Network: &NetworkHealth{State: tc.state, Parts: []NetworkPart{{Part: "names", State: tc.state, Toward: tc.toward}}}}
|
||||||
|
got := NodeHealthOf("anchor", h, nil, machines).Network.Parts[0].Targets
|
||||||
|
if !slices.Equal(got, tc.want) {
|
||||||
|
t.Errorf("%s: targets %v, want %v", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
store := &memHealth{}
|
store := &memHealth{}
|
||||||
var last *NodeHealth
|
var last *NodeHealth
|
||||||
clock := func() time.Time { return since }
|
clock := func() time.Time { return since }
|
||||||
@@ -84,10 +105,10 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
|
|||||||
wrote bool
|
wrote bool
|
||||||
}{
|
}{
|
||||||
{h: r, wrote: true},
|
{h: r, wrote: true},
|
||||||
{h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers)},
|
{h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers, Machines{})},
|
||||||
{h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers), fresh: true},
|
{h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers, Machines{}), fresh: true},
|
||||||
{h: NodeHealthOf("anchor", look(3, "openvpn"), peers), wrote: true},
|
{h: NodeHealthOf("anchor", look(3, "openvpn"), peers, Machines{}), wrote: true},
|
||||||
{h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}), wrote: true},
|
{h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}, Machines{}), wrote: true},
|
||||||
} {
|
} {
|
||||||
if step.fresh {
|
if step.fresh {
|
||||||
last = nil
|
last = nil
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -43,9 +44,43 @@ type (
|
|||||||
// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin).
|
// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin).
|
||||||
const ReasonReloginNeeded = "relogin needed"
|
const ReasonReloginNeeded = "relogin needed"
|
||||||
|
|
||||||
|
// Machines names the mesh's machines for a report's targets: each node by its address on the private network, and the
|
||||||
|
// hub's node. The zero value names none, and every part is then the machine's own.
|
||||||
|
type Machines struct {
|
||||||
|
ByAddress map[string]string
|
||||||
|
Hub string
|
||||||
|
}
|
||||||
|
|
||||||
|
// targets is the machines a part points at, as the controller holds it (mesh-controller machine_network.go): the hub's
|
||||||
|
// node for TowardHub, each resolver's node by its address; none when it points at nothing, or at this machine or an
|
||||||
|
// address no other mesh machine has, as one of them makes the part the machine's own.
|
||||||
|
func (m Machines) targets(node string, toward []string) []string {
|
||||||
|
if len(toward) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, t := range toward {
|
||||||
|
x := m.ByAddress[t]
|
||||||
|
if t == TowardHub {
|
||||||
|
x = m.Hub
|
||||||
|
}
|
||||||
|
if x == "" || x == node {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !slices.Contains(out, x) {
|
||||||
|
out = append(out, x)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// TowardHub is what a part failing toward the hub names in Toward (network.TowardHub).
|
||||||
|
const TowardHub = "hub"
|
||||||
|
|
||||||
// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its
|
// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its
|
||||||
// streaks, its machine units, and each account and directory of its resources that the six alerts read.
|
// streaks, each unhealthy part naming the machines it points at (machines), its machine units, and each account and
|
||||||
func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth {
|
// directory of its resources that the six alerts read.
|
||||||
|
func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) NodeHealth {
|
||||||
out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units}
|
out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units}
|
||||||
if len(peers) > 0 {
|
if len(peers) > 0 {
|
||||||
out.Peers = append([]PeerSeen(nil), peers...)
|
out.Peers = append([]PeerSeen(nil), peers...)
|
||||||
@@ -55,6 +90,10 @@ func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth {
|
|||||||
n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))}
|
n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))}
|
||||||
for _, p := range h.Network.Parts {
|
for _, p := range h.Network.Parts {
|
||||||
p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen
|
p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen
|
||||||
|
p.Targets = nil
|
||||||
|
if p.State == StateUnhealthy {
|
||||||
|
p.Targets = machines.targets(node, p.Toward)
|
||||||
|
}
|
||||||
n.Parts = append(n.Parts, p)
|
n.Parts = append(n.Parts, p)
|
||||||
}
|
}
|
||||||
out.Network = &n
|
out.Network = &n
|
||||||
|
|||||||
@@ -423,6 +423,24 @@ func (c *Composer) takeModuleHave(u agent.Update) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held.
|
// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held.
|
||||||
|
// Machines is, from the placements this composer holds, each node by its address on the private network, and the hub's
|
||||||
|
// node: what names the machine a failing network part points at (delivery.NetworkPart.Targets, issue 774). Empty
|
||||||
|
// before any placement is read.
|
||||||
|
func (c *Composer) Machines() (byAddress map[string]string, hub string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
byAddress = map[string]string{}
|
||||||
|
for node, p := range c.placements {
|
||||||
|
if a, _, _ := strings.Cut(p.Decided.Address, "/"); a != "" {
|
||||||
|
byAddress[a] = node
|
||||||
|
}
|
||||||
|
if p.Decided.Hub {
|
||||||
|
hub = node
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return byAddress, hub
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Composer) inOwnOf(p delivery.Placement) []string {
|
func (c *Composer) inOwnOf(p delivery.Placement) []string {
|
||||||
var out []string
|
var out []string
|
||||||
for _, m := range p.Modules {
|
for _, m := range p.Modules {
|
||||||
|
|||||||
Reference in New Issue
Block a user