Answer the review of hq issue 348's catalogue change
mesh/merge-gate pass: builds dnsmasq, docker, netcheck → ace, g14, novox, shanks; no bus step; 2 wait(s) for a person; every machine composes with the chan…
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery superseded: a newer head of the same pull request

- The dnsmasq watch restarts only a unit that runs, without waiting for
  its start; a unit stopped, starting or failed is held and said (C2).
- dnsmasq_resolve asks only an address the resolver listens on (C3).
- Tests for the watch loop keeping its budget, a watch that wrote nothing
  or stopped looking, and a unit not running (C4).
- A restart is remembered as said only once emitted, restarts made before
  the bundle started are said, and whether one helped is said (C5).
- docker_events reads the runtime's whole buffer: complete when it is not
  full or reaches past the window, never a quiet machine read as lost (C6).
- netcheck: refused read with errors.Is; tests for addresses and ports,
  and for a container named as docker knows it.
This commit is contained in:
2026-10-09 15:25:45 +02:00
parent a0049f3e4e
commit a6a9a818d2
12 changed files with 437 additions and 120 deletions
+2
View File
@@ -7,6 +7,8 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools
modules/messenger/cmd/messenger/messenger
modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher
modules/claude-code/cmd/claude-code/claude-code
modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools
# ...and the same built from the module's root (go build ./cmd/<name>), which names it after the module.
modules/messenger/messenger
modules/mesh-watcher/mesh-watcher
modules/dnsmasq/dnsmasq
+229 -22
View File
@@ -3,6 +3,8 @@ package main
import (
"context"
"encoding/binary"
"errors"
"fmt"
"net"
"os"
"path/filepath"
@@ -11,6 +13,8 @@ import (
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until
@@ -205,15 +209,15 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
var s State
var why string
if s, why = Step(s, failing(t0)); why != "" || s.Failing != 1 {
if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 {
t.Fatalf("one failing look restarted: %q", why)
}
if s, why = Step(s, failing(t0.Add(LookEvery))); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") {
if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") {
t.Fatalf("a second failing look did not restart: %+v %q", s, why)
}
s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why})
// It helped.
s, why = Step(s, healthy(t0.Add(2*LookEvery)))
s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active")
if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped {
t.Fatalf("a healthy look after a restart: %+v %q", s, why)
}
@@ -222,7 +226,7 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
restarts := 1
for at.Add(LookEvery).Sub(t0) < 50*time.Minute {
at = at.Add(LookEvery)
s, why = Step(s, failing(at))
s, why = Step(s, failing(at), "active")
if why == "" {
continue
}
@@ -247,8 +251,8 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) {
// A look that could not ask — no configuration — restarts nothing.
var none State
broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}}
none, _ = Step(none, broken)
if _, why := Step(none, broken); why != "" {
none, _ = Step(none, broken, "active")
if _, why := Step(none, broken, "active"); why != "" {
t.Fatalf("a missing configuration restarted the resolver: %q", why)
}
}
@@ -281,7 +285,7 @@ func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelp
var s State
look := func() {
t.Helper()
next, why := Step(s, m.Look(t.Context()))
next, why := Step(s, m.Look(t.Context()), "active")
if why != "" {
r := Restart{At: now().UTC(), Why: why}
if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil {
@@ -330,23 +334,226 @@ func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelp
}
}
func TestTheAnnouncerSaysWhatChangedAndNothingAtItsStart(t *testing.T) {
var a Announcer
t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
old := State{Restarts: []Restart{{At: t0, Why: "before the bundle started"}}}
if got := a.Next([]Named{{"novox.internal", "10.77.0.1"}}, true, old); len(got) != 0 {
t.Fatalf("said at its start: %+v", got)
// A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came
// back on its own, or by something else.
func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) {
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
s := State{Restarts: []Restart{{At: t0}}}
s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active")
if h := s.Restarts[0].Helped; h == nil || *h {
t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h)
}
if got := a.Next(nil, false, old); len(got) != 0 {
t.Fatalf("an unreadable roster said %+v", got)
s = State{Restarts: []Restart{{At: t0}}}
s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active")
if h := s.Restarts[0].Helped; h == nil || !*h {
t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h)
}
now := State{Restarts: append(old.Restarts, Restart{At: t0.Add(time.Minute), Why: "refused"})}
got := a.Next([]Named{{"ace.internal", "10.77.0.2"}}, true, now)
types := map[string]int{}
for _, e := range got {
types[e.Type]++
}
// A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is
// held, said, and not counted against the budget (review of PR 161, C2).
func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) {
t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
for _, unit := range []string{"inactive", "activating", "failed", ""} {
var s State
var why string
for i := 0; i < 4; i++ {
s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit)
if why != "" {
t.Fatalf("a unit %q was restarted", unit)
}
}
if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 {
t.Fatalf("a unit %q: %+v", unit, s)
}
// Running again and still not answering: restarted, the held looks counted as failing.
if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" {
t.Fatalf("a unit running again after %q, not answering, was not restarted", unit)
}
}
if types["name.added"] != 1 || types["name.removed"] != 1 || types["restarted"] != 1 || len(got) != 3 {
t.Fatalf("said %+v", got)
}
// The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts
// without waiting for the start, and writes what it saw (review of PR 161, C2 and C4).
func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) {
port := freePort(t)
clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC)
m := aResolverMachine(t, port, func() time.Time { return clock })
m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer {
return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"}
}
before := LookEvery
LookEvery = time.Millisecond
t.Cleanup(func() { LookEvery = before })
run := func(spent []Restart) ([]string, State) {
t.Helper()
if err := m.WriteState(State{Restarts: spent}); err != nil {
t.Fatal(err)
}
var asked []string
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
looks := 0
var last State
err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) {
asked = append(asked, name+" "+strings.Join(args, " "))
if args[0] == "show" {
return "active\n", nil
}
return "", nil
}, func(s State) error {
last = s
if looks++; looks == 3 {
cancel()
}
return m.WriteState(s)
})
if err != nil {
t.Fatal(err)
}
return asked, last
}
// Three restarts within the hour before it was restarted: it restarts nothing more.
spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}}
asked, last := run(spent)
for _, a := range asked {
if strings.Contains(a, "restart") {
t.Fatalf("restarted with its budget spent before it started: %v", asked)
}
}
if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 {
t.Fatalf("the budget it kept: %+v", last)
}
// None spent: restarted on its second failing look, not waiting for the start.
asked, last = run(nil)
restarts := 0
for _, a := range asked {
if strings.HasPrefix(a, "systemctl restart") {
restarts++
if a != "systemctl restart --no-block dnsmasq.service" {
t.Fatalf("restarted as %q", a)
}
}
}
if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" {
t.Fatalf("restarts %d: %v %+v", restarts, asked, last)
}
}
// The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running.
func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) {
port := freePort(t)
clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
m := aResolverMachine(t, port, func() time.Time { return clock })
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
startStandIn(t, "127.0.0.2", port, "127.0.0.2")
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") {
t.Fatalf("no watch: %+v", h.Why)
}
if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") {
t.Fatalf("a stale watch: %+v", h.Why)
}
if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); !h.Healthy || h.Why != "" {
t.Fatalf("a watch looking, a resolver answering: %+v", h)
}
if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") {
t.Fatalf("a unit stopped: %+v", h.Why)
}
if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil {
t.Fatal(err)
}
if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") {
t.Fatalf("an unreadable state: %+v", h.Why)
}
}
// dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3).
func TestResolveAsksOnlyAListenAddress(t *testing.T) {
port := freePort(t)
m := aResolverMachine(t, port, time.Now)
startStandIn(t, "127.0.0.2", port, "127.0.0.2")
startStandIn(t, "127.0.0.1", port, "127.0.0.2")
if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") {
t.Fatalf("another address was asked: %v", err)
}
got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp")
if err != nil {
t.Fatal(err)
}
answers := got.(map[string]any)["answers"].([]Answer)
if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" {
t.Fatalf("%+v", answers)
}
if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 {
t.Fatalf("every listen address over both: %+v", got)
}
if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil {
t.Fatal("over quic was accepted")
}
if _, err := m.Resolve(t.Context(), "", "", ""); err == nil {
t.Fatal("no name was accepted")
}
}
// The announcer: names primed silently, restarts made before the bundle started said, an event remembered
// only once emitted, and whether a restart helped said (review of PR 161, C5).
func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) {
port := freePort(t)
t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC)
m := aResolverMachine(t, port, func() time.Time { return t0 })
yes := true
if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil {
t.Fatal(err)
}
var said []string
failing := true
emit := func(kind string, body any, _ ...stdio.EmitOptions) error {
if failing && kind == "restarted" {
return errors.New("no bus")
}
said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"]))
return nil
}
var a Announcer
Tell(m, &a, emit)
if strings.Join(said, "|") != "restart-judged <nil> true" {
t.Fatalf("at the start: %v", said)
}
failing, said = false, nil
Tell(m, &a, emit)
if strings.Join(said, "|") != "restarted <nil> <nil>" {
t.Fatalf("a restart whose emit failed was not said again: %v", said)
}
said = nil
Tell(m, &a, emit)
if len(said) != 0 {
t.Fatalf("said twice: %v", said)
}
// The roster changes: one machine added, one gone. An unreadable roster says nothing.
nodes := filepath.Join(m.Root, NodesFile)
if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil {
t.Fatal(err)
}
Tell(m, &a, emit)
if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") ||
!strings.Contains(got, "name.removed ace.internal") || len(said) != 2 {
t.Fatalf("the roster's change: %v", said)
}
said = nil
if err := os.Remove(nodes); err != nil {
t.Fatal(err)
}
Tell(m, &a, emit)
if len(said) != 0 {
t.Fatalf("an unreadable roster said %v", said)
}
}
+67 -50
View File
@@ -45,84 +45,101 @@ func main() {
// AnnounceEvery is how often the bundle reads the roster and the watch for what to say.
var AnnounceEvery = 30 * time.Second
// Announcer remembers what was said, primed silently so a restart of the bundle does not say every name
// and every restart again.
// Announcer remembers what was said. The names are primed silently, so the bundle starting again does not
// say every name again; the watch's restarts are not — a restart made while the bundle was down is said when
// it starts (the watch keeps the last hour's). Nothing is remembered as said until its emit succeeded.
type Announcer struct {
names map[string]string
restarts map[time.Time]bool
// primed says the names were read once; heard that the watch's restarts were.
primed, heard bool
names map[string]string
primed bool
// said is every restart said, by when it was made; judged every one whose outcome was said.
said, judged map[time.Time]bool
}
// Event is one event to emit.
// Event is one event to emit, and what saying it records.
type Event struct {
Type string
Body map[string]any
said func()
}
// Next is what changed since the last read: names the resolver now answers (`name.added`) or no longer
// does (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own
// events — and restarts the watch made (`restarted`), each with why and whether it helped once known.
//
// read says the roster was read: an unreadable roster says nothing about names, rather than every name
// removed.
// Next is what to say since the last read: names the resolver now answers (`name.added`) or no longer does
// (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own events;
// restarts the watch made (`restarted`), with why; and, once the looks after a restart decide it, whether it
// helped (`restart-judged`). read says the roster was read: an unreadable roster says nothing about names,
// rather than every name removed. An event is remembered by Said, once it was emitted.
func (a *Announcer) Next(names []Named, read bool, s State) []Event {
now := map[string]string{}
for _, n := range names {
now[n.Name] = n.Address
if a.said == nil {
a.said, a.judged, a.names = map[time.Time]bool{}, map[time.Time]bool{}, map[string]string{}
}
var out []Event
if a.primed && read {
for name, address := range now {
if _, had := a.names[name]; !had {
out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}})
if read {
now := map[string]string{}
for _, n := range names {
now[n.Name] = n.Address
}
if !a.primed {
a.names, a.primed = now, true
} else {
for name, address := range now {
if _, had := a.names[name]; !had {
out = append(out, Event{"name.added", map[string]any{"name": name, "address": address},
func() { a.names[name] = address }})
}
}
for name := range a.names {
if _, has := now[name]; !has {
out = append(out, Event{"name.removed", map[string]any{"name": name}, func() { delete(a.names, name) }})
}
}
}
for name := range a.names {
if _, has := now[name]; !has {
out = append(out, Event{"name.removed", map[string]any{"name": name}})
}
}
}
if a.restarts == nil {
a.restarts = map[time.Time]bool{}
}
for _, r := range s.Restarts {
if a.restarts[r.At] {
continue
}
a.restarts[r.At] = true
if a.heard {
body := map[string]any{"unit": Unit, "at": r.At, "why": r.Why}
at := r.At
if !a.said[at] {
body := map[string]any{"unit": Unit, "at": at, "why": r.Why}
if r.Error != "" {
body["error"] = r.Error
}
out = append(out, Event{"restarted", body})
out = append(out, Event{"restarted", body, func() { a.said[at] = true }})
}
if r.Helped != nil && !a.judged[at] {
out = append(out, Event{"restart-judged", map[string]any{"unit": Unit, "at": at, "helped": *r.Helped},
func() { a.judged[at] = true }})
}
}
a.heard = true
if read {
a.names, a.primed = now, true
}
return out
}
// Said remembers an event as said.
func (e Event) Said() {
if e.said != nil {
e.said()
}
}
// Announce says the events for as long as the bundle runs.
func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) {
time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs
var a Announcer
for {
var names []Named
text, err := m.read(NodesFile)
if err == nil {
names = AnsweredNames(text)
}
s, _, _ := m.ReadState()
for _, e := range a.Next(names, err == nil, s) {
if err := emit(e.Type, e.Body); err != nil {
fmt.Fprintf(os.Stderr, "dnsmasq: %s not said: %v\n", e.Type, err)
}
}
Tell(m, &a, emit)
time.Sleep(AnnounceEvery)
}
}
// Tell is one read and what it says: an event that could not be emitted is said again at the next read.
func Tell(m *Machine, a *Announcer, emit func(string, any, ...stdio.EmitOptions) error) {
var names []Named
text, err := m.read(NodesFile)
if err == nil {
names = AnsweredNames(text)
}
s, _, _ := m.ReadState()
for _, e := range a.Next(names, err == nil, s) {
if err := emit(e.Type, e.Body); err != nil {
fmt.Fprintf(os.Stderr, "dnsmasq: %s not said, said again at the next read: %v\n", e.Type, err)
continue
}
e.Said()
}
}
+14 -8
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"slices"
"strings"
"time"
@@ -48,6 +49,9 @@ func (m *Machine) Health(ctx context.Context) Health {
if !l.Healthy {
why = append(why, restartWords(s))
}
if s.Unit != "" && s.Unit != "active" {
why = append(why, "the unit is "+s.Unit)
}
}
h.Why = strings.Join(nonEmpty(why), "; ")
return h
@@ -56,7 +60,7 @@ func (m *Machine) Health(ctx context.Context) Health {
// restartWords is what the watch did about it, in words.
func restartWords(s State) string {
if len(s.Restarts) == 0 {
return "the watch has not restarted it"
return "the watch has not restarted it" + orEmpty("; ", s.Held)
}
r := s.Restarts[len(s.Restarts)-1]
words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC"))
@@ -132,15 +136,17 @@ func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any,
if name == "" {
return nil, fmt.Errorf("name is required")
}
var addresses []string
config, err := m.read(ConfigFile)
if err != nil {
return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err)
}
addresses := ListenAddresses(config)
if address != "" {
addresses = []string{address}
} else {
config, err := m.read(ConfigFile)
if err != nil {
return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err)
if !slices.Contains(addresses, address) {
return nil, fmt.Errorf("address %q is not one the resolver listens on (%s); netcheck_dns asks any resolver",
address, strings.Join(addresses, ", "))
}
addresses = ListenAddresses(config)
addresses = []string{address}
}
overs := []string{"udp", "tcp"}
switch over {
+30 -6
View File
@@ -64,8 +64,11 @@ type State struct {
Since *time.Time `json:"since,omitempty"`
// Restarts are the restarts within the last hour and the last one ever, newest last.
Restarts []Restart `json:"restarts,omitempty"`
// Held is why a restart that was due was not made: the budget is spent.
// Held is why a restart that was due was not made: the budget is spent, or the unit is not running.
Held string `json:"held,omitempty"`
// Unit is the resolver unit's state at the last look, as the service manager said it: active,
// activating, inactive, failed …
Unit string `json:"unit,omitempty"`
}
// Runner runs one command; injected, so the watch is tested over recorded answers.
@@ -83,10 +86,13 @@ func ExecRunner(ctx context.Context, name string, args ...string) (string, error
}
// Step is one look of the watch and what follows from it: the state after, and whether to restart and why.
// Pure but for the look it is given.
func Step(s State, l Look) (State, string) {
// unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of
// mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at
// boot for the private address — or failed is the service manager's, and none of them is a resolver that
// runs and does not answer. Pure.
func Step(s State, l Look, unit string) (State, string) {
now := l.At
s.Last = l
s.Last, s.Unit = l, unit
// What the earlier restarts are now known to have done.
for i := range s.Restarts {
r := &s.Restarts[i]
@@ -124,6 +130,11 @@ func Step(s State, l Look) (State, string) {
// configuration or roster is missing.
return s, ""
}
if unit != "active" {
s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+
"answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit))
return s, ""
}
within := 0
var newest *Restart
for i, r := range s.Restarts {
@@ -155,10 +166,16 @@ func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error
defer tick.Stop()
for {
l := m.Look(ctx)
next, why := Step(s, l)
unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit)
if err != nil {
unit = ""
}
next, why := Step(s, l, strings.TrimSpace(unit))
if why != "" {
r := Restart{At: m.Now().UTC(), Why: why}
if out, err := run(ctx, "systemctl", "restart", Unit); err != nil {
// Not waited for: a start that waits for the private address is the service manager's to time,
// and the next looks say whether it answers.
if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil {
r.Error = strings.TrimSpace(err.Error() + " " + out)
}
fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error))
@@ -178,6 +195,13 @@ func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error
}
}
func orUnknown(s string) string {
if s == "" {
return "not known (the service manager did not say)"
}
return s
}
func orEmpty(prefix, s string) string {
if s == "" {
return ""
+2 -1
View File
@@ -18,7 +18,8 @@
"emits": [
"name.added",
"name.removed",
"restarted"
"restarted",
"restart-judged"
],
"own-secrets": {
"broker": "${dir:mesh-state}/broker"
+35 -20
View File
@@ -1011,15 +1011,24 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind
return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind)
}
}
args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"}
// The whole of what the runtime holds, from the epoch: its buffer, which says where its history really
// begins and whether it is full; the window is cut from it here.
args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"}
out, err := c.docker(ctx, args...)
if err != nil {
return nil, err
}
raw, err := jsonLines[runtimeEvent](out)
all, err := jsonLines[runtimeEvent](out)
if err != nil {
return nil, err
}
start := now.Add(-time.Duration(minutes) * time.Minute)
raw := make([]runtimeEvent, 0, len(all))
for _, e := range all {
if !time.Unix(0, e.TimeNano).Before(start) {
raw = append(raw, e)
}
}
// An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer
// is read by agents and kept in their transcripts, which would make it a second copy of the leak.
envs := &envCache{c: c, ctx: ctx}
@@ -1057,41 +1066,47 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind
events = events[len(events)-limit:]
}
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events,
"held": len(raw)}
"held": len(all)}
if len(redacted) > 0 {
answer["redacted"] = sortedSet(redacted)
answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)"
}
for k, v := range historyReach(raw, now, minutes) {
for k, v := range historyReach(all, start) {
answer[k] = v
}
return answer, nil
}
// historyGap is how much later than the window's start the runtime's oldest held event may be and the
// window still read as covered: events are not made every second.
const historyGap = 2 * time.Minute
// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer
// holding fewer has every event since the runtime started; a full one has lost what came before its oldest.
const RuntimeEventsKept = 256
// historyReach says where the runtime's history begins when it begins after the window asked for: the
// oldest event it still holds, and that nothing before it can be answered from here.
func historyReach(raw []runtimeEvent, now time.Time, minutes int) map[string]any {
start := now.Add(-time.Duration(minutes) * time.Minute)
if len(raw) == 0 {
return map[string]any{"complete": false,
"history": "the runtime holds no events at all for this window: it keeps only its last events in memory, so this says nothing about what happened before now — ask the service manager's journal for the docker unit"}
// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window
// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not
// full — then it holds everything since the runtime started. Otherwise where the history begins, and that
// nothing before it can be answered from here.
func historyReach(all []runtimeEvent, start time.Time) map[string]any {
if len(all) == 0 {
return map[string]any{"complete": true, "oldest_held": nil,
"history": "the runtime holds no events at all: none happened since it started"}
}
oldest := time.Unix(0, raw[0].TimeNano)
for _, e := range raw {
oldest := time.Unix(0, all[0].TimeNano)
for _, e := range all {
if t := time.Unix(0, e.TimeNano); t.Before(oldest) {
oldest = t
}
}
if oldest.Sub(start) <= historyGap {
return map[string]any{"complete": true, "oldest_held": oldest.UTC().Format(time.RFC3339)}
at := oldest.UTC().Format(time.RFC3339)
if len(all) < RuntimeEventsKept {
return map[string]any{"complete": true, "oldest_held": at,
"history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)}
}
return map[string]any{"complete": false, "oldest_held": oldest.UTC().Format(time.RFC3339),
if !oldest.After(start) {
return map[string]any{"complete": true, "oldest_held": at}
}
return map[string]any{"complete": false, "oldest_held": at,
"history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)",
len(raw), oldest.UTC().Format(time.RFC3339), start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
}
func sortedSet(set map[string]bool) []string {
+29 -11
View File
@@ -300,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
{"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2}
`
f := (&fake{}).on("docker events", Ran{Stdout: out})
got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false)
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false)
if err != nil {
t.Fatal(err)
}
@@ -309,7 +309,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
t.Fatalf("%v", evs)
}
// The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348).
if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") {
if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
t.Fatalf("%+v", f.calls)
}
if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil {
@@ -376,15 +376,15 @@ func eventLine(at time.Time, kind, action, name string) string {
func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
var out strings.Builder
for i := 0; i < 250; i++ {
for i := 0; i < RuntimeEventsKept; i++ {
out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail"))
}
got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false)
if err != nil {
t.Fatal(err)
}
if got["count"] != 0 || got["held"] != 250 {
t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"])
if got["count"] != 0 || got["held"] != RuntimeEventsKept {
t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept)
}
if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") ||
got["oldest_held"] != "2026-10-09T11:14:00Z" {
@@ -394,7 +394,10 @@ func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *te
func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
// A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer
// is not full, so it holds every event since the runtime started.
out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") +
eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") +
eventLine(now.Add(-1*time.Minute), "network", "create", "build-net")
f := (&fake{}).on("docker events", Ran{Stdout: out})
@@ -402,14 +405,29 @@ func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *test
if err != nil {
t.Fatal(err)
}
if got["complete"] != true || got["count"] != 2 || got["history"] != nil {
t.Fatalf("a covered window: %v", got)
if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") {
t.Fatalf("a quiet machine's window: %v", got)
}
if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") {
if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
t.Fatal("the type was filtered by the runtime, so the history's start is unknown")
}
// A quiet machine asked for ten minutes, its newest event an hour old: complete, and none.
got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false)
if got["complete"] != true || got["count"] != 0 {
t.Fatalf("a quiet machine read as one whose history was lost: %v", got)
}
// No events at all: none since the runtime started.
got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != false || got["history"] == nil {
t.Fatalf("no events at all reads as quiet: %v", got)
if got["complete"] != true || got["count"] != 0 {
t.Fatalf("an empty buffer: %v", got)
}
// A full buffer whose oldest is before the window: complete.
var full strings.Builder
for i := 0; i < RuntimeEventsKept; i++ {
full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x"))
}
got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != true || got["history"] != nil {
t.Fatalf("a full buffer reaching past the window: %v", got)
}
}
@@ -81,3 +81,16 @@ func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) {
t.Fatalf("an unreadable machine file: %v, %v", got, err)
}
}
// The container is named as docker knows it, never as something docker would read as an option.
func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) {
f := &fake{}
for _, ref := range []string{"--privileged", "a b", "", "x;rm"} {
if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil {
t.Errorf("%q was asked of the runtime", ref)
}
}
if len(f.calls) != 0 {
t.Fatalf("the runtime was asked: %+v", f.calls)
}
}
@@ -5,6 +5,7 @@ import (
"encoding/json"
"strings"
"testing"
"time"
)
// The shape of the leak in hq issue 268: a server password announced at start and a database URI
@@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) {
f := (&fake{}).
on("docker events", Ran{Stdout: execEvents}).
on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"})
got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true)
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true)
if err != nil {
t.Fatal(err)
}
+12
View File
@@ -115,6 +115,18 @@ func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) {
if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" {
t.Fatalf("default port: %s", a)
}
for server, want := range map[string]string{"10.10.0.1:5353": "10.10.0.1:5353", " 127.0.0.1 ": "127.0.0.1:53",
"[::1]:53": "[::1]:53", "::1": "[::1]:53"} {
if a, err := serverAddress(server); err != nil || a != want {
t.Errorf("%q: %q, %v; want %q", server, a, err, want)
}
}
for _, server := range []string{"resolver.example:53", "resolver.example", "10.10.0.1:0", "10.10.0.1:65536",
"10.10.0.1:dns", ""} {
if a, err := serverAddress(server); err == nil {
t.Errorf("%q was taken as %q", server, a)
}
}
}
func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) {
+2 -1
View File
@@ -9,6 +9,7 @@ import (
"sort"
"strconv"
"strings"
"syscall"
"time"
)
@@ -220,7 +221,7 @@ func timeoutOf(args map[string]any) (time.Duration, error) {
func dnsWords(err error, timeout time.Duration) string {
var ne net.Error
switch {
case strings.Contains(err.Error(), "connection refused"):
case errors.Is(err, syscall.ECONNREFUSED):
return "connection refused: nothing listens on that address and port"
case errors.As(err, &ne) && ne.Timeout():
return fmt.Sprintf("no answer within %s", timeout)