diff --git a/.gitignore b/.gitignore index 01c8bb0b..36dbb3ca 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ modules/jetbrains-toolbox/cmd/toolbox-tools/toolbox-tools modules/messenger/cmd/messenger/messenger modules/mesh-watcher/cmd/mesh-watcher/mesh-watcher modules/claude-code/cmd/claude-code/claude-code +modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq-tools # ...and the same built from the module's root (go build ./cmd/), which names it after the module. modules/messenger/messenger modules/mesh-watcher/mesh-watcher +modules/dnsmasq/dnsmasq diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go index 197f0ed4..c9ae3b9a 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/dnsmasq_test.go @@ -3,6 +3,8 @@ package main import ( "context" "encoding/binary" + "errors" + "fmt" "net" "os" "path/filepath" @@ -11,6 +13,8 @@ import ( "sync" "testing" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" ) // A stand-in resolver: answers name's A record with address, on one address, over UDP and TCP, until @@ -205,15 +209,15 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) var s State var why string - if s, why = Step(s, failing(t0)); why != "" || s.Failing != 1 { + if s, why = Step(s, failing(t0), "active"); why != "" || s.Failing != 1 { t.Fatalf("one failing look restarted: %q", why) } - if s, why = Step(s, failing(t0.Add(LookEvery))); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") { + if s, why = Step(s, failing(t0.Add(LookEvery)), "active"); why == "" || !strings.Contains(why, "2 look(s) in a row since 10:57:57 UTC") { t.Fatalf("a second failing look did not restart: %+v %q", s, why) } s.Restarts = append(s.Restarts, Restart{At: t0.Add(LookEvery), Why: why}) // It helped. - s, why = Step(s, healthy(t0.Add(2*LookEvery))) + s, why = Step(s, healthy(t0.Add(2*LookEvery)), "active") if why != "" || s.Failing != 0 || s.Restarts[0].Helped == nil || !*s.Restarts[0].Helped { t.Fatalf("a healthy look after a restart: %+v %q", s, why) } @@ -222,7 +226,7 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { restarts := 1 for at.Add(LookEvery).Sub(t0) < 50*time.Minute { at = at.Add(LookEvery) - s, why = Step(s, failing(at)) + s, why = Step(s, failing(at), "active") if why == "" { continue } @@ -247,8 +251,8 @@ func TestTheWatchRestartsOnASecondFailingLookWithinItsBudget(t *testing.T) { // A look that could not ask — no configuration — restarts nothing. var none State broken := Look{At: t0, Problems: []string{"its configuration cannot be read"}} - none, _ = Step(none, broken) - if _, why := Step(none, broken); why != "" { + none, _ = Step(none, broken, "active") + if _, why := Step(none, broken, "active"); why != "" { t.Fatalf("a missing configuration restarted the resolver: %q", why) } } @@ -281,7 +285,7 @@ func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelp var s State look := func() { t.Helper() - next, why := Step(s, m.Look(t.Context())) + next, why := Step(s, m.Look(t.Context()), "active") if why != "" { r := Restart{At: now().UTC(), Why: why} if _, err := run(t.Context(), "systemctl", "restart", Unit); err != nil { @@ -330,23 +334,226 @@ func TestTheWatchRestartsAResolverThatStoppedAnsweringAndHealthSaysWhetherItHelp } } -func TestTheAnnouncerSaysWhatChangedAndNothingAtItsStart(t *testing.T) { - var a Announcer - t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) - old := State{Restarts: []Restart{{At: t0, Why: "before the bundle started"}}} - if got := a.Next([]Named{{"novox.internal", "10.77.0.1"}}, true, old); len(got) != 0 { - t.Fatalf("said at its start: %+v", got) +// A healthy look later than HelpedWithin after a restart does not say the restart helped: the resolver came +// back on its own, or by something else. +func TestARestartHelpedOnlyWhenTheResolverAnsweredSoonAfterIt(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + s := State{Restarts: []Restart{{At: t0}}} + s, _ = Step(s, healthy(t0.Add(HelpedWithin+time.Second)), "active") + if h := s.Restarts[0].Helped; h == nil || *h { + t.Fatalf("a healthy look %s after the restart said it helped: %v", HelpedWithin+time.Second, h) } - if got := a.Next(nil, false, old); len(got) != 0 { - t.Fatalf("an unreadable roster said %+v", got) + s = State{Restarts: []Restart{{At: t0}}} + s, _ = Step(s, healthy(t0.Add(HelpedWithin)), "active") + if h := s.Restarts[0].Helped; h == nil || !*h { + t.Fatalf("a healthy look within %s after the restart did not say it helped: %v", HelpedWithin, h) } - now := State{Restarts: append(old.Restarts, Restart{At: t0.Add(time.Minute), Why: "refused"})} - got := a.Next([]Named{{"ace.internal", "10.77.0.2"}}, true, now) - types := map[string]int{} - for _, e := range got { - types[e.Type]++ +} + +// A restart is made only of a unit that runs: one stopped on purpose, still starting at boot or failed is +// held, said, and not counted against the budget (review of PR 161, C2). +func TestTheWatchRestartsOnlyAUnitThatRuns(t *testing.T) { + t0 := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + for _, unit := range []string{"inactive", "activating", "failed", ""} { + var s State + var why string + for i := 0; i < 4; i++ { + s, why = Step(s, failing(t0.Add(time.Duration(i)*LookEvery)), unit) + if why != "" { + t.Fatalf("a unit %q was restarted", unit) + } + } + if !strings.Contains(s.Held, "not running") || s.Unit != unit || len(s.Restarts) != 0 { + t.Fatalf("a unit %q: %+v", unit, s) + } + // Running again and still not answering: restarted, the held looks counted as failing. + if _, why = Step(s, failing(t0.Add(5*LookEvery)), "active"); why == "" { + t.Fatalf("a unit running again after %q, not answering, was not restarted", unit) + } } - if types["name.added"] != 1 || types["name.removed"] != 1 || types["restarted"] != 1 || len(got) != 3 { - t.Fatalf("said %+v", got) +} + +// The watch loop: it reads the budget it kept before it was restarted, asks the unit's state, restarts +// without waiting for the start, and writes what it saw (review of PR 161, C2 and C4). +func TestTheWatchLoopKeepsItsBudgetAcrossItsOwnRestart(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 10, 57, 57, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return clock }) + m.Ask = func(_ context.Context, server, over, name string, _ time.Duration) Answer { + return Answer{Server: server, Over: over, Name: name, Failed: "refused: nothing listens there"} + } + before := LookEvery + LookEvery = time.Millisecond + t.Cleanup(func() { LookEvery = before }) + run := func(spent []Restart) ([]string, State) { + t.Helper() + if err := m.WriteState(State{Restarts: spent}); err != nil { + t.Fatal(err) + } + var asked []string + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + looks := 0 + var last State + err := m.Watch(ctx, func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + if args[0] == "show" { + return "active\n", nil + } + return "", nil + }, func(s State) error { + last = s + if looks++; looks == 3 { + cancel() + } + return m.WriteState(s) + }) + if err != nil { + t.Fatal(err) + } + return asked, last + } + // Three restarts within the hour before it was restarted: it restarts nothing more. + spent := []Restart{{At: clock.Add(-50 * time.Minute)}, {At: clock.Add(-30 * time.Minute)}, {At: clock.Add(-10 * time.Minute)}} + asked, last := run(spent) + for _, a := range asked { + if strings.Contains(a, "restart") { + t.Fatalf("restarted with its budget spent before it started: %v", asked) + } + } + if !strings.Contains(last.Held, "the most the watch makes") || len(last.Restarts) != 3 { + t.Fatalf("the budget it kept: %+v", last) + } + // None spent: restarted on its second failing look, not waiting for the start. + asked, last = run(nil) + restarts := 0 + for _, a := range asked { + if strings.HasPrefix(a, "systemctl restart") { + restarts++ + if a != "systemctl restart --no-block dnsmasq.service" { + t.Fatalf("restarted as %q", a) + } + } + } + if restarts != 1 || len(last.Restarts) != 1 || last.Unit != "active" { + t.Fatalf("restarts %d: %v %+v", restarts, asked, last) + } +} + +// The health tool says a watch that wrote nothing, one that stopped looking, and a unit not running. +func TestHealthSaysWhenTheWatchIsNotLookingOrTheUnitIsNotRunning(t *testing.T) { + port := freePort(t) + clock := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return clock }) + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + startStandIn(t, "127.0.0.2", port, "127.0.0.2") + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch has written nothing yet") { + t.Fatalf("no watch: %+v", h.Why) + } + if err := m.WriteState(State{Last: Look{At: clock.Add(-Stale - time.Second), Healthy: true}, Unit: "active"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "the watch last looked 2m1s ago") { + t.Fatalf("a stale watch: %+v", h.Why) + } + if err := m.WriteState(State{Last: Look{At: clock, Healthy: true}, Unit: "active"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); !h.Healthy || h.Why != "" { + t.Fatalf("a watch looking, a resolver answering: %+v", h) + } + if err := m.WriteState(State{Last: Look{At: clock}, Unit: "inactive"}); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); !strings.Contains(h.Why, "the unit is inactive") { + t.Fatalf("a unit stopped: %+v", h.Why) + } + if err := os.WriteFile(filepath.Join(m.Root, StateFile), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if h := m.Health(t.Context()); h.Healthy || !strings.Contains(h.Why, "is not the watch's") { + t.Fatalf("an unreadable state: %+v", h.Why) + } +} + +// dnsmasq_resolve asks only an address the resolver listens on (review of PR 161, C3). +func TestResolveAsksOnlyAListenAddress(t *testing.T) { + port := freePort(t) + m := aResolverMachine(t, port, time.Now) + startStandIn(t, "127.0.0.2", port, "127.0.0.2") + startStandIn(t, "127.0.0.1", port, "127.0.0.2") + if _, err := m.Resolve(t.Context(), "novox.internal", "8.8.8.8", ""); err == nil || !strings.Contains(err.Error(), "not one the resolver listens on") { + t.Fatalf("another address was asked: %v", err) + } + got, err := m.Resolve(t.Context(), "novox.internal", "127.0.0.2", "tcp") + if err != nil { + t.Fatal(err) + } + answers := got.(map[string]any)["answers"].([]Answer) + if len(answers) != 1 || answers[0].Over != "tcp" || answers[0].Code != "NOERROR" || answers[0].Addresses[0] != "127.0.0.2" { + t.Fatalf("%+v", answers) + } + if got, _ := m.Resolve(t.Context(), "novox.internal", "", ""); len(got.(map[string]any)["answers"].([]Answer)) != 4 { + t.Fatalf("every listen address over both: %+v", got) + } + if _, err := m.Resolve(t.Context(), "novox.internal", "", "quic"); err == nil { + t.Fatal("over quic was accepted") + } + if _, err := m.Resolve(t.Context(), "", "", ""); err == nil { + t.Fatal("no name was accepted") + } +} + +// The announcer: names primed silently, restarts made before the bundle started said, an event remembered +// only once emitted, and whether a restart helped said (review of PR 161, C5). +func TestTheAnnouncerSaysWhatChangedAndRemembersOnlyWhatWasSaid(t *testing.T) { + port := freePort(t) + t0 := time.Date(2026, 10, 9, 11, 0, 0, 0, time.UTC) + m := aResolverMachine(t, port, func() time.Time { return t0 }) + yes := true + if err := m.WriteState(State{Restarts: []Restart{{At: t0, Why: "before the bundle started", Helped: &yes}}}); err != nil { + t.Fatal(err) + } + var said []string + failing := true + emit := func(kind string, body any, _ ...stdio.EmitOptions) error { + if failing && kind == "restarted" { + return errors.New("no bus") + } + said = append(said, kind+" "+fmt.Sprint(body.(map[string]any)["name"], body.(map[string]any)["helped"])) + return nil + } + var a Announcer + Tell(m, &a, emit) + if strings.Join(said, "|") != "restart-judged true" { + t.Fatalf("at the start: %v", said) + } + failing, said = false, nil + Tell(m, &a, emit) + if strings.Join(said, "|") != "restarted " { + t.Fatalf("a restart whose emit failed was not said again: %v", said) + } + said = nil + Tell(m, &a, emit) + if len(said) != 0 { + t.Fatalf("said twice: %v", said) + } + // The roster changes: one machine added, one gone. An unreadable roster says nothing. + nodes := filepath.Join(m.Root, NodesFile) + if err := os.WriteFile(nodes, []byte("host-record=novox.internal,127.0.0.2\nhost-record=shanks.internal,10.77.0.4\n"), 0o644); err != nil { + t.Fatal(err) + } + Tell(m, &a, emit) + if got := strings.Join(said, "|"); !strings.Contains(got, "name.added shanks.internal") || + !strings.Contains(got, "name.removed ace.internal") || len(said) != 2 { + t.Fatalf("the roster's change: %v", said) + } + said = nil + if err := os.Remove(nodes); err != nil { + t.Fatal(err) + } + Tell(m, &a, emit) + if len(said) != 0 { + t.Fatalf("an unreadable roster said %v", said) } } diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/main.go b/modules/dnsmasq/cmd/dnsmasq-tools/main.go index 771d3ee5..91f33cc2 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/main.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/main.go @@ -45,84 +45,101 @@ func main() { // AnnounceEvery is how often the bundle reads the roster and the watch for what to say. var AnnounceEvery = 30 * time.Second -// Announcer remembers what was said, primed silently so a restart of the bundle does not say every name -// and every restart again. +// Announcer remembers what was said. The names are primed silently, so the bundle starting again does not +// say every name again; the watch's restarts are not — a restart made while the bundle was down is said when +// it starts (the watch keeps the last hour's). Nothing is remembered as said until its emit succeeded. type Announcer struct { - names map[string]string - restarts map[time.Time]bool - // primed says the names were read once; heard that the watch's restarts were. - primed, heard bool + names map[string]string + primed bool + // said is every restart said, by when it was made; judged every one whose outcome was said. + said, judged map[time.Time]bool } -// Event is one event to emit. +// Event is one event to emit, and what saying it records. type Event struct { Type string Body map[string]any + said func() } -// Next is what changed since the last read: names the resolver now answers (`name.added`) or no longer -// does (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own -// events — and restarts the watch made (`restarted`), each with why and whether it helped once known. -// -// read says the roster was read: an unreadable roster says nothing about names, rather than every name -// removed. +// Next is what to say since the last read: names the resolver now answers (`name.added`) or no longer does +// (`name.removed`) — the mesh's names having reached this resolver, apart from the controller's own events; +// restarts the watch made (`restarted`), with why; and, once the looks after a restart decide it, whether it +// helped (`restart-judged`). read says the roster was read: an unreadable roster says nothing about names, +// rather than every name removed. An event is remembered by Said, once it was emitted. func (a *Announcer) Next(names []Named, read bool, s State) []Event { - now := map[string]string{} - for _, n := range names { - now[n.Name] = n.Address + if a.said == nil { + a.said, a.judged, a.names = map[time.Time]bool{}, map[time.Time]bool{}, map[string]string{} } var out []Event - if a.primed && read { - for name, address := range now { - if _, had := a.names[name]; !had { - out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}}) + if read { + now := map[string]string{} + for _, n := range names { + now[n.Name] = n.Address + } + if !a.primed { + a.names, a.primed = now, true + } else { + for name, address := range now { + if _, had := a.names[name]; !had { + out = append(out, Event{"name.added", map[string]any{"name": name, "address": address}, + func() { a.names[name] = address }}) + } + } + for name := range a.names { + if _, has := now[name]; !has { + out = append(out, Event{"name.removed", map[string]any{"name": name}, func() { delete(a.names, name) }}) + } } } - for name := range a.names { - if _, has := now[name]; !has { - out = append(out, Event{"name.removed", map[string]any{"name": name}}) - } - } - } - if a.restarts == nil { - a.restarts = map[time.Time]bool{} } for _, r := range s.Restarts { - if a.restarts[r.At] { - continue - } - a.restarts[r.At] = true - if a.heard { - body := map[string]any{"unit": Unit, "at": r.At, "why": r.Why} + at := r.At + if !a.said[at] { + body := map[string]any{"unit": Unit, "at": at, "why": r.Why} if r.Error != "" { body["error"] = r.Error } - out = append(out, Event{"restarted", body}) + out = append(out, Event{"restarted", body, func() { a.said[at] = true }}) + } + if r.Helped != nil && !a.judged[at] { + out = append(out, Event{"restart-judged", map[string]any{"unit": Unit, "at": at, "helped": *r.Helped}, + func() { a.judged[at] = true }}) } } - a.heard = true - if read { - a.names, a.primed = now, true - } return out } +// Said remembers an event as said. +func (e Event) Said() { + if e.said != nil { + e.said() + } +} + // Announce says the events for as long as the bundle runs. func Announce(m *Machine, emit func(string, any, ...stdio.EmitOptions) error) { time.Sleep(2 * time.Second) // the SDK reaches the bus once Serve runs var a Announcer for { - var names []Named - text, err := m.read(NodesFile) - if err == nil { - names = AnsweredNames(text) - } - s, _, _ := m.ReadState() - for _, e := range a.Next(names, err == nil, s) { - if err := emit(e.Type, e.Body); err != nil { - fmt.Fprintf(os.Stderr, "dnsmasq: %s not said: %v\n", e.Type, err) - } - } + Tell(m, &a, emit) time.Sleep(AnnounceEvery) } } + +// Tell is one read and what it says: an event that could not be emitted is said again at the next read. +func Tell(m *Machine, a *Announcer, emit func(string, any, ...stdio.EmitOptions) error) { + var names []Named + text, err := m.read(NodesFile) + if err == nil { + names = AnsweredNames(text) + } + s, _, _ := m.ReadState() + for _, e := range a.Next(names, err == nil, s) { + if err := emit(e.Type, e.Body); err != nil { + fmt.Fprintf(os.Stderr, "dnsmasq: %s not said, said again at the next read: %v\n", e.Type, err) + continue + } + e.Said() + } +} diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/tools.go b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go index 3d4d11e2..8d5e5cde 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/tools.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/tools.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "slices" "strings" "time" @@ -48,6 +49,9 @@ func (m *Machine) Health(ctx context.Context) Health { if !l.Healthy { why = append(why, restartWords(s)) } + if s.Unit != "" && s.Unit != "active" { + why = append(why, "the unit is "+s.Unit) + } } h.Why = strings.Join(nonEmpty(why), "; ") return h @@ -56,7 +60,7 @@ func (m *Machine) Health(ctx context.Context) Health { // restartWords is what the watch did about it, in words. func restartWords(s State) string { if len(s.Restarts) == 0 { - return "the watch has not restarted it" + return "the watch has not restarted it" + orEmpty("; ", s.Held) } r := s.Restarts[len(s.Restarts)-1] words := fmt.Sprintf("the watch restarted it at %s", r.At.Format("15:04:05 UTC")) @@ -132,15 +136,17 @@ func (m *Machine) Resolve(ctx context.Context, name, address, over string) (any, if name == "" { return nil, fmt.Errorf("name is required") } - var addresses []string + config, err := m.read(ConfigFile) + if err != nil { + return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err) + } + addresses := ListenAddresses(config) if address != "" { - addresses = []string{address} - } else { - config, err := m.read(ConfigFile) - if err != nil { - return nil, fmt.Errorf("its configuration %s cannot be read: %w", ConfigFile, err) + if !slices.Contains(addresses, address) { + return nil, fmt.Errorf("address %q is not one the resolver listens on (%s); netcheck_dns asks any resolver", + address, strings.Join(addresses, ", ")) } - addresses = ListenAddresses(config) + addresses = []string{address} } overs := []string{"udp", "tcp"} switch over { diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/watch.go b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go index 4989dc8c..fed79191 100644 --- a/modules/dnsmasq/cmd/dnsmasq-tools/watch.go +++ b/modules/dnsmasq/cmd/dnsmasq-tools/watch.go @@ -64,8 +64,11 @@ type State struct { Since *time.Time `json:"since,omitempty"` // Restarts are the restarts within the last hour and the last one ever, newest last. Restarts []Restart `json:"restarts,omitempty"` - // Held is why a restart that was due was not made: the budget is spent. + // Held is why a restart that was due was not made: the budget is spent, or the unit is not running. Held string `json:"held,omitempty"` + // Unit is the resolver unit's state at the last look, as the service manager said it: active, + // activating, inactive, failed … + Unit string `json:"unit,omitempty"` } // Runner runs one command; injected, so the watch is tested over recorded answers. @@ -83,10 +86,13 @@ func ExecRunner(ctx context.Context, name string, args ...string) (string, error } // Step is one look of the watch and what follows from it: the state after, and whether to restart and why. -// Pure but for the look it is given. -func Step(s State, l Look) (State, string) { +// unit is the resolver unit's state at the look. **Only a unit that runs is restarted** (review of +// mesh-catalog PR 161): one stopped or disabled on purpose is a person's, one still starting — waiting at +// boot for the private address — or failed is the service manager's, and none of them is a resolver that +// runs and does not answer. Pure. +func Step(s State, l Look, unit string) (State, string) { now := l.At - s.Last = l + s.Last, s.Unit = l, unit // What the earlier restarts are now known to have done. for i := range s.Restarts { r := &s.Restarts[i] @@ -124,6 +130,11 @@ func Step(s State, l Look) (State, string) { // configuration or roster is missing. return s, "" } + if unit != "active" { + s.Held = fmt.Sprintf("the unit is %s, not running: the watch restarts only a resolver that runs and does not "+ + "answer — one stopped on purpose, still starting or failed is left to the service manager", orUnknown(unit)) + return s, "" + } within := 0 var newest *Restart for i, r := range s.Restarts { @@ -155,10 +166,16 @@ func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error defer tick.Stop() for { l := m.Look(ctx) - next, why := Step(s, l) + unit, err := run(ctx, "systemctl", "show", "--property=ActiveState", "--value", Unit) + if err != nil { + unit = "" + } + next, why := Step(s, l, strings.TrimSpace(unit)) if why != "" { r := Restart{At: m.Now().UTC(), Why: why} - if out, err := run(ctx, "systemctl", "restart", Unit); err != nil { + // Not waited for: a start that waits for the private address is the service manager's to time, + // and the next looks say whether it answers. + if out, err := run(ctx, "systemctl", "restart", "--no-block", Unit); err != nil { r.Error = strings.TrimSpace(err.Error() + " " + out) } fmt.Fprintf(os.Stderr, "dnsmasq-watch: restarted %s: %s%s\n", Unit, why, orEmpty(" — the restart failed: ", r.Error)) @@ -178,6 +195,13 @@ func (m *Machine) Watch(ctx context.Context, run Runner, write func(State) error } } +func orUnknown(s string) string { + if s == "" { + return "not known (the service manager did not say)" + } + return s +} + func orEmpty(prefix, s string) string { if s == "" { return "" diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 9974ead7..d868d771 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -18,7 +18,8 @@ "emits": [ "name.added", "name.removed", - "restarted" + "restarted", + "restart-judged" ], "own-secrets": { "broker": "${dir:mesh-state}/broker" diff --git a/modules/docker/cmd/docker-tools/docker.go b/modules/docker/cmd/docker-tools/docker.go index 7411ac21..88669917 100644 --- a/modules/docker/cmd/docker-tools/docker.go +++ b/modules/docker/cmd/docker-tools/docker.go @@ -1011,15 +1011,24 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind) } } - args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"} + // The whole of what the runtime holds, from the epoch: its buffer, which says where its history really + // begins and whether it is full; the window is cut from it here. + args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"} out, err := c.docker(ctx, args...) if err != nil { return nil, err } - raw, err := jsonLines[runtimeEvent](out) + all, err := jsonLines[runtimeEvent](out) if err != nil { return nil, err } + start := now.Add(-time.Duration(minutes) * time.Minute) + raw := make([]runtimeEvent, 0, len(all)) + for _, e := range all { + if !time.Unix(0, e.TimeNano).Before(start) { + raw = append(raw, e) + } + } // An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer // is read by agents and kept in their transcripts, which would make it a second copy of the leak. envs := &envCache{c: c, ctx: ctx} @@ -1057,41 +1066,47 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind events = events[len(events)-limit:] } answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events, - "held": len(raw)} + "held": len(all)} if len(redacted) > 0 { answer["redacted"] = sortedSet(redacted) answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)" } - for k, v := range historyReach(raw, now, minutes) { + for k, v := range historyReach(all, start) { answer[k] = v } return answer, nil } -// historyGap is how much later than the window's start the runtime's oldest held event may be and the -// window still read as covered: events are not made every second. -const historyGap = 2 * time.Minute +// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer +// holding fewer has every event since the runtime started; a full one has lost what came before its oldest. +const RuntimeEventsKept = 256 -// historyReach says where the runtime's history begins when it begins after the window asked for: the -// oldest event it still holds, and that nothing before it can be answered from here. -func historyReach(raw []runtimeEvent, now time.Time, minutes int) map[string]any { - start := now.Add(-time.Duration(minutes) * time.Minute) - if len(raw) == 0 { - return map[string]any{"complete": false, - "history": "the runtime holds no events at all for this window: it keeps only its last events in memory, so this says nothing about what happened before now — ask the service manager's journal for the docker unit"} +// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window +// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not +// full — then it holds everything since the runtime started. Otherwise where the history begins, and that +// nothing before it can be answered from here. +func historyReach(all []runtimeEvent, start time.Time) map[string]any { + if len(all) == 0 { + return map[string]any{"complete": true, "oldest_held": nil, + "history": "the runtime holds no events at all: none happened since it started"} } - oldest := time.Unix(0, raw[0].TimeNano) - for _, e := range raw { + oldest := time.Unix(0, all[0].TimeNano) + for _, e := range all { if t := time.Unix(0, e.TimeNano); t.Before(oldest) { oldest = t } } - if oldest.Sub(start) <= historyGap { - return map[string]any{"complete": true, "oldest_held": oldest.UTC().Format(time.RFC3339)} + at := oldest.UTC().Format(time.RFC3339) + if len(all) < RuntimeEventsKept { + return map[string]any{"complete": true, "oldest_held": at, + "history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)} } - return map[string]any{"complete": false, "oldest_held": oldest.UTC().Format(time.RFC3339), + if !oldest.After(start) { + return map[string]any{"complete": true, "oldest_held": at} + } + return map[string]any{"complete": false, "oldest_held": at, "history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)", - len(raw), oldest.UTC().Format(time.RFC3339), start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))} + len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))} } func sortedSet(set map[string]bool) []string { diff --git a/modules/docker/cmd/docker-tools/docker_test.go b/modules/docker/cmd/docker-tools/docker_test.go index 41ff8eb7..7bf8042f 100644 --- a/modules/docker/cmd/docker-tools/docker_test.go +++ b/modules/docker/cmd/docker-tools/docker_test.go @@ -300,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) { {"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2} ` f := (&fake{}).on("docker events", Ran{Stdout: out}) - got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false) + got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false) if err != nil { t.Fatal(err) } @@ -309,7 +309,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) { t.Fatalf("%v", evs) } // The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348). - if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") { + if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") { t.Fatalf("%+v", f.calls) } if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil { @@ -376,15 +376,15 @@ func eventLine(at time.Time, kind, action, name string) string { func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) { now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) var out strings.Builder - for i := 0; i < 250; i++ { + for i := 0; i < RuntimeEventsKept; i++ { out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail")) } got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false) if err != nil { t.Fatal(err) } - if got["count"] != 0 || got["held"] != 250 { - t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"]) + if got["count"] != 0 || got["held"] != RuntimeEventsKept { + t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept) } if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") || got["oldest_held"] != "2026-10-09T11:14:00Z" { @@ -394,7 +394,10 @@ func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *te func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) { now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC) - out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") + + // A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer + // is not full, so it holds every event since the runtime started. + out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") + + eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") + eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") + eventLine(now.Add(-1*time.Minute), "network", "create", "build-net") f := (&fake{}).on("docker events", Ran{Stdout: out}) @@ -402,14 +405,29 @@ func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *test if err != nil { t.Fatal(err) } - if got["complete"] != true || got["count"] != 2 || got["history"] != nil { - t.Fatalf("a covered window: %v", got) + if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") { + t.Fatalf("a quiet machine's window: %v", got) } - if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") { + if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") { t.Fatal("the type was filtered by the runtime, so the history's start is unknown") } + // A quiet machine asked for ten minutes, its newest event an hour old: complete, and none. + got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false) + if got["complete"] != true || got["count"] != 0 { + t.Fatalf("a quiet machine read as one whose history was lost: %v", got) + } + // No events at all: none since the runtime started. got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false) - if got["complete"] != false || got["history"] == nil { - t.Fatalf("no events at all reads as quiet: %v", got) + if got["complete"] != true || got["count"] != 0 { + t.Fatalf("an empty buffer: %v", got) + } + // A full buffer whose oldest is before the window: complete. + var full strings.Builder + for i := 0; i < RuntimeEventsKept; i++ { + full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x")) + } + got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false) + if got["complete"] != true || got["history"] != nil { + t.Fatalf("a full buffer reaching past the window: %v", got) } } diff --git a/modules/docker/cmd/docker-tools/resolv_test.go b/modules/docker/cmd/docker-tools/resolv_test.go index 7512c7c3..6c1285e8 100644 --- a/modules/docker/cmd/docker-tools/resolv_test.go +++ b/modules/docker/cmd/docker-tools/resolv_test.go @@ -81,3 +81,16 @@ func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) { t.Fatalf("an unreadable machine file: %v, %v", got, err) } } + +// The container is named as docker knows it, never as something docker would read as an option. +func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) { + f := &fake{} + for _, ref := range []string{"--privileged", "a b", "", "x;rm"} { + if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil { + t.Errorf("%q was asked of the runtime", ref) + } + } + if len(f.calls) != 0 { + t.Fatalf("the runtime was asked: %+v", f.calls) + } +} diff --git a/modules/docker/cmd/docker-tools/secrets_test.go b/modules/docker/cmd/docker-tools/secrets_test.go index ef2fa257..bd8baaf8 100644 --- a/modules/docker/cmd/docker-tools/secrets_test.go +++ b/modules/docker/cmd/docker-tools/secrets_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "strings" "testing" + "time" ) // The shape of the leak in hq issue 268: a server password announced at start and a database URI @@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) { f := (&fake{}). on("docker events", Ran{Stdout: execEvents}). on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"}) - got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true) + got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true) if err != nil { t.Fatal(err) } diff --git a/modules/netcheck/cmd/netcheck/ask_test.go b/modules/netcheck/cmd/netcheck/ask_test.go index 19ca738a..1ad2c293 100644 --- a/modules/netcheck/cmd/netcheck/ask_test.go +++ b/modules/netcheck/cmd/netcheck/ask_test.go @@ -115,6 +115,18 @@ func TestDNSSaysARefusedPortAndRefusesANameAsServer(t *testing.T) { if a, _ := serverAddress("10.10.0.1"); a != "10.10.0.1:53" { t.Fatalf("default port: %s", a) } + for server, want := range map[string]string{"10.10.0.1:5353": "10.10.0.1:5353", " 127.0.0.1 ": "127.0.0.1:53", + "[::1]:53": "[::1]:53", "::1": "[::1]:53"} { + if a, err := serverAddress(server); err != nil || a != want { + t.Errorf("%q: %q, %v; want %q", server, a, err, want) + } + } + for _, server := range []string{"resolver.example:53", "resolver.example", "10.10.0.1:0", "10.10.0.1:65536", + "10.10.0.1:dns", ""} { + if a, err := serverAddress(server); err == nil { + t.Errorf("%q was taken as %q", server, a) + } + } } func TestListeningReadsTheKernelsTablesAndSaysWhatItCouldNotSee(t *testing.T) { diff --git a/modules/netcheck/cmd/netcheck/checks.go b/modules/netcheck/cmd/netcheck/checks.go index 8fbf74a9..db7ca08c 100644 --- a/modules/netcheck/cmd/netcheck/checks.go +++ b/modules/netcheck/cmd/netcheck/checks.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "syscall" "time" ) @@ -220,7 +221,7 @@ func timeoutOf(args map[string]any) (time.Duration, error) { func dnsWords(err error, timeout time.Duration) string { var ne net.Error switch { - case strings.Contains(err.Error(), "connection refused"): + case errors.Is(err, syscall.ECONNREFUSED): return "connection refused: nothing listens on that address and port" case errors.As(err, &ne) && ne.Timeout(): return fmt.Sprintf("no answer within %s", timeout)