Answer the review of hq issue 348's catalogue change
mesh/merge-gate pass: builds dnsmasq, docker, netcheck → ace, g14, novox, shanks; no bus step; 2 wait(s) for a person; every machine composes with the chan…
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery superseded: a newer head of the same pull request

- The dnsmasq watch restarts only a unit that runs, without waiting for
  its start; a unit stopped, starting or failed is held and said (C2).
- dnsmasq_resolve asks only an address the resolver listens on (C3).
- Tests for the watch loop keeping its budget, a watch that wrote nothing
  or stopped looking, and a unit not running (C4).
- A restart is remembered as said only once emitted, restarts made before
  the bundle started are said, and whether one helped is said (C5).
- docker_events reads the runtime's whole buffer: complete when it is not
  full or reaches past the window, never a quiet machine read as lost (C6).
- netcheck: refused read with errors.Is; tests for addresses and ports,
  and for a container named as docker knows it.
This commit is contained in:
2026-10-09 15:25:45 +02:00
parent a0049f3e4e
commit a6a9a818d2
12 changed files with 437 additions and 120 deletions
+35 -20
View File
@@ -1011,15 +1011,24 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind
return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind)
}
}
args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"}
// The whole of what the runtime holds, from the epoch: its buffer, which says where its history really
// begins and whether it is full; the window is cut from it here.
args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"}
out, err := c.docker(ctx, args...)
if err != nil {
return nil, err
}
raw, err := jsonLines[runtimeEvent](out)
all, err := jsonLines[runtimeEvent](out)
if err != nil {
return nil, err
}
start := now.Add(-time.Duration(minutes) * time.Minute)
raw := make([]runtimeEvent, 0, len(all))
for _, e := range all {
if !time.Unix(0, e.TimeNano).Before(start) {
raw = append(raw, e)
}
}
// An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer
// is read by agents and kept in their transcripts, which would make it a second copy of the leak.
envs := &envCache{c: c, ctx: ctx}
@@ -1057,41 +1066,47 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind
events = events[len(events)-limit:]
}
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events,
"held": len(raw)}
"held": len(all)}
if len(redacted) > 0 {
answer["redacted"] = sortedSet(redacted)
answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)"
}
for k, v := range historyReach(raw, now, minutes) {
for k, v := range historyReach(all, start) {
answer[k] = v
}
return answer, nil
}
// historyGap is how much later than the window's start the runtime's oldest held event may be and the
// window still read as covered: events are not made every second.
const historyGap = 2 * time.Minute
// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer
// holding fewer has every event since the runtime started; a full one has lost what came before its oldest.
const RuntimeEventsKept = 256
// historyReach says where the runtime's history begins when it begins after the window asked for: the
// oldest event it still holds, and that nothing before it can be answered from here.
func historyReach(raw []runtimeEvent, now time.Time, minutes int) map[string]any {
start := now.Add(-time.Duration(minutes) * time.Minute)
if len(raw) == 0 {
return map[string]any{"complete": false,
"history": "the runtime holds no events at all for this window: it keeps only its last events in memory, so this says nothing about what happened before now — ask the service manager's journal for the docker unit"}
// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window
// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not
// full — then it holds everything since the runtime started. Otherwise where the history begins, and that
// nothing before it can be answered from here.
func historyReach(all []runtimeEvent, start time.Time) map[string]any {
if len(all) == 0 {
return map[string]any{"complete": true, "oldest_held": nil,
"history": "the runtime holds no events at all: none happened since it started"}
}
oldest := time.Unix(0, raw[0].TimeNano)
for _, e := range raw {
oldest := time.Unix(0, all[0].TimeNano)
for _, e := range all {
if t := time.Unix(0, e.TimeNano); t.Before(oldest) {
oldest = t
}
}
if oldest.Sub(start) <= historyGap {
return map[string]any{"complete": true, "oldest_held": oldest.UTC().Format(time.RFC3339)}
at := oldest.UTC().Format(time.RFC3339)
if len(all) < RuntimeEventsKept {
return map[string]any{"complete": true, "oldest_held": at,
"history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)}
}
return map[string]any{"complete": false, "oldest_held": oldest.UTC().Format(time.RFC3339),
if !oldest.After(start) {
return map[string]any{"complete": true, "oldest_held": at}
}
return map[string]any{"complete": false, "oldest_held": at,
"history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)",
len(raw), oldest.UTC().Format(time.RFC3339), start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
}
func sortedSet(set map[string]bool) []string {
+29 -11
View File
@@ -300,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
{"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2}
`
f := (&fake{}).on("docker events", Ran{Stdout: out})
got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false)
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false)
if err != nil {
t.Fatal(err)
}
@@ -309,7 +309,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
t.Fatalf("%v", evs)
}
// The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348).
if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") {
if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
t.Fatalf("%+v", f.calls)
}
if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil {
@@ -376,15 +376,15 @@ func eventLine(at time.Time, kind, action, name string) string {
func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
var out strings.Builder
for i := 0; i < 250; i++ {
for i := 0; i < RuntimeEventsKept; i++ {
out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail"))
}
got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false)
if err != nil {
t.Fatal(err)
}
if got["count"] != 0 || got["held"] != 250 {
t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"])
if got["count"] != 0 || got["held"] != RuntimeEventsKept {
t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept)
}
if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") ||
got["oldest_held"] != "2026-10-09T11:14:00Z" {
@@ -394,7 +394,10 @@ func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *te
func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) {
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
// A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer
// is not full, so it holds every event since the runtime started.
out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") +
eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") +
eventLine(now.Add(-1*time.Minute), "network", "create", "build-net")
f := (&fake{}).on("docker events", Ran{Stdout: out})
@@ -402,14 +405,29 @@ func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *test
if err != nil {
t.Fatal(err)
}
if got["complete"] != true || got["count"] != 2 || got["history"] != nil {
t.Fatalf("a covered window: %v", got)
if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") {
t.Fatalf("a quiet machine's window: %v", got)
}
if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") {
if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
t.Fatal("the type was filtered by the runtime, so the history's start is unknown")
}
// A quiet machine asked for ten minutes, its newest event an hour old: complete, and none.
got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false)
if got["complete"] != true || got["count"] != 0 {
t.Fatalf("a quiet machine read as one whose history was lost: %v", got)
}
// No events at all: none since the runtime started.
got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != false || got["history"] == nil {
t.Fatalf("no events at all reads as quiet: %v", got)
if got["complete"] != true || got["count"] != 0 {
t.Fatalf("an empty buffer: %v", got)
}
// A full buffer whose oldest is before the window: complete.
var full strings.Builder
for i := 0; i < RuntimeEventsKept; i++ {
full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x"))
}
got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
if got["complete"] != true || got["history"] != nil {
t.Fatalf("a full buffer reaching past the window: %v", got)
}
}
@@ -81,3 +81,16 @@ func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) {
t.Fatalf("an unreadable machine file: %v, %v", got, err)
}
}
// The container is named as docker knows it, never as something docker would read as an option.
func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) {
f := &fake{}
for _, ref := range []string{"--privileged", "a b", "", "x;rm"} {
if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil {
t.Errorf("%q was asked of the runtime", ref)
}
}
if len(f.calls) != 0 {
t.Fatalf("the runtime was asked: %+v", f.calls)
}
}
@@ -5,6 +5,7 @@ import (
"encoding/json"
"strings"
"testing"
"time"
)
// The shape of the leak in hq issue 268: a server password announced at start and a database URI
@@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) {
f := (&fake{}).
on("docker events", Ran{Stdout: execEvents}).
on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"})
got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true)
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true)
if err != nil {
t.Fatal(err)
}