Answer the review of hq issue 348's catalogue change
mesh/merge-gate pass: builds dnsmasq, docker, netcheck → ace, g14, novox, shanks; no bus step; 2 wait(s) for a person; every machine composes with the chan…
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate pass: builds dnsmasq, docker, netcheck → ace, g14, novox, shanks; no bus step; 2 wait(s) for a person; every machine composes with the chan…
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-check-typescript.sh (typescript)
mesh/delivery superseded: a newer head of the same pull request
- The dnsmasq watch restarts only a unit that runs, without waiting for its start; a unit stopped, starting or failed is held and said (C2). - dnsmasq_resolve asks only an address the resolver listens on (C3). - Tests for the watch loop keeping its budget, a watch that wrote nothing or stopped looking, and a unit not running (C4). - A restart is remembered as said only once emitted, restarts made before the bundle started are said, and whether one helped is said (C5). - docker_events reads the runtime's whole buffer: complete when it is not full or reaches past the window, never a quiet machine read as lost (C6). - netcheck: refused read with errors.Is; tests for addresses and ports, and for a container named as docker knows it.
This commit is contained in:
@@ -1011,15 +1011,24 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind
|
||||
return nil, fmt.Errorf("type %q: container, image, network, volume, daemon, plugin or builder", kind)
|
||||
}
|
||||
}
|
||||
args := []string{"events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s", "--format", "{{json .}}"}
|
||||
// The whole of what the runtime holds, from the epoch: its buffer, which says where its history really
|
||||
// begins and whether it is full; the window is cut from it here.
|
||||
args := []string{"events", "--since", "0", "--until", "0s", "--format", "{{json .}}"}
|
||||
out, err := c.docker(ctx, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := jsonLines[runtimeEvent](out)
|
||||
all, err := jsonLines[runtimeEvent](out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start := now.Add(-time.Duration(minutes) * time.Minute)
|
||||
raw := make([]runtimeEvent, 0, len(all))
|
||||
for _, e := range all {
|
||||
if !time.Unix(0, e.TimeNano).Before(start) {
|
||||
raw = append(raw, e)
|
||||
}
|
||||
}
|
||||
// An exec's command line is shown without the secrets it carried (novox/hq issue 282): this answer
|
||||
// is read by agents and kept in their transcripts, which would make it a second copy of the leak.
|
||||
envs := &envCache{c: c, ctx: ctx}
|
||||
@@ -1057,41 +1066,47 @@ func (c *Client) eventsAt(ctx context.Context, now time.Time, minutes int, kind
|
||||
events = events[len(events)-limit:]
|
||||
}
|
||||
answer := map[string]any{"minutes": minutes, "count": total, "shown": len(events), "events": events,
|
||||
"held": len(raw)}
|
||||
"held": len(all)}
|
||||
if len(redacted) > 0 {
|
||||
answer["redacted"] = sortedSet(redacted)
|
||||
answer["leak"] = "exec command lines carried secrets, which the runtime keeps in its events; docker_secrets_in_events names them (novox/hq issue 282)"
|
||||
}
|
||||
for k, v := range historyReach(raw, now, minutes) {
|
||||
for k, v := range historyReach(all, start) {
|
||||
answer[k] = v
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// historyGap is how much later than the window's start the runtime's oldest held event may be and the
|
||||
// window still read as covered: events are not made every second.
|
||||
const historyGap = 2 * time.Minute
|
||||
// RuntimeEventsKept is how many events the runtime keeps in memory (moby's events.eventsLimit): a buffer
|
||||
// holding fewer has every event since the runtime started; a full one has lost what came before its oldest.
|
||||
const RuntimeEventsKept = 256
|
||||
|
||||
// historyReach says where the runtime's history begins when it begins after the window asked for: the
|
||||
// oldest event it still holds, and that nothing before it can be answered from here.
|
||||
func historyReach(raw []runtimeEvent, now time.Time, minutes int) map[string]any {
|
||||
start := now.Add(-time.Duration(minutes) * time.Minute)
|
||||
if len(raw) == 0 {
|
||||
return map[string]any{"complete": false,
|
||||
"history": "the runtime holds no events at all for this window: it keeps only its last events in memory, so this says nothing about what happened before now — ask the service manager's journal for the docker unit"}
|
||||
// historyReach says how far back the runtime's buffer reaches, read from all of it: complete for a window
|
||||
// that starts at or after the oldest event a full buffer holds, or for any window when the buffer is not
|
||||
// full — then it holds everything since the runtime started. Otherwise where the history begins, and that
|
||||
// nothing before it can be answered from here.
|
||||
func historyReach(all []runtimeEvent, start time.Time) map[string]any {
|
||||
if len(all) == 0 {
|
||||
return map[string]any{"complete": true, "oldest_held": nil,
|
||||
"history": "the runtime holds no events at all: none happened since it started"}
|
||||
}
|
||||
oldest := time.Unix(0, raw[0].TimeNano)
|
||||
for _, e := range raw {
|
||||
oldest := time.Unix(0, all[0].TimeNano)
|
||||
for _, e := range all {
|
||||
if t := time.Unix(0, e.TimeNano); t.Before(oldest) {
|
||||
oldest = t
|
||||
}
|
||||
}
|
||||
if oldest.Sub(start) <= historyGap {
|
||||
return map[string]any{"complete": true, "oldest_held": oldest.UTC().Format(time.RFC3339)}
|
||||
at := oldest.UTC().Format(time.RFC3339)
|
||||
if len(all) < RuntimeEventsKept {
|
||||
return map[string]any{"complete": true, "oldest_held": at,
|
||||
"history": fmt.Sprintf("the runtime holds %d events, fewer than the %d it keeps: every event since it started", len(all), RuntimeEventsKept)}
|
||||
}
|
||||
return map[string]any{"complete": false, "oldest_held": oldest.UTC().Format(time.RFC3339),
|
||||
if !oldest.After(start) {
|
||||
return map[string]any{"complete": true, "oldest_held": at}
|
||||
}
|
||||
return map[string]any{"complete": false, "oldest_held": at,
|
||||
"history": fmt.Sprintf("the runtime keeps only its last %d events in memory and its oldest is from %s; the window asked starts at %s, so what happened before %s is gone from here, not absent — the images' health checks fill the history with exec events (hq research 032)",
|
||||
len(raw), oldest.UTC().Format(time.RFC3339), start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
|
||||
len(all), at, start.UTC().Format(time.RFC3339), oldest.UTC().Format("15:04:05Z"))}
|
||||
}
|
||||
|
||||
func sortedSet(set map[string]bool) []string {
|
||||
|
||||
@@ -300,7 +300,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
|
||||
{"Type":"container","Action":"die","Actor":{"ID":"aaaaaaaaaaaaaaaa","Attributes":{"name":"web","mesh-host.id":"hello-web.server","exitCode":"137"}},"timeNano":2}
|
||||
`
|
||||
f := (&fake{}).on("docker events", Ran{Stdout: out})
|
||||
got, err := client(f, 1000).Events(context.Background(), 30, "container", 10, false)
|
||||
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 3), 30, "container", 10, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -309,7 +309,7 @@ func TestEventsAreABoundedWindowWithoutExecNoise(t *testing.T) {
|
||||
t.Fatalf("%v", evs)
|
||||
}
|
||||
// The type is filtered here, not by the runtime: the history's start is only known from all of it (issue 348).
|
||||
if !f.ran("docker events --since 30m --until 0s --format {{json .}}") || f.ran("docker events --since 30m --until 0s --format {{json .}} --filter") {
|
||||
if !f.ran("docker events --since 0 --until 0s --format {{json .}}") || f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
|
||||
t.Fatalf("%+v", f.calls)
|
||||
}
|
||||
if _, err := client(f, 1000).Events(context.Background(), 30, "secret", 10, false); err == nil {
|
||||
@@ -376,15 +376,15 @@ func eventLine(at time.Time, kind, action, name string) string {
|
||||
func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
|
||||
var out strings.Builder
|
||||
for i := 0; i < 250; i++ {
|
||||
for i := 0; i < RuntimeEventsKept; i++ {
|
||||
out.WriteString(eventLine(now.Add(-10*time.Minute+time.Duration(i)*2*time.Second), "container", "exec_die", "mailu-webmail"))
|
||||
}
|
||||
got, err := client((&fake{}).on("docker events", Ran{Stdout: out.String()}), 1000).eventsAt(context.Background(), now, 180, "", 200, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["count"] != 0 || got["held"] != 250 {
|
||||
t.Fatalf("count %v held %v, want 0 shown of 250 held", got["count"], got["held"])
|
||||
if got["count"] != 0 || got["held"] != RuntimeEventsKept {
|
||||
t.Fatalf("count %v held %v, want 0 shown of %d held", got["count"], got["held"], RuntimeEventsKept)
|
||||
}
|
||||
if got["complete"] != false || !strings.Contains(fmt.Sprint(got["history"]), "gone from here, not absent") ||
|
||||
got["oldest_held"] != "2026-10-09T11:14:00Z" {
|
||||
@@ -394,7 +394,10 @@ func TestEventsSayWhereTheRuntimesHistoryBeginsWhenTheWindowReachesFurther(t *te
|
||||
|
||||
func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *testing.T) {
|
||||
now := time.Date(2026, 10, 9, 11, 24, 0, 0, time.UTC)
|
||||
out := eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
|
||||
// A quiet machine: three events in the whole buffer, the oldest inside the window. Complete — the buffer
|
||||
// is not full, so it holds every event since the runtime started.
|
||||
out := eventLine(now.Add(-90*time.Minute), "container", "start", "old") +
|
||||
eventLine(now.Add(-59*time.Minute), "network", "destroy", "build-net") +
|
||||
eventLine(now.Add(-30*time.Minute), "container", "die", "build-bus") +
|
||||
eventLine(now.Add(-1*time.Minute), "network", "create", "build-net")
|
||||
f := (&fake{}).on("docker events", Ran{Stdout: out})
|
||||
@@ -402,14 +405,29 @@ func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *test
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["complete"] != true || got["count"] != 2 || got["history"] != nil {
|
||||
t.Fatalf("a covered window: %v", got)
|
||||
if got["complete"] != true || got["count"] != 2 || got["held"] != 4 || !strings.Contains(fmt.Sprint(got["history"]), "every event since it started") {
|
||||
t.Fatalf("a quiet machine's window: %v", got)
|
||||
}
|
||||
if f.ran("docker events --since 60m --until 0s --format {{json .}} --filter") {
|
||||
if f.ran("docker events --since 0 --until 0s --format {{json .}} --filter") {
|
||||
t.Fatal("the type was filtered by the runtime, so the history's start is unknown")
|
||||
}
|
||||
// A quiet machine asked for ten minutes, its newest event an hour old: complete, and none.
|
||||
got, _ = client(f, 1000).eventsAt(context.Background(), now.Add(time.Hour), 10, "", 200, false)
|
||||
if got["complete"] != true || got["count"] != 0 {
|
||||
t.Fatalf("a quiet machine read as one whose history was lost: %v", got)
|
||||
}
|
||||
// No events at all: none since the runtime started.
|
||||
got, _ = client((&fake{}).on("docker events", Ran{}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
|
||||
if got["complete"] != false || got["history"] == nil {
|
||||
t.Fatalf("no events at all reads as quiet: %v", got)
|
||||
if got["complete"] != true || got["count"] != 0 {
|
||||
t.Fatalf("an empty buffer: %v", got)
|
||||
}
|
||||
// A full buffer whose oldest is before the window: complete.
|
||||
var full strings.Builder
|
||||
for i := 0; i < RuntimeEventsKept; i++ {
|
||||
full.WriteString(eventLine(now.Add(-2*time.Hour+time.Duration(i)*time.Second), "container", "exec_die", "x"))
|
||||
}
|
||||
got, _ = client((&fake{}).on("docker events", Ran{Stdout: full.String()}), 1000).eventsAt(context.Background(), now, 60, "", 200, false)
|
||||
if got["complete"] != true || got["history"] != nil {
|
||||
t.Fatalf("a full buffer reaching past the window: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,3 +81,16 @@ func TestAResolverFileThatCannotBeReadIsSaidPlainly(t *testing.T) {
|
||||
t.Fatalf("an unreadable machine file: %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The container is named as docker knows it, never as something docker would read as an option.
|
||||
func TestAResolverFileIsAskedOnlyOfAContainerRef(t *testing.T) {
|
||||
f := &fake{}
|
||||
for _, ref := range []string{"--privileged", "a b", "", "x;rm"} {
|
||||
if _, err := client(f, 1000).ResolvConfOf(context.Background(), ref); err == nil {
|
||||
t.Errorf("%q was asked of the runtime", ref)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Fatalf("the runtime was asked: %+v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The shape of the leak in hq issue 268: a server password announced at start and a database URI
|
||||
@@ -210,7 +211,7 @@ func TestEventsShowAnExecsCommandLineWithoutItsSecrets(t *testing.T) {
|
||||
f := (&fake{}).
|
||||
on("docker events", Ran{Stdout: execEvents}).
|
||||
on("docker container inspect --format {{json .Config.Env}}", Ran{Stdout: "[]\n"})
|
||||
got, err := client(f, 1000).Events(context.Background(), 30, "", 100, true)
|
||||
got, err := client(f, 1000).eventsAt(context.Background(), time.Unix(0, 1791320130000000000), 30, "", 100, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user