diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 2fdfbddd..bb78c718 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "path" + "regexp" + "runtime/debug" "slices" "sort" "strings" @@ -246,9 +248,11 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop if dir == "mesh-controller" { beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]} if e.Source.Seat != "" { - if lab, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository, - "mesh-lab")}); err == nil { - beside["mesh-lab"] = link.CheckedOut{Repository: lab, Ref: refs["mesh-lab"]} + for _, sibling := range []string{"mesh-lab", "mesh-sdk"} { + if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository, + sibling)}); err == nil { + beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]} + } } } } @@ -271,17 +275,54 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop // catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge // builds from; and beside the controller its main, for a judge the running controller predates, and the // lab's main, whose replays every check runs. **One rule, read by the check the controller asks for and by -// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. +// the facts snapshot** (Facts.Beside), so a check run by hand clones what the build seat clones. Beside the +// controller also the SDK, checked out at the commit the running controller's go.mod pins (sdkPinned), not +// one a desktop holds (novox/hq issue 449). The checkout only places the clone: the conformance test reads the +// fixtures at the pin of the tree under check, from the clone's history, so a pull request moving the SDK is +// judged against the SDK it moves to (internal/link/conformance_test.go). func besideRefs(dir, running string) map[string]string { switch dir { case "mesh-catalog": return map[string]string{dir: "main"} case "mesh-controller": - return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main"} + return map[string]string{dir: running, "mesh-controller-main": "main", "mesh-lab": "main", + "mesh-sdk": sdkPinned()} } return map[string]string{dir: running} } +// sdkModule is the Go module of the SDK the controller is built against. +const sdkModule = "git.novox.be/novox/mesh-sdk/go" + +// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970. +var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`) + +// sdkPinned is the ref of the SDK repository this controller was built from, as its go.mod pins it and its +// build records it: a pseudo-version's commit, or a release's tag (the SDK tags its Go module under go/). +// "main" only for a binary that records no SDK version — a local replace — which a running controller is +// not (novox/hq issue 449). +func sdkPinned() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "main" + } + for _, dep := range info.Deps { + if dep.Path != sdkModule { + continue + } + if dep.Replace != nil { + dep = dep.Replace + } + if m := pseudoCommit.FindStringSubmatch(dep.Version); m != nil { + return m[1] + } + if strings.HasPrefix(dep.Version, "v") { + return "go/" + dep.Version + } + } + return "main" +} + // siblingOf is another repository of the same owner: novox/mesh-controller → novox/mesh-lab. func siblingOf(repository, name string) string { if cut := strings.LastIndex(repository, "/"); cut >= 0 { diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index d24cc913..50b891cf 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -514,6 +514,10 @@ func composedAndValidated(ctx context.Context, open *stores, node string, gens m if declared.Epoch, err = open.inventory.SentEpoch(ctx, record.ID); err != nil { return sendable{}, nil, err } + // And the generation it was last sent, as the would-send is (novox/hq issue 234). + if declared.Generation, err = open.inventory.SentGeneration(ctx, record.ID); err != nil { + return sendable{}, nil, err + } body, err := declared.Body() if err != nil { return sendable{}, nil, err diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go new file mode 100644 index 00000000..15f79cbb --- /dev/null +++ b/cmd/mesh-controller/generation.go @@ -0,0 +1,222 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The assignment generation a declaration was composed from, and the record of every send (novox/hq +// issue 234). +// +// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the +// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and +// cannot tell a later send that carries an older view of the assignments. So a declaration now carries +// the generation of the assignments it was composed from — a counter the store raises in the same +// transaction as every assignment change — and a machine refuses one older than it applied, unless it is +// a gate's put-back. And every send is written down with who sent it, from which generation and naming +// which modules: the controller logged no send then, and which process sent the stale declaration could +// not be read back from anything the mesh kept. + +// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from. +const kindOlderGeneration = "older-generation" + +// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. +const generationRefusalsSaid = advisoryQuiet + +// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and +// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the +// record of the send whether or not the machine is sent them. +// +// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its +// declaration is allotted here like any other and carries the generation as it stands — never older than +// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never +// needs to send one. +func (o order) stamp(d *sendable) { + d.Sequence, d.Epoch = o.sequence, o.epoch + d.composedFrom, d.actingEpoch = o.generation, o.acting + d.Generation = 0 + if o.readsGeneration && o.generation > 0 { + d.Generation = o.generation + } +} + +// sentRecord is what the record of a send keeps beside its digest. +type sentRecord struct { + sequence int64 + epoch uint64 + generation int64 + // toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine + // whose node-engine has not said it reads one. + toldGeneration int64 + sender string + modules []string +} + +// sentRecordOf is a composed send's record: its sender is the caller this process acts for. +func sentRecordOf(ctx context.Context, d sendable) sentRecord { + return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, + toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} +} + +// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the +// caller the controller ran this command for, or the account at the shell. +func callerOf(ctx context.Context) string { + if c := link.CallerIn(ctx); c != "" { + return c + } + return link.Caller() +} + +// senderOf is a send's sender in words: who asked, and the process and build that composed it — the +// answer issue 234 could not find, because two controllers and a one-shot push were all sending then. +func senderOf(caller string) string { + host, _ := os.Hostname() + return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version) +} + +// namedModules are the modules a declaration names: its machine's set, less what was left out of it. +func namedModules(plan catalogue.Resolution, leftOut map[string]string) []string { + out := make([]string, 0, len(plan.Modules)) + for _, m := range plan.Modules { + if _, left := leftOut[m.Module]; !left { + out = append(out, m.Module) + } + } + sort.Strings(out) + return out +} + +// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender +// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that +// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the +// mesh cannot attribute is the louder fact, not a quieter one. +// +// **And raises the mesh's counter past what the machine applied, when the refused send was composed from +// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind +// the machine, which is a store put back from a backup — and every send after would be refused for ever. A +// stale sender's generation is below the counter, and the counter is left alone for it. +// +// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that +// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185. +// S20 names `push ` for that case instead. +func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { + r := report.OlderGeneration + if r == nil || inv == nil || report.Node == "" { + return + } + node, err := inv.NodeByName(ctx, report.Node) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+ + "read, so it is not raised: %v\n", report.Node, err) + return + } + var raised int64 + if now, err := inv.AssignmentGeneration(ctx); err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+ + "read: %v\n", report.Node, err) + } else if counterBehind(*r, now) { + if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+ + "and could not be raised: %v\n", now, report.Node, r.Applied, err) + raised = 0 + } else { + fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+ + "a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n", + now, report.Node, r.Applied, raised, report.Node) + } + } + send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised) + if err == nil && !found { + send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence, + Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied, + CounterRaisedTo: raised}) + } + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ + "kept, so it is not raised: %v\n", report.Node, report.Sequence, err) + return + } + fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ + "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) +} + +// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the +// refused send carried the counter as it stands now, and the machine applied more than that. +func counterBehind(r link.GenerationRefusal, now int64) bool { + return r.Generation >= now && r.Applied > r.Generation +} + +// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was +// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal. +func watchGenerationRefusals(f *signalFacts) []conditions.Observation { + seen := map[string]bool{} + var out []conditions.Observation + for _, s := range f.refusedSends { + if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] { + continue + } + seen[s.NodeName] = true + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, + Machine: s.NodeName, Severity: conditions.Warning, + Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ + "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ + "past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, + modulesWords(s.Modules)), + Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)), + Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), + Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ + "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), + Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)} + if s.CounterRaisedTo > 0 { + // Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and + // was raised; nothing has sent the machine its declaration since, so a person is asked to. + o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+ + "the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+ + "put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now", + s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo, + s.NodeName) + o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+ + "update and kept what it had. The record is repaired; a push sends the update again.", + s.NodeName, s.NodeName, s.NodeName) + } + out = append(out, o) + } + return out +} + +// modulesWords is a send's modules as a sentence says them. +func modulesWords(modules []string) string { + if len(modules) == 0 { + return "no module the mesh recorded" + } + return strings.Join(modules, ", ") +} + +// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234): +// nothing when the mesh has not recorded a send to it. +func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { + s, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + return err + } + generation := "no generation recorded" + if s.Generation > 0 { + generation = fmt.Sprintf("assignment generation %d", s.Generation) + } + fmt.Fprintf(w, "%s was last sent sequence %d at %s by %s, composed from %s, naming %s\n", node, s.Sequence, s.SentAt.Local().Format("2006-01-02 15:04:05"), s.Sender, generation, modulesWords(s.Modules)) + if s.RefusedAt != nil { + fmt.Fprintf(w, " and refused it at %s: it had applied generation %d\n", + s.RefusedAt.Local().Format("2006-01-02 15:04:05"), s.RefusedApplied) + } + return nil +} diff --git a/cmd/mesh-controller/generation_test.go b/cmd/mesh-controller/generation_test.go new file mode 100644 index 00000000..a2372bcc --- /dev/null +++ b/cmd/mesh-controller/generation_test.go @@ -0,0 +1,148 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The assignment generation a declaration was composed from (novox/hq issue 234). + +func bodyKeys(t *testing.T, s sendable) map[string]any { + t.Helper() + raw, err := s.Body() + if err != nil { + t.Fatal(err) + } + var keys map[string]any + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + return keys +} + +// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older +// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate +// composes its put-back afresh, with the generation as it stands. +func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) { + resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}} + reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57} + var told sendable + told.Resources = resources + reads.stamp(&told) + keys := bodyKeys(t, told) + if keys["generation"] != float64(40) || keys["sequence"] != float64(12) { + t.Fatalf("a machine that reads a generation was sent %v", keys) + } + if _, there := keys["put_back"]; there { + t.Fatalf("a put-back mark was sent: %v", keys) + } + if told.composedFrom != 40 || told.actingEpoch != 57 { + t.Errorf("the send does not keep what it was composed from for its record: %+v", told) + } + + older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57} + var untold sendable + untold.Resources = resources + older.stamp(&untold) + if _, there := bodyKeys(t, untold)["generation"]; there { + t.Fatal("a machine whose node-engine never said it reads a generation was sent one") + } + if untold.composedFrom != 40 { + t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold) + } +} + +// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the +// process and build that composed it. +func TestASendNamesItsSender(t *testing.T) { + said := senderOf("g14.node-tools, through the mesh-controller seat") + if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") { + t.Fatalf("the sender reads %q", said) + } +} + +// refusedSend is a send a machine refused for its generation at a moment. +func refusedSend(at time.Time) inventory.Send { + return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40, + Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)", + Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true} +} + +// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation, +// against which the machine applied, and its sequence — the facts that took a morning to look for. +func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))} + got := watchGenerationRefusals(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" { + t.Errorf("raised as %s about %q", o.Key(), o.Machine) + } + for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not say %q: %s", want, o.Summary) + } + } + if o.Headline == "" || o.Explanation == "" || o.Resolved == "" { + t.Errorf("the condition is not worded for the operator: %+v", o) + } +} + +// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never +// only a line on stderr. +func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + s := refusedSend(now.Add(-time.Minute)) + s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil + f.refusedSends = []inventory.Send{s} + got := watchGenerationRefusals(f) + if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") { + t.Fatalf("an unattributed refusal was not raised as one: %+v", got) + } +} + +// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was +// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do. +func TestACounterBehindTheMachineAsksForAPush(t *testing.T) { + now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) + f := calm(now) + s := refusedSend(now.Add(-time.Minute)) + s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41 + f.refusedSends = []inventory.Send{s} + got := watchGenerationRefusals(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") || + strings.Contains(got[0].Explanation, "Nothing for you to do") { + t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation) + } +} + +// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below +// it, and the counter is left alone for that. +func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) { + for _, c := range []struct { + refused, applied, now int64 + behind bool + }{ + {refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40 + {refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past + {refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is + } { + r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied} + if got := counterBehind(r, c.now); got != c.behind { + t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind) + } + } +} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index bcb0fe94..f9d2ba06 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -128,7 +128,7 @@ func (r *recordedDelivery) grant(context.Context, []readyNode) error { return ni func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { r.declared = append(r.declared, s.node) - return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch) + return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared)) } // aResolver is a module built from a repository, at a commit, with something on the machine that diff --git a/cmd/mesh-controller/issue_204_test.go b/cmd/mesh-controller/issue_204_test.go index feda54d8..1779292a 100644 --- a/cmd/mesh-controller/issue_204_test.go +++ b/cmd/mesh-controller/issue_204_test.go @@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { } cancel() // the sender is going away: its context is cancelled between the send and the record body := []byte(`{"declaration":1,"resources":[]}`) - digest, err := recordSent(ctx, inv, "anchor", body, nil, 0) + digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) if err != nil { // NodeByName on the cancelled context may itself refuse; the record must still be possible // through the detached context, so look the node up again on a live one. diff --git a/cmd/mesh-controller/merge_gate_test.go b/cmd/mesh-controller/merge_gate_test.go index 49792733..00289ff2 100644 --- a/cmd/mesh-controller/merge_gate_test.go +++ b/cmd/mesh-controller/merge_gate_test.go @@ -390,9 +390,10 @@ func TestAWithheldPathIsStoodInForByAPath(t *testing.T) { // controller's ask and by the facts — and finds the repository it checks from its origin. func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) { for dir, refs := range map[string]map[string]string{ - "mesh-catalog": {"mesh-catalog": "main"}, - "mesh-host": {"mesh-host": "c0ffee"}, - "mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main"}, + "mesh-catalog": {"mesh-catalog": "main"}, + "mesh-host": {"mesh-host": "c0ffee"}, + "mesh-controller": {"mesh-controller": "c0ffee", "mesh-controller-main": "main", "mesh-lab": "main", + "mesh-sdk": sdkPinnedByGoMod(t)}, } { got := besideRefs(dir, "c0ffee") for d, ref := range refs { @@ -412,3 +413,24 @@ func TestACheckByHandClonesWhatTheSeatClones(t *testing.T) { } } } + +// sdkPinnedByGoMod is the SDK commit this tree's go.mod pins, read from the file rather than the build, so +// sdkPinned is held to what the repository says (novox/hq issue 449). +func sdkPinnedByGoMod(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "go.mod")) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(raw), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == sdkModule { + if m := pseudoCommit.FindStringSubmatch(fields[1]); m != nil { + return m[1] + } + return "go/" + fields[1] + } + } + t.Fatalf("go.mod requires no %s", sdkModule) + return "" +} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 82f0fbb4..2ec829b1 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -420,7 +420,8 @@ func declarationWith(ctx context.Context, open *stores, node string, out := sendable{Resources: composed.Resources, Adoption: adoption, Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers, LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld, - unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced} + unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, + modules: namedModules(plan, composed.LeftOut)} // And which build of each module it carries, for the send to record (novox/hq issue 259, ADR // 0221). Read only on the send path: a question about what would be sent records nothing. if choosing == Allocating { @@ -1290,6 +1291,11 @@ func planCommand(ctx context.Context, args []string) error { return nil } + // What the machine was last told, by whom and from which assignment generation (novox/hq issue 234), + // beside what it would be told now. A record that cannot be read is said, not taken for none. + if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil { + fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err) + } // Which modules a push would leave out, and why — said before the plan, since the plan is of // what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan` // without --json allocates nothing. diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index a8377bc1..df28926e 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -288,13 +288,24 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { "those walks answer them; a newer merge, or `rebuild `, builds again", p.ID) } } + // Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier + // that failed after the plan did is as failed as the one that failed it. What toRetry says is the + // failed set every step below works from. + var failed []string + if p.Tier < len(p.Tiers) { + recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier]) + if err != nil { + return "", err + } + failed = toRetry(&p, recorded, byID) + } if err := retryRefusal(p, plans); err != nil { return "", err } - if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 { + if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 { return retryTierWhole(ctx, open, &p, again) } - if len(failedIn(p)) == 0 { + if len(failed) == 0 { return retryRollouts(ctx, open, &p) } entries, err := inv.Catalogued(ctx) @@ -305,7 +316,6 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { for _, e := range entries { byName[e.Manifest.Module] = e } - failed := failedIn(p) var asked []string for _, m := range failed { askModule(ctx, &p, m, byName) @@ -525,3 +535,16 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again func sendAgain(s *inventory.PlanModule) { s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", "" } + +// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state +// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its +// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build +// records. Read from the plan alone, a retry asked only the build that failed first, and the records +// then failed the plan again on the next: each failed build of a tier took a retry of its own. What +// still runs is left asked, and its outcome is the plan's once the retry sets it building. +func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string { + if p.Tier < len(p.Tiers) { + settleFromRecords(p, p.Tiers[p.Tier], recorded, byID) + } + return failedIn(*p) +} diff --git a/cmd/mesh-controller/plan_retry_tier_test.go b/cmd/mesh-controller/plan_retry_tier_test.go new file mode 100644 index 00000000..7efd44a8 --- /dev/null +++ b/cmd/mesh-controller/plan_retry_tier_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "reflect" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea +// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan, +// and plex's, arriving after, found no open plan and was kept only in the build records. The first +// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked +// plex. +func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) { + asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) + failedAt := asked.Add(2 * time.Minute) + p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, + Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{ + "gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"}, + "plex": {State: "asked", AskedAt: &asked, Build: "build-plex"}, + }} + byID := map[string]inventory.Build{ + "build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"}, + } + + if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) { + t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got) + } +} + +// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's +// once the retry sets it building, and one that is recorded built is taken as built. +func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) { + asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) + builtAt := asked.Add(3 * time.Minute) + p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, + Tiers: [][]string{{"a", "b", "c"}}, + Modules: map[string]*inventory.PlanModule{ + "a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"}, + "b": {State: "asked", AskedAt: &asked, Build: "build-b"}, + "c": {State: "asked", AskedAt: &asked, Build: "build-c"}, + }} + byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}} + + if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) { + t.Fatalf("asks %v, want a alone", got) + } + if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" { + t.Errorf("c, recorded built, is %+v", s) + } + if s := p.Modules["b"]; s.State != "asked" { + t.Errorf("b, still building, is %+v", s) + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 9779b442..9f3c1434 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -356,10 +356,16 @@ func declare(ctx context.Context, args []string) error { // the mesh did not compose it, so a push that does not name this machine treats it as held. // The epoch it carried, if a person wrote one in, is what the machine heard. var carried struct { - Epoch uint64 `json:"epoch"` + Epoch uint64 `json:"epoch"` + Sequence int64 `json:"sequence"` + Generation int64 `json:"generation"` } _ = json.Unmarshal(raw, &carried) - if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch); err != nil { + // And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it + // named are not known, since the mesh did not compose it. + if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence, + epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation, + sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil { return err } fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) @@ -798,7 +804,7 @@ func composeEach(names []string, allot func(node string) (order, error), refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch + numbered.stamp(&declared) if len(declared.Resources) == 0 { // Sent, not skipped (novox/hq issue 127). A node whose declaration composes to // nothing may have HELD something before — the broker opening a placement gave it, @@ -1016,7 +1022,8 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri } // After it is away, not before. A digest recorded for something that failed to send would make // the machine look current for a declaration it never received. - digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch) + digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch, + sentRecordOf(ctx, s.declared)) if err != nil { return "", err } @@ -1228,7 +1235,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) continue } - declared.Sequence, declared.Epoch = numbered.sequence, numbered.epoch + numbered.stamp(&declared) reportLeftOut(name, declared) sending = append(sending, readyNode{name, declared}) } @@ -1393,6 +1400,11 @@ func wouldSendFrom(ctx context.Context, open *stores, if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { return nil, err } + // And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment + // elsewhere in the mesh is not a change of this machine. + if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { + return nil, err + } body, err := declared.Body() if err != nil { return nil, err @@ -1531,6 +1543,12 @@ var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.e type order struct { sequence int64 epoch uint64 + // generation is the assignment generation read before the composition, and readsGeneration whether the + // machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts + // under, whether or not the machine is sent it — both kept for the record of the send. + generation int64 + readsGeneration bool + acting uint64 } // allot takes the next sequence for a machine — the number its next declaration carries — under the @@ -1544,6 +1562,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e if err != nil { return order{}, err } + acting := epoch if epoch > 0 { reads, err := inv.ReadsEpoch(ctx, record.ID) if err != nil { @@ -1553,11 +1572,24 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e epoch = 0 } } + // **The generation is read here, before the composition reads a single assignment** (novox/hq issue + // 234). A generation only grows, so one read before is never newer than the view composed after it: + // the declaration may claim a generation older than its content, never newer — and a claim newer than + // the content is exactly the stale send the machine must be able to refuse. + generation, err := inv.AssignmentGeneration(ctx) + if err != nil { + return order{}, err + } + readsGeneration, err := inv.ReadsGeneration(ctx, record.ID) + if err != nil { + return order{}, err + } seq, err := inv.NextSequence(ctx, record.ID) if err != nil { return order{}, err } - return order{sequence: seq, epoch: epoch}, nil + return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration, + acting: acting}, nil } // recordSent writes down what a machine was just sent, and returns the digest. @@ -1572,7 +1604,7 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, e // And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known: // what tells a machine held back by a policy or a plan from one a push left behind. func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte, - builds map[string]string, epoch uint64) (string, error) { + builds map[string]string, epoch uint64, sent sentRecord) (string, error) { kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() record, err := inv.NodeByName(kept, node) @@ -1580,7 +1612,21 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body return "", err } digest := digestOf(body) - if err := inv.RecordSentUnder(kept, record.ID, digest, builds, epoch); err != nil { + // The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the + // would-send is never stamped with a generation the machine was not sent; and the send itself, who sent + // it and from which generation, beside them. A record of the send that cannot be written does not make a + // send that is away look failed: it is said, loudly, and the machine's own record stands. + err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{ + Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, + Digest: digest, Modules: sent.modules}) + var unrecorded *inventory.SendNotRecordedError + switch { + case errors.As(err, &unrecorded): + fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+ + "sent it and from which generation could not be written down, so a refusal of it will name no sender "+ + "(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err) + fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err) + case err != nil: return "", err } // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 76fbd95b..7adc089c 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -649,26 +649,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // the controller in its first tier: the build that produced the new one is recorded, and the // plan never hears it. The record is the fact; a build recorded after the ask is that tier's // outcome, whoever was listening. - recorded := map[string][]inventory.Build{} - byID := map[string]inventory.Build{} - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.State == "asked" { - builds, err := inv.Builds(ctx, m, 5) - if err != nil { - return false, err - } - recorded[m] = builds - // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). - if s.Build != "" { - b, found, err := inv.BuildByID(ctx, s.Build) - if err != nil { - return false, err - } - if found { - byID[s.Build] = b - } - } - } + recorded, byID, err := recordsOfAsked(ctx, inv, p, tier) + if err != nil { + return false, err } if settleFromRecords(p, tier, recorded, byID) { return true, nil @@ -1776,6 +1759,34 @@ func splitList(s string) []string { return out } +// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last +// builds and the record of its own ask, by id. +func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) ( + map[string][]inventory.Build, map[string]inventory.Build, error) { + recorded := map[string][]inventory.Build{} + byID := map[string]inventory.Build{} + for _, m := range tier { + if s := p.Modules[m]; s != nil && s.State == "asked" { + builds, err := inv.Builds(ctx, m, 5) + if err != nil { + return nil, nil, err + } + recorded[m] = builds + // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). + if s.Build != "" { + b, found, err := inv.BuildByID(ctx, s.Build) + if err != nil { + return nil, nil, err + } + if found { + byID[s.Build] = b + } + } + } + } + return recorded, byID, nil +} + // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214). // Newest first, as Builds answers: the first record after the ask is the outcome of that ask. diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index 2311522a..0c1e2116 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -27,6 +27,19 @@ type sendable struct { // said it reads one is sent none, because an older node-engine refuses a key it does not know, whole // (link/order.go, the contract). Epoch uint64 + // Generation is the assignment generation it was composed from (novox/hq issue 234): a machine that + // applied a later one refuses it, so a send composed from a view of the assignments the mesh has moved + // past cannot undeclare what is still assigned. Zero is not sent at all — every machine whose + // node-engine has not said it reads one is sent none, for the reason the epoch is not (an older + // node-engine refuses a key it does not know, whole). + Generation int64 + // composedFrom is the generation read before this declaration was composed, and actingEpoch the + // lease epoch its sender acted under — whether or not the machine is sent either — for the record of + // the send; never on the wire. + composedFrom int64 + actingEpoch uint64 + // modules are the modules this declaration names, for the record of the send; never on the wire. + modules []string // Adoption is nil for a converged node, and then the body is byte for byte what it was before // adoption existed: an older host parses the envelope strictly and would refuse the key. Adoption *adoptionEnvelope @@ -94,6 +107,9 @@ func (s sendable) Body() ([]byte, error) { if len(s.LeftOut) > 0 { envelope["left_out"] = s.LeftOut } + if s.Generation > 0 { + envelope["generation"] = s.Generation + } // An empty declaration is deliberate here — the node owns nothing the mesh put there // (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness // is meant, so a truncated or mis-composed body is never mistaken for "own nothing". diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index ab3e8a93..34a6781c 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -226,6 +226,20 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed }) }}, + {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", + Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", + Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + + "machine applied; cleared an hour after the last", + Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2, + needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals, + newest: func(f *signalFacts) time.Time { + return newestOf(f.refusedSends, func(s inventory.Send) time.Time { + if s.RefusedAt == nil { + return time.Time{} + } + return *s.RefusedAt + }) + }}, {Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan", Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)", Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " + diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index c69a9d02..9abf6ce8 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -178,6 +178,14 @@ var suppressions = map[string]suppression{ commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}} }, }, + // A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at + // once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago. + "S20": { + inside: func(f *signalFacts) { + f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))} + }, + past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, + }, // Twice by hand within a fortnight is a healer wanted; once, or the first of two a day too old, is not. "S15": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 7974abd4..a81a0b8e 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -196,6 +196,11 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } + // A send refused for the generation it was composed from is kept on the send it refused, so S20 names + // its sender (novox/hq issue 234). + if report.OlderGeneration != nil { + heardGenerationRefusal(ctx, l.Enrolment.Inventory, report) + } // What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its // probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey. if report.Superseded == "" && report.Rekey == nil && report.Node != "" { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 35c2c951..7ccb352d 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -55,6 +55,10 @@ type signalFacts struct { waits []waitFacts // batches are the batches not yet cut (S18, S19, novox/hq ADR 0276). batches []batchFacts + // refusedSends are the sends a machine refused within the hour for the generation they were composed + // from, each naming its sender (S20, novox/hq issue 234). + refusedSends []inventory.Send + refusedSendsErr error loop loopFacts loopErr error @@ -364,6 +368,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) + f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) f.facts.taken, _, f.facts.began, f.facts.err = exportedFacts.last() return f } diff --git a/go.mod b/go.mod index 067a2f55..cdf300d5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 @@ -35,4 +35,4 @@ require ( // committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and // fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without // `go mod vendor` fails loudly, at once, everywhere. -replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 +replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f diff --git a/go.sum b/go.sum index c7f30d1a..e7436cbb 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 h1:pzVzwF5VMWaTECxu8+Pd1dNoOHNEm7upC5wPadQTkBw= -git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812/go.mod h1:K3/xEzVgmrNKLMV2vv4M80MwmPnQNXqvQ4C5Jj0fJT4= -git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI= -git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU= +git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/beside/beside.go b/internal/beside/beside.go index 3b281765..b64f67cc 100644 --- a/internal/beside/beside.go +++ b/internal/beside/beside.go @@ -1,5 +1,5 @@ // Package beside is where a test finds another repository of the mesh it reads: the catalogue's manifests, -// the node-engine's genesis template (novox/hq issue 432). +// the node-engine's genesis template (novox/hq issue 432), the SDK's conformance fixtures (issue 449). // // A test used to read the checkout beside this one, `../../../mesh-catalog`, so its verdict depended on // whatever sat on the machine running it: a stale or dirty checkout failed it on a desktop, and where none @@ -10,7 +10,8 @@ // test judges against those clones, so agreement with the other repository is checked where // `mesh/repo-check` runs; a repository missing there fails the test, never skips it. Which clones a // check gets is chosen from the inventory: mesh-catalog by the source of the `nats` module, mesh-host -// by the source of `mesh-host`. In a mesh where either module has no source repository nothing is +// by the source of `mesh-host`, and mesh-sdk as the controller's sibling at the commit the controller's +// go.mod pins. In a mesh where either module has no source repository nothing is // cloned, and these tests fail loudly with "not beside this check": a cause in the setup, not in the // change. And in a delivery group that holds a mesh-catalog pull request, these tests read the // catalogue's main, not the group's head. diff --git a/internal/broker/callers.go b/internal/broker/callers.go new file mode 100644 index 00000000..24392b58 --- /dev/null +++ b/internal/broker/callers.go @@ -0,0 +1,117 @@ +package broker + +import ( + "regexp" + "strings" +) + +// A tool call names its caller (novox/hq issue 365, by ADR 0259 §3's precedent for asks). +// +// mesh.mod..call.[.]. +// mesh.seat..call.[.]. +// +// The last token is the bus user that published the call, and each user is granted these subjects with its own +// name there and no other — the caller is a fact the server enforces, and the tool runtime hands it to the +// module from the subject a call arrived on (mesh-tools node-tools internal/bus caller.go holds the same shape). +// +// **A kind of its own, `call`, not the `tool` subject with a token appended.** Every grant to call a tool is a +// wildcard over the `tool` kind — `.tool..*` for the instance on any machine, `mesh.mod.*.tool.>` for every +// tool — and either would match a `tool` subject with any caller appended, so a caller could name another. +// Under `call` nothing is granted but the caller-named subjects. No stream's filter covers it: a tool call is +// never persisted (design 25 §3). +// +// **Derived from the `tool` grants, in one place** (callerNamed): wherever a principal may call or answer a +// tool, it may call it in its own name, or answer it naming any caller — so no kind of principal is left +// unable to call in its own name, and a new grant to call is one in both shapes by construction. +// +// The `tool` grants stay beside these for one release, so every caller and every runtime moves without a gap: +// their retirement is hq issue 464. +const CallKind = "call" + +var userName = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`) + +// CallerToken is a bus user's name as the last token of a call it publishes: each dot written `~`, which no part +// of a user's name may hold (safeSubject), so the token names that user and no other. "" for a name that is not +// a user's. +func CallerToken(user string) string { + if !userName.MatchString(user) { + return "" + } + return strings.ReplaceAll(user, ".", "~") +} + +// toolGrant splits a grant on the `tool` kind — `mesh.mod..tool.` or `mesh.seat..tool.`, any +// part possibly a wildcard — at the kind; ok is false for any other subject. +func toolGrant(subject string) (head, rest string, ok bool) { + parts := strings.SplitN(subject, ".", 5) + if len(parts) != 5 || parts[0] != "mesh" || (parts[1] != "mod" && parts[1] != "seat") || parts[3] != "tool" || + parts[2] == "" || parts[4] == "" { + return "", "", false + } + return parts[0] + "." + parts[1] + "." + parts[2], parts[4], true +} + +// CalledSubject is where a call to a tool subject goes naming its caller; "" when the subject is not a tool's +// or the user is not a bus user. +func CalledSubject(subject, user string) string { + head, rest, ok := toolGrant(subject) + token := CallerToken(user) + if !ok || token == "" || strings.ContainsAny(rest, "*>") { + return "" + } + return head + "." + CallKind + "." + rest + "." + token +} + +// CalledPattern is what a holder answering a tool subject also subscribes, to hear the calls that name their +// caller: the same address under the `call` kind, any caller last. "" for a subject that is not a tool's. +func CalledPattern(subject string) string { + head, rest, ok := toolGrant(subject) + if !ok || strings.ContainsAny(rest, "*>") { + return "" + } + return head + "." + CallKind + "." + rest + ".*" +} + +// calledPublish is a grant to call on the `tool` kind, as the same grant in the caller's own name: its last +// token the caller's, and nothing that reaches past it. A `>` is every tail a call carries — the tool alone or +// the tool and the machine — so it becomes both, each ending in the caller. +func calledPublish(grant, token string) []string { + head, rest, ok := toolGrant(grant) + if !ok || token == "" { + return nil + } + base := head + "." + CallKind + "." + switch { + case rest == ">": + return []string{base + "*." + token, base + "*.*." + token} + case strings.HasSuffix(rest, ".>"): + return []string{base + strings.TrimSuffix(rest, ">") + "*." + token} + } + return []string{base + rest + "." + token} +} + +// calledSubscribe is a grant to answer on the `tool` kind, as the same grant for the calls naming any caller. +func calledSubscribe(grant string) []string { + head, rest, ok := toolGrant(grant) + if !ok { + return nil + } + base := head + "." + CallKind + "." + if strings.HasSuffix(rest, ">") { + return []string{base + rest} + } + return []string{base + rest + ".*"} +} + +// callerNamed adds, beside a principal's grants on the `tool` kind, the same grants under `call`: to publish in +// its own name, to subscribe naming anybody. +func callerNamed(user string, pub, sub []string) ([]string, []string) { + token := CallerToken(user) + for _, g := range pub { + pub = append(pub, calledPublish(g, token)...) + } + for _, g := range sub { + sub = append(sub, calledSubscribe(g)...) + } + return unique(pub), unique(sub) +} diff --git a/internal/broker/callers_live_test.go b/internal/broker/callers_live_test.go new file mode 100644 index 00000000..cacb0424 --- /dev/null +++ b/internal/broker/callers_live_test.go @@ -0,0 +1,100 @@ +package broker + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "golang.org/x/crypto/bcrypt" +) + +// **On a real server, as composed** (novox/hq issue 365): a machine's runtime calls in its own name, and the +// server refuses it a call naming another — the grant, not the runtime, is what makes the caller a fact. +func TestAServerComposedFromTheGrantsRefusesACallNamingAnother(t *testing.T) { + hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + ledger := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}} + accounts, err := ComposeAccounts([]Principal{ + {Kind: KindNodeTools, Node: "novox", Module: RuntimeModule, PasswordHash: string(hash), + Carries: []Declared{{Module: "mesh-issues", Holds: []Seat{ledger}}}}, + {Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, PasswordHash: string(hash)}, + }) + if err != nil { + t.Fatal(err) + } + conf := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+ + `"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatalf("the composed accounts do not parse: %v", err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + defer s.Shutdown() + + holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.node-tools", "pw")) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + heard := make(chan string, 4) + sub, err := holder.Subscribe("mesh.seat.issue-tracker.call.open.*", func(m *nats.Msg) { + heard <- m.Subject + _ = m.Respond([]byte(`{"result":{}}`)) + }) + if err != nil { + t.Fatal(err) + } + _ = holder.Flush() + if !sub.IsValid() { + t.Fatal("the holder may not hear the caller-named calls to its seat") + } + + refusals := make(chan error, 4) + caller, err := nats.Connect(s.ClientURL(), nats.UserInfo("shanks.node-tools", "pw"), + nats.CustomInboxPrefix("_INBOX.shanks.node-tools"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err })) + if err != nil { + t.Fatal(err) + } + defer caller.Close() + if _, err := caller.Request("mesh.seat.issue-tracker.call.open.shanks~node-tools", []byte(`{}`), 3*time.Second); err != nil { + t.Fatalf("a call in the caller's own name was not answered: %v", err) + } + if got := <-heard; got != "mesh.seat.issue-tracker.call.open.shanks~node-tools" { + t.Fatalf("heard %s", got) + } + if err := caller.Publish("mesh.seat.issue-tracker.call.open.novox~node-tools", []byte(`{}`)); err != nil { + t.Fatal(err) + } + _ = caller.Flush() + select { + case err := <-refusals: + if !errors.Is(err, nats.ErrPermissionViolation) { + t.Errorf("the server said %v, want a permissions violation", err) + } + case <-time.After(3 * time.Second): + t.Error("the server did not refuse a call naming another caller") + } + select { + case got := <-heard: + t.Errorf("a call naming another reached the holder: %s", got) + case <-time.After(200 * time.Millisecond): + } +} diff --git a/internal/broker/callers_test.go b/internal/broker/callers_test.go new file mode 100644 index 00000000..f11cac30 --- /dev/null +++ b/internal/broker/callers_test.go @@ -0,0 +1,119 @@ +package broker + +import ( + "strings" + "testing" +) + +// **A tool call names its caller, and the bus lets each user name itself alone** (novox/hq issue 365, by ADR +// 0259 §3's precedent for asks): wherever a principal may call a tool, it may call it on the caller-named +// subject — kind `call`, its own bus user last, dots written `~` — and on no subject naming anybody else. +func TestEachCredentialMayCallOnlyInItsOwnName(t *testing.T) { + ledger := Seat{Name: "node-desk", Scope: "node", Serves: []string{"who"}} + tracker := Seat{Name: "issue-tracker", Scope: "mesh", Serves: []string{"open"}} + for _, c := range []struct { + p Principal + may []string + mayNot []string + subject []string // what it subscribes, to answer calls naming any caller + }{ + {p: Principal{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}}, + may: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.anchor.person~jochen", + "mesh.seat.issue-tracker.call.open.person~jochen", "mesh.seat.node-desk.call.who.anchor.person~jochen"}, + mayNot: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.mod.ledger.call.who.anchor.controller", + "mesh.seat.issue-tracker.call.open.person~somebody", "mesh.mod.ledger.call.who.person"}}, + {p: Principal{Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule, + Carries: []Declared{{Module: "ledger", Holds: []Seat{ledger, tracker}}}}, + may: []string{"mesh.mod.ledger.call.who.shanks~node-tools", "mesh.seat.issue-tracker.call.open.shanks~node-tools"}, + mayNot: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.issue-tracker.call.open.controller"}, + subject: []string{"mesh.mod.ledger.call.who.novox~node-tools", "mesh.seat.node-desk.call.who.shanks.person~jochen", "mesh.seat.issue-tracker.call.open.controller"}}, + {p: Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who", "seat:issue-tracker.open"}}, + may: []string{"mesh.mod.ledger.call.who.two~shop", "mesh.mod.ledger.call.who.anchor.two~shop", + "mesh.seat.issue-tracker.call.open.two~shop"}, + mayNot: []string{"mesh.mod.ledger.call.other.two~shop", "mesh.mod.ledger.call.who.one~shop", + "mesh.seat.issue-tracker.call.open.one~telegram"}}, + {p: Principal{Kind: KindNode, Node: "one", Checks: []string{"ledger.health"}}, + may: []string{"mesh.mod.ledger.call.health.one.node~one"}, + mayNot: []string{"mesh.mod.ledger.call.health.two.node~one", "mesh.mod.ledger.call.health.one.node~two"}}, + {p: Principal{Kind: KindModule, Node: "one", Module: "ledger", Holds: []Seat{ledger, tracker}}, + subject: []string{"mesh.mod.ledger.call.who.person~jochen", "mesh.mod.ledger.call.who.one.controller", + "mesh.seat.node-desk.call.who.one.two~shop", "mesh.seat.issue-tracker.call.open.two~shop"}}, + } { + perms, err := PermissionsFor(c.p) + if err != nil { + t.Fatalf("%s: %v", c.p.Username(), err) + } + for _, s := range c.may { + if !MayPublish(perms, s) { + t.Errorf("%s may not call %s, in its own name", c.p.Username(), s) + } + } + for _, s := range c.mayNot { + if MayPublish(perms, s) { + t.Errorf("%s may call %s, naming somebody else", c.p.Username(), s) + } + } + for _, s := range c.subject { + if !MaySubscribe(perms, s) { + t.Errorf("%s does not hear %s, a call to what it serves", c.p.Username(), s) + } + } + } +} + +// The subjects that name no caller stay granted beside the caller-named ones for one release, so a caller +// moves over without a gap (their retirement: hq issue 464). +func TestTheSubjectsThatNameNoCallerStayGrantedForOneRelease(t *testing.T) { + perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "shop", Invokes: []string{"ledger.who"}}) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{"mesh.mod.ledger.tool.who", "mesh.mod.ledger.tool.who.anchor"} { + if !MayPublish(perms, s) { + t.Errorf("the old subject %s is no longer granted", s) + } + } +} + +// The desk's hidden prompt is the controller's alone on the caller-named subjects too (the review of +// 2026-10-09, M4): a grant of every tool does not reach it there either. +func TestTheDesksPromptIsTheControllersAloneUnderCallToo(t *testing.T) { + for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}}, + {Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} { + perms, err := PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + token := CallerToken(p.Username()) + for _, s := range []string{"mesh.seat.node-launcher.call.secret.shanks." + token, + "mesh.mod.shell.call.node-launcher.secret.shanks." + token, "mesh.mod.shell.call.node-launcher.secret." + token} { + if MayPublish(perms, s) { + t.Errorf("%s may publish %s, the desk's hidden prompt", p.Username(), s) + } + } + } +} + +// The controller's grants are the installer's first user list too, so they move with hq issue 464: this release +// it calls and serves on the subjects that name no caller alone, and nobody may call in its name. +func TestTheControllerKeepsTheSubjectsThatNameNoCallerThisRelease(t *testing.T) { + perms, err := PermissionsFor(Principal{Kind: KindController}) + if err != nil { + t.Fatal(err) + } + for _, s := range append(perms.Publish, perms.Subscribe...) { + if strings.Contains(s, "."+CallKind+".") { + t.Errorf("the controller is granted %s, which the installer's first user list does not carry", s) + } + } + for _, p := range []Principal{{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}}, + {Kind: KindNodeTools, Node: "shanks", Module: RuntimeModule}} { + perms, err := PermissionsFor(p) + if err != nil { + t.Fatal(err) + } + if MayPublish(perms, "mesh.mod.ledger.call.who.controller") || MayPublish(perms, "mesh.seat.mesh-controller.call.status.controller") { + t.Errorf("%s may call in the controller's name", p.Username()) + } + } +} diff --git a/internal/broker/invokes_test.go b/internal/broker/invokes_test.go index 3d1e86ca..c22c12e8 100644 --- a/internal/broker/invokes_test.go +++ b/internal/broker/invokes_test.go @@ -42,8 +42,9 @@ func TestInvokingGrantsNothingButTheCall(t *testing.T) { if strings.Contains(p, ".event.") { t.Errorf("a module that only invokes may publish %q, an event it never declared", p) } - // A role's tools are tools (ADR 0132); a role's work queue and events are not. - if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") { + // A role's tools are tools (ADR 0132), named by their caller or not (novox/hq issue 365); a role's work + // queue and events are not. + if strings.HasPrefix(p, "mesh.seat.") && !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") { t.Errorf("a module that only invokes may publish %q, a seat it neither holds nor uses", p) } } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index d46caf7f..659a9cc8 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -212,6 +212,10 @@ func ControllerOnly() []string { for _, base := range []string{"mesh.seat." + v.Seat + ".tool." + v.Verb, "mesh.mod.*.tool." + v.Seat + "." + v.Verb} { out = append(out, base, base+".*") } + // And where a call names its caller (novox/hq issue 365): any machine, any caller. + for _, base := range []string{"mesh.seat." + v.Seat + "." + CallKind + "." + v.Verb, "mesh.mod.*." + CallKind + "." + v.Seat + "." + v.Verb} { + out = append(out, base+".>") + } } return out } @@ -875,6 +879,19 @@ func PermissionsFor(p Principal) (Permissions, error) { pub = append(pub, "$JS.ACK."+consumerStream(p)+"."+consumerDurable(p)+".>") } + // **And every tool grant again, naming its caller** (novox/hq issue 365): a call in this principal's own + // name, and an answer to a call naming anybody. The `tool` grants above stay for one release beside these, + // so callers and runtimes move without a gap; their retirement is hq issue 464. + // + // **Not the controller's, this release.** Its grants are also the installer's first user list + // (TestTheInstallersFirstUserListIsWhatTheControllerWouldCompose), judged against the node-engine the mesh + // runs, so a change to them waits on a node-engine delivery. It calls and serves on the subjects that name + // no caller meanwhile — a caller-named call to its seat reaches nobody and is asked again on those at once — + // and moves with issue 464. + if p.Kind != KindController { + pub, sub = callerNamed(p.Username(), pub, sub) + } + sort.Strings(pub) sort.Strings(sub) // One writer per piece of state (novox/hq to-be 45 §1): a grant that would make a second is diff --git a/internal/broker/nats_test.go b/internal/broker/nats_test.go index 23d3f8d3..bfd52cfb 100644 --- a/internal/broker/nats_test.go +++ b/internal/broker/nats_test.go @@ -240,9 +240,9 @@ func TestAPersonReachesNothingButTools(t *testing.T) { perms, _ := PermissionsFor(Principal{Kind: KindPerson, Module: "jo", Invokes: []string{"*"}, PasswordHash: "x"}) for _, p := range perms.Publish { - // A tool call, or asking what answers (novox/hq ADR 0197) — a question every service - // answers about itself, which claims nothing and controls nothing. - if !strings.Contains(p, ".tool.") && !strings.HasPrefix(p, "$SRV.") { + // A tool call — named by its caller or not (novox/hq issue 365) — or asking what answers (novox/hq + // ADR 0197), a question every service answers about itself, which claims nothing and controls nothing. + if !strings.Contains(p, ".tool.") && !strings.Contains(p, ".call.") && !strings.HasPrefix(p, "$SRV.") { t.Errorf("a person may publish %q, which is not a tool call", p) } } diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 0919b2b8..ea22d3c5 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -43,17 +43,17 @@ accounts { } } { user: "one.telegram", password: "$2a$11$tttttttttttttttttttttt", permissions: { publish: { allow: ["$JS.ACK.EVENTS.one_telegram.>", "$JS.ACK.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker.>", "$JS.API.CONSUMER.INFO.EVENTS.one_telegram", "$JS.API.CONSUMER.INFO.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.one_telegram", "$JS.API.CONSUMER.MSG.NEXT.SEAT_TELEGRAM_SENDER.SEAT_TELEGRAM_SENDER_worker", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.one.telegram", "mesh.seat.telegram-sender.event.delivered", "mesh.seat.telegram-sender.event.failed"] } - subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.telegram", "$SRV.INFO.telegram.>", "$SRV.PING", "$SRV.PING.telegram", "$SRV.PING.telegram.>", "$SRV.STATS", "$SRV.STATS.telegram", "$SRV.STATS.telegram.>", "_INBOX.one.telegram.>", "mesh.assignment.one.telegram", "mesh.mod.telegram.call.>", "mesh.mod.telegram.tool.>", "mesh.seat.telegram-sender.accept.send"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "two.audit", password: "$2a$11$aaaaaaaaaaaaaaaaaaaaaa", permissions: { publish: { allow: ["$JS.ACK.EVENTS.two_audit.>", "$JS.API.CONSUMER.INFO.EVENTS.two_audit", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_audit", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.audit"] } - subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.audit", "$SRV.INFO.audit.>", "$SRV.PING", "$SRV.PING.audit", "$SRV.PING.audit.>", "$SRV.STATS", "$SRV.STATS.audit", "$SRV.STATS.audit.>", "_INBOX.two.audit.>", "mesh.assignment.two.audit", "mesh.mod.audit.call.>", "mesh.mod.audit.tool.>", "mesh.mod.shop.event.order.placed"] } allow_responses: { max: 1, ttl: "1m" } } } { user: "two.shop", password: "$2a$11$ssssssssssssssssssssss", permissions: { publish: { allow: ["$JS.ACK.EVENTS.two_shop.>", "$JS.API.CONSUMER.INFO.EVENTS.two_shop", "$JS.API.CONSUMER.MSG.NEXT.EVENTS.two_shop", "$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.two.shop", "mesh.mod.shop.event.order.placed", "mesh.seat.telegram-sender.accept.send"] } - subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.tool.>"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.shop", "$SRV.INFO.shop.>", "$SRV.PING", "$SRV.PING.shop", "$SRV.PING.shop.>", "$SRV.STATS", "$SRV.STATS.shop", "$SRV.STATS.shop.>", "_INBOX.two.shop.>", "mesh.assignment.two.shop", "mesh.mod.shop.call.>", "mesh.mod.shop.tool.>"] } allow_responses: { max: 1, ttl: "1m" } } } ] diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 2c38e005..e142ef29 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -7,8 +7,10 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" + "io/fs" "os" "os/exec" "path" @@ -159,17 +161,18 @@ func build(ctx context.Context, run Runner, publish Publisher, } // The credential is a file git reads, never an argument: a URL carrying a password in argv // would be readable by anything that can list processes for as long as a clone runs. - credentials := "" - if forge.URL != "" { - credentials = filepath.Join(workspace, "git-credentials") - if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil { - return Result{}, err - } + credentials, forget, err := storeCredential(workspace, forge) + if err != nil { + return Result{}, err } + defer forget() tree := filepath.Join(workspace, "source") if err := os.RemoveAll(tree); err != nil { return Result{}, err } + // Removed when the build ends, whatever happens: the tree holds the .npmrc a build may be handed, in the + // workspace a later check's container mounts as its HOME (novox/hq issue 462). + defer os.RemoveAll(tree) // A fresh clone every time rather than a fetch into a tree that is already there. A build // that reuses a working tree can succeed because of something a previous build left behind, // and that is a build nobody can reproduce. @@ -177,6 +180,9 @@ func build(ctx context.Context, run Runner, publish Publisher, say("clone", "FAILED: %v", err) return Result{}, fmt.Errorf("cannot clone %s: %w", repository, err) } + if err := recordedWithoutUserinfo(ctx, run, tree, repository); err != nil { + return Result{}, err + } say("clone", "done") if ref != "" { if _, err := run(ctx, tree, "git", "checkout", "--quiet", ref); err != nil { @@ -248,6 +254,8 @@ func build(ctx context.Context, run Runner, publish Publisher, if err := os.WriteFile(npmrcPath, []byte(content), 0o600); err != nil { return Result{}, fmt.Errorf("cannot write the package-registry credential for the build: %w", err) } + // Only for as long as the build: a later check's container mounts this workspace (novox/hq issue 462). + defer os.Remove(npmrcPath) say("packages", "resolving %s from the mesh's package registry", npmrc.Scope) src.notPinned("it resolves packages from the mesh's registry at build time") } @@ -441,6 +449,9 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil { return "", fmt.Errorf("cannot clone %s: %w", url, err) } + if err := recordedWithoutUserinfo(ctx, run, dir, url); err != nil { + return "", err + } if from.Ref != "" { if _, err := run(ctx, dir, "git", "checkout", "--quiet", from.Ref); err != nil { return "", fmt.Errorf("%s has no %s: %w", from.Repository, from.Ref, err) @@ -467,6 +478,96 @@ func contextURL(from catalogue.ArtifactContext, seats map[string]string) (string return strings.TrimRight(base, "/") + "/" + strings.TrimSuffix(strings.Trim(from.Repository, "/"), ".git") + ".git", nil } +// storeCredential writes the forge credential where git's credential store reads it, and the function that +// removes it again; "" and nothing to remove when the builder holds none. +// +// **Never inside the workspace** (novox/hq issue 462): a merge check's toolchain container mounts the +// workspace as its HOME, so a credential kept there — even one a build left behind — is readable by any pull +// request's merge-check.sh, and what it prints is kept on the bus. So it lives in a directory of its own +// outside the workspace, made private, and a credential an older builder left in the workspace is removed. +func storeCredential(workspace string, forge GitCredential) (string, func(), error) { + if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) { + return "", nil, fmt.Errorf("a credential left in the workspace cannot be removed: %w", err) + } + if forge.URL == "" { + return "", func() {}, nil + } + dir, err := os.MkdirTemp("", "mesh-forge-credential-") + if err != nil { + return "", nil, err + } + forget := func() { os.RemoveAll(dir) } + if withinDir(workspace, dir) { + forget() + return "", nil, fmt.Errorf("the temporary directory %s is inside the workspace %s, which a check's "+ + "container mounts: the forge credential is not written where it could read it", dir, workspace) + } + path := filepath.Join(dir, "git-credentials") + if err := os.WriteFile(path, []byte(forge.URL+"\n"), 0o600); err != nil { + forget() + return "", nil, err + } + return path, forget, nil +} + +// recordedWithoutUserinfo makes a clone record the URL it came from without userinfo: git keeps it as given in +// the clone's .git/config, inside the workspace a check's container mounts (novox/hq issue 462). +func recordedWithoutUserinfo(ctx context.Context, run Runner, clone, repository string) error { + bare, carried := withoutUserinfo(repository) + if !carried { + return nil + } + if _, err := run(ctx, clone, "git", "remote", "set-url", "origin", bare); err != nil { + return fmt.Errorf("the clone of %s keeps its credential: %w", bare, err) + } + return nil +} + +// forgetLeftCredentials removes what an earlier build or an older builder left in the workspace that holds +// a credential: the forge's git-credentials, and every .npmrc a build wrote into a tree it cloned. Run +// before a check, whose container mounts the workspace as its HOME (novox/hq issue 462). The Go caches are +// not walked: no build writes a credential there, and they hold more files than everything else. +func forgetLeftCredentials(workspace string) error { + if err := os.Remove(filepath.Join(workspace, "git-credentials")); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return filepath.WalkDir(workspace, func(p string, d fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if d.IsDir() && p != workspace && (d.Name() == "go-cache" || d.Name() == "go-modules") && + filepath.Dir(p) == workspace { + return filepath.SkipDir + } + if !d.IsDir() && d.Name() == ".npmrc" { + if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("a package-registry credential left at %s cannot be removed: %w", p, err) + } + } + return nil + }) +} + +// withinDir is whether path is dir or under it, both made absolute and resolved. +func withinDir(dir, path string) bool { + d, err1 := filepath.Abs(dir) + p, err2 := filepath.Abs(path) + if err1 != nil || err2 != nil { + return true + } + if r, err := filepath.EvalSymlinks(d); err == nil { + d = r + } + if r, err := filepath.EvalSymlinks(p); err == nil { + p = r + } + rel, err := filepath.Rel(d, p) + return err != nil || rel == "." || filepath.IsLocal(rel) +} + // cloneWith is a git invocation that may offer a stored credential. // // The first `-c credential.helper=` clears every helper the environment might carry, so exactly diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index da362f2c..afe57572 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -438,30 +438,34 @@ func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) { if err != nil { t.Fatal(err) } - stored := filepath.Join(workspace, "git-credentials") clone := r.ran[0] - if !strings.Contains(clone, "credential.helper=store --file="+stored) { - t.Fatalf("the clone does not name the credential store: %s", clone) - } + stored := storeNamedIn(t, clone) for _, line := range r.ran { if strings.Contains(line, "sw0rdfi5h") { t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line) } } - raw, err := os.ReadFile(stored) - if err != nil { - t.Fatal(err) + // Outside the workspace a check's container mounts, and gone once the build is (novox/hq issue 462); what + // it held while git read it is checked with the check's clones (TestACheckContainerSeesNoForgeCredential). + if withinDir(workspace, stored) { + t.Fatalf("the credential store %s is inside the workspace %s", stored, workspace) } - if strings.TrimSpace(string(raw)) != "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000" { - t.Fatalf("the store does not hold the credential as given: %q", raw) + if _, err := os.Stat(stored); !os.IsNotExist(err) { + t.Fatalf("the credential store outlives the build: %v", err) } - info, err := os.Stat(stored) - if err != nil { - t.Fatal(err) + if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { + t.Fatal("a credential file is left in the workspace") } - if info.Mode().Perm() != 0o600 { - t.Fatalf("the credential file is readable beyond its owner: %v", info.Mode()) +} + +// storeNamedIn is the credential store a git command line offers. +func storeNamedIn(t *testing.T, line string) string { + t.Helper() + _, after, ok := strings.Cut(line, "credential.helper=store --file=") + if !ok { + t.Fatalf("the clone does not name the credential store: %s", line) } + return strings.Fields(after)[0] } // Without a credential, a clone is exactly the invocation it always was, and no credential file @@ -506,7 +510,7 @@ func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) { if err != nil { t.Fatal(err) } - stored := filepath.Join(workspace, "git-credentials") + stored := storeNamedIn(t, r.ran[0]) var contextClone string for _, line := range r.ran { if strings.Contains(line, "clone") && strings.Contains(line, "source.git") { diff --git a/internal/builder/check.go b/internal/builder/check.go index 773f6702..50b3a476 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -146,6 +146,9 @@ type Layer struct { Verdict string Summary string Modules []string + // Failed is what the repository's own check printed that names what failed, picked from the whole of + // its output and said at the end of the verdict's report (novox/hq issue 460). Empty when it passed. + Failed string } // CheckScript is what a repository declares its own merge check as: run from its root, with the @@ -226,7 +229,13 @@ func ScriptToolchain(script []byte) string { // Check runs one merge check. An error is that it could not run; the verdict is then "error". func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry string, forge GitCredential, log Log) (CheckVerdict, error) { - say := logging(log) + // Everything a check says goes to the build's log, which the bus keeps: said redacted (novox/hq issue 462). + redact := redactorFor(forge) + say := logging(func(step, message string) { + if log != nil { + log(step, redact.redact(message)) + } + }) began := time.Now() ctx, stop := context.WithTimeout(ctx, CheckTimeout) defer stop() @@ -239,20 +248,26 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry return CheckVerdict{}, err } defer os.RemoveAll(root) - credentials := "" - if forge.URL != "" { - credentials = filepath.Join(workspace, "git-credentials") - if err := os.WriteFile(credentials, []byte(forge.URL+"\n"), 0o600); err != nil { - return CheckVerdict{}, err - } + // The forge credential lives outside the workspace the check's containers mount, and only while the + // check clones (novox/hq issue 462). + credentials, forget, err := storeCredential(workspace, forge) + if err != nil { + return CheckVerdict{}, err + } + defer forget() + if err := forgetLeftCredentials(workspace); err != nil { + return CheckVerdict{}, err } clone := func(repository, ref, dir string) error { if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil { - return fmt.Errorf("cannot clone %s: %w", repository, err) + return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error())) + } + if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil { + return err } if ref != "" { if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil { - return fmt.Errorf("%s has no %s: %w", repository, ref, err) + return fmt.Errorf("%s has no %s: %w", redact.redact(repository), ref, err) } } return nil @@ -320,6 +335,9 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } } + // Every clone is made: the credential is gone before anything of the check runs (novox/hq issue 462). + forget() + // The facts, and the versions they say the mesh runs. if registry == "" { return CheckVerdict{}, errors.New("no artifact store to read the facts snapshot from") @@ -476,8 +494,11 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } } + // What the check printed travels in the verdict to the forge and the controller: redacted as the log is. + v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary), + redact.redact(v.Repo.Failed) v.Verdict, v.Summary = v.Gate.Verdict, v.Gate.Summary - v.Report, v.Took = out.String(), time.Since(began) + v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began) say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary, strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second)) return v, nil @@ -515,17 +536,29 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri say("check", "running its %s in the mesh's %s toolchain", part.Script, part.Toolchain) fmt.Fprintf(out, "--- its %s (%s toolchain)\n", part.Script, part.Toolchain) var own tail + var picked failures + logged := &toTheLog{say: say} cmd := command(image, part.Script) inItsOwnGroup(cmd) - w := io.MultiWriter(out, &own) + w := io.MultiWriter(out, &own, &picked, logged) cmd.Stdout, cmd.Stderr = w, w - switch err := cmd.Run(); { + err := cmd.Run() + logged.close(picked.String()) + switch { case timedOut(): - return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout)} + return &Layer{Verdict: "error", Summary: fmt.Sprintf("its %s ran past %s and was ended", part.Script, CheckTimeout), + Failed: picked.String()} case ctx.Err() != nil: return nil case err != nil: - return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + whatFailed(own.String())} + // Named from the whole run, not the tail: what failed may be thousands of lines from the end + // (novox/hq issue 460). + said := picked.said() + if said == "" { + said = whatFailed(own.String()) + } + return &Layer{Verdict: "fail", Summary: passedSoFar(ran) + "its " + part.Script + " failed: " + said, + Failed: picked.String()} } ran = append(ran, fmt.Sprintf("%s (%s)", part.Script, part.Toolchain)) } @@ -535,6 +568,86 @@ func ownCheck(ctx context.Context, spec CheckSpec, parts []ScriptPart, tree stri return &Layer{Verdict: "pass", Summary: "its " + CheckScript + " passed, each part in its toolchain: " + strings.Join(ran, ", ")} } +// withWhatFailed is a check's report with what its repository's own check names as failed said last, whole: +// the report's tail is the last reportLines lines, and the controller keeps the end of a report it cuts, so +// what failed travels whatever came before or after it (novox/hq issue 460). +func withWhatFailed(report string, repo *Layer) string { + if repo == nil || repo.Failed == "" { + return report + } + return report + "\n--- what failed, picked from the whole of its output (the whole is in the build's log)\n" + + repo.Failed +} + +// logLines bounds the lines of a repository's own check that go into the build's log. The bus keeps 10 000 +// messages per subject (EVENTS' MaxMsgsPerSubject), and a build's log is one subject: past that, its first +// lines would be lost. The build's own lines are a few hundred at most, so the check's output takes 8 000. +const logLines = 8000 + +// logLineBytes bounds one line of it, so a line that pastes a document does not fill a message. +const logLineBytes = 4 << 10 + +// toTheLog says each line a repository's own check prints in the build's log, as it prints it — the whole +// output kept for the build and read with `builds` and its id (novox/hq issue 460), where before only the +// verdict's tail of it was kept. Past logLines it says how many lines it left out, and the lines that name +// what failed after them. +type toTheLog struct { + say func(step, format string, args ...any) + partial []byte + said int + dropped int +} + +func (l *toTheLog) Write(p []byte) (int, error) { + l.partial = append(l.partial, p...) + for { + i := bytes.IndexByte(l.partial, '\n') + if i < 0 { + break + } + l.line(string(bytes.TrimRight(l.partial[:i], "\r"))) + l.partial = l.partial[i+1:] + } + if len(l.partial) > logLineBytes { + l.line(string(l.partial)) + l.partial = nil + } + return len(p), nil +} + +func (l *toTheLog) line(line string) { + if l.said >= logLines { + l.dropped++ + return + } + if len(line) > logLineBytes { + line = line[:logLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-logLineBytes) + } + l.said++ + // Redacted by its shape before the bus keeps it (novox/hq issue 462); Check's own say adds the secrets + // the builder knows. + l.say("output", "%s", redactor{}.redact(line)) +} + +// close says the last line, and, when lines were left out, how many and what failed. +func (l *toTheLog) close(failed string) { + if len(l.partial) > 0 { + l.line(string(l.partial)) + l.partial = nil + } + if l.dropped == 0 { + return + } + l.say("output", "… %d more line(s) of its output are not in this log, which keeps its first %d", l.dropped, logLines) + if failed == "" { + return + } + l.say("output", "--- what failed, picked from the whole of its output") + for _, line := range strings.Split(failed, "\n") { + l.say("output", "%s", redactor{}.redact(line)) + } +} + // passedSoFar is what of a check's parts passed before the one that did not, said first. func passedSoFar(ran []string) string { if len(ran) == 0 { @@ -1018,9 +1131,17 @@ func whatFailed(s string) string { return "not gofmt'd by the toolchain's gofmt: " + strings.Join(files, ", ") } } - for _, prefix := range []string{"--- FAIL:", "FAIL\t", "panic:"} { + // A panic, a data race or a build error outside any test says more than the package's bare FAIL line + // (novox/hq issue 460). + for _, said := range []func(string) bool{ + func(l string) bool { return strings.HasPrefix(l, "--- FAIL:") }, + func(l string) bool { return strings.HasPrefix(l, "panic:") || strings.HasPrefix(l, "fatal error:") }, + func(l string) bool { return l == "WARNING: DATA RACE" }, + goError.MatchString, + func(l string) bool { return strings.HasPrefix(l, "FAIL\t") }, + } { for _, line := range lines { - if line = strings.TrimSpace(line); strings.HasPrefix(line, prefix) { + if line = strings.TrimSpace(line); said(line) { return line } } diff --git a/internal/builder/check_credential_test.go b/internal/builder/check_credential_test.go new file mode 100644 index 00000000..7fc52aa9 --- /dev/null +++ b/internal/builder/check_credential_test.go @@ -0,0 +1,251 @@ +package builder + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io/fs" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/facts" +) + +// **A check's container never sees the forge credential** (novox/hq issue 462): the toolchain container +// mounts the workspace as HOME, so a credential kept there — or a clone's .git/config carrying one — is +// readable by any pull request's merge-check.sh, and printed, kept on the bus for days. + +const ( + forgeSecret = "sw0rdfi5h-forge" + forgeURL = "http://mesh_novox_builder:" + forgeSecret + "@forge.invalid:20000" + besideSecret = "b3side-t0ken" + npmSecret = "npm-s3cret-t0ken" +) + +// aFactsRegistry is an artifact store holding the facts snapshot, and nothing else. +func aFactsRegistry(t *testing.T) string { + t.Helper() + body, err := json.Marshal(facts.Facts{Format: facts.Format, Taken: time.Now().UTC(), + Versions: facts.Versions{Bus: "2.11.17", Store: "17.11"}, Machines: []facts.Machine{{Name: "abcdef", Length: 6}}}) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + manifest, _ := json.Marshal(map[string]any{"schemaVersion": 2, "layers": []map[string]any{ + {"mediaType": facts.MediaType, "digest": digest, "size": len(body)}}}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.Contains(r.URL.Path, "/manifests/"): + w.Write(manifest) + case strings.HasSuffix(r.URL.Path, "/blobs/"+digest): + w.Write(body) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://") +} + +// bareURL is a URL with its userinfo left out. +func bareURL(t *testing.T, raw string) string { + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + u.User = nil + return u.String() +} + +func TestACheckContainerSeesNoForgeCredential(t *testing.T) { + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) + besideRepo, besideHead := aCheckedRepository(t, map[string]string{"README": "beside"}) + // A clone source that carries userinfo, as a forge's clone URL may: git records it as given in the + // clone's .git/config, which the container reads. + besideURL := "file://beside-user:" + besideSecret + "@" + besideRepo + workspace := t.TempDir() + + var stores []string + reached := false + var leaks []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + switch name { + case "git": + for _, a := range args { + if f, ok := strings.CutPrefix(a, "credential.helper=store --file="); ok { + stores = append(stores, f) + raw, err := os.ReadFile(f) + if err != nil || strings.TrimSpace(string(raw)) != forgeURL { + t.Errorf("git is offered a store that does not hold the credential as given: %q, %v", raw, err) + } + if info, err := os.Stat(f); err == nil && info.Mode().Perm() != 0o600 { + t.Errorf("the credential store is readable beyond its owner: %v", info.Mode()) + } + } + } + if len(args) >= 2 && args[len(args)-3] == "--quiet" && hasString(args, "clone") { + source := args[len(args)-2] + if u, err := url.Parse(source); err == nil && u.User != nil { + // Git cannot reach a file:// URL with userinfo; clone it without, then record it as git + // would have: as given. + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, filepath.Join(dir, args[len(args)-1]), "git", "remote", "set-url", "origin", source) + } + } + return Command(ctx, dir, name, args...) + case "docker": + if !reached { + reached = true + // The first container: everything the workspace holds is what the toolchain container sees. + filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + raw, _ := os.ReadFile(path) + s := string(raw) + if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) || + strings.Contains(s, npmSecret) || d.Name() == "git-credentials" || d.Name() == ".npmrc" || + strings.Contains(s, "credential.helper") { + leaks = append(leaks, path) + } + return nil + }) + for _, f := range stores { + if _, err := os.Stat(f); !errors.Is(err, os.ErrNotExist) { + leaks = append(leaks, f+" (still there when the first container runs)") + } + if rel, err := filepath.Rel(workspace, f); err == nil && !strings.HasPrefix(rel, "..") { + leaks = append(leaks, f+" (inside the workspace the container mounts)") + } + } + } + if len(args) > 0 && args[0] == "ps" { + return "", nil + } + return "", errors.New("no container runtime in this test") + } + return "", errors.New("unexpected command " + name) + } + // A credential an older builder left in the workspace is removed too: the forge's, and the .npmrc a + // build wrote into the tree it cloned. + if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, dir := range []string{"source/x", "context-server"} { + if err := os.MkdirAll(filepath.Join(workspace, dir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspace, dir, ".npmrc"), + []byte("//forge.invalid/api/packages/novox/npm/:_authToken="+npmSecret+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + _, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{ + "mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t), + GitCredential{URL: forgeURL}, nil) + if err == nil { + t.Fatal("the check ran past its first container in a test with none") + } + if !reached { + t.Fatalf("the check never reached its first container: %v", err) + } + if len(stores) == 0 { + t.Fatal("no clone was offered the forge credential") + } + if len(leaks) > 0 { + t.Fatalf("the check's container sees the credential:\n%s", strings.Join(leaks, "\n")) + } +} + +// Every line a repository's own check prints is published to the build's log redacted. +func TestACheckLinePublishedToTheLogIsRedacted(t *testing.T) { + var said []string + say := func(step, format string, args ...any) { + if step == "output" && len(args) > 0 { + said = append(said, args[0].(string)) + } + } + var out tail + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "golang"}, []ScriptPart{{Toolchain: "go", Script: CheckScript}}, + t.TempDir(), &out, func() bool { return false }, func(string, string) *exec.Cmd { + return exec.CommandContext(t.Context(), "sh", "-c", "echo cloning http://mesh_builder:t0ps3cret-forge@forge.invalid/novox/x.git; "+ + "echo token ghp_abcdefghijklmnopqrstuvwxyz0123456789") + }, say) + if layer == nil || layer.Verdict != "pass" { + t.Fatalf("the check answered %+v\n%s", layer, out.String()) + } + joined := strings.Join(said, "\n") + if strings.Contains(joined, "t0ps3cret-forge") || strings.Contains(joined, "ghp_abcdef") { + t.Fatalf("a secret the check printed is published to the build's log:\n%s", joined) + } + if !strings.Contains(joined, "http://mesh_builder:[redacted: a password in a URI]@forge.invalid/novox/x.git") { + t.Fatalf("the line is not said with what was there named:\n%s", joined) + } +} + +func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) { + r := redactorFor(GitCredential{URL: forgeURL}) + for in, want := range map[string]string{ + "the secret alone: " + forgeSecret: "the secret alone: [redacted: the forge credential]", + "go test ./... ok": "go test ./... ok", + "--password hunter22 and done": "--password [redacted: the word after --password] and done", + "commit 3b6b54a0c1d2e3f4a5b6c7d8e9f0": "commit 3b6b54a0c1d2e3f4a5b6c7d8e9f0", + } { + if got := r.redact(in); got != want { + t.Errorf("%q redacted as %q, want %q", in, got, want) + } + } +} + +// A build's clone of a URL carrying userinfo records it without, so no build leaves a credential in +// workspace/source/.git/config while it runs either (novox/hq issue 462); the tree itself is gone when the +// build ends. +func TestABuildsCloneRecordsNoUserinfo(t *testing.T) { + repo, _ := aCheckedRepository(t, map[string]string{ManifestName: `{"module":"plain","version":"1"}`}) + source := "file://build-user:" + besideSecret + "@" + repo + workspace := t.TempDir() + tree := filepath.Join(workspace, "source") + var configs []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "git" && hasString(args, "clone") && args[len(args)-2] == source { + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, args[len(args)-1], "git", "remote", "set-url", "origin", source) + } + if name == "git" && len(args) > 0 && args[0] == "rev-parse" { + raw, _ := os.ReadFile(filepath.Join(tree, ".git", "config")) + configs = append(configs, string(raw)) + } + return Command(ctx, dir, name, args...) + } + if _, err := Build(t.Context(), run, &recorded{}, source, "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if len(configs) == 0 { + t.Fatal("the build never read its clone") + } + for _, c := range configs { + if strings.Contains(c, besideSecret) || strings.Contains(c, "build-user") { + t.Fatalf("the build's clone records the credential it was cloned with:\n%s", c) + } + } + if _, err := os.Stat(tree); !os.IsNotExist(err) { + t.Fatalf("the build's tree outlives the build: %v", err) + } +} diff --git a/internal/builder/failures.go b/internal/builder/failures.go new file mode 100644 index 00000000..3e5e0e12 --- /dev/null +++ b/internal/builder/failures.go @@ -0,0 +1,285 @@ +package builder + +import ( + "bytes" + "fmt" + "regexp" + "strings" +) + +// **What failed is picked from the whole of a check's output, as it streams** (novox/hq issue 460). +// +// A repository's own check kept only the last reportLines lines of what it printed, and named what failed +// from those. A long store-backed run prints its logs after the failure: the `--- FAIL:` line of mesh-controller +// #218 was pushed out of the tail by the package's own log lines, and the verdict said only +// "FAIL 268.072s" — the test that failed could not be named, and the fix was a guess. #220's failure +// came a few dozen lines before the end, inside the tail, and was named. The length of what came after the +// failure decided it, nothing else. +// +// So every line is looked at as it is written, and the lines that name what failed are kept whole, wherever +// in the output they are: each `--- FAIL:` line with its indented message lines, a `panic:` with its first +// frames, a `WARNING: DATA RACE` report, each `FAIL\t` line, a build error with its package, and +// the files gofmt lists. The rest is cut as before. + +// Bounds on what is picked, so a run that fails everywhere still travels in a verdict: the controller cuts a +// report to its last 60 KiB (maxCheckReport), and what failed is said at the report's end, so it stays inside. +const ( + // failureLines and failureBytes bound everything picked from one run. + failureLines = 400 + failureBytes = 32 << 10 + // failureLineBytes bounds one picked line: a message that pastes a whole document is cut, said. + failureLineBytes = 1 << 10 + // The lines kept after the line that opens each kind of block. + failMessageLines = 30 // a --- FAIL's messages + panicLines = 40 // a panic's first frames + raceLines = 80 // a data race report, to its closing rule + buildLines = 40 // a package's build errors + gofmtLines = 50 // the files gofmt lists + // failedNames bounds how many further failing tests the summary names after the first. + failedNames = 5 +) + +// goError is a compiler's or vet's line: a Go file, a line, and what is wrong there. +var goError = regexp.MustCompile(`^\S+\.go:\d+(:\d+)?: `) + +// raceRule is the line the race detector opens and closes its report with. +const raceRule = "==================" + +type failureBlock int + +const ( + noBlock failureBlock = iota + inFail + inPanic + inRace + inBuild + inGofmt +) + +// failures keeps the lines of a check's output that name what failed. It is an io.Writer, fed the same +// bytes as the report's tail. +type failures struct { + partial []byte + kept []string + size int + dropped int + + block failureBlock + indent int // a --- FAIL line's indentation: its messages are indented deeper + left int // lines the open block may still keep + headers []string + tests []string // every failing test's --- FAIL line, in order + lastRule bool // the line before was the race detector's rule, which opens its report +} + +func (f *failures) Write(p []byte) (int, error) { + f.partial = append(f.partial, p...) + for { + i := bytes.IndexByte(f.partial, '\n') + if i < 0 { + break + } + f.line(strings.TrimRight(string(f.partial[:i]), "\r")) + f.partial = f.partial[i+1:] + } + // A line with no end, longer than any line is kept, is judged by its start. + if len(f.partial) > failureLineBytes*4 { + f.line(string(f.partial)) + f.partial = nil + } + return len(p), nil +} + +// flush judges what is left of a last line with no newline. +func (f *failures) flush() { + if len(f.partial) > 0 { + f.line(strings.TrimRight(string(f.partial), "\r")) + f.partial = nil + } +} + +func indentOf(line string) int { + return len(line) - len(strings.TrimLeft(line, " \t")) +} + +func (f *failures) keep(line string) { + if f.size >= failureBytes || len(f.kept) >= failureLines { + f.dropped++ + return + } + if len(line) > failureLineBytes { + line = line[:failureLineBytes] + fmt.Sprintf(" … (%d bytes more)", len(line)-failureLineBytes) + } + f.kept = append(f.kept, line) + f.size += len(line) + 1 +} + +func (f *failures) open(block failureBlock, lines int) { + f.block, f.left = block, lines +} + +func (f *failures) line(line string) { + trimmed := strings.TrimSpace(line) + rule := trimmed == raceRule + + // A line that opens a block ends whatever block was open. + if f.opens(line, trimmed) { + f.lastRule = rule + return + } + switch f.block { + case inFail: + if trimmed != "" && indentOf(line) > f.indent && f.left > 0 { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + case inPanic: + if f.left > 0 && !strings.HasPrefix(line, "FAIL") && !strings.HasPrefix(line, "ok \t") && + !strings.HasPrefix(line, "exit status") { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + case inRace: + if f.left > 0 { + f.left-- + f.keep(line) + if rule { + f.block = noBlock + } + f.lastRule = rule + return + } + case inBuild: + if f.left > 0 && trimmed != "" && !strings.HasPrefix(line, "FAIL") && !strings.HasPrefix(line, "ok \t") && + !strings.HasPrefix(line, "? \t") { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + case inGofmt: + if f.left > 0 && trimmed != "" { + f.left-- + f.keep(line) + f.lastRule = rule + return + } + } + f.block = noBlock + + switch { + case strings.HasPrefix(line, "# "): + // A package's name before its build errors — kept only when an error follows it. + f.headers = append(f.headers, line) + if len(f.headers) > 4 { + f.headers = f.headers[1:] + } + case len(f.headers) > 0 && goError.MatchString(line): + for _, h := range f.headers { + f.keep(h) + } + f.headers = nil + f.keep(line) + f.open(inBuild, buildLines) + case strings.HasPrefix(line, "FAIL\t"): + f.headers = nil + f.keep(line) + default: + f.headers = nil + } + f.lastRule = rule +} + +// opens keeps a line that opens a block of what failed, and opens the block. False for any other line. +func (f *failures) opens(line, trimmed string) bool { + switch { + case strings.HasPrefix(trimmed, "--- FAIL:"): + f.headers = nil + f.keep(line) + f.tests = append(f.tests, trimmed) + f.indent = indentOf(line) + f.open(inFail, failMessageLines) + case strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:"): + f.headers = nil + f.keep(line) + f.open(inPanic, panicLines) + case trimmed == "WARNING: DATA RACE": + f.headers = nil + if f.lastRule { + f.keep(raceRule) + } + f.keep(line) + f.open(inRace, raceLines) + case strings.HasPrefix(trimmed, "not gofmt'd:"): + f.headers = nil + f.keep(line) + f.open(inGofmt, gofmtLines) + default: + return false + } + return true +} + +// String is every line picked, in the order the run printed them, and how many more were left out. +func (f *failures) String() string { + f.flush() + s := strings.Join(f.kept, "\n") + if f.dropped > 0 { + s += fmt.Sprintf("\n… %d more line(s) naming what failed, past the %d lines or %d KiB kept", f.dropped, + failureLines, failureBytes>>10) + } + return s +} + +// said is what the repository layer's summary says failed: the first thing that failed, whole, then the +// further failing tests by name. Empty when nothing was picked. +func (f *failures) said() string { + picked := f.String() + if picked == "" { + return "" + } + first := whatFailed(picked) + var more []string + for _, t := range f.tests { + if t == first { + continue + } + // A subtest's parent fails with it; its name is in the subtest's. + name := strings.Fields(strings.TrimPrefix(t, "--- FAIL:")) + if len(name) > 0 { + more = append(more, name[0]) + } + } + more = withoutParents(more) + switch { + case len(more) == 0: + return first + case len(more) > failedNames: + return fmt.Sprintf("%s; and %s and %d more", first, strings.Join(more[:failedNames], ", "), + len(more)-failedNames) + default: + return first + "; and " + strings.Join(more, ", ") + } +} + +// withoutParents drops a test whose subtest is also named. +func withoutParents(names []string) []string { + var out []string + for _, n := range names { + parent := false + for _, m := range names { + if strings.HasPrefix(m, n+"/") { + parent = true + break + } + } + if !parent { + out = append(out, n) + } + } + return out +} diff --git a/internal/builder/failures_test.go b/internal/builder/failures_test.go new file mode 100644 index 00000000..e6278dfa --- /dev/null +++ b/internal/builder/failures_test.go @@ -0,0 +1,182 @@ +package builder + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// ownCheckOf runs a repository's own check whose script prints a captured output and fails, as the build +// seat runs it, and answers the layer and every line it said in the build's log. +func ownCheckOf(t *testing.T, printed string) (*Layer, []string) { + t.Helper() + tree := t.TempDir() + if err := os.WriteFile(filepath.Join(tree, "printed.txt"), []byte(printed), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, CheckScript), []byte("cat printed.txt\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + var out tail + var logged []string + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "go-image"}, []ScriptPart{{"go", CheckScript}}, tree, &out, + func() bool { return false }, + func(_, script string) *exec.Cmd { + cmd := exec.CommandContext(t.Context(), "sh", script) + cmd.Dir = tree + return cmd + }, func(step, format string, args ...any) { + if step == "output" { + logged = append(logged, fmt.Sprintf(format, args...)) + } + }) + if layer == nil || layer.Verdict != "fail" { + t.Fatalf("a failing check answered %+v", layer) + } + return layer, logged +} + +func captured(t *testing.T, name string) string { + t.Helper() + body, err := os.ReadFile(filepath.Join("testdata", "issue460", name)) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +// **novox/hq issue 460**: mesh-controller #218's check failed with "FAIL 268.072s" and no test +// name: the package's own logs, printed after its `--- FAIL:` lines, pushed them out of the last 200 lines, +// which was all the summary was named from. The first failing test is named, and the others after it, +// however much came after them. +func TestAFailureEarlyInALongRunIsStillNamed(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") { + t.Errorf("a failure 600 lines from the end is said as %q", layer.Summary) + } + if !strings.Contains(layer.Summary, "TestSubtests/the_hub") { + t.Errorf("the second failing test is not named: %q", layer.Summary) + } +} + +// One -race run of every package: the first failure is the earliest, not the last within the tail. +func TestARaceRunOfEveryPackageNamesItsFirstFailure(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-race-run-of-every-package.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") || + !strings.Contains(layer.Summary, "TestAMapIsNil") || !strings.Contains(layer.Summary, "TestACounterIsShared") { + t.Errorf("a run failing in four packages is said as %q", layer.Summary) + } +} + +// A package that does not build is named by its error, not by its bare "[build failed]". +func TestABuildErrorIsNamedByTheError(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-build-error.txt")) + if !strings.HasSuffix(layer.Summary, "failed: broken/broken.go:3:28: undefined: undefinedThing") { + t.Errorf("a build error is said as %q", layer.Summary) + } +} + +// The whole of what the check printed is in the build's log, line by line — where before none of it was. +func TestTheWholeOutputIsInTheBuildsLog(t *testing.T) { + printed := captured(t, "a-long-run-failing-early.txt") + _, logged := ownCheckOf(t, printed) + want := strings.Split(strings.TrimRight(printed, "\n"), "\n") + if len(logged) != len(want) { + t.Fatalf("the build's log holds %d line(s) of the %d printed", len(logged), len(want)) + } + for i := range want { + if logged[i] != want[i] { + t.Fatalf("line %d is logged as %q, printed as %q", i+1, logged[i], want[i]) + } + } +} + +// What failed is kept whole in the verdict: each --- FAIL with its messages, a panic with its first frames, +// a data race report from rule to rule — and none of the log lines around them. +func TestTheVerdictKeepsWhatFailedWhole(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + want := "--- FAIL: TestTheEnvelopeIsPinned (0.00s)\n" + + " early_test.go:9: the envelope's subject is \"mesh.a\", not \"mesh.b\"\n" + + " early_test.go:10: a second line of the same failure\n" + + "--- FAIL: TestSubtests (0.00s)\n" + + " --- FAIL: TestSubtests/the_hub (0.00s)\n" + + " early_test.go:14: the hub does not compose\n" + + "FAIL\texample.com/cap/early\t" + if !strings.HasPrefix(layer.Failed, want) || strings.Contains(layer.Failed, "kept what home-server says") { + t.Errorf("the early failure is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-panic.txt")) + for _, line := range []string{"--- FAIL: TestAMapIsNil (0.00s)", "panic: assignment to entry in nil map", + "[running]:", "example.com/cap/panics.TestAMapIsNil(", "/src/cap/panics/panics_test.go:7", + "FAIL\texample.com/cap/panics\t"} { + if !strings.Contains(layer.Failed, line) { + t.Errorf("a panic is kept without %q:\n%s", line, layer.Failed) + } + } + + layer, _ = ownCheckOf(t, captured(t, "a-data-race.txt")) + race := captured(t, "a-data-race.txt") + report := race[:strings.Index(race, "--- FAIL:")] + if !strings.HasPrefix(layer.Failed, report) || + !strings.Contains(layer.Failed, "--- FAIL: TestACounterIsShared (0.00s)\n testing.go:1865: race detected") { + t.Errorf("a data race is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-build-error.txt")) + if layer.Failed != "# example.com/cap/broken\nbroken/broken.go:3:28: undefined: undefinedThing\n"+ + "FAIL\texample.com/cap/broken [build failed]" { + t.Errorf("a build error is kept as:\n%s", layer.Failed) + } +} + +// A run that fails everywhere is bounded in what it keeps, and what it keeps survives the controller's cut +// of a report to its last 60 KiB, said at the report's end. +func TestWhatFailedIsBoundedAndSurvivesTheReportsCut(t *testing.T) { + var b strings.Builder + for i := range 3000 { + fmt.Fprintf(&b, "--- FAIL: TestNumber%d (0.00s)\n x_test.go:1: %s\n", i, strings.Repeat("why ", 600)) + fmt.Fprintf(&b, "2026/10/11 01:30:03 a log line %d\n", i) + } + b.WriteString("FAIL\tx/everything\t1s\nFAIL\n") + layer, logged := ownCheckOf(t, b.String()) + if len(layer.Failed) > failureBytes+2*failureLineBytes || !strings.Contains(layer.Failed, "more line(s) naming what failed") { + t.Errorf("what failed in a run that fails everywhere is %d bytes, ending %q", len(layer.Failed), + layer.Failed[max(0, len(layer.Failed)-200):]) + } + if !strings.Contains(layer.Summary, "--- FAIL: TestNumber0 (0.00s); and TestNumber1, ") || + !strings.Contains(layer.Summary, "and 2994 more") { + t.Errorf("a run failing 3000 tests is said as %q", layer.Summary) + } + + var out tail + out.Write([]byte(b.String())) + report := withWhatFailed(out.String(), layer) + const maxCheckReport = 60 << 10 // as the controller cuts it, from the front + if len(report) > maxCheckReport { + report = report[len(report)-maxCheckReport:] + } + if !strings.Contains(report, "--- what failed") || !strings.Contains(report, "--- FAIL: TestNumber0 (0.00s)") { + t.Error("what failed did not survive the controller's cut of the report") + } + + // The build's log keeps its first logLines lines, says how many it left out, then what failed. + if len(logged) < logLines+2 || logged[logLines] != "… 1002 more line(s) of its output are not in this log, which keeps its first 8000" || + logged[logLines+1] != "--- what failed, picked from the whole of its output" || + logged[logLines+2] != "--- FAIL: TestNumber0 (0.00s)" { + t.Errorf("the log past its bound says %q", logged[logLines:min(len(logged), logLines+3)]) + } +} + +// A passing check says nothing of what failed. +func TestAPassingCheckReportsAsBefore(t *testing.T) { + if got := withWhatFailed("ok\tx\t1s", &Layer{Verdict: "pass"}); got != "ok\tx\t1s" { + t.Errorf("a passing report became %q", got) + } + if got := withWhatFailed("r", nil); got != "r" { + t.Errorf("no repository layer became %q", got) + } +} diff --git a/internal/builder/mirror.go b/internal/builder/mirror.go index 8840e3d5..4a2da87e 100644 --- a/internal/builder/mirror.go +++ b/internal/builder/mirror.go @@ -429,6 +429,11 @@ func (r Registry) copyBlob(ctx context.Context, src *source, where upstream, dig if response.ContentLength > 0 { put.ContentLength = response.ContentLength } + // **Not waited for if refused** (novox/hq issue 457): the body streams from upstream and cannot be + // read twice, so a registry held still between the POST above and this PUT fails the copy with + // "its body cannot be read twice" rather than waiting. Accepted: the POST a moment before already + // waited the registry out, so the window is the length of one upstream fetch, and the build fails + // loudly, to be asked again, rather than buffering every base blob in memory. done, err := r.client().Do(put) if err != nil { return fmt.Errorf("cannot upload blob %s: %w", digest, err) diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go index a87f19fd..7cccbf16 100644 --- a/internal/builder/packages_test.go +++ b/internal/builder/packages_test.go @@ -70,7 +70,16 @@ func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) { func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"}) n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - if _, err := Build(context.Background(), r.run, r, + npmrc := filepath.Join(workspace, "source", ".npmrc") + inContext := false + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "build" { + _, err := os.Stat(npmrc) + inContext = err == nil + } + return r.run(ctx, dir, name, args...) + } + if _, err := Build(context.Background(), run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { t.Fatalf("the build failed: %v", err) } @@ -90,11 +99,14 @@ func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { if !strings.Contains(build, "--network host") { t.Fatalf("the build was not given the host network to reach the registry: %s", build) } - // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it. - tree := filepath.Join(workspace, "source") - npmrc := filepath.Join(tree, ".npmrc") - if _, err := os.Stat(npmrc); err != nil { - t.Fatalf("the credential was not written into the build context: %v", err) + // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it, and + // removed with the tree when the build ends: a later check's container mounts the workspace (novox/hq + // issue 462). + if !inContext { + t.Fatal("the credential was not in the build context when the image was built") + } + if _, err := os.Stat(npmrc); !os.IsNotExist(err) { + t.Fatalf("the credential outlives the build in the workspace: %v", err) } } diff --git a/internal/builder/redact.go b/internal/builder/redact.go new file mode 100644 index 00000000..84205614 --- /dev/null +++ b/internal/builder/redact.go @@ -0,0 +1,191 @@ +package builder + +// **Every line of a check's output is redacted before it is kept** (novox/hq issue 462). +// +// A repository's own check prints into the build's log, which the bus keeps for days and anyone who may read +// its events reads, and into the verdict, which the forge shows on the pull request. Software prints what it +// was given — a URL carrying a password, a token in a flag — and a pull request may print on purpose. +// So a line is said only after every secret the builder knows (the forge credential's password) and every +// value whose shape says it is one is replaced by a mark naming what was there, as the journal verb does. +// +// Copied from the journal tool's redactor (mesh-catalog, modules/systemd/cmd/systemd-tools/secrets.go, +// itself a copy of the docker module's), narrowed to a line's shapes, with the token shapes a check's output +// may carry added. A third copy: sharing them through mesh-sdk is novox/hq issue 471. + +import ( + "net/url" + "regexp" + "strings" +) + +// secretName is a variable name that says its value is a secret. +var secretName = regexp.MustCompile(`(?i)(pass(word|wd|phrase)?|secret|token|api_?key|private_?key|access_?key|credential|auth)`) + +// notAValue is a name that says its value is where a secret is, not the secret: a file or a path. +var notAValue = regexp.MustCompile(`(?i)(_FILE|FILE|_PATH|_DIR)$`) + +// uriPassword is a URI carrying a password in its userinfo: scheme://user:password@. +var uriPassword = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://[^\s/:@'"]*:([^\s/@'"]+)@`) + +// tokenShaped are tokens recognised by their own prefix, whatever surrounds them: a forge's or a host's +// access token, a JSON web token, a NATS seed. +var tokenShaped = []struct { + name string + re *regexp.Regexp +}{ + {"an access token", regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abpr]-[A-Za-z0-9-]{10,}|sk-ant-[A-Za-z0-9_-]{20,})`)}, + {"a JSON web token", regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+`)}, + {"a NATS seed", regexp.MustCompile(`\bS[ACNOU][A-Z2-7]{56}\b`)}, +} + +// masked is a password a program already hid: ***, xxx, , [REDACTED]. +var masked = regexp.MustCompile(`^(\*+|x+|X+|<[^>]*>|\[[^\]]*\]|%2A+)$`) + +// ordinary is a value under a secret's name that is not one: a path, an address, a number, a switch. +var ordinary = regexp.MustCompile(`^(/.*|[A-Za-z][A-Za-z0-9+.-]*://.*|[0-9.]+[a-z]?|(?i:true|false|yes|no|on|off|none|null))$`) + +// leastSecret is the shortest value compared as a secret: a shorter one matches ordinary words. +const leastSecret = 6 + +// passwordFlags take a secret as their next word, or after `=`, whatever the program. +var passwordFlags = map[string]bool{ + "-P": true, "--password": true, "--pass": true, "--passwd": true, "--secret": true, "--secret-key": true, + "--token": true, "--api-key": true, "--apikey": true, "--auth": true, +} + +// knownSecret is one value the builder holds, by the name it is said under. +type knownSecret struct { + Name string + Value string +} + +// redactor hides the secrets it knows and those a line's shapes say are secrets. +type redactor struct{ known []knownSecret } + +// redactorFor knows the forge credential's password, and its user's name with it, in every form git or a +// program may print them. +func redactorFor(forge GitCredential) redactor { + var r redactor + if forge.URL == "" { + return r + } + for _, m := range uriPassword.FindAllStringSubmatch(forge.URL, -1) { + r.add("the forge credential", m[1]) + if dec, err := url.PathUnescape(m[1]); err == nil && dec != m[1] { + r.add("the forge credential", dec) + } + } + return r +} + +func (r *redactor) add(name, value string) { + if len(value) < leastSecret || masked.MatchString(value) { + return + } + for _, k := range r.known { + if k.Value == value { + return + } + } + r.known = append(r.known, knownSecret{name, value}) +} + +// redact is a text with every known secret, every value its shape says is one, and every password inside a +// URI replaced by a mark naming what was there. Line by line: a shape is judged within its line. +func (r redactor) redact(text string) string { + if !strings.ContainsAny(text, "\n") { + return r.line(text) + } + lines := strings.Split(text, "\n") + for i, l := range lines { + lines[i] = r.line(l) + } + return strings.Join(lines, "\n") +} + +func (r redactor) line(line string) string { + replace := func(s knownSecret) { + for _, f := range forms(s.Value) { + line = strings.ReplaceAll(line, f, "[redacted: "+s.Name+"]") + } + } + for _, s := range r.known { + replace(s) + } + for _, s := range shaped(line) { + replace(s) + } + line = uriPassword.ReplaceAllStringFunc(line, func(m string) string { + sub := uriPassword.FindStringSubmatch(m) + if masked.MatchString(sub[1]) || strings.HasPrefix(sub[1], "[redacted") { + return m + } + return strings.TrimSuffix(m, sub[1]+"@") + "[redacted: a password in a URI]@" + }) + for _, t := range tokenShaped { + line = t.re.ReplaceAllString(line, "[redacted: "+t.name+"]") + } + return line +} + +// forms are the ways a value may appear printed: as given, and URL-encoded. +func forms(value string) []string { + out := []string{value} + for _, f := range []string{url.QueryEscape(value), url.PathEscape(value)} { + if f != value && !hasString(out, f) { + out = append(out, f) + } + } + return out +} + +func hasString(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +// shaped are the values a line carries by their shape: the word after a password flag, or the value of one +// given with `=`, and a NAME=value whose name says secret. +func shaped(line string) []knownSecret { + var out []knownSecret + add := func(name, value string) { + value = strings.Trim(value, `"',;`) + if len(value) < leastSecret || masked.MatchString(value) || ordinary.MatchString(value) || + strings.HasPrefix(value, "[redacted") { + return + } + out = append(out, knownSecret{name, value}) + } + words := strings.Fields(line) + for i, w := range words { + if flag, value, ok := strings.Cut(w, "="); ok && strings.HasPrefix(flag, "-") { + if passwordFlags[flag] { + add("the value of "+flag, value) + } + continue + } + if name, value, ok := strings.Cut(w, "="); ok && name != "" && secretName.MatchString(name) && + !notAValue.MatchString(name) && !strings.ContainsAny(name, "/:") { + add("the value of "+name, value) + continue + } + if i+1 < len(words) && passwordFlags[w] { + add("the word after "+w, words[i+1]) + } + } + return out +} + +// withoutUserinfo is a URL with its userinfo left out, and whether it carried any. +func withoutUserinfo(raw string) (string, bool) { + u, err := url.Parse(raw) + if err != nil || u.User == nil { + return raw, false + } + u.User = nil + return u.String(), true +} diff --git a/internal/builder/registry.go b/internal/builder/registry.go index 219b4fe5..4f7ecd11 100644 --- a/internal/builder/registry.go +++ b/internal/builder/registry.go @@ -52,7 +52,11 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string) if _, err := r.Run(ctx, "", "docker", "tag", localTag, remote); err != nil { return "", err } - if _, err := r.Run(ctx, "", "docker", "push", remote); err != nil { + // The push waits for a registry held still, as every request to it does (novox/hq issue 457). + if err := waitForRegistry(ctx, r.Address, "docker push "+remote, func() error { + _, err := r.Run(ctx, "", "docker", "push", remote) + return err + }); err != nil { return "", err } out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{index .RepoDigests 0}}", remote) @@ -169,11 +173,13 @@ func (r Registry) has(ctx context.Context, url string, accept ...string) (bool, return response.StatusCode == http.StatusOK, nil } +// client is the client for the registry and for upstream, whose requests to the registry wait out a +// registry held still (novox/hq issue 457). func (r Registry) client() *http.Client { if r.HTTP != nil { - return r.HTTP + return waiting(r.HTTP, r.Address) } - return http.DefaultClient + return waiting(http.DefaultClient, r.Address) } // separator is whether the upload location already carries a query. diff --git a/internal/builder/registry_wait.go b/internal/builder/registry_wait.go new file mode 100644 index 00000000..f2b10bae --- /dev/null +++ b/internal/builder/registry_wait.go @@ -0,0 +1,137 @@ +package builder + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "syscall" + "time" +) + +// A build waits out a registry that refuses connections, for a bounded time (novox/hq issue 457). +// +// **Why waiting, and why here.** The store's nightly collection holds the registry still for its run — +// about a minute and a half, measured on 2026-10-11 — and a build that reached the registry in that +// window failed on "connection refused", and its whole delivery plan with it: a plan failed for a +// pause the mesh itself scheduled. The other design weighed was the collection telling the controller +// it holds the registry, and the controller holding build asks while it runs. Waiting here is smaller +// and covers more: it is local to the one place that talks to the registry, needs no new message +// between modules, and also carries a build over any other short outage — a registry restarted by its +// own update, say. A refusal is the one error waited for: nothing was sent, so trying again cannot +// do anything twice, and it is what a registry that is stopped answers. +// +// **Bounded, and loud past the bound.** registryWait is longer than the collection holds the registry +// (five minutes against about one and a half), so the pause the mesh schedules is always waited out, +// and a registry that is really down still fails the build — saying how long it was refused — rather +// than holding a build machine for ever. Every wait is said in the build's log, with why, and so is +// the registry answering again. + +var ( + // registryWait is how long a build waits for a registry that refuses, per call that found it so. + registryWait = 5 * time.Minute + // registryFirstPause is the first pause between tries; each pause doubles, up to registryMostPause. + registryFirstPause = time.Second +) + +// registryMostPause is the longest pause between two tries: short enough that a build goes on within +// seconds of the registry answering again. +const registryMostPause = 10 * time.Second + +// refused is whether an error is a connection refused: from a dial here, or as a command such as docker +// said it in its output. +func refused(err error) bool { + return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) +} + +// waitForRegistry runs try, and while it fails because the registry at address refuses connections, +// tries again with a growing pause until registryWait has passed. what names the call, for the log. +func waitForRegistry(ctx context.Context, address, what string, try func() error) error { + err := try() + if !refused(err) { + return err + } + started := time.Now() + pause := registryFirstPause + tell("registry", "%s: refused; the build waits for the registry at %s, for up to %s — it is held still while "+ + "the store's nightly collection runs, about a minute and a half (novox/hq issue 457)", + what, address, registryWait) + for { + left := registryWait - time.Since(started) + if left <= 0 { + tell("registry", "%s: the registry at %s still refuses after %s; the build fails", what, address, + time.Since(started).Round(time.Second)) + return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly "+ + "collection holds it still, so it is down, not paused: %w", + address, time.Since(started).Round(time.Millisecond), err) + } + wait := min(pause, left) + select { + case <-ctx.Done(): + return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", address, ctx.Err(), err) + case <-time.After(wait): + } + pause = min(pause*2, registryMostPause) + if err = try(); !refused(err) { + // Said as it is: the registry answering is only the build going on when the call worked. + if err == nil { + tell("registry", "%s: the registry at %s answers again after %s; the build goes on", what, address, + time.Since(started).Round(time.Millisecond)) + } else { + tell("registry", "%s: the registry at %s no longer refuses after %s, and answered with: %v", what, + address, time.Since(started).Round(time.Millisecond), err) + } + return err + } + } +} + +// waitingTransport waits for the registry on every request to it, and on none to anywhere else: the +// same client copies from upstream registries, whose refusals are theirs to answer. +type waitingTransport struct { + base http.RoundTripper + address string +} + +func (t waitingTransport) RoundTrip(request *http.Request) (*http.Response, error) { + if request.URL.Host != t.address { + return t.base.RoundTrip(request) + } + var response *http.Response + tries := 0 + err := waitForRegistry(request.Context(), t.address, request.Method+" "+request.URL.Path, func() error { + attempt := request + if tries > 0 && request.Body != nil && request.Body != http.NoBody { + // A body is sent again only when it can be read again; one that cannot is not retried. + if request.GetBody == nil { + return fmt.Errorf("%s %s cannot be sent again: its body cannot be read twice", request.Method, request.URL) + } + body, err := request.GetBody() + if err != nil { + return err + } + attempt = request.Clone(request.Context()) + attempt.Body = body + } + tries++ + var err error + response, err = t.base.RoundTrip(attempt) + return err + }) + return response, err +} + +// waiting is a client like c whose requests to the registry wait for it. +func waiting(c *http.Client, address string) *http.Client { + if _, already := c.Transport.(waitingTransport); already { + return c + } + copied := *c + base := c.Transport + if base == nil { + base = http.DefaultTransport + } + copied.Transport = waitingTransport{base: base, address: address} + return &copied +} diff --git a/internal/builder/registry_wait_test.go b/internal/builder/registry_wait_test.go new file mode 100644 index 00000000..8bb9be85 --- /dev/null +++ b/internal/builder/registry_wait_test.go @@ -0,0 +1,154 @@ +package builder + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "net" + "net/http" + "strings" + "sync" + "testing" + "time" +) + +// A registry held still for a while (novox/hq issue 457): the store's nightly collection stops it for +// about a minute and a half, and a build in that window was failed, and its whole plan with it, for a +// pause the mesh itself scheduled. A build waits it out — for a bound longer than the collection +// holds it — says so in its log, and still fails, loudly, past the bound. + +// heldStill is a registry address that refuses every connection until it starts answering after +// pause, or never when pause is negative. +func heldStill(t *testing.T, f *fakeRegistry, pause time.Duration) string { + t.Helper() + handler := f.serve(t).Config.Handler + reserved, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + address := reserved.Addr().String() + reserved.Close() // refused from here on: nothing listens + if pause < 0 { + return address + } + server := &http.Server{Handler: handler} + go func() { + time.Sleep(pause) + l, err := net.Listen("tcp", address) + if err != nil { + t.Errorf("cannot answer at %s again: %v", address, err) + return + } + _ = server.Serve(l) + }() + t.Cleanup(func() { _ = server.Close() }) + return address +} + +// saying collects what a build says, as the build machine's per-build Said does. +func saying(t *testing.T) func() []string { + t.Helper() + var mu sync.Mutex + var lines []string + was := Said + Said = func(step, message string) { + mu.Lock() + defer mu.Unlock() + lines = append(lines, step+": "+message) + } + t.Cleanup(func() { Said = was }) + return func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), lines...) + } +} + +// waitingFor shortens the bound and the first pause, so a test waits for milliseconds. +func waitingFor(t *testing.T, bound time.Duration) { + t.Helper() + wasBound, wasFirst := registryWait, registryFirstPause + registryWait, registryFirstPause = bound, 20*time.Millisecond + t.Cleanup(func() { registryWait, registryFirstPause = wasBound, wasFirst }) +} + +func TestABuildWaitsForARegistryHeldStillAndGoesOn(t *testing.T) { + waitingFor(t, 5*time.Second) + said := saying(t) + f := &fakeRegistry{} + r := Registry{Address: heldStill(t, f, 400*time.Millisecond)} + body := []byte("a theme") + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil { + t.Fatalf("a registry refusing for 400ms failed the build: %v", err) + } + if string(f.blobs[digest]) != "a theme" { + t.Fatalf("the registry holds %q", f.blobs[digest]) + } + log := strings.Join(said(), "\n") + if !strings.Contains(log, "waits for the registry at "+r.Address) || !strings.Contains(log, "nightly collection") { + t.Errorf("the build's log does not say it waited for the registry, and why:\n%s", log) + } + if !strings.Contains(log, "answers again") { + t.Errorf("the build's log does not say the registry came back:\n%s", log) + } +} + +func TestABuildFailsLoudlyOnARegistryRefusingPastTheBound(t *testing.T) { + waitingFor(t, 300*time.Millisecond) + said := saying(t) + f := &fakeRegistry{} + r := Registry{Address: heldStill(t, f, -1)} + body := []byte("a theme") + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + + started := time.Now() + _, err := r.PublishArchive(context.Background(), "shell/config", body, digest) + if err == nil { + t.Fatal("a registry that never answered published the archive") + } + if !strings.Contains(err.Error(), "refused every connection for") || !strings.Contains(err.Error(), "connection refused") { + t.Errorf("the failure does not say it waited and was refused throughout: %v", err) + } + if waited := time.Since(started); waited < 300*time.Millisecond { + t.Errorf("failed after %s, before the bound", waited) + } + if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") { + t.Errorf("the build's log does not say it waited:\n%s", log) + } +} + +func TestAnImagePushWaitsForARegistryHeldStill(t *testing.T) { + waitingFor(t, 5*time.Second) + said := saying(t) + pushes := 0 + run := func(_ context.Context, _ string, name string, args ...string) (string, error) { + switch args[0] { + case "push": + pushes++ + if pushes < 3 { + return "", errorString("docker push: dial tcp 127.0.0.1:5000: connect: connection refused") + } + case "inspect": + return "127.0.0.1:5000/m/server@sha256:abc\n", nil + } + return "", nil + } + r := Registry{Address: "127.0.0.1:5000", Run: run} + if _, err := r.PublishImage(context.Background(), "local", "m/server"); err != nil { + t.Fatalf("a push refused twice failed the build: %v", err) + } + if pushes != 3 { + t.Errorf("pushed %d times, want 3", pushes) + } + if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") { + t.Errorf("the build's log does not say it waited:\n%s", log) + } +} + +type errorString string + +func (e errorString) Error() string { return string(e) } diff --git a/internal/builder/testdata/issue460/CAPTURED b/internal/builder/testdata/issue460/CAPTURED new file mode 100644 index 00000000..84084363 --- /dev/null +++ b/internal/builder/testdata/issue460/CAPTURED @@ -0,0 +1,7 @@ +Captured 2026-10-11 from real `go test` runs (go1.27.1, linux/amd64) of a throwaway module whose +packages fail each way a repository's own check fails: a test failing early in a run its own logs +then fill (early: 600 log lines after the failures), a panic, a data race under -race, a build error, +and one -race run of every package. Only the module's directory was rewritten to /src/cap. + +They are the output shapes of novox/hq issue 460: mesh-controller #218's check printed its +`--- FAIL:` line before the store-backed package's logs, which pushed it out of the report's tail. diff --git a/internal/builder/testdata/issue460/a-build-error.txt b/internal/builder/testdata/issue460/a-build-error.txt new file mode 100644 index 00000000..5e37bdd0 --- /dev/null +++ b/internal/builder/testdata/issue460/a-build-error.txt @@ -0,0 +1,5 @@ +# example.com/cap/broken +broken/broken.go:3:28: undefined: undefinedThing +FAIL example.com/cap/broken [build failed] +ok example.com/cap/fine (cached) +FAIL diff --git a/internal/builder/testdata/issue460/a-data-race.txt b/internal/builder/testdata/issue460/a-data-race.txt new file mode 100644 index 00000000..e56cf202 --- /dev/null +++ b/internal/builder/testdata/issue460/a-data-race.txt @@ -0,0 +1,31 @@ +================== +WARNING: DATA RACE +Read at 0x00c000018398 by goroutine 8: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x7b + +Previous write at 0x00c000018398 by goroutine 9: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x8d + +Goroutine 8 (running) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 + +Goroutine 9 (finished) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 +================== +--- FAIL: TestACounterIsShared (0.00s) + testing.go:1865: race detected during execution of test +FAIL +FAIL example.com/cap/race 0.008s +FAIL diff --git a/internal/builder/testdata/issue460/a-long-run-failing-early.txt b/internal/builder/testdata/issue460/a-long-run-failing-early.txt new file mode 100644 index 00000000..b58ac0be --- /dev/null +++ b/internal/builder/testdata/issue460/a-long-run-failing-early.txt @@ -0,0 +1,609 @@ +--- FAIL: TestTheEnvelopeIsPinned (0.00s) + early_test.go:9: the envelope's subject is "mesh.a", not "mesh.b" + early_test.go:10: a second line of the same failure +--- FAIL: TestSubtests (0.00s) + --- FAIL: TestSubtests/the_hub (0.00s) + early_test.go:14: the hub does not compose +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d0 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d1 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d2 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d3 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d4 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d5 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d6 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d7 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d8 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d9 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d10 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d11 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d12 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d13 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d14 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d15 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d16 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d17 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d18 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d19 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d20 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d21 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d22 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d23 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d24 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d25 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d26 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d27 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d28 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d29 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d30 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d31 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d32 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d33 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d34 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d35 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d36 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d37 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d38 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d39 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d40 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d41 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d42 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d43 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d44 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d45 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d46 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d47 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d48 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d49 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d50 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d51 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d52 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d53 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d54 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d55 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d56 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d57 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d58 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d59 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d60 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d61 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d62 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d63 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d64 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d65 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d66 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d67 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d68 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d69 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d70 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d71 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d72 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d73 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d74 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d75 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d76 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d77 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d78 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d79 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d80 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d81 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d82 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d83 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d84 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d85 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d86 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d87 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d88 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d89 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d90 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d91 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d92 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d93 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d94 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d95 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d96 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d97 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d98 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d99 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d100 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d101 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d102 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d103 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d104 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d105 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d106 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d107 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d108 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d109 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d110 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d111 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d112 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d113 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d114 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d115 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d116 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d117 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d118 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d119 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d120 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d121 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d122 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d123 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d124 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d125 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d126 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d127 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d128 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d129 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d130 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d131 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d132 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d133 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d134 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d135 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d136 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d137 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d138 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d139 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d140 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d141 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d142 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d143 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d144 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d145 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d146 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d147 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d148 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d149 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d150 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d151 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d152 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d153 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d154 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d155 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d156 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d157 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d158 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d159 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d160 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d161 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d162 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d163 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d164 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d165 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d166 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d167 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d168 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d169 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d170 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d171 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d172 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d173 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d174 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d175 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d176 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d177 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d178 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d179 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d180 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d181 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d182 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d183 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d184 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d185 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d186 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d187 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d188 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d189 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d190 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d191 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d192 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d193 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d194 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d195 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d196 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d197 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d198 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d199 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d200 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d201 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d202 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d203 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d204 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d205 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d206 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d207 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d208 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d209 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d210 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d211 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d212 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d213 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d214 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d215 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d216 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d217 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d218 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d219 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d220 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d221 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d222 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d223 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d224 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d225 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d226 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d227 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d228 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d229 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d230 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d231 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d232 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d233 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d234 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d235 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d236 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d237 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d238 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d239 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d240 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d241 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d242 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d243 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d244 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d245 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d246 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d247 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d248 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d249 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d250 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d251 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d252 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d253 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d254 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d255 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d256 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d257 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d258 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d259 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d260 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d261 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d262 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d263 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d264 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d265 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d266 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d267 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d268 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d269 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d270 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d271 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d272 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d273 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d274 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d275 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d276 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d277 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d278 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d279 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d280 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d281 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d282 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d283 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d284 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d285 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d286 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d287 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d288 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d289 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d290 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d291 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d292 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d293 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d294 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d295 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d296 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d297 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d298 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d299 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d300 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d301 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d302 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d303 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d304 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d305 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d306 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d307 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d308 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d309 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d310 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d311 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d312 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d313 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d314 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d315 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d316 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d317 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d318 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d319 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d320 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d321 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d322 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d323 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d324 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d325 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d326 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d327 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d328 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d329 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d330 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d331 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d332 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d333 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d334 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d335 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d336 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d337 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d338 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d339 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d340 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d341 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d342 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d343 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d344 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d345 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d346 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d347 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d348 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d349 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d350 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d351 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d352 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d353 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d354 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d355 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d356 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d357 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d358 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d359 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d360 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d361 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d362 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d363 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d364 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d365 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d366 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d367 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d368 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d369 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d370 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d371 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d372 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d373 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d374 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d375 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d376 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d377 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d378 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d379 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d380 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d381 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d382 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d383 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d384 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d385 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d386 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d387 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d388 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d389 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d390 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d391 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d392 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d393 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d394 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d395 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d396 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d397 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d398 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d399 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d400 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d401 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d402 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d403 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d404 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d405 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d406 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d407 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d408 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d409 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d410 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d411 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d412 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d413 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d414 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d415 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d416 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d417 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d418 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d419 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d420 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d421 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d422 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d423 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d424 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d425 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d426 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d427 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d428 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d429 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d430 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d431 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d432 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d433 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d434 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d435 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d436 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d437 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d438 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d439 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d440 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d441 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d442 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d443 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d444 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d445 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d446 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d447 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d448 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d449 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d450 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d451 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d452 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d453 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d454 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d455 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d456 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d457 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d458 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d459 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d460 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d461 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d462 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d463 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d464 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d465 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d466 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d467 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d468 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d469 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d470 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d471 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d472 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d473 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d474 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d475 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d476 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d477 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d478 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d479 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d480 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d481 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d482 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d483 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d484 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d485 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d486 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d487 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d488 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d489 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d490 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d491 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d492 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d493 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d494 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d495 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d496 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d497 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d498 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d499 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d500 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d501 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d502 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d503 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d504 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d505 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d506 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d507 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d508 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d509 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d510 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d511 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d512 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d513 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d514 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d515 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d516 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d517 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d518 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d519 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d520 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d521 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d522 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d523 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d524 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d525 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d526 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d527 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d528 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d529 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d530 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d531 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d532 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d533 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d534 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d535 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d536 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d537 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d538 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d539 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d540 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d541 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d542 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d543 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d544 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d545 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d546 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d547 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d548 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d549 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d550 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d551 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d552 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d553 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d554 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d555 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d556 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d557 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d558 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d559 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d560 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d561 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d562 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d563 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d564 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d565 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d566 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d567 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d568 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d569 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d570 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d571 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d572 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d573 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d574 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d575 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d576 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d577 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d578 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d579 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d580 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d581 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d582 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d583 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d584 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d585 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d586 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d587 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d588 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d589 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d590 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d591 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d592 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d593 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d594 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d595 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d596 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d597 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d598 +2026/10/11 04:22:00 kept what home-server says about the machine, and refused its account of declaration d599 +FAIL +FAIL example.com/cap/early 0.003s +FAIL diff --git a/internal/builder/testdata/issue460/a-panic.txt b/internal/builder/testdata/issue460/a-panic.txt new file mode 100644 index 00000000..ad18a6ac --- /dev/null +++ b/internal/builder/testdata/issue460/a-panic.txt @@ -0,0 +1,18 @@ +--- FAIL: TestAMapIsNil (0.00s) +panic: assignment to entry in nil map [recovered, repanicked] + +goroutine 18 [running]: +testing.tRunner.func1.2({0x6b6810, 0x6ee000}) + /usr/lib/go/src/testing/testing.go:2123 +0x232 +testing.tRunner.func1() + /usr/lib/go/src/testing/testing.go:2126 +0x329 +panic({0x6b6810?, 0x6ee000?}) + /usr/lib/go/src/runtime/panic.go:859 +0x125 +example.com/cap/panics.TestAMapIsNil(0x1e97d59dc248?) + /src/cap/panics/panics_test.go:7 +0x28 +testing.tRunner(0x1e97d59dc248, 0x6d41f8) + /usr/lib/go/src/testing/testing.go:2193 +0xea +created by testing.(*T).Run in goroutine 1 + /usr/lib/go/src/testing/testing.go:2258 +0x4d4 +FAIL example.com/cap/panics 0.004s +FAIL diff --git a/internal/builder/testdata/issue460/a-race-run-of-every-package.txt b/internal/builder/testdata/issue460/a-race-run-of-every-package.txt new file mode 100644 index 00000000..fa816310 --- /dev/null +++ b/internal/builder/testdata/issue460/a-race-run-of-every-package.txt @@ -0,0 +1,660 @@ +# example.com/cap/broken +broken/broken.go:3:28: undefined: undefinedThing +FAIL example.com/cap/broken [build failed] +--- FAIL: TestTheEnvelopeIsPinned (0.00s) + early_test.go:9: the envelope's subject is "mesh.a", not "mesh.b" + early_test.go:10: a second line of the same failure +--- FAIL: TestSubtests (0.00s) + --- FAIL: TestSubtests/the_hub (0.00s) + early_test.go:14: the hub does not compose +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d0 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d1 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d2 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d3 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d4 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d5 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d6 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d7 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d8 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d9 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d10 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d11 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d12 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d13 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d14 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d15 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d16 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d17 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d18 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d19 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d20 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d21 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d22 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d23 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d24 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d25 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d26 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d27 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d28 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d29 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d30 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d31 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d32 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d33 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d34 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d35 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d36 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d37 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d38 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d39 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d40 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d41 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d42 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d43 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d44 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d45 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d46 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d47 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d48 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d49 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d50 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d51 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d52 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d53 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d54 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d55 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d56 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d57 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d58 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d59 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d60 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d61 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d62 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d63 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d64 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d65 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d66 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d67 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d68 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d69 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d70 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d71 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d72 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d73 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d74 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d75 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d76 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d77 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d78 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d79 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d80 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d81 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d82 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d83 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d84 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d85 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d86 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d87 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d88 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d89 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d90 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d91 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d92 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d93 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d94 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d95 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d96 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d97 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d98 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d99 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d100 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d101 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d102 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d103 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d104 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d105 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d106 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d107 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d108 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d109 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d110 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d111 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d112 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d113 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d114 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d115 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d116 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d117 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d118 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d119 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d120 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d121 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d122 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d123 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d124 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d125 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d126 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d127 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d128 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d129 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d130 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d131 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d132 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d133 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d134 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d135 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d136 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d137 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d138 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d139 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d140 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d141 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d142 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d143 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d144 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d145 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d146 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d147 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d148 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d149 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d150 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d151 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d152 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d153 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d154 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d155 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d156 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d157 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d158 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d159 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d160 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d161 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d162 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d163 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d164 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d165 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d166 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d167 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d168 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d169 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d170 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d171 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d172 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d173 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d174 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d175 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d176 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d177 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d178 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d179 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d180 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d181 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d182 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d183 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d184 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d185 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d186 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d187 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d188 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d189 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d190 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d191 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d192 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d193 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d194 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d195 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d196 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d197 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d198 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d199 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d200 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d201 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d202 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d203 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d204 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d205 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d206 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d207 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d208 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d209 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d210 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d211 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d212 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d213 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d214 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d215 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d216 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d217 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d218 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d219 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d220 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d221 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d222 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d223 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d224 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d225 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d226 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d227 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d228 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d229 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d230 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d231 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d232 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d233 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d234 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d235 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d236 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d237 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d238 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d239 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d240 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d241 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d242 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d243 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d244 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d245 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d246 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d247 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d248 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d249 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d250 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d251 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d252 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d253 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d254 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d255 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d256 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d257 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d258 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d259 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d260 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d261 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d262 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d263 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d264 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d265 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d266 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d267 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d268 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d269 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d270 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d271 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d272 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d273 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d274 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d275 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d276 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d277 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d278 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d279 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d280 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d281 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d282 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d283 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d284 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d285 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d286 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d287 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d288 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d289 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d290 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d291 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d292 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d293 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d294 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d295 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d296 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d297 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d298 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d299 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d300 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d301 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d302 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d303 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d304 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d305 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d306 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d307 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d308 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d309 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d310 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d311 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d312 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d313 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d314 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d315 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d316 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d317 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d318 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d319 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d320 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d321 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d322 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d323 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d324 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d325 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d326 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d327 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d328 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d329 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d330 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d331 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d332 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d333 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d334 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d335 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d336 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d337 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d338 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d339 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d340 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d341 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d342 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d343 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d344 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d345 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d346 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d347 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d348 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d349 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d350 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d351 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d352 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d353 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d354 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d355 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d356 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d357 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d358 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d359 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d360 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d361 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d362 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d363 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d364 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d365 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d366 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d367 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d368 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d369 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d370 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d371 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d372 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d373 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d374 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d375 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d376 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d377 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d378 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d379 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d380 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d381 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d382 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d383 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d384 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d385 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d386 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d387 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d388 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d389 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d390 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d391 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d392 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d393 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d394 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d395 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d396 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d397 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d398 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d399 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d400 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d401 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d402 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d403 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d404 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d405 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d406 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d407 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d408 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d409 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d410 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d411 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d412 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d413 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d414 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d415 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d416 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d417 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d418 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d419 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d420 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d421 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d422 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d423 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d424 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d425 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d426 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d427 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d428 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d429 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d430 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d431 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d432 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d433 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d434 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d435 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d436 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d437 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d438 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d439 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d440 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d441 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d442 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d443 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d444 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d445 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d446 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d447 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d448 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d449 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d450 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d451 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d452 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d453 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d454 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d455 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d456 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d457 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d458 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d459 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d460 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d461 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d462 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d463 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d464 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d465 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d466 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d467 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d468 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d469 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d470 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d471 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d472 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d473 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d474 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d475 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d476 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d477 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d478 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d479 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d480 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d481 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d482 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d483 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d484 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d485 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d486 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d487 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d488 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d489 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d490 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d491 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d492 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d493 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d494 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d495 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d496 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d497 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d498 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d499 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d500 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d501 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d502 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d503 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d504 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d505 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d506 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d507 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d508 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d509 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d510 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d511 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d512 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d513 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d514 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d515 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d516 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d517 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d518 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d519 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d520 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d521 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d522 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d523 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d524 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d525 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d526 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d527 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d528 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d529 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d530 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d531 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d532 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d533 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d534 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d535 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d536 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d537 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d538 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d539 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d540 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d541 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d542 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d543 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d544 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d545 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d546 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d547 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d548 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d549 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d550 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d551 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d552 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d553 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d554 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d555 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d556 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d557 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d558 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d559 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d560 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d561 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d562 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d563 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d564 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d565 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d566 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d567 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d568 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d569 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d570 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d571 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d572 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d573 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d574 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d575 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d576 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d577 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d578 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d579 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d580 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d581 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d582 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d583 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d584 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d585 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d586 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d587 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d588 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d589 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d590 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d591 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d592 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d593 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d594 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d595 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d596 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d597 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d598 +2026/10/11 04:21:53 kept what home-server says about the machine, and refused its account of declaration d599 +FAIL +FAIL example.com/cap/early 0.013s +ok example.com/cap/fine (cached) +--- FAIL: TestAMapIsNil (0.00s) +panic: assignment to entry in nil map [recovered, repanicked] + +goroutine 35 [running]: +testing.tRunner.func1.2({0x7c6e60, 0x806640}) + /usr/lib/go/src/testing/testing.go:2123 +0x419 +testing.tRunner.func1() + /usr/lib/go/src/testing/testing.go:2126 +0x65f +panic({0x7c6e60?, 0x806640?}) + /usr/lib/go/src/runtime/panic.go:859 +0x125 +example.com/cap/panics.TestAMapIsNil(0xc0001d6248?) + /src/cap/panics/panics_test.go:7 +0x32 +testing.tRunner(0xc0001d6248, 0x7e5f78) + /usr/lib/go/src/testing/testing.go:2193 +0x21d +created by testing.(*T).Run in goroutine 1 + /usr/lib/go/src/testing/testing.go:2258 +0xb13 +FAIL example.com/cap/panics 0.011s +================== +WARNING: DATA RACE +Read at 0x00c000018398 by goroutine 9: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x7b + +Previous write at 0x00c000018398 by goroutine 8: + example.com/cap/race.TestACounterIsShared.func1() + /src/cap/race/race_test.go:13 +0x8d + +Goroutine 9 (running) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 + +Goroutine 8 (finished) created at: + example.com/cap/race.TestACounterIsShared() + /src/cap/race/race_test.go:13 +0x78 + testing.tRunner() + /usr/lib/go/src/testing/testing.go:2193 +0x21c + testing.(*T).Run.gowrap1() + /usr/lib/go/src/testing/testing.go:2258 +0x38 +================== +--- FAIL: TestACounterIsShared (0.00s) + testing.go:1865: race detected during execution of test +FAIL +FAIL example.com/cap/race 0.009s +FAIL diff --git a/internal/catalogue/resolver_manifests_test.go b/internal/catalogue/resolver_manifests_test.go index 8ea3e32b..7d149596 100644 --- a/internal/catalogue/resolver_manifests_test.go +++ b/internal/catalogue/resolver_manifests_test.go @@ -194,8 +194,31 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) { if err := json.Unmarshal([]byte(runtime["content"].(string)), &keys); err != nil { t.Fatalf("the runtime's keys are not JSON: %v", err) } - if len(keys) != 1 || keys["live-restore"] != true { - t.Errorf("the runtime is given %v; live-restore and nothing else", keys) + // What the runtime is given is what its own module declares, and nothing a layer or another module + // adds. This used to say "live-restore and nothing else", to hold the line ADR 0196 drew when `dns` + // left the file; since the docker module gained log rotation as a preference (mesh-catalog #205 and + // #217, novox/hq issue 452) that frozen copy failed every check (novox/hq issue 498). So the keys are + // judged against the module's own daemon.json instead: live-restore is given, `dns` never is, and no + // key the module does not declare appears. + if keys["live-restore"] != true { + t.Errorf("the runtime is given %v; a restart must keep every container running (live-restore)", keys) + } + if _, ok := keys["dns"]; ok { + t.Errorf("the runtime is given a resolver of its own, %v; a container copies its machine's (ADR 0196)", keys["dns"]) + } + declared := daemonKeysDeclared(t) + for key := range keys { + if !declared[key] { + t.Errorf("the runtime is given %q, which the docker module does not declare: %v", key, keys) + } + } + // The log rotation preference, at its defaults, wherever the module declares it. The copy captured in + // testdata/beside predates it, so outside a merge check it is not there to judge. + if declared["log-driver"] { + opts, _ := keys["log-opts"].(map[string]any) + if keys["log-driver"] != "json-file" || opts["max-size"] != "100m" || opts["max-file"] != "10" { + t.Errorf("the runtime is given %v; its containers' logs are cut at 100m and ten are kept (novox/hq issue 452)", keys) + } } // The runtime is reloaded when that file changes, and never restarted: a restart stops every // container on the machine (ADR 0102), and a reload is what turns live-restore on. @@ -275,3 +298,26 @@ func TestTheResolverOnAMachineOffTheNetworkIsRefused(t *testing.T) { t.Fatalf("a machine off the network was refused for another reason: %v", err) } } + +// daemonKeysDeclared is the top-level keys of the docker module's own daemon.json, read from its +// manifest: what the runtime may be given (novox/hq issue 498). A placeholder sits inside a JSON +// string there, so the content parses as it is. +func daemonKeysDeclared(t *testing.T) map[string]bool { + t.Helper() + for _, r := range catalogueManifest(t, "docker").Resources { + if r["id"] != "daemon" { + continue + } + var keys map[string]any + if err := json.Unmarshal([]byte(r["content"].(string)), &keys); err != nil { + t.Fatalf("the docker module's daemon.json is not JSON: %v", err) + } + declared := map[string]bool{} + for key := range keys { + declared[key] = true + } + return declared + } + t.Fatal("the docker module declares no daemon.json") + return nil +} diff --git a/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql new file mode 100644 index 00000000..52dde45b --- /dev/null +++ b/internal/inventory/migrations/0094-a-declaration-says-the-generation-it-came-from.sql @@ -0,0 +1,72 @@ +-- A declaration says the assignment generation it was composed from, and every send is recorded +-- (novox/hq issue 234). +-- +-- On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the +-- assignments held, and the machine undeclared all four. The sequence orders arrival; it cannot tell a +-- later send that carries an older view of the assignments. And nothing the mesh kept said which process +-- sent it, from what view: the node row holds only the digest of the last send. + +-- The assignment generation: one counter for the mesh, raised in the same transaction as every change to +-- what is assigned where. A trigger, not a line in each writer: an assignment is also taken by a node's +-- removal (on delete cascade) and renamed with its module (on update cascade), which no writer in Go +-- sees, and a rule kept only by the writers that remember it is the rule this issue found broken. Seeded +-- at 1, so every declaration composed after this claims a generation and none claims zero ("none"). +create table assignment_generation ( + one boolean primary key default true check (one), + generation bigint not null +); +insert into assignment_generation (one, generation) values (true, 1); + +create function raise_assignment_generation() returns trigger language plpgsql as $$ +begin + update assignment_generation set generation = generation + 1; + return null; +end +$$; + +-- Per row, so a statement that changes nothing raises nothing: an assignment repeated (`on conflict do +-- nothing` inserts no row) and an update that leaves a row as it was (the WHEN below; a plain UPDATE fires a +-- row trigger whether or not it changed anything). A statement that changes several rows raises once per +-- row, which only ever moves it forward. Two triggers, because only an update has both OLD and NEW. +create trigger assignment_generation_raised + after insert or delete on assignment + for each row execute function raise_assignment_generation(); +create trigger assignment_generation_raised_by_update + after update on assignment + for each row when (old is distinct from new) execute function raise_assignment_generation(); + +-- The generation a machine was last sent, written in the same transaction as the digest of that send, so +-- what the mesh WOULD send is stamped the same way and reads as byte for byte what it DID send when +-- nothing else changed (as sent_epoch, migration 0068). Null for a send without one. +alter table node add column sent_generation bigint; +-- Whether the machine's node-engine said it reads a generation (its reports' `reads_generation`): until +-- it has, it is sent none, because an older node-engine refuses a key it does not know, whole. +alter table node add column reads_generation boolean not null default false; + +-- Every send: the machine, its sequence, who sent it — the lease epoch the sender acted under and the +-- caller, process and build — the generation it was composed from, its digest and the modules it named. +-- Written after the declaration is away, with the node row's record of it. A refusal by the machine of a +-- send for its generation is kept on the send it refused, so the condition it raises names the sender +-- from here; a refusal of a sequence the mesh has no send for is kept as a row of its own, `recorded` +-- false, whose sender is said to be unknown. `counter_raised_to` is the generation the mesh's counter was +-- raised to when the refusal showed the counter behind the machine (a store put back from a backup). The +-- newest 200 per machine are kept. +create table declaration_send ( + id bigserial primary key, + node uuid not null references node (id) on delete cascade, + sequence bigint, + epoch bigint, + sender text not null, + generation bigint, + digest text not null, + modules text[] not null default '{}', + sent_at timestamptz not null default now(), + refused_at timestamptz, + -- refused_applied is the generation the machine said it had applied when it refused this send. + refused_applied bigint, + counter_raised_to bigint, + recorded boolean not null default true +); +create index declaration_send_node on declaration_send (node, id desc); +create index declaration_send_sequence on declaration_send (node, sequence); +create index declaration_send_refused on declaration_send (refused_at) where refused_at is not null; diff --git a/internal/inventory/people_test.go b/internal/inventory/people_test.go index 7df4b15c..5919607e 100644 --- a/internal/inventory/people_test.go +++ b/internal/inventory/people_test.go @@ -44,15 +44,17 @@ func TestAPersonMayCallToolsAndNothingElse(t *testing.T) { } // The one tool, both ways it is addressed (novox/hq ADR 0159): to whichever instance // answers, and to the instance on one machine. Nothing else. - // And asking what answers (novox/hq ADR 0197), which claims nothing and calls nothing. + // And asking what answers (novox/hq ADR 0197), which claims nothing and calls nothing. Each way again + // naming ada as the caller and nobody else (novox/hq issue 365). var tools []string for _, s := range perms.Publish { if !strings.HasPrefix(s, "$SRV.") { tools = append(tools, s) } } - if len(tools) != 2 || tools[0] != "mesh.mod.mesh-catalog.tool.catalog_tools" || - tools[1] != "mesh.mod.mesh-catalog.tool.catalog_tools.*" { + want := []string{"mesh.mod.mesh-catalog.call.catalog_tools.*.person~ada", "mesh.mod.mesh-catalog.call.catalog_tools.person~ada", + "mesh.mod.mesh-catalog.tool.catalog_tools", "mesh.mod.mesh-catalog.tool.catalog_tools.*"} + if strings.Join(tools, " ") != strings.Join(want, " ") { t.Errorf("ada may publish %v, which should be the one tool, both ways addressed, and nothing else", perms.Publish) } for _, s := range perms.Publish { diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go new file mode 100644 index 00000000..8a6efa36 --- /dev/null +++ b/internal/inventory/sends.go @@ -0,0 +1,287 @@ +package inventory + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +// The assignment generation and the record of every send (novox/hq issue 234, migration 0094). +// +// A declaration carries the generation of the assignments it was composed from, and a machine refuses one +// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later +// send carrying an older view. And every send is written down with who sent it, from which generation and +// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at. + +// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes +// on a busy machine; the record is for finding who sent what, not an archive. +const sendsKept = 200 + +// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same +// transaction as every change to what is assigned where (a trigger on the assignment table). +func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) { + var g int64 + if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil { + return 0, fmt.Errorf("reading the assignment generation: %w", err) + } + return g, nil +} + +// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more +// than that, or what it already was when it is past it. It never lowers it. +// +// For one case only: a machine refused a send composed from the generation the mesh holds now, so the +// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after +// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed +// from the assignments the store holds now. A stale sender never meets this: its generation is below the +// counter, and the counter is not touched for it. +func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) { + var g int64 + err := i.store.Pool().QueryRow(ctx, + `update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g) + if err != nil { + return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err) + } + return g, nil +} + +// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the +// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed. +func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) { + var g *int64 + if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil { + return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err) + } + if g == nil { + return 0, nil + } + return *g, nil +} + +// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id. +func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) { + var reads bool + if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil { + return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err) + } + return reads, nil +} + +// RecordReadsGeneration keeps what a machine's latest report said of reading a generation. +func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error { + _, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads) + return err +} + +// Send is one declaration sent to a machine, as the mesh records it. +type Send struct { + // Node is the machine's id; NodeName its name, filled where it is read back. + Node string + NodeName string + Sequence int64 + // Epoch is the lease epoch its sender acted under, zero for one that acted under none. + Epoch int64 + // Sender is who sent it, in words: the caller, and the process and build that composed it. + Sender string + // Generation is the assignment generation it was composed from, zero when not known. + Generation int64 + Digest string + // Modules are the modules it named. + Modules []string + SentAt time.Time + // RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the + // generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to + // because the refusal showed it behind the machine, zero when it was not. + RefusedAt *time.Time + RefusedApplied int64 + CounterRaisedTo int64 + // Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown. + Recorded bool +} + +// unknownSender is the sender of a refused sequence the mesh has no record of sending. +const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " + + "hand, before sends were recorded, or by a controller whose record was not written)" + +// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it +// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the +// caller says this loudly and does not take the send for failed. +type SendNotRecordedError struct{ Err error } + +func (e *SendNotRecordedError) Error() string { + return "the record of the send was not written: " + e.Err.Error() +} +func (e *SendNotRecordedError) Unwrap() error { return e.Err } + +// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and +// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue +// 234), in one transaction. The digest and the generation are written together, so the would-send is never +// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if +// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest +// sends beyond the newest 200 for the machine are let go. +func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64, + generation int64, s Send) error { + var sentEpoch *int64 + if epoch > 0 { + e := int64(epoch) + sentEpoch = &e + } + var carried *string + if builds != nil { + raw, err := json.Marshal(builds) + if err != nil { + return err + } + text := string(raw) + carried = &text + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + if _, err := tx.Exec(ctx, + `update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb, sent_epoch = $4, sent_generation = $5 + where id = $1`, node, digest, carried, sentEpoch, nullIfZero(generation)); err != nil { + return err + } + recordErr := recordSend(ctx, tx, node, s) + if err := tx.Commit(ctx); err != nil { + return err + } + if recordErr != nil { + return &SendNotRecordedError{Err: recordErr} + } + return nil +} + +// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written. +func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error { + sp, err := tx.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = sp.Rollback(ctx) }() + modules := s.Modules + if modules == nil { + modules = []string{} + } + if _, err := sp.Exec(ctx, + `insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules) + values ($1, $2, $3, $4, $5, $6, $7)`, + node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest, + modules); err != nil { + return err + } + if err := pruneSends(ctx, sp, node); err != nil { + return err + } + return sp.Commit(ctx) +} + +// pruneSends lets go of a machine's sends beyond the newest 200. +func pruneSends(ctx context.Context, q queries, node string) error { + _, err := q.Exec(ctx, + `delete from declaration_send where node = $1 and id not in + (select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept) + return err +} + +// sendColumns are a send's columns as scanSends reads them. +const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0), + s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0), + s.recorded` + +func scanSends(rows pgx.Rows) ([]Send, error) { + defer rows.Close() + var out []Send + for rows.Next() { + var s Send + if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, + &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} + +// LastSend is the last declaration a machine was sent, by its name; false when none was recorded. A refusal +// of a sequence the mesh has no send for is not a send, and is not it. +func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { + rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` + from declaration_send s join node n on n.id = s.node + where n.name = $1 and s.recorded order by s.id desc limit 1`, name) + if err != nil { + return Send{}, false, err + } + sends, err := scanSends(rows) + if err != nil || len(sends) == 0 { + return Send{}, false, err + } + return sends[0], true, nil +} + +// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and +// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal +// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may +// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none. +func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) { + rows, err := i.store.Pool().Query(ctx, `with refused as ( + update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4 + where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded + order by id desc limit 1) + returning *) + select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied, + nullIfZero(raisedTo)) + if err != nil { + return Send{}, false, err + } + sends, err := scanSends(rows) + if err != nil || len(sends) == 0 { + return Send{}, false, err + } + return sends[0], true, nil +} + +// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its +// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can +// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the +// refused declaration carried, and the refusal's applied generation and counter raise. +func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) { + rows, err := i.store.Pool().Query(ctx, `with refused as ( + insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied, + counter_raised_to, recorded) + values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *) + select `+sendColumns+` from refused s join node n on n.id = s.node`, + s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest, + s.RefusedApplied, nullIfZero(s.CounterRaisedTo)) + if err != nil { + return Send{}, err + } + sends, err := scanSends(rows) + if err != nil { + return Send{}, err + } + if len(sends) == 0 { + return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence) + } + if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil { + return Send{}, err + } + return sends[0], nil +} + +// RefusedSendsSince is every send refused for its generation since a moment, newest first. +func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) { + rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` + from declaration_send s join node n on n.id = s.node + where s.refused_at >= $1 order by s.refused_at desc`, since) + if err != nil { + return nil, err + } + return scanSends(rows) +} diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go new file mode 100644 index 00000000..96c98d15 --- /dev/null +++ b/internal/inventory/sends_test.go @@ -0,0 +1,207 @@ +package inventory + +import ( + "errors" + "slices" + "testing" + "time" +) + +// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration +// newer in sequence than every other named four fewer modules than the assignments held, a machine applied +// it, and nothing the mesh kept could say who sent it or from what view. + +func generationNow(t *testing.T, inv *Inventory) int64 { + t.Helper() + g, err := inv.AssignmentGeneration(t.Context()) + if err != nil { + t.Fatal(err) + } + return g +} + +// **Every change to what is assigned where raises the generation, in its own transaction** — an +// assignment, an unassignment, and one taken by the machine's removal, which no writer in Go makes — and +// a repeated assignment, which changes nothing, does not. +func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) { + inv, node := aNodeWithModules(t, "postgres", "web") + ctx := t.Context() + start := generationNow(t, inv) + if start < 1 { + t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start) + } + if _, err := inv.Assign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + assigned := generationNow(t, inv) + if assigned <= start { + t.Fatalf("an assignment left the generation at %d", assigned) + } + if _, err := inv.Assign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + if again := generationNow(t, inv); again != assigned { + t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again) + } + if err := inv.Unassign(ctx, node, "postgres"); err != nil { + t.Fatal(err) + } + unassigned := generationNow(t, inv) + if unassigned <= assigned { + t.Fatalf("an unassignment left the generation at %d", unassigned) + } + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + before := generationNow(t, inv) + if _, err := inv.RemoveNodeForTest(ctx, node); err != nil { + t.Fatal(err) + } + if removed := generationNow(t, inv); removed <= before { + t.Errorf("a machine's removal took its assignments and left the generation at %d", removed) + } +} + +// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter +// is behind it** — a store put back from a backup — so the next send is not refused for ever. +func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) { + inv, _ := aNodeWithModules(t) + now := generationNow(t, inv) + if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 { + t.Fatalf("raised past %d to %d (%v)", now+40, raised, err) + } + if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 { + t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err) + } +} + +// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it +// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation +// written with the digest, and a refusal is kept on the send it refused, naming its sender. +func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", + Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}} + if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil { + t.Fatal(err) + } + stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", + Generation: 38, Digest: "d12", Modules: []string{"docker"}} + if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil { + t.Fatal(err) + } + last, found, err := inv.LastSend(ctx, node) + if err != nil || !found { + t.Fatalf("the last send is not kept: %v", err) + } + if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded || + !slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() { + t.Fatalf("the last send reads %+v", last) + } + if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 { + t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err) + } + + refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0) + if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 || + refused.CounterRaisedTo != 0 { + t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err) + } + since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 { + t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) + } + if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found { + t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err) + } +} + +// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it; +// it is not the machine's last send, and it says what the counter was raised to. +func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41, + Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45}) + if err != nil { + t.Fatal(err) + } + if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 || + kept.NodeName != node { + t.Fatalf("the refusal reads %+v", kept) + } + since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(since) != 1 || since[0].Sequence != 99 { + t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) + } + if _, found, err := inv.LastSend(ctx, node); err != nil || found { + t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err) + } +} + +// **The machine's record and the send's are written together, and a send record that cannot be written +// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a +// send that is away must not read as failed, nor the machine as behind. +func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + // A NUL byte is text PostgreSQL refuses: the send's own row cannot be written. + broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"} + err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken) + var unrecorded *SendNotRecordedError + if !errors.As(err, &unrecorded) { + t.Fatalf("a send record that could not be written read as %v", err) + } + if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 { + t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err) + } + if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" { + t.Fatalf("the machine's digest was not written: %q (%v)", sent, err) + } +} + +// **An update that leaves an assignment as it was raises nothing** (the trigger's WHEN). +func TestAnUpdateThatChangesNoAssignmentRaisesNothing(t *testing.T) { + inv, node := aNodeWithModules(t, "web") + ctx := t.Context() + if _, err := inv.Assign(ctx, node, "web"); err != nil { + t.Fatal(err) + } + before := generationNow(t, inv) + if _, err := inv.store.Pool().Exec(ctx, `update assignment set module = module`); err != nil { + t.Fatal(err) + } + if after := generationNow(t, inv); after != before { + t.Errorf("an update that changed nothing moved the generation from %d to %d", before, after) + } +} + +// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. +func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { + inv, node := aNodeWithModules(t) + record, err := inv.NodeByName(t.Context(), node) + if err != nil { + t.Fatal(err) + } + for _, reads := range []bool{false, true, false} { + if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil { + t.Fatal(err) + } + if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads { + t.Fatalf("recorded %v, read %v (%v)", reads, got, err) + } + } +} diff --git a/internal/link/conformance_sources_test.go b/internal/link/conformance_sources_test.go new file mode 100644 index 00000000..06a0fcd7 --- /dev/null +++ b/internal/link/conformance_sources_test.go @@ -0,0 +1,137 @@ +package link + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Where the conformance fixtures are read from, held on a small SDK repository of the test's own: two +// commits, the clone checked out at the older as the build seat leaves it at the running controller's pin +// (novox/hq issue 449). The contents are the test's, because what is judged is which commit is read. +type sdkBed struct { + root, goMod, captured, capturedLog string + older, newer string +} + +const fixtureName = "events/module-event.json" + +func newSDKBed(t *testing.T) sdkBed { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Fatalf("git is needed to read the SDK at a pin, as a merge check does: %v", err) + } + b := sdkBed{root: t.TempDir()} + clone := filepath.Join(b.root, "mesh-sdk") + git := func(args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", clone, "-c", "user.name=t", "-c", "user.email=t@t", + "-c", "commit.gpgsign=false"}, args...)...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + write := func(dir, body string) { + t.Helper() + file := filepath.Join(dir, "conformance", filepath.FromSlash(fixtureName)) + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(file, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.MkdirAll(clone, 0o755); err != nil { + t.Fatal(err) + } + git("init", "--quiet") + write(clone, "older\n") + git("add", "-A") + git("commit", "--quiet", "-m", "older") + b.older = git("rev-parse", "HEAD") + write(clone, "newer\n") + git("commit", "--quiet", "-am", "newer") + b.newer = git("rev-parse", "HEAD") + git("checkout", "--quiet", b.older) + + other := t.TempDir() + b.captured = filepath.Join(other, "mesh-sdk") + write(b.captured, "older\n") + b.capturedLog = filepath.Join(other, "CAPTURED") + if err := os.WriteFile(b.capturedLog, []byte("mesh-sdk "+b.older+" conformance/events\n"), 0o644); err != nil { + t.Fatal(err) + } + b.goMod = filepath.Join(other, "go.mod") + b.pin(t, b.newer) + return b +} + +func (b sdkBed) pin(t *testing.T, commit string) { + t.Helper() + mod := "module x\n\nrequire (\n\t" + sdkModule + " v0.1.11-0.20261009143344-" + commit[:12] + "\n)\n" + if err := os.WriteFile(b.goMod, []byte(mod), 0o644); err != nil { + t.Fatal(err) + } +} + +// A pull request moving the SDK is judged against the SDK it moves to: the clone is read at the tree's own +// pin, not at what it is checked out at, the running controller's. +func TestAMergeCheckReadsTheSDKAtTheTreesOwnPin(t *testing.T) { + b := newSDKBed(t) + raw, from, err := sdkFixture(b.root, b.goMod, b.captured, b.capturedLog, fixtureName) + if err != nil { + t.Fatal(err) + } + if string(raw) != "newer\n" { + t.Errorf("read %q from %s; the tree pins the newer commit", raw, from) + } +} + +// A captured copy that is not the SDK's at the commit CAPTURED names fails a merge check. +func TestAMergeCheckFailsACapturedCopyEditedByHand(t *testing.T) { + b := newSDKBed(t) + if err := os.WriteFile(filepath.Join(b.captured, "conformance", filepath.FromSlash(fixtureName)), + []byte("edited\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := sdkFixture(b.root, b.goMod, b.captured, b.capturedLog, fixtureName); err == nil || + !strings.Contains(err.Error(), "is not mesh-sdk's at") { + t.Errorf("a hand-edited captured copy was accepted: %v", err) + } +} + +// A pin the clone does not hold fails loudly and names it. +func TestAMergeCheckFailsWithoutTheSDKAtItsPin(t *testing.T) { + b := newSDKBed(t) + b.pin(t, "0123456789ab0123456789ab0123456789ab0123") + if _, _, err := sdkFixture(b.root, b.goMod, b.captured, b.capturedLog, fixtureName); err == nil || + !strings.Contains(err.Error(), "0123456789ab") { + t.Errorf("a pin the clone lacks was not named: %v", err) + } +} + +// Until mesh-controller #220 has rolled out the seat places no mesh-sdk beside a check: the captured copy is +// read, and where it was read from says plainly that the clone did not judge it (novox/hq issue 449). +func TestAMergeCheckWithoutTheCloneSaysItReadTheCapturedCopy(t *testing.T) { + b := newSDKBed(t) + raw, from, err := sdkFixture(t.TempDir(), b.goMod, b.captured, b.capturedLog, fixtureName) + if err != nil || string(raw) != "older\n" { + t.Fatalf("read %q, %v; the captured copy holds the older", raw, err) + } + if !strings.HasPrefix(from, "NOT JUDGED AGAINST THE CLONE") || !strings.Contains(from, b.older) { + t.Errorf("the log does not say the clone was missing and which copy was read: %q", from) + } +} + +// Away from a merge check the captured copy is read, and nothing else. +func TestAwayFromACheckTheCapturedCopyIsRead(t *testing.T) { + b := newSDKBed(t) + raw, _, err := sdkFixture("", b.goMod, b.captured, b.capturedLog, fixtureName) + if err != nil || string(raw) != "older\n" { + t.Errorf("read %q, %v; the captured copy holds the older", raw, err) + } +} diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go index 50f60dac..048ba4f3 100644 --- a/internal/link/conformance_test.go +++ b/internal/link/conformance_test.go @@ -1,19 +1,40 @@ package link import ( + "bytes" "encoding/json" + "fmt" "os" + "os/exec" + "path" "path/filepath" "regexp" + "strings" + "sync" "testing" "time" + + "github.com/novox/mesh-controller/internal/beside" ) // The Go implementation, held to the shared fixtures (novox/hq ADR 0074, design 19). // -// **Read from the sdk's conformance directory by sibling path**, the way the lab finds its -// siblings — deliberately not copied here. A fixture copied into each implementation is two -// fixtures, and two fixtures drift, which is the exact failure the suite exists to prevent. +// **Read from the SDK's own conformance directory, never copied by hand**: a fixture written into each +// implementation is two fixtures, and two fixtures drift, which is the failure the suite exists to prevent. +// Which SDK is read is the one this tree's go.mod pins, never the one a desktop happens to hold beside this +// checkout (novox/hq issue 449): +// +// - in a merge check, the fixture as it stands at that pin in the mesh-sdk clone the build seat places +// beside the check — read with `git show`, because the clone is checked out at the *running* +// controller's pin, and a pull request moving the SDK must be judged against the SDK it moves to, or it +// passes the check and fails everywhere after it rolls out. A pin the clone lacks fails the test; a +// missing clone reads the captured copy and says so loudly, until mesh-controller #220 has rolled out +// and the build seat clones mesh-sdk (then its follow-up makes a missing clone fail again); +// - elsewhere, the copy captured in testdata/beside at the commit testdata/beside/CAPTURED names, held to +// go.mod by TestTheCapturedSDKIsTheOneGoModPins. +// +// In a merge check the captured copy is also compared with the clone at the captured commit, byte for byte, +// so a hand-edited copy cannot pass for the SDK's. type fixture struct { Name string `json:"name"` Given struct { @@ -30,12 +51,27 @@ type fixture struct { } `json:"wire"` } +// sdkModule is the Go module of the SDK the controller is built against. +const sdkModule = "git.novox.be/novox/mesh-sdk/go" + +// The tree's go.mod, and the captured copy, as this package finds them. +var ( + goModFile = filepath.Join("..", "..", "go.mod") + capturedSDK = filepath.Join(beside.Captured(), "mesh-sdk") + capturedLog = filepath.Join(beside.Captured(), "CAPTURED") +) + func loadFixture(t *testing.T, name string) fixture { t.Helper() - path := filepath.Join("..", "..", "..", "mesh-sdk", "conformance", name) - raw, err := os.ReadFile(path) + raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name) if err != nil { - t.Skipf("the sdk's conformance fixtures are not beside this checkout: %v", err) + // Failed, never skipped: a skip here passed the suite with nothing judged. + t.Fatal(err) + } + t.Logf("%s: %s", name, from) + if strings.HasPrefix(from, notJudged) { + // Said on the check's own output too, where `go test` without -v prints no log of a passing test. + saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) }) } var f fixture if err := json.Unmarshal(raw, &f); err != nil { @@ -44,6 +80,141 @@ func loadFixture(t *testing.T, name string) fixture { return f } +// notJudged opens what a merge check without the SDK's clone says; saidNotJudged says it once on stderr. +const notJudged = "NOT JUDGED AGAINST THE CLONE" + +var saidNotJudged sync.Once + +// sdkFixture is one conformance fixture and where it was read: from the mesh-sdk clone in root (a merge +// check's MESH_CHECK_BESIDE) at the pin of goMod, or, with no root, from the captured copy. +func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) { + file := path.Join("conformance", name) + readCaptured := func() ([]byte, error) { + raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, fmt.Errorf("the captured SDK's %s: %w", file, err) + } + return raw, nil + } + if root == "" { + raw, err := readCaptured() + return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " + + "at this tree's pin", err + } + clone := filepath.Join(root, "mesh-sdk") + if _, err := os.Stat(clone); err != nil { + // A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the + // check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the + // captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this + // a failure again. + at, cerr := capturedCommit(capturedLog) + if cerr != nil { + return nil, "", cerr + } + raw, rerr := readCaptured() + return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+ + "%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env, + root, at), rerr + } + ref, err := sdkRef(goMod) + if err != nil { + return nil, "", err + } + raw, err := gitShow(clone, ref, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+ + "go.mod pins: %w", file, ref, err) + } + // The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so. + at, err := capturedCommit(capturedLog) + if err != nil { + return nil, "", err + } + theirs, err := gitShow(clone, at, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err) + } + ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) + } + if !bytes.Equal(ours, theirs) { + return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+ + "edited or captured wrong; capture it again as CAPTURED says", file, at) + } + return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil +} + +// gitShow is one file of a repository at a ref. safe.directory, because the clone is the build seat's and +// the test may run as another user. +func gitShow(repository, ref, file string) ([]byte, error) { + cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970. +var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`) + +// sdkRef is the SDK repository's ref a go.mod pins: a pseudo-version's commit, or a release's tag (the SDK +// tags its Go module under go/). +func sdkRef(goMod string) (string, error) { + raw, err := os.ReadFile(goMod) + if err != nil { + return "", err + } + version := "" + for _, line := range strings.Split(string(raw), "\n") { + fields := strings.Fields(strings.TrimPrefix(strings.TrimSpace(line), "require ")) + if len(fields) >= 2 && fields[0] == sdkModule { + version = fields[1] + } + } + if m := pseudoCommit.FindStringSubmatch(version); m != nil { + return m[1], nil + } + if strings.HasPrefix(version, "v") { + return "go/" + version, nil + } + return "", fmt.Errorf("%s pins no version of %s that names a commit or a tag", goMod, sdkModule) +} + +// capturedCommit is the commit testdata/beside/CAPTURED names for mesh-sdk. +func capturedCommit(capturedLog string) (string, error) { + raw, err := os.ReadFile(capturedLog) + if err != nil { + return "", err + } + at := regexp.MustCompile(`(?m)^mesh-sdk\s+([0-9a-f]{40})\s`).FindSubmatch(raw) + if at == nil { + return "", fmt.Errorf("%s names no commit for mesh-sdk", capturedLog) + } + return string(at[1]), nil +} + +// The captured SDK is the one this controller is built against: when go.mod moves the SDK, the copy moves +// with it, or the tests away from a merge check judge an SDK the controller no longer uses (novox/hq issue +// 449). +func TestTheCapturedSDKIsTheOneGoModPins(t *testing.T) { + pinned, err := sdkRef(goModFile) + if err != nil { + t.Fatal(err) + } + at, err := capturedCommit(capturedLog) + if err != nil { + t.Fatal(err) + } + if strings.HasPrefix(pinned, "go/") || !strings.HasPrefix(at, pinned) { + t.Errorf("the SDK is captured at %s but go.mod pins %s: capture it again at the pinned commit, as "+ + "testdata/beside/CAPTURED says", at, pinned) + } +} + // Every header the fixture requires is one this implementation actually sets. func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { f := loadFixture(t, "events/module-event.json") diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 6c3cab6b..de36001f 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -447,6 +447,13 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil { return false, err } + // And whether it reads a generation (novox/hq issue 234), the same way: only from a node-engine that + // orders its reports, so a one-shot report (a rekey) does not say an engine stopped reading one. + if report.Ordered() { + if err := e.Inventory.RecordReadsGeneration(ctx, node.ID, report.ReadsGeneration); err != nil { + return false, err + } + } if report.Ordered() { account := AccountOf(report) news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing, diff --git a/internal/link/generation_test.go b/internal/link/generation_test.go new file mode 100644 index 00000000..7a25283b --- /dev/null +++ b/internal/link/generation_test.go @@ -0,0 +1,33 @@ +package link + +import ( + "encoding/json" + "testing" +) + +// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it: +// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal +// of a declaration composed from an older generation than the machine applied. +func TestTheGenerationOnTheWire(t *testing.T) { + raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` + + `"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`) + var r Report + if err := json.Unmarshal(raw, &r); err != nil { + t.Fatal(err) + } + if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) || + r.Sequence != 12 { + t.Fatalf("a node-engine's generation refusal reads as %+v", r) + } + // Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender. + if r.StaleRefusalOf() { + t.Errorf("a generation refusal reads as a stale writer's") + } + var older Report + if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil { + t.Fatal(err) + } + if older.ReadsGeneration || older.OlderGeneration != nil { + t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older) + } +} diff --git a/internal/link/protocol.go b/internal/link/protocol.go index b64a9a72..63349ece 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -197,6 +197,15 @@ type Report struct { // RefusedOlder is how many declarations the node-engine has refused as older, ever, on every report: // a refusal whose own report was lost is still counted from the next. RefusedOlder int64 `json:"refused_older,omitempty"` + // ReadsGeneration says the node-engine reads a declaration's assignment generation and its put-back + // mark (novox/hq issue 234), on every report it makes: the mesh sends them only to a machine that said + // so, because an older node-engine refuses a key it does not know, whole. Mirrors mesh-host + // internal/link/messages.go. + ReadsGeneration bool `json:"reads_generation,omitempty"` + // OlderGeneration is set on a report refusing a declaration composed from an older assignment + // generation than the machine applied (novox/hq issue 234). The refused declaration is the report's + // own Declared and Order, and its sender is read from the record of sends by that sequence. + OlderGeneration *GenerationRefusal `json:"older_generation,omitempty"` // Held is what an adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. @@ -605,3 +614,10 @@ type HealthSaid struct { Node string `json:"node"` Health Health `json:"health"` } + +// GenerationRefusal is a declaration refused for the assignment generation it was composed from: that +// generation, and the highest the machine applied (novox/hq issue 234; mesh-host's GenerationRefusal). +type GenerationRefusal struct { + Generation int64 `json:"generation"` + Applied int64 `json:"applied"` +} diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 6b819622..f922ff40 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,6 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog b9de001833b1b61b297182b8e3bcdae1cbdeace9 modules/*/module.json, modules/nats/Dockerfile mesh-host bd5cc6980419c1bd4824be6d58dfebd2381a9de3 examples/foundation-first-node-nats.lock +mesh-sdk 33917f91ac3cc213b879b32f3c70e2aa70395f98 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: @@ -19,4 +20,8 @@ To move them, from this repository's root, with the two repositories checked out find testdata/beside -name module.json -exec mv {} {}.captured \; git -C ../mesh-host archive examples/foundation-first-node-nats.lock | tar -x -C testdata/beside/mesh-host -and write the commits here. +and write the commits here. The SDK is captured at the commit go.mod pins for git.novox.be/novox/mesh-sdk/go +(the last part of its pseudo-version; novox/hq issue 449), which internal/link's conformance test holds it to: + + rm -rf testdata/beside/mesh-sdk && mkdir -p testdata/beside/mesh-sdk + git -C ../mesh-sdk archive conformance/events | tar -x -C testdata/beside/mesh-sdk diff --git a/testdata/beside/mesh-sdk/conformance/events/module-event.json b/testdata/beside/mesh-sdk/conformance/events/module-event.json new file mode 100644 index 00000000..8de1169b --- /dev/null +++ b/testdata/beside/mesh-sdk/conformance/events/module-event.json @@ -0,0 +1,44 @@ +{ + "capability": "events", + "name": "a module emits an event", + "why": "The envelope is what two implementations can disagree about without either failing: a missing header, a header spelled differently, or a body nested where metadata belongs. None of those stop a mesh running; they stop it reacting.", + "given": { + "module": "shop", + "node": "one", + "key": "order.placed", + "body": { "id": "a1", "total": 12 }, + "headers": { + "x-event-id": "0123456789abcdef0123456789abcdef", + "x-source": "shop", + "x-node": "one", + "x-time": "2026-09-26T12:00:00Z", + "content-type": "application/json" + } + }, + "wire": { + "subject": "mesh.mod.shop.event.order.placed", + "requiredHeaders": ["x-event-id", "x-source", "x-node", "x-time", "content-type"], + "optionalHeaders": ["x-causation-id", "x-schema"], + "headerFormats": { + "x-time": "RFC3339", + "content-type": "application/json", + "x-event-id": "^[0-9a-f]{32}$" + }, + "payloadIs": "the body alone, not the envelope", + "keyRecoveredFrom": "the subject, after the .event. token" + }, + "refuses": [ + { + "what": "an envelope nested in the payload", + "why": "an implementation that publishes the whole envelope as the body passes all of its own tests and is unreadable to every other" + }, + { + "what": "a missing x-event-id", + "why": "delivery is at-least-once and only the emitter can say which of two messages is a redelivery" + }, + { + "what": "an x-source that differs from the subject's module", + "why": "the bus enforces the namespace, so a disagreement means the envelope is lying about its origin" + } + ] +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go index 81737ac4..008e69af 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/asks/asks.go @@ -154,7 +154,9 @@ func canonical(b *strings.Builder, v string) { // // It is over the ask's named fields in a fixed order, each written canonically, and the expiry as UTC // RFC 3339 to the nanosecond — never over a language's encoding of the struct, so a field added to Ask -// later changes no digest until it is added here, on purpose. +// later changes no digest until it is added here, on purpose. Whole and Details (novox/hq issue 383) follow +// the options only when the ask gives either: an ask without them digests as it did before they existed, so +// a router and an asker of different builds still agree on every such ask. func (a Ask) Digest() string { var b strings.Builder b.WriteString("novox.ask.v1\n") @@ -168,6 +170,10 @@ func (a Ask) Digest() string { canonical(&b, v) } } + if a.Whole != "" || a.Details != "" { + canonical(&b, a.Whole) + canonical(&b, a.Details) + } sum := sha256.Sum256([]byte(b.String())) return "sha256:" + hex.EncodeToString(sum[:]) } @@ -190,6 +196,15 @@ type Ask struct { // About is what the ask is about (a condition's key): a newer ask about it replaces the older. About string `json:"about,omitempty"` Urgent bool `json:"urgent,omitempty"` + // Whole is the explanation with its exact values whole, shown in place of Explanation on a channel kind + // that proves who answers (verified-sender) and carries the ask's answers (novox/hq issue 383): what the + // person approves — a mount point, a share, a private address — must be readable where they approve it. + // The asker withholds a value shaped like a secret in it, and the router refuses the ask when one is left; + // Explanation stays under the whole content rule everywhere else. Empty, Explanation is shown everywhere. + Whole string `json:"whole,omitempty"` + // Details is what the Details answer on the ask's message shows, line by line under the content rule: a + // fingerprint, how to read the proposal whole at the terminal. Empty, an ask's own message offers no Details. + Details string `json:"details,omitempty"` } // The bounds of an ask (novox/hq ADR 0234 §8, ADR 0259 §4). diff --git a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go index ea03cf57..50db892e 100644 --- a/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go +++ b/vendor/github.com/novox/mesh-host/internal/declaration/declaration.go @@ -1378,6 +1378,20 @@ type Declaration struct { // what it wrote for it — its resources are absent from the declaration, and absence would // otherwise read as removal. LeftOut []string + + // Generation is the assignment generation this declaration was composed from: the controller's + // counter, raised in the same transaction as every change to what is assigned where (novox/hq + // issue 234). The sequence orders arrival and cannot tell a later send that carries an older view + // of the assignments; this can. A node-engine refuses a declaration composed from a generation + // older than the highest it applied — unless it is a put-back — because applying it would + // undeclare what the mesh still assigns. Zero is a declaration from a controller that claims none, + // and is applied as before. + Generation int64 + + // PutBack says this declaration is a gate putting a machine back (novox/hq issue 234): it carries + // the generation of what it puts back, which may be older than what the machine applied, and is + // not refused for it. + PutBack bool } // LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the @@ -1542,6 +1556,11 @@ type envelope struct { Epoch int64 `json:"epoch,omitempty"` // LeftOut is optional on the wire too, and absent when nothing was left out (ADR 0163). LeftOut []string `json:"left_out,omitempty"` + // Generation and PutBack are optional on the wire too (novox/hq issue 234): absent is a controller + // that claims no generation. **An older host refuses these keys**, decoding strictly; a controller + // sends them only to a host whose reports say `reads_generation`. + Generation int64 `json:"generation,omitempty"` + PutBack bool `json:"put_back,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1559,8 +1578,17 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { } d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, - Epoch: env.Epoch, LeftOut: env.LeftOut} + Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack} var problems []string + if env.Generation < 0 { + // Below zero would read as "none claimed" and pass every refusal of an older generation. + problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+ + "mesh assigns", env.Generation)) + } + if env.PutBack && allowActions { + // A put-back is a gate's, sent by the mesh; a carried bundle puts nothing back. + problems = append(problems, "a carried bundle says it is a put-back, and only the mesh's gate can say that") + } if env.Sequence < 0 || env.Epoch < 0 { // Below zero is no order any controller assigns, and read as "none claimed" it would let the // declaration past every refusal of what is older. diff --git a/vendor/modules.txt b/vendor/modules.txt index 486530b9..bb6a448e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c ## explicit; go 1.22 git.novox.be/novox/mesh-sdk/go/asks # github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op @@ -78,7 +78,7 @@ github.com/nats-io/nkeys # github.com/nats-io/nuid v1.0.1 ## explicit github.com/nats-io/nuid -# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 +# github.com/novox/mesh-host v0.0.0 => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f ## explicit; go 1.26.0 github.com/novox/mesh-host/internal/declaration github.com/novox/mesh-host/rootsearch @@ -135,4 +135,4 @@ golang.org/x/text/width # golang.org/x/time v0.15.0 ## explicit; go 1.25.0 golang.org/x/time/rate -# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261009231844-b8c854611812 +# github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f