diff --git a/checks/readable/go.mod b/checks/readable/go.mod new file mode 100644 index 00000000..6226f1c1 --- /dev/null +++ b/checks/readable/go.mod @@ -0,0 +1,5 @@ +module checks/readable + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/checks/readable/go.sum b/checks/readable/go.sum new file mode 100644 index 00000000..f8bfbd2b --- /dev/null +++ b/checks/readable/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/checks/readable/main.go b/checks/readable/main.go new file mode 100644 index 00000000..8fa81fba --- /dev/null +++ b/checks/readable/main.go @@ -0,0 +1,333 @@ +// Command readable refuses a catalogue where a module reads another module's state that the owner does not announce +// readable (novox/hq ADR 0495 clause 3, issue 1790): the controller grants a read only from the owner's announcement, so +// such a read is refused on the live mesh, said only in the controller's journal. The sweep of 2026-10-30 (issue 774) +// found three: gitea's `bound` (read by mesh-provisioner, so no forge account was bound), disk-load's `readings` (read by +// restic, nfs-server and mounts) and restic's `good` (read by the self-check). The merge gate's check of reads against +// the owner's manifest went with clause 5, and announcetest holds a module to its own list, not to its readers. +// +// An owner announces from its code. A Go module announces when its code imports mesh-sdk's go/announce: every +// announce.State literal there is read, its Name a string, a constant of the package, or a constant of mesh-sdk this +// check knows (Known). A TypeScript module announces when its code says the event `states` or calls the SDK's keeps: +// every `{ name: "...", readable: true }` there is read. An owner that announces nothing from code is read as before +// on the mesh, and is not judged; neither is an owner outside the catalogue (another repository's). +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Known are the mesh-sdk constants an announcement may name a state by, by the selector the code writes. +var Known = map[string]string{ + "health.GoodState": health.GoodState, + "health.ReadingsState": health.ReadingsState, +} + +const announcePkg = `"git.novox.be/novox/mesh-sdk/go/announce"` + +// Announced is what one owner's code announces: each state, readable or not. +type Announced map[string]bool + +// Findings is every refusal in the catalogue at root, sorted. +func Findings(root string) ([]string, error) { + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + dirs := map[string]string{} + reads := map[string][]string{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Reads []json.RawMessage `json:"reads"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + dirs[m.Module] = filepath.Dir(path) + for _, r := range m.Reads { + var s string + if json.Unmarshal(r, &s) == nil { + reads[m.Module] = append(reads[m.Module], s) + } + } + } + var out []string + owners := map[string]Announced{} + for reader, rs := range reads { + for _, r := range rs { + owner, state, ok := strings.Cut(r, ".") + dir, inCatalogue := dirs[owner] + if !ok || !inCatalogue || owner == reader { + continue + } + a, seen := owners[owner] + if !seen { + var problems []string + a, problems, err = AnnouncedBy(dir) + if err != nil { + return nil, err + } + owners[owner] = a + for _, p := range problems { + out = append(out, owner+": "+p) + } + } + if a == nil { + continue // announces nothing from code: read as before + } + readable, keeps := a[state] + switch { + case !keeps: + out = append(out, fmt.Sprintf("%s reads %s.%s, which %s's code does not announce: the read is refused on the mesh", reader, owner, state, owner)) + case !readable: + out = append(out, fmt.Sprintf("%s reads %s.%s, which %s's code announces not readable: the read is refused on the mesh; announce it readable", reader, owner, state, owner)) + } + } + } + sort.Strings(out) + return dedupe(out), nil +} + +func dedupe(in []string) []string { + var out []string + for i, s := range in { + if i == 0 || s != in[i-1] { + out = append(out, s) + } + } + return out +} + +var ( + tsAnnounces = regexp.MustCompile(`emit\(\s*"states"|\bkeeps\(`) + tsState = regexp.MustCompile(`\{\s*name:\s*"([a-z0-9][a-z0-9-]*)"\s*(,\s*readable:\s*(true|false)\s*)?\}`) +) + +// AnnouncedBy is what the module in dir announces from its code, nil when it announces nothing; problems are +// announcements whose state's name cannot be read. +func AnnouncedBy(dir string) (Announced, []string, error) { + var out Announced + var problems []string + pkgs := map[string][]*ast.File{} + fset := token.NewFileSet() + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if n := d.Name(); path != dir && (n == "node_modules" || n == "dist" || n == "test" || n == "testdata" || strings.HasPrefix(n, ".")) { + return filepath.SkipDir + } + return nil + } + name := d.Name() + switch { + case strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go"): + f, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + return err + } + pkgs[filepath.Dir(path)] = append(pkgs[filepath.Dir(path)], f) + case strings.HasSuffix(name, ".ts") && !strings.HasSuffix(name, ".d.ts") && !strings.HasSuffix(name, ".test.ts"): + raw, err := os.ReadFile(path) + if err != nil { + return err + } + if !tsAnnounces.Match(raw) && !strings.Contains(string(raw), "KEPT") { + return nil + } + for _, m := range tsState.FindAllStringSubmatch(string(raw), -1) { + if out == nil { + out = Announced{} + } + out[m[1]] = out[m[1]] || m[3] == "true" + } + } + return nil + }) + if err != nil { + return nil, nil, err + } + for _, files := range pkgs { + imports := false + consts := map[string]ast.Expr{} + for _, f := range files { + for _, im := range f.Imports { + if im.Path.Value == announcePkg && (im.Name == nil || im.Name.Name == "announce") { + imports = true + } + } + for _, decl := range f.Decls { + g, ok := decl.(*ast.GenDecl) + if !ok || g.Tok != token.CONST { + continue + } + for _, spec := range g.Specs { + v := spec.(*ast.ValueSpec) + for i, n := range v.Names { + if i < len(v.Values) { + consts[n.Name] = v.Values[i] + } + } + } + } + } + if !imports { + continue + } + if out == nil { + out = Announced{} + } + for _, f := range files { + ast.Inspect(f, func(n ast.Node) bool { + lit, ok := n.(*ast.CompositeLit) + if !ok { + return true + } + states := []*ast.CompositeLit{} + switch { + case isAnnounceState(lit.Type): + states = append(states, lit) + case isStateSlice(lit.Type): + for _, e := range lit.Elts { + if c, ok := e.(*ast.CompositeLit); ok && (c.Type == nil || isAnnounceState(c.Type)) { + states = append(states, c) + } + } + default: + return true + } + for _, s := range states { + name, readable, problem := stateOf(s, consts) + if problem != "" { + problems = append(problems, fmt.Sprintf("%s: %s", fset.Position(s.Pos()), problem)) + continue + } + out[name] = out[name] || readable + } + return false + }) + } + } + return out, problems, nil +} + +func isAnnounceState(e ast.Expr) bool { + s, ok := e.(*ast.SelectorExpr) + if !ok { + return false + } + x, ok := s.X.(*ast.Ident) + return ok && x.Name == "announce" && s.Sel.Name == "State" +} + +func isStateSlice(e ast.Expr) bool { + a, ok := e.(*ast.ArrayType) + return ok && isAnnounceState(a.Elt) +} + +// stateOf is one announce.State literal's name and whether it is readable (Readable true, or Readers named). +func stateOf(lit *ast.CompositeLit, consts map[string]ast.Expr) (string, bool, string) { + var name string + var readable bool + named := false + for _, e := range lit.Elts { + kv, ok := e.(*ast.KeyValueExpr) + if !ok { + return "", false, "an announce.State without field names cannot be read; name its fields" + } + key, _ := kv.Key.(*ast.Ident) + if key == nil { + continue + } + switch key.Name { + case "Name": + v, ok := resolve(kv.Value, consts, 0) + if !ok { + return "", false, fmt.Sprintf("the state's name %s cannot be read: name it by a string, a constant of the package, or a mesh-sdk constant in checks/readable's Known", exprString(kv.Value)) + } + name, named = v, true + case "Readable": + if id, ok := kv.Value.(*ast.Ident); ok { + readable = readable || id.Name == "true" + } + case "Readers": + readable = true + } + } + if !named { + return "", false, "an announce.State without a Name" + } + return name, readable, "" +} + +func resolve(e ast.Expr, consts map[string]ast.Expr, depth int) (string, bool) { + if depth > 8 { + return "", false + } + switch v := e.(type) { + case *ast.BasicLit: + if v.Kind == token.STRING { + s, err := strconv.Unquote(v.Value) + return s, err == nil + } + case *ast.Ident: + if c, ok := consts[v.Name]; ok { + return resolve(c, consts, depth+1) + } + case *ast.SelectorExpr: + s, ok := Known[exprString(v)] + return s, ok + } + return "", false +} + +func exprString(e ast.Expr) string { + switch v := e.(type) { + case *ast.Ident: + return v.Name + case *ast.SelectorExpr: + return exprString(v.X) + "." + v.Sel.Name + case *ast.BasicLit: + return v.Value + } + return fmt.Sprintf("%T", e) +} + +func main() { + root := "." + if len(os.Args) > 1 { + root = os.Args[1] + } + found, err := Findings(root) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + os.Exit(1) + } +} diff --git a/checks/readable/main_test.go b/checks/readable/main_test.go new file mode 100644 index 00000000..3a15e322 --- /dev/null +++ b/checks/readable/main_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A read of another module's state is refused here when the owner's code announces that state not readable, or not at +// all (the sweep of 2026-10-30, issue 774); an owner that announces nothing from code, or lives in another repository, +// is not judged. +func TestFindings(t *testing.T) { + const goOwner = "package main\n\nimport \"git.novox.be/novox/mesh-sdk/go/announce\"\n\n" + for _, row := range []struct { + name string + files map[string]string + want string // empty: clean + }{ + {"go, readable by a constant", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.s"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/cmd/o/main.go": goOwner + "const S = \"s\"\n\nvar _ = announce.State{Name: S, Readable: true}\n", + "modules/o/cmd/o/main_test.go": goOwner + "var _ = announce.State{Name: \"s\"}\n"}, ""}, + {"go, not readable", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.s"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/cmd/o/main.go": goOwner + "var Kept = []announce.State{{Name: \"t\", Readable: true}, {Name: \"s\"}}\n"}, + "r reads o.s, which o's code announces not readable"}, + {"go, a known mesh-sdk name and readers", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.good","o.s"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/cmd/o/main.go": goOwner + "var _ = []announce.State{{Name: health.GoodState, Readable: true}, {Name: \"s\", Readers: []string{\"r\"}}}\n"}, ""}, + {"go, a name that cannot be read", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.s"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/cmd/o/main.go": goOwner + "var _ = announce.State{Name: other.S, Readable: true}\n"}, + "the state's name other.S cannot be read"}, + {"go, a state not announced", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.s"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/cmd/o/main.go": goOwner + "var _ = announce.State{Name: \"t\"}\n"}, + "r reads o.s, which o's code does not announce"}, + {"typescript, not readable", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.bound"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/kept.ts": `export const KEPT = [{ name: "announced" }, { name: "bound" }];`}, + "r reads o.bound, which o's code announces not readable"}, + {"typescript, readable", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.bound"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/index.ts": `await keeps([{ name: "bound", readable: true }]);`}, ""}, + {"an owner that announces nothing, and one outside the catalogue", map[string]string{ + "modules/r/module.json": `{"module":"r","reads":["o.s","mesh-controller.lease"]}`, + "modules/o/module.json": `{"module":"o"}`, + "modules/o/cmd/o/main.go": "package main\n\nvar _ = 1\n"}, ""}, + } { + t.Run(row.name, func(t *testing.T) { + root := t.TempDir() + for path, text := range row.files { + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := Findings(root) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(got, "\n") + if row.want == "" && len(got) != 0 || row.want != "" && !strings.Contains(joined, row.want) { + t.Errorf("findings %q, want %q", joined, row.want) + } + }) + } +} + +// The catalogue itself: every state a module reads of another module that announces from code is announced readable. +func TestTheCatalogueReadsOnlyWhatItsOwnersAnnounceReadable(t *testing.T) { + got, err := Findings(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + for _, f := range got { + t.Error(f) + } +} diff --git a/checks/readable/reads-outside b/checks/readable/reads-outside new file mode 100644 index 00000000..6cf17846 --- /dev/null +++ b/checks/readable/reads-outside @@ -0,0 +1,6 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +# every manifest, for what it reads; and every module's Go and TypeScript code, for the states it announces. +modules/*/module.json +modules/*/cmd +modules/*/*.go +modules/*/*.ts diff --git a/modules/claude-code/cmd/claude-code/forge.go b/modules/claude-code/cmd/claude-code/forge.go index 112c033c..fda20225 100644 --- a/modules/claude-code/cmd/claude-code/forge.go +++ b/modules/claude-code/cmd/claude-code/forge.go @@ -24,6 +24,8 @@ import ( "os" "path/filepath" "strings" + + "git.novox.be/novox/mesh-sdk/go/asks" ) // ForgeProvision is the provision whose credential this is, and the file this module's manifest has it delivered to. @@ -32,7 +34,59 @@ const ( forgeSecret = "gitea-read.secret" ) -func (p Paths) forgeSecret() string { return filepath.Join(p.State, forgeSecret) } +func (p Paths) forgeSecret() string { + if p.Forge != "" { + return p.Forge + } + return filepath.Join(p.State, forgeSecret) +} + +// ForgeHealth is what this module says to the operator of its forge credential (the sweep of 2026-10-30, issue 774: +// the file was root's, 0600, and the runtime, the operator's account, could not read it; said only in the journal, +// some 870 times a machine). raised is the reason last raised, "" when none. +type ForgeHealth struct { + Node string + raised string + looked bool +} + +// forgeConditionKey is the condition's key: its last part is its kind. +func (f *ForgeHealth) key() string { return "claude-code.machine." + f.Node + ".forge-credential" } + +// Look is the condition to say after one placing of the credential, nil when nothing changed: raised on a failure, again +// when its reason changes, cleared when it places again; on the first look a success clears what a run before raised. +func (f *ForgeHealth) Look(err error) *asks.Condition { + first := !f.looked + f.looked = true + subject := asks.Scope{Scope: "machine", ID: f.Node, Machine: f.Node} + if err == nil { + if f.raised == "" && !first { + return nil + } + f.raised = "" + return &asks.Condition{Key: f.key(), Subject: subject, State: asks.StateCleared} + } + why := err.Error() + if why == f.raised { + return nil + } + f.raised = why + needs := "read claude-code's journal on " + f.Node + " and the owner of its forge credential file" + return &asks.Condition{Key: f.key(), Severity: asks.SeverityWarning, State: asks.StateRaised, Subject: subject, + Headline: "Agents on " + f.Node + " cannot use the forge credential", + Summary: why, Needs: needs, + Explanation: asks.NeedsYou + needs + ". " + sentence(why) + " Until it is placed, an agent there cannot " + + "clone or fetch the mesh's private repositories.", + Resolved: "The forge's read credential is placed again on " + f.Node + "."} +} + +func sentence(s string) string { + s = strings.TrimSpace(s) + if s == "" || strings.HasSuffix(s, ".") { + return s + } + return strings.ToUpper(s[:1]) + s[1:] + "." +} // forgeCredentials is where the agent's git reads the credential: under the agent's home, in a directory of the // mesh's own. diff --git a/modules/claude-code/cmd/claude-code/forge_test.go b/modules/claude-code/cmd/claude-code/forge_test.go index 8c504620..551ad774 100644 --- a/modules/claude-code/cmd/claude-code/forge_test.go +++ b/modules/claude-code/cmd/claude-code/forge_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "net/http" "net/http/cgi" "net/http/httptest" @@ -10,6 +11,8 @@ import ( "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-sdk/go/asks" ) const agentToken = "0123456789abcdef0123456789abcdef01234567" @@ -260,3 +263,72 @@ func TestForgeCredentialLinesAreReadStrictly(t *testing.T) { } } } + +// The forge credential's file is named by a word of the bundle, so the runtime's account owns it and reads it (the +// sweep of 2026-10-30: root's 0600 file, permission denied ~870 times a machine), and a failure to place it is said to +// the operator as a condition, raised once per reason, cleared when it places again, and cleared at a first good look. +func TestTheForgeCredentialIsTheRuntimesAndItsFailureIsAConditionOfTheModule(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Secrets map[string]string `json:"secrets"` + Build struct { + Artifacts []struct { + Kind string `json:"kind"` + Env map[string]string `json:"env"` + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + named := false + for _, a := range m.Build.Artifacts { + if a.Kind == "bundle" && a.Env["MESH_CLAUDE_CODE_FORGE"] == m.Secrets[ForgeProvision] && m.Secrets[ForgeProvision] != "" { + named = true + } + } + if !named { + t.Errorf("no word of the bundle names the forge credential's file %q: the runtime cannot read it", m.Secrets[ForgeProvision]) + } + p, _ := PathsFrom(envMap{"MESH_CLAUDE_CODE_STATE": "/s", "MESH_CLAUDE_CODE_FORGE": "/s/forge"}.get) + if p.forgeSecret() != "/s/forge" { + t.Errorf("the word is not read: %s", p.forgeSecret()) + } + + f := &ForgeHealth{Node: "anchor"} + denied := errors.New("the forge's read credential cannot be read: open /s/gitea-read.secret: permission denied") + for i, row := range []struct { + err error + want string // "" nothing said, else the state said + }{ + {nil, "cleared"}, // a first good look clears what a run before raised + {nil, ""}, + {denied, "raised"}, + {denied, ""}, + {errors.New("the forge credential could not be written: disk full"), "raised"}, + {nil, "cleared"}, + {nil, ""}, + } { + c := f.Look(row.err) + got := "" + if c != nil { + got = c.State + if _, err := asks.EncodeCondition(*c); err != nil { + t.Errorf("look %d: the condition is refused: %v", i, err) + } + if c.Key != "claude-code.machine.anchor.forge-credential" { + t.Errorf("look %d: key %s", i, c.Key) + } + } + if got != row.want { + t.Errorf("look %d: said %q, want %q", i, got, row.want) + } + } +} + +type envMap map[string]string + +func (e envMap) get(k string) string { return e[k] } diff --git a/modules/claude-code/cmd/claude-code/main.go b/modules/claude-code/cmd/claude-code/main.go index d879bbae..9e0d6c43 100644 --- a/modules/claude-code/cmd/claude-code/main.go +++ b/modules/claude-code/cmd/claude-code/main.go @@ -22,6 +22,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" ) func say(format string, args ...any) { @@ -177,6 +178,19 @@ func (s memoryStateOf) Delete(key string) error { return s.s.Delete(key) } // memoryCarrier carries this node's memory folders (ADR 0343); nil until the long-running half starts it. var memoryCarrier *MemoryCarrier +// sayCondition says a condition on the operator-channel seat under this module's name (novox/hq ADR 0437 §9). +func sayCondition(c asks.Condition) error { + subject, err := asks.ConditionSubject("claude-code") + if err != nil { + return err + } + raw, err := asks.EncodeCondition(c) + if err != nil { + return err + } + return stdio.SeatPublish(subject, json.RawMessage(raw), "") +} + // notifyOperator tells the operator through the operator channel's notify verb (ADR 0253). func notifyOperator(args map[string]any) error { _, err := ask("seat:operator-channel.notify", args) @@ -599,12 +613,21 @@ func run(p Paths, view *ServerView, configView *ConfigView) { // The forge's read credential (forge.go, ADR 0365 §4): placed as held, and put back when git or a person changed // it; the sessions' environment names it only while one is held. forgeHeld := ForgeCredentialHeld(p) + forgeHealth := &ForgeHealth{Node: p.Node} placeForge := func() { - if line, err := PlaceForgeCredential(p); err != nil { + line, err := PlaceForgeCredential(p) + if err != nil { say("%v", err) } else if line != "" { say("%s", line) } + // Said to the operator as a condition, raised once per reason and cleared when it places again. + if c := forgeHealth.Look(err); c != nil { + if serr := sayCondition(*c); serr != nil { + say("the forge credential's condition could not be said (%s): %v", c.State, serr) + forgeHealth = &ForgeHealth{Node: p.Node} // said again at the next look + } + } if held := ForgeCredentialHeld(p); held != forgeHeld { forgeHeld = held if _, err := RenderNow(p, writeManaged); err != nil { diff --git a/modules/claude-code/cmd/claude-code/node.go b/modules/claude-code/cmd/claude-code/node.go index 12868291..8b04d109 100644 --- a/modules/claude-code/cmd/claude-code/node.go +++ b/modules/claude-code/cmd/claude-code/node.go @@ -54,14 +54,17 @@ func Fingerprint(s string) string { // AgentHome and OperatorHome are the runtime's words as given, read to judge whose home the home scope writes // (novox/hq issue 340). type Paths struct { - State, Facts, Settings, Home, Node string + State, Facts, Settings, Home, Node string + // Forge is the file the forge's read credential arrives in (MESH_CLAUDE_CODE_FORGE): named by a word of the + // bundle, so the runtime's account owns it and can read it (catalogue givenTo); empty is gitea-read.secret in State. + Forge string Agent, Operator, AgentHome, OperatorHome string } // PathsFrom reads them, or answers false outside a launch. func PathsFrom(env func(string) string) (Paths, bool) { p := Paths{State: env("MESH_CLAUDE_CODE_STATE"), Facts: env("MESH_CLAUDE_CODE_FACTS"), - Settings: env("MESH_CLAUDE_CODE_SETTINGS"), Home: env("MESH_AGENT_HOME"), Node: env("MESH_NODE"), + Settings: env("MESH_CLAUDE_CODE_SETTINGS"), Forge: env("MESH_CLAUDE_CODE_FORGE"), Home: env("MESH_AGENT_HOME"), Node: env("MESH_NODE"), Agent: env("MESH_AGENT_ACCOUNT"), Operator: env("MESH_OPERATOR_ACCOUNT"), AgentHome: env("MESH_AGENT_HOME"), OperatorHome: env("MESH_OPERATOR_HOME")} if (p.Home == "") != (p.Agent == "") { diff --git a/modules/claude-code/go.mod b/modules/claude-code/go.mod index cc52e29f..4aa79572 100644 --- a/modules/claude-code/go.mod +++ b/modules/claude-code/go.mod @@ -1,7 +1,16 @@ module claude-code -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 require gopkg.in/yaml.v3 v3.0.1 + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/claude-code/go.sum b/modules/claude-code/go.sum index c26c88fc..26b38621 100644 --- a/modules/claude-code/go.sum +++ b/modules/claude-code/go.sum @@ -1,11 +1,17 @@ -git.novox.be/novox/mesh-sdk/go v0.1.13 h1:Su4JYpZ+zhNovkGA2DgxiZdcuHpjw2tPJzc/7PljhXg= -git.novox.be/novox/mesh-sdk/go v0.1.13/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629 h1:7TYFV1Q6Mcz+lsLP/sds87tysET4rKCn3rRIQV8gNmA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d h1:HEJyqorKktrkJPeeJdwEo0kdrJjH3fm/YVS9MGftCqY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/modules/claude-code/module.json b/modules/claude-code/module.json index 3b62d219..9f85df0c 100644 --- a/modules/claude-code/module.json +++ b/modules/claude-code/module.json @@ -236,7 +236,8 @@ "env": { "MESH_CLAUDE_CODE_STATE": "${dir:state}", "MESH_CLAUDE_CODE_FACTS": "${dir:state}/facts.json", - "MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json" + "MESH_CLAUDE_CODE_SETTINGS": "${dir:state}/settings.json", + "MESH_CLAUDE_CODE_FORGE": "${dir:state}/gitea-read.secret" } }, { diff --git a/modules/disk-load/cmd/disk-load/main.go b/modules/disk-load/cmd/disk-load/main.go index 2eeff0b8..578539fc 100644 --- a/modules/disk-load/cmd/disk-load/main.go +++ b/modules/disk-load/cmd/disk-load/main.go @@ -24,7 +24,8 @@ import ( func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) } func main() { - if err := stdio.Keeps(announce.State{Name: StateReadings}); err != nil { + // Readable: restic, nfs-server, mounts and the self-check read it (their manifests' `reads`); checks/readable holds it. + if err := stdio.Keeps(announce.State{Name: StateReadings, Readable: true}); err != nil { logf("%v", err) os.Exit(1) } diff --git a/modules/gitea/check.ts b/modules/gitea/check.ts index 63800bf5..3af49049 100644 --- a/modules/gitea/check.ts +++ b/modules/gitea/check.ts @@ -83,6 +83,12 @@ export function checkStatusOf(change: OwedChange): { owner: string; repo: string return { owner, repo, sha, revision, status }; } +/** Whether the forge's refusal will not change by asking again: it forbids the write (403) or knows no such repository + * or commit (404), as the client words it ("Gitea API : ..."). */ +export function refusedForGood(err: unknown): boolean { + return /Gitea API \S+: (403|404) /.test(err instanceof Error ? err.message : String(err)); +} + /** How many mesh/check writes this module has at the forge at once. */ export const WRITES_AT_ONCE = 4; @@ -115,6 +121,12 @@ export class CheckStatuses { try { asked = checkStatusOf(change); } catch (err) { + // A deleted key is owed no more: its revision is seen, so a write of it still asked again ends (askAgain). + if (change?.op === "delete" && typeof change.key === "string" && Number.isInteger(change.revision)) { + const seen = this.seen.get(change.key); + if (seen === undefined || change.revision > seen) this.remember(this.seen, change.key, change.revision); + return "dropped"; + } this.say(`[gitea] ${CHECKER_OWED} ${change?.key} at revision ${change?.revision} writes nothing: ${err instanceof Error ? err.message : err}`); return "dropped"; } @@ -172,6 +184,12 @@ export class CheckStatuses { private async askAgain(change: OwedChange, first: unknown): Promise { let err = first; for (let attempt = 1; ; attempt++) { + if (refusedForGood(err)) { + // A repository the forge forbids or does not know answers the same at every ask (the sweep of 2026-10-30, issue + // 774: hal and novadock, 119 asks each): said once, never asked again; a newer revision of the key asks anew. + this.say(`[gitea] ${change.key}: mesh/check at revision ${change.revision} is refused for good (${err instanceof Error ? err.message : err}); not asked again`); + return; + } const wait = bindWait(attempt); this.say(`[gitea] ${change.key}: mesh/check at revision ${change.revision} could not be written (${err instanceof Error ? err.message : err}); asked again in ${wait / 1000}s`); await this.sleep(wait); diff --git a/modules/gitea/kept.ts b/modules/gitea/kept.ts index fccc5514..5f83f73a 100644 --- a/modules/gitea/kept.ts +++ b/modules/gitea/kept.ts @@ -1,10 +1,11 @@ // The states this module keeps on the bus and whether other modules may read them (novox/hq ADR 0495 clause 2; issue // 1790): the module says it itself, from its code, as it starts (index.ts calls keeps with this list), and a reader is -// granted what it said (clause 3). A state is readable when another module's manifest reads it; none reads gitea's. -// test/kept.test.ts holds the list to the manifest's `state`. +// granted what it said (clause 3). A state is readable when another module's manifest reads it: mesh-provisioner reads +// `bound` (the forge accounts it binds). test/kept.test.ts holds the list to the manifest's `state`, and the catalogue's +// checks/readable every state another manifest reads to a readable announcement. export const KEPT: readonly { readonly name: string; readonly readable?: boolean }[] = [ { name: "announced" }, - { name: "bound" }, + { name: "bound", readable: true }, ]; /** The announcement of these states, in the layout mesh-sdk's go/announce reads (novox/hq ADR 0495 clause 2): version 1, diff --git a/modules/gitea/test/check-owed.test.ts b/modules/gitea/test/check-owed.test.ts index 5ec9e22d..afd43584 100644 --- a/modules/gitea/test/check-owed.test.ts +++ b/modules/gitea/test/check-owed.test.ts @@ -175,3 +175,33 @@ test("the writes a watch's start hands over go to the forge a few at a time, nev assert.equal(done, 50); assert.equal(most, WRITES_AT_ONCE); }); + +// The sweep of 2026-10-30 (issue 774): a status the forge refuses with 403 or 404 was asked again every 30 s forever (hal +// and novadock, 119 times each), and a key the checker deleted kept its loop running. Now a 403 or 404 is said once and +// never asked again, a 5xx is still asked again, and a delete ends a loop that is running. +test("a refusal for good is said once and not asked again; a delete ends a running ask again", async () => { + const { CheckStatuses, refusedForGood } = await import("../check.ts"); + for (const [err, forGood] of [["Gitea API /repos/jschoubben/hal/statuses/x: 403 forbidden", true], + ["Gitea API /repos/jschoubben/novadock/statuses/x: 404 not found", true], ["Gitea API /repos/x/statuses: 502 bad gateway", false], + ["Gitea API /repos/x: no answer in 10 s, the request was cut", false]] as [string, boolean][]) { + assert.equal(refusedForGood(new Error(err)), forGood, err); + } + const said: string[] = []; + const forbidden = { setCommitStatus: async () => { throw new Error("Gitea API /repos/jschoubben/hal/statuses/x: 403 forbidden"); } }; + const checks = new CheckStatuses(forbidden, (l) => said.push(l), async () => { throw new Error("asked again"); }); + await checks.take(put(key(sha, "jschoubben/hal"), owed(), 3)); + await checks.settled(); + assert.equal(said.length, 1); + assert.match(said[0], /refused for good.*403.*not asked again/); + // A 502 asks again; the key's delete meanwhile ends the loop before its next ask. + const g = forge(); + let release: () => void = () => {}; + const gate = new Promise((r) => { release = r; }); + const later = new CheckStatuses(g, () => {}, () => gate); + g.fail(5); + await later.take(put(key(), owed(), 8)); + await later.take({ state: "mesh-delivery-checker.owed", key: key(), op: "delete", revision: 9, current: false } as any); + release(); + await later.settled(); + assert.deepEqual(g.sent, [], "a deleted key is not written"); +}); diff --git a/modules/gitea/test/kept.test.ts b/modules/gitea/test/kept.test.ts index 5702f8e1..f0ca9ec9 100644 --- a/modules/gitea/test/kept.test.ts +++ b/modules/gitea/test/kept.test.ts @@ -2,20 +2,21 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { readFileSync } from "node:fs"; -// 1790: the states the module announces are the states its manifest declares, no more, no fewer; none is readable -// because no other module's manifest reads one of them. +// 1790: the states the module announces are the states its manifest declares, no more, no fewer; `bound` is readable, +// because mesh-provisioner's manifest reads it (the sweep of 2026-10-30, issue 774: announced unreadable, the read was +// refused and no forge account was bound). test("the states the module announces are the states its manifest declares", async () => { const { KEPT } = await import("../kept.ts"); const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8")); const declared = (manifest.state as (string | { name: string })[]).map((s) => (typeof s === "string" ? s : s.name)).sort(); assert.deepEqual(KEPT.map((k) => k.name).sort(), declared); - assert.ok(KEPT.every((k) => !k.readable), "no other module's manifest reads a state of gitea"); + assert.deepEqual(KEPT.filter((k) => k.readable).map((k) => k.name), ["bound"], "bound alone is readable: mesh-provisioner reads it"); }); test("the announcement has the layout mesh-sdk's go/announce reads: version 1, the module, the node, the states sorted by name", async () => { const { statesAnnouncement } = await import("../kept.ts"); - assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound" }] }); - assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound" }] }); + assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound", readable: true }] }); + assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound", readable: true }] }); }); test("index.ts says the announcement as the event `states` before the tools are served", () => { diff --git a/modules/grafana/index.ts b/modules/grafana/index.ts index 6e174b23..ab541dd3 100644 --- a/modules/grafana/index.ts +++ b/modules/grafana/index.ts @@ -9,6 +9,7 @@ import { emit } from "@novox/mesh-sdk/events"; import { GrafanaClient, type GrafanaAlert } from "./client.js"; +import { Refusal } from "./refused.js"; // Constructed lazily so an unconfigured node (no auth) loads this entrypoint without crashing the // events host — it simply watches nothing. @@ -51,7 +52,13 @@ async function pollAlerts(client: GrafanaClient): Promise { if (grafana) { const client = grafana; - const run = (): void => void pollAlerts(client).catch((err) => console.error(`[grafana] ${err}`)); + // A refused credential is said once and asked again hourly (refused.ts), not every 30 s. + const refusal = new Refusal(); + const say = (line: string | undefined): void => { if (line) console.error(line); }; + const run = (): void => { + if (!refusal.due(Date.now())) return; + void pollAlerts(client).then(() => say(refusal.after(Date.now())), (err) => say(refusal.after(Date.now(), err))); + }; setInterval(run, 30_000); run(); console.log("[grafana] watching for firing alerts"); diff --git a/modules/grafana/refused.ts b/modules/grafana/refused.ts new file mode 100644 index 00000000..1610a265 --- /dev/null +++ b/modules/grafana/refused.ts @@ -0,0 +1,46 @@ +// What the alert watcher does when Grafana refuses the vault's admin password (the sweep of 2026-10-30, issue 774: +// a 401 every 30 s on ace since at least 2026-10-05, 145 times in the sweep's window, said only in the journal). +// +// The cause on ace: Grafana keeps the admin password it was first given in its own database, and applies +// GF_SECURITY_ADMIN_PASSWORD__FILE only when it creates that database. ace's Grafana was adopted with its database +// before the mesh minted `admin` (2026-09-30), so the minted value never took (the same kind as keycloak's admin, +// hq issue 179). Asking again every 30 s changes nothing until the password in Grafana's database is the vault's. +// So a refusal is said once, with that cause and what ends it, and asked again hourly, not every look; an answer +// after it is said too. + +/** How long a refused credential waits before the watcher asks again. */ +export const AskAgainRefusedMs = 60 * 60_000; + +/** Whether err is Grafana refusing the credential (the client's "Grafana API : 401 ..."). */ +export function refusedCredential(err: unknown): boolean { + return /^Error: Grafana API \S+: 401 |^Grafana API \S+: 401 /.test(String(err)); +} + +export class Refusal { + private since: number | undefined; + private lastAsk = 0; + + /** Whether the watcher asks Grafana now. */ + due(now: number): boolean { + return this.since === undefined || now - this.lastAsk >= AskAgainRefusedMs; + } + + /** What to say after one ask at now: its error, the refusal once, the recovery once; undefined for nothing. */ + after(now: number, err?: unknown): string | undefined { + this.lastAsk = now; + if (err === undefined) { + if (this.since === undefined) return undefined; + this.since = undefined; + return "[grafana] Grafana accepts the vault's admin password again: watching for firing alerts every 30 s"; + } + if (!refusedCredential(err)) return `[grafana] ${err}`; + if (this.since !== undefined) return undefined; + this.since = now; + return ( + "[grafana] Grafana refuses the vault's admin password (401), so no firing alert is seen; asked again hourly. " + + "Grafana keeps the admin password it was first given in its database and takes the vault's only when it creates " + + "that database: an install adopted with its database keeps its old one. It ends when Grafana's admin password is " + + "the vault's (grafana cli admin reset-admin-password in the container, with the value of own//grafana/admin)." + ); + } +} diff --git a/modules/grafana/test/refused.test.ts b/modules/grafana/test/refused.test.ts new file mode 100644 index 00000000..8d88f736 --- /dev/null +++ b/modules/grafana/test/refused.test.ts @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +// The sweep of 2026-10-30 (issue 774): a refused admin password was said every 30 s, forever. Now it is said once with +// its cause, asked again hourly, and its end is said; any other error is still said each time. +test("a refused credential is said once, asked again hourly, and its end said", async () => { + const { Refusal, AskAgainRefusedMs } = await import("../refused.ts"); + const r = new Refusal(); + const refused = new Error('Grafana API /api/prometheus/grafana/api/v1/alerts: 401 {"message":"Invalid username or password"}'); + const rows: [number, Error | undefined, boolean, RegExp | undefined][] = [ + // at, error of the ask, due before it, what is said + [0, refused, true, /refuses the vault's admin password.*reset-admin-password/], + [30_000, undefined, false, undefined], + [AskAgainRefusedMs, refused, true, undefined], + [AskAgainRefusedMs + 1, new Error("Grafana API /api/health: 502 bad gateway"), false, undefined], + [2 * AskAgainRefusedMs, undefined, true, /accepts the vault's admin password again/], + [2 * AskAgainRefusedMs + 30_000, new Error("Grafana API /x: 502 bad gateway"), true, /502 bad gateway/], + [2 * AskAgainRefusedMs + 60_000, new Error("Grafana API /x: 502 bad gateway"), true, /502 bad gateway/], + ]; + for (const [i, [at, err, due, said]] of rows.entries()) { + assert.equal(r.due(at), due, `row ${i}: due`); + if (!due) continue; + const line = r.after(at, err); + if (said) assert.match(line ?? "", said, `row ${i}`); + else assert.equal(line, undefined, `row ${i}`); + } +}); diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 1d51cdb8..7765b80c 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index c8ee2177..ff5a0f48 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index cb908ec3..ff2f5221 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index a2b8e018..61230834 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index cd83b38b..d22b9a4c 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index 5f0e39b2..9f9ca2a0 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index c0bba92e..c0bff6a7 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -134,16 +134,8 @@ func resolving(c *Config) { // every change of what a module declares where (issue 1953); the builds themselves are read, not watched. for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead, CandidatesRead, MirrorDeclaredRead, TakeoverState} { - go func(read string) { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - time.Sleep(wait) - if err := stdio.State(read).Watch("", poke); err != nil { - logf("[mesh-config] %s is not watched yet (%v); asking again in %s", read, err, wait) - continue - } - return - } - }(read) + // Asked again until the runtime takes it, each give-up said (mesh-sdk's Follow). + go stdio.State(read).Follow("", poke) } tick := time.NewTicker(5 * time.Minute) defer tick.Stop() diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index 0c8a0a52..191237ac 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,7 +2,7 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 require ( github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index 05084f7c..991de7e5 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index 6e48bfba..fb1e50b5 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -52,18 +52,6 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state). func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } -// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. -func watched(state, key string, handle func(stdio.StateChange) error) { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - time.Sleep(wait) - if err := stdio.State(state).Watch(key, handle); err != nil { - logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) - continue - } - return - } -} - func main() { h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, @@ -71,16 +59,16 @@ func main() { // The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended // while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry. var started sync.Once - go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { + go stdio.State(LeaseRead).Follow(LeaseKey, func(c stdio.StateChange) error { h.LeaseHeard(c.Value, c.Op == "put", nil) started.Do(func() { go h.Rearm() }) return nil }) - go watched(WakesState, "", func(c stdio.StateChange) error { + go stdio.State(WakesState).Follow("", func(c stdio.StateChange) error { h.WakeHeard(c.Key, c.Op) return nil }) - go watched(HandActsRead, "", func(c stdio.StateChange) error { + go stdio.State(HandActsRead).Follow("", func(c stdio.StateChange) error { h.HandActHeard(c.Key, c.Op, c.Value) return nil }) diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod index 89b3df38..d0ba8bc3 100644 --- a/modules/mesh-healers/go.mod +++ b/modules/mesh-healers/go.mod @@ -3,7 +3,7 @@ module mesh-healers go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum index a2b8e018..61230834 100644 --- a/modules/mesh-healers/go.sum +++ b/modules/mesh-healers/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-issues/cmd/mesh-issues/main.go b/modules/mesh-issues/cmd/mesh-issues/main.go index ce7a0a4a..08654a2d 100644 --- a/modules/mesh-issues/cmd/mesh-issues/main.go +++ b/modules/mesh-issues/cmd/mesh-issues/main.go @@ -132,14 +132,8 @@ func main() { // reads the bucket, and a write carries the issue it changes first, so no write reaches the log before its // issue's carry. func start(t *Tracker) { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - time.Sleep(wait) - if err := stdio.State(IssuesState).Watch("", t.Watched); err != nil { - logf("[mesh-issues] the bucket is not watched yet (%v); asking again in %s", err, wait) - continue - } - break - } + // Asked again until the runtime takes it, each give-up said (mesh-sdk's Follow). + stdio.State(IssuesState).Follow("", t.Watched) for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { s, err := t.Start() if err == nil { diff --git a/modules/mesh-issues/go.mod b/modules/mesh-issues/go.mod index ea63f374..4333f0eb 100644 --- a/modules/mesh-issues/go.mod +++ b/modules/mesh-issues/go.mod @@ -3,7 +3,7 @@ module mesh-issues go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-issues/go.sum b/modules/mesh-issues/go.sum index a2b8e018..61230834 100644 --- a/modules/mesh-issues/go.sum +++ b/modules/mesh-issues/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go index e6815a3e..5c4eb2a3 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go @@ -80,6 +80,26 @@ type Report struct { // Items are the collections the module's items are recorded in on this node (design 57 L1). Items []string `json:"items,omitempty"` At time.Time `json:"at"` + // FailingSince is, on a failed report, when the module's standing failure began: kept while it stays failed, however + // its reason's words change, so the controller's apply-failing alarm judges the failure's age, not its newest words + // (the sweep of 2026-10-30, issue 774). Empty on any other status. + FailingSince time.Time `json:"failing-since,omitzero"` +} + +// failingSince is when the failure r says began: before's when the module's last report was a failure too, whatever +// its words; else now. Zero when r is no failure. +func failingSince(r Report, before *Report, now time.Time) time.Time { + switch { + case r.Status != StatusFailed: + return time.Time{} + case before == nil || before.Status != StatusFailed: + return now + case !before.FailingSince.IsZero(): + return before.FailingSince + case !before.At.IsZero(): + return before.At // a report from before the field: its time is the oldest known + } + return now } // FileApplied is one file as the node-engine wrote it. @@ -139,6 +159,7 @@ type state struct { config map[string]map[string]json.RawMessage applied map[string]bool // module → a report is kept reported map[string][]byte // module → the last report said, undated + last map[string]Report // module → the last report said, dated secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere // items is every item recorded for this node, by collection and contributor: what a collector is filled from. items map[string]map[string]itemValue @@ -940,6 +961,7 @@ func (c *Client) report(ctx context.Context, kvs map[string]jetstream.KeyValue, return } r.At = c.now() + r.FailingSince = failingSince(r, c.lastReport(ctx, kvs, s, r.Module), r.At) body, err := json.Marshal(r) if err != nil { logf("[provisioner-client] the report of %s: %v", r.Module, err) @@ -952,6 +974,10 @@ func (c *Client) report(ctx context.Context, kvs map[string]jetstream.KeyValue, } c.publish(r) s.reported[r.Module] = undated + if s.last == nil { + s.last = map[string]Report{} + } + s.last[r.Module] = r switch r.Status { case StatusFailed: logf("[provisioner-client] %s failed: %s", r.Module, r.Reason) @@ -960,6 +986,27 @@ func (c *Client) report(ctx context.Context, kvs map[string]jetstream.KeyValue, } } +// lastReport is the module's last report: this run's, else the one kept in the applied bucket by a run before; nil when +// none can be read. +func (c *Client) lastReport(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string) *Report { + if r, ok := s.last[module]; ok { + return &r + } + kv := kvs[AppliedBucket] + if kv == nil { + return nil + } + e, err := kv.Get(ctx, c.Node+"."+module) + if err != nil { + return nil + } + var r Report + if json.Unmarshal(e.Value(), &r) != nil { + return nil + } + return &r +} + func (c *Client) publish(r Report) { body, err := json.Marshal(r) if err != nil { diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go index fb0734ee..39bbd12b 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go @@ -826,3 +826,32 @@ func planned(assignment string) ([]byte, bool) { raw, _ := json.Marshal(map[string]any{"v": 1, "node": a.Node, "module": a.Module, "plan": a.Provisioning}) return raw, true } + +// A standing failure keeps when it began while only its words change, so the controller's apply-failing alarm judges +// its age (the sweep of 2026-10-30, issue 774: novox's reason changed at 16:56 and the alarm cleared for five minutes +// while every module still failed); it begins anew after any other status, and a report from before the field gives +// its time. +func TestAFailureKeepsWhenItBeganWhileItsWordsChange(t *testing.T) { + t0 := time.Date(2026, 10, 30, 15, 0, 0, 0, time.UTC) + now := t0.Add(time.Hour) + failed := func(why string, since time.Time) *Report { + return &Report{Status: StatusFailed, Reason: why, At: since.Add(time.Minute), FailingSince: since} + } + for _, row := range []struct { + name string + r Report + before *Report + want time.Time + }{ + {"a first failure", Report{Status: StatusFailed, Reason: "a"}, nil, now}, + {"the same failure in other words", Report{Status: StatusFailed, Reason: "the vault refused it"}, failed("bus refused", t0), t0}, + {"a failure after an apply", Report{Status: StatusFailed, Reason: "a"}, &Report{Status: StatusApplied, At: t0}, now}, + {"a failure after waiting", Report{Status: StatusFailed, Reason: "a"}, &Report{Status: StatusWaiting, At: t0}, now}, + {"a report from before the field", Report{Status: StatusFailed, Reason: "b"}, &Report{Status: StatusFailed, Reason: "a", At: t0}, t0}, + {"an apply", Report{Status: StatusApplied}, failed("a", t0), time.Time{}}, + } { + if got := failingSince(row.r, row.before, now); !got.Equal(row.want) { + t.Errorf("%s: failing since %v, want %v", row.name, got, row.want) + } + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go index c8a8b357..96e4bc13 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -37,12 +37,22 @@ import ( "git.novox.be/novox/mesh-sdk/go/provisions" ) -// The states it keeps, beside Registry, Needs, Applied and Collect. +// The states it keeps, beside Registry, Needs, Applied and Collect. RefusedState holds, by `.`, why a +// placed module has no plan composed (issue 774): written while it is held, deleted once it composes again. const ( ProvisionsState = provisions.ProvisionsState PlansState = provisions.PlansState + RefusedState = "refused" ) +// Refusal is one module held on a node, as RefusedState keeps it. +type Refusal struct { + Node string `json:"node"` + Module string `json:"module"` + Why string `json:"why"` + Since time.Time `json:"since"` +} + // The reads it composes from. const ( PlacementRead = delivery.PlacementRead @@ -63,8 +73,11 @@ var BoundReads = []string{"gitea", "keycloak", "postgres", "route-adapter", "mes func boundRead(module string) string { return module + ".bound" } // Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names -// (module → fingerprint), the newest build of every other placed module. -type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) +// (module → fingerprint), the newest build of every other placed module. A module whose build or plan refuses is held +// (frombus.PlansHeld; issue 774, one bad module must not drop its node): in held with why, none of its plan in plans. +// err is a refusal that is no single module's. +type Composer func(ctx context.Context, node string, keptAt map[string]string) (plans map[string]catalogue.Plan, + held map[string]string, err error) // Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's // record of it. @@ -86,12 +99,15 @@ type Holder struct { // Composed is what one node's composition did. type Composed struct { - Node string `json:"node"` - Plans []string `json:"plans-written,omitempty"` - Gone []string `json:"plans-removed,omitempty"` - Changed []string `json:"provisions-changed,omitempty"` - Asked []string `json:"retiring-asked,omitempty"` - Problems []string `json:"problems,omitempty"` + Node string `json:"node"` + Plans []string `json:"plans-written,omitempty"` + Gone []string `json:"plans-removed,omitempty"` + Changed []string `json:"provisions-changed,omitempty"` + Asked []string `json:"retiring-asked,omitempty"` + // Held is each module placed on the node whose plan is not composed, with why: its last plan and its provisions + // stay as they were, and every other module is composed (issue 774). + Held map[string]string `json:"modules-held,omitempty"` + Problems []string `json:"problems,omitempty"` } // ComposeNode composes one node: its plans and its provisions. @@ -110,15 +126,28 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) wanted[module] = fp } } - atWants, err := h.Compose(ctx, node, wanted) + atWants, heldAtWants, err := h.Compose(ctx, node, wanted) if err != nil { return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err) } - newest, err := h.Compose(ctx, node, nil) + newest, heldNewest, err := h.Compose(ctx, node, nil) if err != nil { return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err) } - if err := h.writePlans(node, atWants, &out); err != nil { + // A module held at its wanted build has no plan composed: its last plan and its provisions are kept, said in + // `refused`. One held at its newest build alone keeps its plan at its want; what its newest build needs ahead waits. + if len(heldAtWants) > 0 { + out.Held = heldAtWants + } + for _, m := range sortedKeys(heldNewest) { + if _, ok := heldAtWants[m]; !ok { + out.Problems = append(out.Problems, fmt.Sprintf("%s at its newest build is not composed: %s", m, heldNewest[m])) + } + } + if err := h.writeRefused(node, heldAtWants); err != nil { + out.Problems = append(out.Problems, err.Error()) + } + if err := h.writePlans(node, atWants, heldAtWants, &out); err != nil { out.Problems = append(out.Problems, err.Error()) } needed := map[string]provisions.Provision{} @@ -166,10 +195,59 @@ func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) } } h.judgeElsewhere(elsewhere, &out) - h.judgeNode(node, needed, &out) + h.judgeNode(node, needed, heldAtWants, &out) return out, nil } +// writeRefused keeps, for each module held on node, why, and deletes the refusal of one that composes again: a change +// of the state is the event a watcher hears. +func (h *Holder) writeRefused(node string, held map[string]string) error { + var problems []error + for _, module := range sortedKeys(held) { + key, err := provisions.PlanKey(node, module) + if err != nil { + problems = append(problems, err) + continue + } + r := Refusal{Node: node, Module: module, Why: held[module], Since: h.Now().UTC()} + if e, err := h.Store.Get(RefusedState, key); err == nil && e != nil { + var was Refusal + if json.Unmarshal(e.Value, &was) == nil && was.Why == r.Why { + continue + } + } + raw, err := json.Marshal(r) + if err != nil { + problems = append(problems, err) + continue + } + if err := h.Store.Put(RefusedState, key, json.RawMessage(raw)); err != nil { + problems = append(problems, fmt.Errorf("the refusal %s: %w", key, err)) + continue + } + h.Logf("[%s] %s on %s has no plan composed: %s", Module, module, node, r.Why) + } + keys, err := h.Store.Keys(RefusedState) + if err != nil { + return errors.Join(append(problems, err)...) + } + for _, k := range keys { + n, module, ok := strings.Cut(k, ".") + if !ok || n != node { + continue + } + if _, still := held[module]; still { + continue + } + if err := h.Store.Delete(RefusedState, k); err != nil { + problems = append(problems, err) + continue + } + h.Logf("[%s] %s on %s composes again", Module, module, node) + } + return errors.Join(problems...) +} + // providerPlan is the plan of a provision's provider on its node: from this composition when it is on this node, else // as kept in `plans`; nil when the provider is not handed. func (h *Holder) providerPlan(p catalogue.PlanProvision, node string, here map[string]catalogue.Plan) *catalogue.Plan { @@ -297,8 +375,9 @@ func (h *Holder) runningOn(node string) map[string]string { return out } -// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there. -func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error { +// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there; a held +// module's plan is kept as it was (issue 774). +func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, held map[string]string, out *Composed) error { var problems []error for _, module := range sortedKeys(plans) { key, err := provisions.PlanKey(node, module) @@ -332,6 +411,9 @@ func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *C if _, still := plans[module]; still { continue } + if _, kept := held[module]; kept { + continue + } if err := h.Store.Delete(PlansState, k); err != nil { problems = append(problems, err) continue @@ -341,9 +423,17 @@ func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *C return errors.Join(problems...) } -// judgeNode brings the node's provision keys to what is needed and what each provider said. -func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) { +// judgeNode brings the node's provision keys to what is needed and what each provider said. A key of a module held on +// the node (heldModules) that its composition no longer names is left as it is: not composed is not unneeded. +func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, heldModules map[string]string, out *Composed) { held := h.held(node) + for key, have := range held { + if _, isNeeded := needed[key]; !isNeeded && have.Consumer.Node == node { + if _, notComposed := heldModules[have.Consumer.Module]; notComposed { + delete(held, key) + } + } + } // A key of this node another node's provision depends on is needed (a provider's one shared credential). for _, dep := range h.dependedOn(node) { if have, ok := held[dep]; ok { @@ -902,7 +992,7 @@ func orSaid(words ...string) string { // credential is read (compose.Planning). func FromBus() Composer { r := fromruntime.Reader() - return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { - return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) { + return frombus.PlansHeld(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go index 9aa766c3..4b650470 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go @@ -168,8 +168,8 @@ func TestTheHolderComposesProvidesAndAsksOnTheBus(t *testing.T) { } a := &asked{} h := &Holder{Store: s, Asks: a, Now: time.Now, Logf: t.Logf, - Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { - return frombus.Plans(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) { + return frombus.PlansHeld(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) }} state := func(key string) provisions.Provision { e, _ := s.Get(ProvisionsState, key) diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go index 238b589c..c26fd22e 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -56,21 +56,23 @@ type mesh struct { r *router atWant map[string]map[string]catalogue.Plan atNewest map[string]map[string]catalogue.Plan + heldAt map[string]map[string]string // by node, the modules its composition at the wanted builds holds composed []string } func newMesh() *mesh { m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}}, - atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}} + atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}, + heldAt: map[string]map[string]string{}} now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {}, Now: func() time.Time { now = now.Add(time.Second); return now }, - Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, map[string]string, error) { m.composed = append(m.composed, node) if keptAt != nil { - return m.atWant[node], nil + return m.atWant[node], m.heldAt[node], nil } - return m.atNewest[node], nil + return m.atNewest[node], nil, nil }} // What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their // have. @@ -314,6 +316,56 @@ func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { } } +// Issue 774, one bad module must not drop its node: a module whose manifest or plan refuses is held, named in `refused` +// with why, its last plan and its provisions kept; every other module on the node is composed and written; once it +// composes again its refusal goes. +func TestOneBadModuleDoesNotDropItsNode(t *testing.T) { + m := newMesh() + why := "the built manifest of nats at src2:6258f3 cannot be used" + steps := []struct { + name string + plans map[string]catalogue.Plan + held map[string]string + written []string // plans the step writes + refused bool // g14.nats in refused + }{ + {"a node new to the seat with one bad module: the others are composed", + map[string]catalogue.Plan{"gitea": planOf(pgDB)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true}, + {"the bad one composes: its plan is written and its refusal goes", + map[string]catalogue.Plan{"gitea": planOf(pgDB), "nats": planOf(oidc)}, nil, []string{"g14.nats"}, false}, + {"it refuses again: its last plan and its provision are kept, the others composed", + map[string]catalogue.Plan{"gitea": planOf(pgDB, split)}, map[string]string{"nats": why}, []string{"g14.gitea"}, true}, + } + for _, s := range steps { + m.atWant["g14"], m.atNewest["g14"], m.heldAt["g14"] = s.plans, s.plans, s.held + c := m.compose(t, "g14") + if !slices.Equal(c.Plans, s.written) || len(c.Gone) != 0 || len(c.Asked) != 0 { + t.Errorf("%s: %+v", s.name, c) + } + e, _ := m.s.Get(RefusedState, "g14.nats") + var r Refusal + if e != nil { + _ = json.Unmarshal(e.Value, &r) + } + if (e != nil) != s.refused || (s.refused && (r.Module != "nats" || r.Why != why || c.Held["nats"] != why)) { + t.Errorf("%s: refused %+v, held %v", s.name, r, c.Held) + } + if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { + t.Errorf("%s: gitea has no plan", s.name) + } + } + if e, _ := m.s.Get(PlansState, "g14.nats"); e == nil { + t.Error("the held module's last plan was removed") + } + if p := m.state(t, "g14.nats.oidc-client"); p.State != provisions.Asked { + t.Errorf("the held module's provision is %s (%s), not kept", p.State, p.Why) + } + got, err := m.h.read(RefusedState, "", "g14", "") + if rows := got.(map[string]any)[RefusedState].([]map[string]any); err != nil || len(rows) != 1 || rows[0]["why"] != why { + t.Errorf("the refusal reads %v, %v", got, err) + } +} + // Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build // its have says it runs, so the holder writes it a plan and its client keeps filling it. func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) { @@ -322,12 +374,12 @@ func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) { m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`) m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`) var keptAt map[string]string - m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, error) { + m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, map[string]string, error) { if at != nil { keptAt = at - return map[string]catalogue.Plan{"nats": {}}, nil + return map[string]catalogue.Plan{"nats": {}}, nil, nil } - return nil, nil + return nil, nil, nil } c := m.compose(t, "novox") if keptAt["nats"] != fp || len(keptAt) != 1 { diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go index 30a16446..8690cd73 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go @@ -37,10 +37,22 @@ func holderTools(h *Holder) []stdio.Tool { }}, {Name: PlansTool, Description: "Every module's provisioning plan the seat composed, ., narrowed by node, with how many " + - "keys and bytes the state holds; or one plan whole.", + "keys and bytes the state holds, and each placed module held with no plan composed and why (refused: its " + + "manifest or its plan refuses, the node's other modules composed); or one plan whole.", Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, ."), "node": str("a node")}}, - Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }}, + Run: func(a map[string]any) (any, error) { + out, err := h.read(PlansState, arg(a, "key"), arg(a, "node"), "") + if err != nil || arg(a, "key") != "" { + return out, err + } + refused, err := h.read(RefusedState, "", arg(a, "node"), "") + if err != nil { + return nil, err + } + out.(map[string]any)[RefusedState] = refused.(map[string]any)[RefusedState] + return out, nil + }}, {Name: ComposeTool, Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " + "was written, removed, changed and asked.", @@ -98,6 +110,14 @@ func (h *Holder) read(state, key, node, want string) (any, error) { row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why } } + if state == RefusedState { + var r Refusal + if err := json.Unmarshal(e.Value, &r); err != nil { + row["unreadable"] = err.Error() + } else { + row["module"], row["why"], row["since"] = r.Module, r.Why, r.Since + } + } rows = append(rows, row) } return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index cca4a622..95d795be 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -103,34 +103,22 @@ func (w *work) run(h *Holder) { } } -// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. -func watched(state string, handle func(stdio.StateChange) error) { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - time.Sleep(wait) - if err := stdio.State(state).Watch("", handle); err != nil { - logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) - continue - } - return - } -} - func main() { s := runtimeStore{} h := &Holder{Store: s, Compose: FromBus(), Asks: runtimeAsks{}, Now: time.Now, Logf: logf} w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)} go w.run(h) go starting(h) - go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil }) - go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil }) - go watched(ModulesRead, func(c stdio.StateChange) error { + go stdio.State(PlacementRead).Follow("", func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil }) + go stdio.State(WantsRead).Follow("", func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil }) + go stdio.State(ModulesRead).Follow("", func(c stdio.StateChange) error { if module, err := delivery.ParseModulesKey(c.Key); err == nil { w.add(h.NodesOfModule(module)...) } return nil }) for _, provider := range BoundReads { - go watched(boundRead(provider), func(c stdio.StateChange) error { + go stdio.State(boundRead(provider)).Follow("", func(c stdio.StateChange) error { if said, err := h.Rejudge(c.Key); err != nil { logf("[%s] %s is not judged again: %v", Module, c.Key, err) } else if said != "" { @@ -139,7 +127,7 @@ func main() { return nil }) } - go watched(Registry, func(c stdio.StateChange) error { + go stdio.State(Registry).Follow("", func(c stdio.StateChange) error { if c.Op != "put" { if _, _, err := Publish(s); err != nil { logf("[%s] the registry is not written back: %v", Module, err) diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index dd0ccde3..f717ea06 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -305,7 +305,7 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { t.Errorf("the claim: %+v", m.Claims) } // needs stays declared this release so its first start can empty it (ADR 0570 clause 1). - if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) { + if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState, RefusedState}; !slices.Equal(m.State, want) { t.Errorf("the state is %v, want %v", m.State, want) } // Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index db34240f..27360793 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -3,7 +3,7 @@ module mesh-provisioner go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum index a2b8e018..61230834 100644 --- a/modules/mesh-provisioner/go.sum +++ b/modules/mesh-provisioner/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 1aba09af..b417b373 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -40,7 +40,8 @@ "applied", "collect", "provisions", - "plans" + "plans", + "refused" ], "own-secrets": { "broker": "${dir:state}/broker" diff --git a/modules/mesh-vault/.gitignore b/modules/mesh-vault/.gitignore new file mode 100644 index 00000000..512a0107 --- /dev/null +++ b/modules/mesh-vault/.gitignore @@ -0,0 +1 @@ +/mesh-vault diff --git a/modules/mesh-vault/cmd/mesh-vault/accounts.go b/modules/mesh-vault/cmd/mesh-vault/accounts.go index c102337e..2a024ccb 100644 --- a/modules/mesh-vault/cmd/mesh-vault/accounts.go +++ b/modules/mesh-vault/cmd/mesh-vault/accounts.go @@ -187,6 +187,12 @@ func (v *Vault) HashAccount(secret string, force bool) (Hashed, error) { if err != nil { return Hashed{}, err } + if !v.busAccount(secret, value) { + // A broker- no process declares with a bus, holding no account (mesh-controller's broker-address): nothing + // to hash, and nothing owed. + v.hashes.paid(secret) + return Hashed{}, errNotABusAccount{secret} + } if rec.Retired != nil { v.hashes.paid(secret) return Hashed{}, fmt.Errorf("%s is retired (%s): its hash is written again only when it is wanted again", secret, rec.Retired.Why) @@ -245,12 +251,22 @@ func (v *Vault) openHeld(secret string) (string, Record, error) { return value, rec, nil } +// errNotABusAccount is a covered name that holds no bus account and names no declared one. +type errNotABusAccount struct{ secret string } + +func (e errNotABusAccount) Error() string { + return e.secret + " is no bus account: no process declares it with a bus, and its value is no account" +} + // hashKept is what keeping a covered account does after it is kept: its hash written, or the failure said and owed. func (v *Vault) hashKept(secret string) { if v.Accounts == nil || !Covered(secret) { return } if _, err := v.HashAccount(secret, false); err != nil { + if errors.As(err, new(errNotABusAccount)) { + return + } v.hashes.owe(secret, err.Error()) v.log("THE HASH OF %s IS NOT WRITTEN: %v", secret, err) } @@ -273,6 +289,9 @@ func (v *Vault) HashHeld() (written []Hashed, failed map[string]string) { continue } h, err := v.HashAccount(r.Name, false) + if errors.As(err, new(errNotABusAccount)) { + continue + } if err != nil { v.hashes.owe(r.Name, err.Error()) failed[r.Name] = err.Error() @@ -299,7 +318,7 @@ func (v *Vault) HashOwed() { v.hashes.paid(n) continue } - if _, err := v.HashAccount(n, false); err != nil { + if _, err := v.HashAccount(n, false); err != nil && !errors.As(err, new(errNotABusAccount)) { still[n] = err.Error() } } diff --git a/modules/mesh-vault/cmd/mesh-vault/core_test.go b/modules/mesh-vault/cmd/mesh-vault/core_test.go index 0e98843e..a22f7d5f 100644 --- a/modules/mesh-vault/cmd/mesh-vault/core_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/core_test.go @@ -50,7 +50,7 @@ func pemOf(t *testing.T, raw string) string { return p } -const brokerValue = "pv-anchors-provisioner-broker-password-2025" +const brokerValue = `{"user":"provisioner.anchor","password":"pv-anchors-provisioner-broker-password-2025"}` func clientsBroker(sealingKey string) MoveOf { return MoveOf{Node: "anchor", Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", diff --git a/modules/mesh-vault/cmd/mesh-vault/intake.go b/modules/mesh-vault/cmd/mesh-vault/intake.go index b887916b..42bb41cd 100644 --- a/modules/mesh-vault/cmd/mesh-vault/intake.go +++ b/modules/mesh-vault/cmd/mesh-vault/intake.go @@ -250,6 +250,14 @@ func Covered(secret string) bool { return name == "broker" || name == "provisioner-broker" } +// NamedAccount says secret is a bus account by its name alone: a module's own `broker` or a provisioner client's +// `provisioner-broker`. A `broker-` is one only when a process of that id is declared with a bus, which its name +// cannot say (mesh-controller's broker-address is the bus's address, no account): Vault.DeclaredAccount. +func NamedAccount(secret string) bool { + parts := strings.Split(secret, "/") + return len(parts) == 4 && parts[0] == "own" && (parts[3] == "broker" || parts[3] == "provisioner-broker") +} + // AcceptOpen opens a call to take a value given at the controller's terminal for secret, which the terminal // seals to the call's key before it leaves it (ADR 0349 clause 8: the vault alone opens an accepted value). // diff --git a/modules/mesh-vault/cmd/mesh-vault/mint.go b/modules/mesh-vault/cmd/mesh-vault/mint.go index 06bb80a7..a56b71d1 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint.go @@ -132,6 +132,33 @@ func (v *Vault) noLongerHeld(user string) { delete(m.heldBack, user) } +// DeclaredAccount says secret is a bus account the mesh declares: one by its name (NamedAccount), or the own secret of +// an account the grants part wants, which it composes from the manifests' bus processes (catalogue.BusAccounts, as +// catalogue.ToolsBusSecret names them). Never by a `broker-` prefix alone (the sweep of 2026-10-30, issue 774: +// mesh-controller's broker-address is the bus's address, read by the controller to issue tokens). +func (v *Vault) DeclaredAccount(secret string) bool { + if NamedAccount(secret) { + return true + } + if !Covered(secret) { + return false + } + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + for _, w := range v.minting.wants { + if s, err := AccountSecret(w); err == nil && s == secret { + return true + } + } + return false +} + +// busAccount says the held secret is a bus account: declared, or a `broker-` whose value is an account (one the +// controller issued before its want was heard). Anything else is not hashed and not owed. +func (v *Vault) busAccount(secret, value string) bool { + return v.DeclaredAccount(secret) || (Covered(secret) && BusAccountValue(secret, value)) +} + // AccountSecret is the vault's name of the own secret a wanted account is held in. func AccountSecret(w buscompose.WantedAccount) (string, error) { return OwnSecret(w.Node, w.Module, w.OwnSecret()) @@ -189,12 +216,17 @@ func (v *Vault) mintOnce(w buscompose.WantedAccount, by string) (Minted, error) v.holdBack(w.User, err.Error()) return Minted{}, err } - if _, held, err := v.Store.Read(secret); err != nil { + replacing := false + if rec, held, err := v.Store.Read(secret); err != nil { return Minted{}, err } else if held { - // Its value is held (the controller's, handed in, or one the vault minted): never minted anew (§4). - v.noLongerHeld(w.User) - return Minted{}, errHeldBack{secret + " is held already: an account the vault holds a value for is never minted anew (ADR 0561 §4)"} + // A held value that is no account (kept before the vault judged it, the sweep of 2026-10-30) has no hash and + // logs in nowhere on the bus: it is replaced by a minted account. Any other held value is never minted anew (§4). + if value, err := openAtRest(rec, v); err != nil || BusAccountValue(secret, value) { + v.noLongerHeld(w.User) + return Minted{}, errHeldBack{secret + " is held already: an account the vault holds a value for is never minted anew (ADR 0561 §4)"} + } + replacing = true } if v.Accounts == nil { v.holdBack(w.User, "the vault has no state of bus accounts to write the hash to") @@ -230,11 +262,25 @@ func (v *Vault) mintOnce(w buscompose.WantedAccount, by string) (Minted, error) v.holdBack(w.User, err.Error()) return Minted{}, err } - c, err := v.keepLocked(secret, string(value), "the vault", "", OriginMade, keepOptions{neverReplace: true, quiet: true}) + opt := keepOptions{neverReplace: true, quiet: true} + if replacing { + // Only the value that is no account is replaced, judged again under the store's lock; the one replaced is kept. + opt = keepOptions{quiet: true, keepPrevious: true, refuse: func(before Record) string { + if was, err := openAtRest(before, v); err != nil || BusAccountValue(secret, was) { + return secret + " is held as an account meanwhile: it is never minted anew (ADR 0561 §4)" + } + return "" + }} + } + c, err := v.keepLocked(secret, string(value), "the vault", "", OriginMade, opt) if err != nil { v.holdBack(w.User, "its value could not be kept: "+err.Error()) return Minted{}, err } + if replacing { + v.log("%s held a value that is no bus account, which no hash was taken of and which logs in nowhere: replaced by a "+ + "minted account (generation %d), the value it replaces kept beside it", secret, c.Generation) + } out := Minted{User: w.User, Secret: secret, Generation: c.Generation, At: c.At} // The hash first, then account-issued, then secret-changed: the grants part composes the user before the node asks. if _, err := v.HashAccount(secret, false); err != nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_test.go index e36454e5..9f7d2687 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint_test.go @@ -242,3 +242,30 @@ func TestAFailedMintIsTriedAgainAtThePass(t *testing.T) { t.Errorf("not minted at the pass after the failure: %+v", v.MintState()) } } + +// A held value that is no account (kept before the vault judged it: broker-nats-tools, broker-address and +// broker-management on 2026-10-30, issue 774) is replaced by a minted account when the grants part wants its user; +// a held account still never is. +func TestAHeldValueThatIsNoAccountIsReplacedByAMintedOne(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + w := aWant("mod-x") + if _, err := v.Put("own/ace/mod-x/broker", `{"password":"pw"}`, "the test"); err != nil { + t.Fatal(err) + } + heldPlain(t, v.Vault, "own/ace/mod-x/broker", "a-random-value-the-controller-made") + v.WantHeard(w.User, wantRaw(t, w), false) + rec, _, _ := v.Store.Read("own/ace/mod-x/broker") + value, err := openAtRest(rec, v.Vault) + if err != nil || !BusAccountValue("own/ace/mod-x/broker", value) || rec.Generation != 3 || rec.Previous == nil { + t.Fatalf("not replaced by a minted account: generation %d, previous kept %v, %v", rec.Generation, rec.Previous != nil, err) + } + var got struct{ Password string } + _ = json.Unmarshal([]byte(value), &got) + if !accounts.hashOf(w.User, got.Password, 3) { + t.Errorf("%s's hash is not the minted password's: %+v", w.User, accounts.keys[w.User]) + } + v.WantHeard(w.User, wantRaw(t, w), false) + if again, _, _ := v.Store.Read("own/ace/mod-x/broker"); again.Generation != 3 { + t.Errorf("a held account was minted anew: generation %d", again.Generation) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/move.go b/modules/mesh-vault/cmd/mesh-vault/move.go index 8db5f853..be9dab1b 100644 --- a/modules/mesh-vault/cmd/mesh-vault/move.go +++ b/modules/mesh-vault/cmd/mesh-vault/move.go @@ -74,8 +74,22 @@ type MovedIn struct { // ReadBack is the value the vault's store holds now, sealed to the key the second step sent (NaCl anonymous // box, base64). ReadBack string `json:"read_back"` + // Refused is set, with no read-back, when nothing was taken because the vault holds another value for the name: its + // reason ("holds-another-value") and the held value's record, never the value (the sweep of 2026-10-30, issue 774: + // the refusal was words alone). Why is the same in words. + Refused *MoveRefused `json:"refused,omitempty"` + Why string `json:"why,omitempty"` } +// MoveRefused is why a move took nothing, as fields. +type MoveRefused struct { + Reason string `json:"reason"` + Held *HeldAnother `json:"held,omitempty"` +} + +// RefusedHeldAnother is MoveRefused's reason when the vault holds another value for the name. +const RefusedHeldAnother = "holds-another-value" + // PairSecret is the vault's name for one row of the controller's pair credentials. func PairSecret(provider, provision, consumer, consumerModule, local string) (string, error) { parts := map[string]string{"the provider's machine": provider, "the provision": provision, @@ -213,6 +227,14 @@ func (v *Vault) MoveTake(caller, call, sealed, readBackTo string) (MovedIn, erro m := c.move changed, err := v.keepWith(c.secret, string(value), caller, "", movedOrigins[m.Origin], keepOptions{ neverReplace: true, madeAt: m.MadeAt, movedFrom: &MovedFrom{By: caller, Generation: m.Generation}}) + var another *HeldAnother + if errors.As(err, &another) { + // Answered, not failed: the caller reads the reason and the held record. A caller that reads no `refused` finds + // no read-back and fails as loudly as before. + v.log("did not move in %s from %s: %v", c.secret, caller, err) + return MovedIn{Name: c.secret, Generation: another.Generation, + Refused: &MoveRefused{Reason: RefusedHeldAnother, Held: another}, Why: err.Error()}, nil + } if err != nil { return MovedIn{}, err } diff --git a/modules/mesh-vault/cmd/mesh-vault/move_test.go b/modules/mesh-vault/cmd/mesh-vault/move_test.go index 4da010aa..b6270be4 100644 --- a/modules/mesh-vault/cmd/mesh-vault/move_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/move_test.go @@ -198,9 +198,13 @@ func TestTheMoveNeverReplacesAValueTheVaultHolds(t *testing.T) { if _, err := v.Rotate(ControllerCaller, "pair/anchor/database/shanks/gitea", "jochen", "a test"); err != nil { t.Fatal(err) } - if _, err := moveIn(t, v, "node.anchor", pair, "the-pair-value"); err == nil || - !strings.Contains(err.Error(), "never replaces") { - t.Fatalf("a move over a value the vault changed since: %v", err) + // Refused as fields, not words alone (the sweep of 2026-10-30, issue 774): the reason and the held record, no + // read-back, so a caller that reads no `refused` still fails. + refused, err := moveIn(t, v, "node.anchor", pair, "the-pair-value") + if err != nil || refused.Refused == nil || refused.Refused.Reason != RefusedHeldAnother || refused.ReadBack != "" || + refused.Refused.Held == nil || refused.Refused.Held.Generation != 2 || refused.Refused.Held.Origin != OriginMade || + !strings.Contains(refused.Why, "never replaces") { + t.Fatalf("a move over a value the vault changed since: %+v, %v", refused, err) } rec, _, _ := v.Store.Read("pair/anchor/database/shanks/gitea") if rec.Generation != 2 || rec.Origin != OriginMade { diff --git a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go index aca4fc1d..3e5681bb 100644 --- a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go @@ -41,7 +41,11 @@ func TestANodeEngineStatesItsOwnKeyAndAChangeRaisesItsCoreSecrets(t *testing.T) {Node: "ace", Module: "postgres", Name: "superuser-password", Origin: "made", Generation: "g-1"}, {Node: "g14", Module: "nats", Name: "broker", Origin: "made", Generation: "g-1"}, } { - if _, err := moveIn(t, v, "node."+m.Node, m, movedValue); err != nil { + value := movedValue + if m.Name != "superuser-password" { + value = brokerValue // a bus account is kept only as one + } + if _, err := moveIn(t, v, "node."+m.Node, m, value); err != nil { t.Fatal(err) } } diff --git a/modules/mesh-vault/cmd/mesh-vault/pair.go b/modules/mesh-vault/cmd/mesh-vault/pair.go index ca0293a2..cc343e98 100644 --- a/modules/mesh-vault/cmd/mesh-vault/pair.go +++ b/modules/mesh-vault/cmd/mesh-vault/pair.go @@ -55,6 +55,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/seal" ) @@ -145,23 +146,21 @@ type binds struct { tools map[string]bool } -// The credential forms a plan entry names (issue 768), as the provider's offer declares them. +// The credential forms a plan entry names (issue 768), as the provider's offer declares them: mesh-sdk's one list +// (catalogue.CredentialForms, issue 774), which the provisioner client reads too, so the two cannot drift. const ( // FormShared is one value every holder shares (ADR 0158): the offer's credential is one of its own secrets. - FormShared = "shared" + FormShared = catalogue.CredentialFormShared // FormPair is a login of its own for each consumer (ADR 0048, ADR 0158 §11): the offer says no credential. - FormPair = "pair" + FormPair = catalogue.CredentialFormPair // FormIssued is a value the provider's software issues (ADR 0305): the mesh never makes it. - FormIssued = "issued" + FormIssued = catalogue.CredentialFormIssued // FormGiven is a value only a person gives (ADR 0337): the mesh never makes it. - FormGiven = "given" + FormGiven = catalogue.CredentialFormGiven // FormNone is an offer whose consumers present no credential: the vault mints nothing for it. - FormNone = "none" + FormNone = catalogue.CredentialFormNone ) -// credentialForms are every form a plan entry may name. -var credentialForms = []string{FormShared, FormPair, FormIssued, FormGiven, FormNone} - // form is the one form the plans name for k, or why the vault cannot trust one. func (b *binds) form(k pairKey) (string, string) { if b == nil || len(b.forms) == 0 || b.forms[""] { @@ -169,9 +168,9 @@ func (b *binds) form(k pairKey) (string, string) { } var named []string for f := range b.forms { - if !slices.Contains(credentialForms, f) { + if !catalogue.IsCredentialForm(f) { return "", fmt.Sprintf("a plan names %q for %s from %s, which is not a credential form the vault knows "+ - "(%s); it makes nothing", f, k.provision, k.provider, strings.Join(credentialForms, ", ")) + "(%s); it makes nothing", f, k.provision, k.provider, strings.Join(catalogue.CredentialForms(), ", ")) } named = append(named, f) } diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index 416e63cf..0f575e8f 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -924,14 +924,16 @@ func ownAnswer(v *Vault, node, module, prefix string) (map[string]any, error) { } fp := "" var note any + account := v.DeclaredAccount(r.Name) if value, err := seal.Open(r.AtRest, v.Own.PrivateKey); err != nil { note = "the vault could not open its copy: " + err.Error() } else { fp = Fingerprint(value) + account = v.busAccount(r.Name, value) } row := map[string]any{"name": r.Name, "node": parts[1], "module": parts[2], "secret": parts[3], "generation": r.Generation, "fingerprint": fp, "made": r.MadeAt, "changed": r.ChangedAt, - "changedBy": r.ChangedBy, "origin": r.Origin, "busAccount": Covered(r.Name)} + "changedBy": r.ChangedBy, "origin": r.Origin, "busAccount": account} if r.ChangedVia != "" { row["changedVia"] = r.ChangedVia } @@ -941,7 +943,7 @@ func ownAnswer(v *Vault, node, module, prefix string) (map[string]any, error) { if note != nil { row["error"] = note } - if Covered(r.Name) && fp != "" { + if account && fp != "" { hashedRow(v, r, fp, row) } rows = append(rows, row) diff --git a/modules/mesh-vault/cmd/mesh-vault/tools_test.go b/modules/mesh-vault/cmd/mesh-vault/tools_test.go index 415bf5d9..00612307 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools_test.go @@ -157,15 +157,15 @@ func TestSecretOwnListsFingerprintsOfOwnSecretsOnly(t *testing.T) { v := newVault(t, AllowList{}) acc := &fakeAccounts{} v.Accounts = acc - for name, value := range map[string]string{"own/ace/mesh-build-go/broker": "pw-one", "own/ace/mesh-build-go/token": "tok-secret", + for name, value := range map[string]string{"own/ace/mesh-build-go/broker": pwOne, "own/ace/mesh-build-go/token": "tok-secret", "own/g14/dunst/broker": `{"user":"g14.dunst","password":"pw-two"}`, "pair/postgres/gitea": "pair-value"} { if _, err := v.Put(name, value, "the test"); err != nil { t.Fatal(err) } } - if _, err := v.Put("own/ace/mesh-build-go/broker", "pw-one-b", "the test"); err != nil { - t.Fatal(err) - } + // A value held from before the vault kept bus accounts only as accounts (generation 2, plain), as a store may still + // hold one: written past keep, which refuses it now. + heldPlain(t, v.Vault, "own/ace/mesh-build-go/broker", "pw-one-b") own := toolNamed(t, definitions(&Env{Vault: v.Vault}), "secret_own") for _, c := range []struct { args map[string]any @@ -214,7 +214,7 @@ func TestSecretOwnListsFingerprintsOfOwnSecretsOnly(t *testing.T) { } hash := toolNamed(t, definitions(&Env{Vault: v.Vault}), "account_hash") got, err := hash.Run(map[string]any{"user": "g14.dunst"}) - if h, _ := got.(Hashed); err != nil || !h.Written || h.Generation != 1 || acc.puts != 2 { + if h, _ := got.(Hashed); err != nil || !h.Written || h.Generation != 1 || acc.puts != 3 { // mesh-build-go's first value, an account, was hashed too t.Errorf("account_hash = %+v, %v (%d writes)", got, err, acc.puts) } if _, err := hash.Run(map[string]any{"user": "ace.mesh-build-go"}); err == nil { @@ -330,3 +330,24 @@ func TestSecretOwnListsFingerprintsOfOwnSecretsOnly(t *testing.T) { t.Errorf("secret_own narrowed to a module lists adopted accounts: %v", narrowed) } } + +// The bus account values of TestSecretOwnListsFingerprintsOfOwnSecretsOnly: a bus account is kept only as one. +const pwOne = `{"password":"pw-one"}` + +// heldPlain writes value as the next generation of name straight into the store, past keep: a value kept before keep +// refused a bus account that is no account. +func heldPlain(t *testing.T, v *Vault, name, value string) { + t.Helper() + rec, held, err := v.Store.Read(name) + if err != nil || !held { + t.Fatalf("%s is not held: %v", name, err) + } + if rec.AtRest, err = seal.Seal(value, v.Own.PublicKey); err != nil { + t.Fatal(err) + } + rec.Generation++ + rec.ChangedAt = v.Now().UTC() + if err := v.Store.Write(rec); err != nil { + t.Fatal(err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index 9b5634ad..b9f7585e 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -147,6 +147,29 @@ func (v *Vault) keepWith(name, value, by, via, origin string, opt keepOptions) ( return c, err } +// HeldAnother is the refusal of a value the vault never replaces: it holds another value for the name. Its fields are the +// held value's record, never the value, so a caller reads them rather than the words. +type HeldAnother struct { + Name string `json:"name"` + Generation int `json:"generation"` + ChangedBy string `json:"changed_by"` + ChangedAt time.Time `json:"changed_at"` + Origin string `json:"origin,omitempty"` +} + +func (h *HeldAnother) Error() string { + return fmt.Sprintf("the vault already holds another value for %s (generation %d, changed by %s at %s), and the move "+ + "never replaces a value the vault holds: nothing was taken", h.Name, h.Generation, h.ChangedBy, + h.ChangedAt.Format(time.RFC3339)) +} + +// BusAccountValue says value is a bus account as secret names it: the controller's or the vault's JSON, with its user +// (or one its name gives) and a password (accountOf). +func BusAccountValue(secret, value string) bool { + _, _, err := accountOf(secret, value) + return err == nil +} + // openAtRest opens a record's value with the vault's own key. func openAtRest(r Record, v *Vault) (string, error) { return seal.Open(r.AtRest, v.Own.PrivateKey) } @@ -160,6 +183,14 @@ func (v *Vault) keepLocked(name, value, by, via, origin string, opt keepOptions) if privateKeyIn(value) { return Changed{}, fmt.Errorf("%s: that is a private key, and a private key stays where it was made and never reaches the vault (ADR 0113 §2)", name) } + // A bus account is kept only as one (the sweep of 2026-10-30, issue 774): the controller had moved in random values + // for broker-nats-tools, broker-address and broker-management, which no hash could be taken of, so the grants part + // left those users out of the bus. Such a name is never kept with a value that is no account: not issued. + if v.DeclaredAccount(name) && !BusAccountValue(name, value) { + return Changed{}, fmt.Errorf("%s is not issued: it names a bus account, and the value handed in is no account "+ + "(the JSON the controller hands or the vault mints, with a user and a password); the vault keeps a bus "+ + "account only as one, so nothing was kept", name) + } if by == "" { by = "the vault" } @@ -189,9 +220,8 @@ func (v *Vault) keepLocked(name, value, by, via, origin string, opt keepOptions) } } if opt.neverReplace { - return Changed{}, fmt.Errorf("the vault already holds another value for %s (generation %d, changed by %s "+ - "at %s), and the move never replaces a value the vault holds: nothing was taken", name, - before.Generation, before.ChangedBy, before.ChangedAt.Format(time.RFC3339)) + return Changed{}, &HeldAnother{Name: name, Generation: before.Generation, ChangedBy: before.ChangedBy, + ChangedAt: before.ChangedAt, Origin: before.Origin} } } atRest, err := seal.Seal(value, v.Own.PublicKey) diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go index c3f22821..d1ab87c9 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -22,6 +22,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/seal" "golang.org/x/crypto/bcrypt" "golang.org/x/crypto/nacl/box" @@ -905,3 +906,68 @@ func TestTheRecordOfHandOutsIsBounded(t *testing.T) { t.Errorf("the newest are not read back: %+v", h) } } + +// A bus account is kept only as one, and a name is one only when it is declared (the sweep of 2026-10-30, issue 774): +// broker, provisioner-broker, or the own secret of an account the grants part wants. mesh-controller's broker-address is +// the bus's address, never an account: kept as given, never refused, never replaced, never owed a hash. nats's +// broker-nats-tools, wanted as a process account, has the random value the controller moved in replaced by a minted +// account, and a value that is no account is refused for it from then on. +func TestABusAccountIsKeptOnlyAsAnAccount(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + const tools, address = "own/novox/nats/broker-nats-tools", "own/novox/mesh-controller/broker-address" + for _, row := range []struct { + name, value string + kept bool + }{ + {tools, "Xq3-a-random-value-the-controller-made", true}, // not wanted yet: no declared account + {address, "nats://novox.mesh:4222", true}, + {"own/novox/nats/broker", "Xq3-a-random-value-the-controller-made", false}, + {"own/novox/nats/broker", `{"password":"pw"}`, true}, + {"own/novox/postgres/superuser-password", "Xq3-a-random-value-the-controller-made", true}, + } { + _, err := v.Put(row.name, row.value, "the test") + if (err == nil) != row.kept || (err != nil && !strings.Contains(err.Error(), "is not issued")) { + t.Errorf("%s: kept %v, err %v", row.name, err == nil, err) + } + } + w := buscompose.WantedAccount{User: "novox.nats.nats-tools", Kind: buscompose.KindProcess, Node: "novox", + Module: "nats", Process: "nats-tools", URL: "nats://novox.mesh:4222", Fingerprint: "ab:cd"} + if err := w.Valid(); err != nil { + t.Fatal(err) + } + v.WantHeard(w.User, wantRaw(t, w), false) + v.HashHeld() + v.HashOwed() + for _, row := range []struct { + name string + generation int + account bool + }{ + {tools, 2, true}, + {address, 1, false}, + } { + rec, _, _ := v.Store.Read(row.name) + value, err := openAtRest(rec, v.Vault) + if err != nil || rec.Generation != row.generation || BusAccountValue(row.name, value) != row.account { + t.Errorf("%s: generation %d, an account %v, %v", row.name, rec.Generation, BusAccountValue(row.name, value), err) + } + } + if owed := v.hashes.list(); len(owed) != 0 { + t.Errorf("hashes owed for names that are no accounts: %v", owed) + } + if _, hashed := accounts.keys["novox.nats.nats-tools"]; !hashed { + t.Errorf("the minted nats-tools account is not hashed") + } + for _, row := range []struct { + name, value string + kept bool + }{ + {tools, "another-random-value", false}, + {address, "nats://novox.mesh:4223", true}, + } { + _, err := v.Put(row.name, row.value, "the test") + if (err == nil) != row.kept { + t.Errorf("%s again: kept %v, err %v", row.name, err == nil, err) + } + } +} diff --git a/modules/mesh-vault/go.mod b/modules/mesh-vault/go.mod index cfda18d1..78371b50 100644 --- a/modules/mesh-vault/go.mod +++ b/modules/mesh-vault/go.mod @@ -3,7 +3,7 @@ module mesh-vault go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/modules/mesh-vault/go.sum b/modules/mesh-vault/go.sum index a2b8e018..61230834 100644 --- a/modules/mesh-vault/go.sum +++ b/modules/mesh-vault/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/messenger/cmd/messenger/answers.go b/modules/messenger/cmd/messenger/answers.go index 6a3fbb5a..ab0159f2 100644 --- a/modules/messenger/cmd/messenger/answers.go +++ b/modules/messenger/cmd/messenger/answers.go @@ -133,7 +133,7 @@ func pruneGrants(grants map[string]Grant, kind string, now time.Time, keep int) // stands, closes the ask with a warrant for its asker (asks.go); Details are shown where they were asked. // Anything else is refused, and said there. The same answer is acted on once. func (h *Holder) Chosen(kind string, c asks.Choice) (string, error) { - // The machines' engines' root-free word is asked before the work is taken (the confirmation review of 2026-10-09): + // The root-free word is asked before the work is taken (the confirmation review of 2026-10-09): // up to RootFreeWithin, which no ask, answer or link should wait behind. Asked fresh here, for this answer, // and judged with in it. root := h.rootWordFor(kind) diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index b719e68f..1fd236c8 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -337,20 +337,36 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { kept, ok := h.rootWord[key] h.mu.Unlock() var verdicts map[string]RootVerdict + fresh := ok && kept.verdicts != nil && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0 switch { - case cached && ok && now.Sub(kept.at) < RootFreeKept && now.Sub(kept.at) >= 0: + case cached && fresh: verdicts = kept.verdicts case cached: // Read inside the work, which never waits on the controller: no recent word is no pass. warmRootFree - // asks outside it. - return "the machines' engines' word on whether an agent can become root there is not recent" + // asks outside it; when its last ask failed, that failure is the reason, so a held ask says it. + if ok && kept.failed != "" { + return "the word on whether an agent can become root there is not recent: asking it at " + + kept.failedAt.UTC().Format("15:04:05") + " UTC failed: " + kept.failed + } + return "the word on whether an agent can become root there is not recent" default: got, err := h.askRootFree(machines) if err != nil { - // A word that could not be asked again is not kept from before: no recent word, no pass. + // The failure is said and kept beside the last word. That word is not thrown away: it still renders + // until it is RootFreeKept old, and an answer is never judged on it (this path is not a pass). h.mu.Lock() - delete(h.rootWord, key) + if h.rootWord == nil { + h.rootWord = map[string]rootWord{} + } + w := h.rootWord[key] + w.failed, w.failedAt = err.Error(), now + h.rootWord[key] = w h.mu.Unlock() + age := "no word was kept" + if fresh { + age = "the word of " + now.Sub(kept.at).Round(time.Second).String() + " ago is kept for rendering" + } + h.Logf("[messenger] could not ask whether an agent can become root on %s: %v (%s)", key, err, age) return "whether an agent can become root where it runs could not be asked: " + err.Error() } verdicts = got @@ -358,6 +374,10 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { if h.rootWord == nil { h.rootWord = map[string]rootWord{} } + if w := h.rootWord[key]; w.failed != "" { + h.Logf("[messenger] the word on whether an agent can become root on %s was asked again after it failed at %s", + key, w.failedAt.UTC().Format(time.RFC3339)) + } h.rootWord[key] = rootWord{at: now, verdicts: got} h.mu.Unlock() } @@ -378,7 +398,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { return "" } -// warmRootFree asks the machines' engines' root-free word for every kind that proves its sender, outside the work, +// warmRootFree asks the root-free word for every kind that proves its sender, outside the work, // so what is rendered and offered inside it reads a recent word. // force asks even a recent word again: taking an ask, whose refusal its asker keeps until something changes. func (h *Holder) warmRootFree(force bool) { @@ -404,7 +424,7 @@ func (h *Holder) warmRootFree(force bool) { } } -// rootWordFor asks the machines' engines' root-free word for a kind fresh, before the work is taken, and answers +// rootWordFor asks the root-free word for a kind fresh, before the work is taken, and answers // what judging an answer of that kind then reads: the word for that very kind's placement, or why there is none. // A kind that does not prove its sender is asked nothing: its answers never approve. func (h *Holder) rootWordFor(kind string) func(KindInfo) string { @@ -424,10 +444,13 @@ func (h *Holder) rootWordFor(kind string) func(KindInfo) string { } } -// rootWord is the controller's last word on a set of machines, and when it was asked. +// rootWord is the controller's last word on a set of machines, and when it was asked; failed and failedAt the last +// ask that failed since, and why. type rootWord struct { at time.Time verdicts map[string]RootVerdict + failed string + failedAt time.Time } // approveHolds is why an answer on a kind could not approve now, or "": the kind verifies its sender, an @@ -704,6 +727,8 @@ func (h *Holder) releaseHeld() { rec.Tickets = nil if why := h.holdWhy(rec.Ask); why != rec.HeldWhy { rec.HeldWhy = why + h.Logf("[messenger] %s's ask %q (%s) is still held, since %s: %s", rec.Asker, rec.Ask.Headline, rec.Ask.ID, + rec.HeldSince.UTC().Format(time.RFC3339), why) h.saveHeld(key, rec, rev) } continue diff --git a/modules/messenger/cmd/messenger/review_test.go b/modules/messenger/cmd/messenger/review_test.go index 9c042edc..0ecb1b39 100644 --- a/modules/messenger/cmd/messenger/review_test.go +++ b/modules/messenger/cmd/messenger/review_test.go @@ -5,6 +5,7 @@ package main import ( "encoding/json" "errors" + "fmt" "os" "path/filepath" "regexp" @@ -30,6 +31,8 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) { k.Node = "" h.Bench.SetKinds([]KindInfo{k}) }, "has not said which machine"}, + // The word asked before it, still fresh, renders until it is RootFreeKept old (a failed ask throws no fresh + // word away; the sweep of 2026-10-30); the answer itself is refused all the same. "a controller that does not answer": {func(h *Holder) { h.RootFree = func([]string) (map[string]RootVerdict, error) { return nil, errors.New("no responders") } }, "could not be asked: no responders"}, @@ -60,6 +63,10 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) { if len(w.of("mesh-delivery")) != 0 { t.Error("a warrant was issued") } + if strings.Contains(tc.want, "could not be asked") { + c.pass(RootFreeKept) + stand(h, c) + } if got := h.carriers(asks.Approve); len(got) != 0 { t.Errorf("approval still carried on %v", got) } @@ -108,6 +115,41 @@ func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) { } } +// A failed ask of the root word is said with its error, keeps the word still fresh for rendering, never passes an +// answer, and is the reason a held ask carries (the sweep of 2026-10-30, issue 774: the engines did not answer, the +// word was deleted silently and the held reason said only "not recent"). +func TestAFailedRootAskIsSaidAndKeepsAFreshWord(t *testing.T) { + h, _, _, c, store, _ := newRouter(t) + link(h, store, c, "42", c.now().Add(-2*time.Hour)) + var logs []string + h.Logf = func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) } + info, _ := h.Bench.Kind("telegram") + if why := h.rootFree(info, false); why != "" { + t.Fatalf("free not read: %s", why) + } + h.RootFree = func([]string) (map[string]RootVerdict, error) { + return nil, errors.New("anchor's engine: no responders") + } + c.pass(RootFreeKept / 3) + if why := h.rootFree(info, false); !strings.Contains(why, "could not be asked: anchor's engine: no responders") { + t.Errorf("a failed ask judged an answer: %q", why) + } + if len(logs) != 1 || !strings.Contains(logs[0], "no responders") || !strings.Contains(logs[0], "kept for rendering") { + t.Errorf("the failure was not said with its error: %v", logs) + } + if why := h.rootFree(info, true); why != "" { + t.Errorf("one failed ask threw away a word still fresh: %q", why) + } + c.pass(RootFreeKept) + stand(h, c) + if why := h.rootFree(info, true); !strings.Contains(why, "failed: anchor's engine: no responders") { + t.Errorf("a stale word passed, or its reason hid the failure: %q", why) + } + if why := h.holdWhy(heldAsk(c, "d-9")); !strings.Contains(why, "no responders") { + t.Errorf("a held ask does not say why: %q", why) + } +} + // H3: a message rendered while root is not free offers no approval; a word kept for rendering is never the // word an answer is judged on. func TestAMessageRenderedWhileRootIsNotFreeOffersNoApproval(t *testing.T) { diff --git a/modules/power/cmd/power/templates_test.go b/modules/power/cmd/power/templates_test.go index 1ba16b8c..71c9b1bd 100644 --- a/modules/power/cmd/power/templates_test.go +++ b/modules/power/cmd/power/templates_test.go @@ -84,7 +84,9 @@ func TestEveryFileTakesItsTextFromATemplateOutsideTheUnpackedFiles(t *testing.T) // The bar's blocks are two items of the collection the bar places (design 57 L3, issue 781): the text i3status-rust's // bars hold for the battery and for the power draw, byte for byte as the bar's own template rendered them before, the -// draw's key being the machine's own name. +// draw's key being the machine's own name. The draw's item says the state its text reads (shows: state, state: +// power.draw), as the bar's contribution said it before, so the bar's holder is granted this machine's key of it (the +// sweep of 2026-10-30, issue 774: moved to an item, it said nothing, and i3status-rust was issued no read). func TestTheBarBlocksAreTwoItemsOfTheBarsCollection(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -106,19 +108,20 @@ func TestTheBarBlocksAreTwoItemsOfTheBarsCollection(t *testing.T) { if len(m.Contributions) != 0 || len(m.Items) != 2 { t.Fatalf("%d contributions and %d items", len(m.Contributions), len(m.Items)) } - want := []struct{ name, capability, order, template, text string }{ - {"battery", "battery", "40", "templates/bar-battery", "[[block]]\nblock = \"battery\"\n" + + want := []struct{ name, capability, order, reads, template, text string }{ + {"battery", "battery", "40", "", "templates/bar-battery", "[[block]]\nblock = \"battery\"\n" + "format = \"$icon $percentage{ $time|}\"\ncharging_format = \"$icon $percentage{ $time|}\"\n" + "not_charging_format = \"$icon $percentage\"\nfull_format = \"$icon $percentage\"\n" + "empty_format = \"$icon $percentage\"\nmissing_format = \"\"\n\n"}, - {"power-draw", "power-meter", "45", "templates/bar-power-draw", "[[block]]\nblock = \"custom\"\n" + + {"power-draw", "power-meter", "45", "power.draw", "templates/bar-power-draw", "[[block]]\nblock = \"custom\"\n" + "command = \"~/.local/bin/i3status-value power.draw ${machine:name}\"\ninterval = 5\nhide_when_empty = true\n\n"}, } for i, w := range want { it := m.Items[i] if it.Collection != "contribution:node-bar:block" || it.Name != w.name || it.IfCapability != w.capability || it.Template != w.template || it.Attributes["bar"] != "bottom" || it.Attributes["place"] != "status" || - it.Attributes["order"] != w.order || len(it.Attributes) != 3 { + it.Attributes["order"] != w.order || it.Attributes["state"] != w.reads || + (w.reads != "") != (it.Attributes["shows"] == "state") || len(it.Attributes) != map[bool]int{false: 3, true: 5}[w.reads != ""] { t.Errorf("item %d is %+v", i, it) } text, err := os.ReadFile(filepath.Join("../..", w.template)) diff --git a/modules/power/module.json b/modules/power/module.json index 909499ad..c43e601c 100644 --- a/modules/power/module.json +++ b/modules/power/module.json @@ -442,7 +442,9 @@ "attributes": { "bar": "bottom", "place": "status", - "order": "45" + "order": "45", + "shows": "state", + "state": "power.draw" }, "template": "templates/bar-power-draw" } diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 20c0a76b..7da0a7cb 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -56,7 +56,8 @@ func main() { case "awake": counting(b) case "": - if err := stdio.Keeps(announce.State{Name: health.GoodState}); err != nil { + // Readable: the self-check reads it (its manifest's `reads`); checks/readable holds it. + if err := stdio.Keeps(announce.State{Name: health.GoodState, Readable: true}); err != nil { say("%v", err) os.Exit(1) }