Show placement candidates and each node's answer in indexer_placement, add indexer_placement_reread, and drop a group's candidates when its head changes no placement file, so a missed message or a revert is recoverable by tool (issue 1126, ADR 0559 §3, §7, §8)

This commit is contained in:
2026-10-27 02:18:21 +01:00
parent ad60ba3a46
commit 04bb2d1398
17 changed files with 493 additions and 22 deletions
@@ -65,7 +65,9 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
{Name: view.PlacementReadTool,
Description: "Where modules run, as the indexer read it from the catalogue's placement files at the last merge (placement.<node>, novox/hq ADR 0549): " +
"each node's modules in order with their host ports, bindings and settings placement, its pins and the seats its modules hold, and the catalogue revision. " +
"With node, that node's; with module, every node placing it; with neither, every node's. Keys that do not read are listed as unreadable. Read-only.",
"With node, that node's; with module, every node placing it; with neither, every node's. Keys that do not read are listed as unreadable. " +
"Beside it, candidates: each open catalogue pull request's placement of a node that differs from the trunk's (placement-candidates <group>.<node>, issue 1126), " +
"narrowed the same way, with the node's answer from its <node>.node have (answered: same, differs, refused, stale for an earlier head, none when it has not answered). Read-only.",
Input: map[string]any{"type": "object", "properties": nodeModule},
Run: func(args map[string]any) (any, error) {
node, module := nodeModuleOf(args)
@@ -202,8 +204,9 @@ func read(state stdio.KeptState) view.StateRead {
func main() {
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")),
ReachState: read(stdio.State("reach")),
PlacementState: read(stdio.State("mesh-delivery-indexer.placement"))}
ReachState: read(stdio.State("reach")),
PlacementState: read(stdio.State("mesh-delivery-indexer.placement")),
PlacementCandidatesState: read(stdio.State("mesh-delivery-indexer.placement-candidates"))}
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")),
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
for _, lang := range delivery.AskLanguages {
@@ -131,6 +131,50 @@ func TestModulesToolAnswersEveryKeyAsStored(t *testing.T) {
if _, err := run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"}); err == nil || !strings.Contains(err.Error(), "no placement of novox") {
t.Errorf("a node with no key: %v", err)
}
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); len(c) != 0 {
t.Errorf("with no candidate open: %v", c)
}
// ADR 0559 §7 (issue 1126): beside the trunk's, each open candidate key and the node's answer from its <node>.node:
// answered at the key's head and revision, at an earlier head (stale), or not at all (none); a node the trunk does not
// place yet is shown by its candidate alone.
const h1, h2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222"
cand := func(node, head string, placed bool) string {
pl := `,"placement":{"v":1,"node":"` + node + `","modules":[{"module":"plex"}]}`
if !placed {
pl = ""
}
return `{"v":1,"group":"feat/p","pull":{"repository":"novox/mesh-catalog","number":9,"head":"` + head + `"}` + pl + `}`
}
cands := fake{"feat/p.ace": cand("ace", h2, true), "feat/p.g14": cand("g14", h2, false), "feat/p.novox": cand("novox", h2, true)}
revs := map[string]uint64{"feat/p.ace": 4, "feat/p.g14": 5, "feat/p.novox": 6}
p.PlacementCandidatesState = view.StateRead{Keys: cands.reader().State.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
raw, _ := cands.reader().State.Get(key)
return raw, revs[key], nil
}}
p.HavesState = fake{
"ace.node": `{"v":1,"composed":"x","candidates":{"feat/p":{"head":"` + h2 + `","revision":4,"verdict":"same","same":["plex"]}}}`,
"g14.node": `{"v":1,"composed":"x","candidates":{"feat/p":{"head":"` + h1 + `","revision":2,"verdict":"same"}}}`,
}.reader().State
out, err = run(t, p, view.PlacementReadTool, map[string]any{})
if err != nil {
t.Fatal(err)
}
answered := map[string]string{}
for _, c := range out.(map[string]any)["candidates"].([]view.CandidateRow) {
answered[c.Node] = c.Answered
}
if want := map[string]string{"ace": "same", "g14": view.AnswerStale, "novox": view.AnswerNone}; fmt.Sprint(answered) != fmt.Sprint(want) {
t.Errorf("answers %v, want %v", answered, want)
}
out, err = run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"})
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); err != nil || len(c) != 1 || c[0].Pull.Head != h2 || len(out.(map[string]any)["placement"].(map[string]json.RawMessage)) != 0 {
t.Errorf("a node placed by its candidate alone: %v, %v", out, err)
}
out, _ = run(t, p, view.PlacementReadTool, map[string]any{"module": "plex"})
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); len(c) != 2 {
t.Errorf("the candidates placing plex: %v (g14's removes its file)", c)
}
}
const (
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 h1:/c6ncICWjq55nBf8TlDM0ROe/PuH+qZIdnuVbtg8UuM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f h1:aWqn4HHTPUp/+28h7b8uFqQptAYvQFs05CALMHN7SkI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -43,6 +43,8 @@ type Reader struct {
HavesState StateRead // the node-engine's haves
ReachState StateRead // the view's own reach (ADR 0556)
PlacementState StateRead // the indexer's placement
// PlacementCandidatesState is the indexer's placement candidates (issue 1126); unset reads none.
PlacementCandidatesState StateRead
}
// ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556).
@@ -652,11 +654,19 @@ func (r Reader) Haves(node, module string) (KeptAnswer, error) {
// Placement is the indexer's placement as stored (ADR 0549 §2): one node's key with node, every node placing a module
// with module (each node's key whole, so its pins, ports and bindings are beside the module), and every node's key with
// neither. A key that does not read as a placement is listed as unreadable, never dropped.
//
// Beside it, candidates (issue 1126, ADR 0549 §4): each open catalogue pull request's placement of a node that differs from
// the trunk's (the indexer's placement-candidates `<group>.<node>`), narrowed the same way, each with the node's answer from
// its `<node>.node` have and whether that answer is for the candidate as it stands (its head and key revision).
func (r Reader) Placement(node, module string) (map[string]any, error) {
node, module = strings.TrimSpace(node), strings.TrimSpace(module)
if r.PlacementState.Keys == nil || r.PlacementState.Get == nil {
return nil, fmt.Errorf("the placement state cannot be read here")
}
candidates, err := r.PlacementCandidates(node, module)
if err != nil {
return nil, err
}
if node != "" {
key, err := delivery.PlacementKey(node)
if err != nil {
@@ -666,10 +676,14 @@ func (r Reader) Placement(node, module string) (map[string]any, error) {
if err != nil {
return nil, err
}
if len(raw) == 0 {
if len(raw) == 0 && len(candidates) == 0 {
return nil, fmt.Errorf("no placement of %s: the catalogue holds no placement file for it, or the indexer has read none yet", node)
}
return map[string]any{"placement": map[string]json.RawMessage{node: raw}}, nil
out := map[string]json.RawMessage{}
if len(raw) > 0 {
out[node] = raw
}
return map[string]any{"placement": out, "candidates": candidates}, nil
}
keys, err := r.PlacementState.Keys()
if err != nil {
@@ -698,5 +712,102 @@ func (r Reader) Placement(node, module string) (map[string]any, error) {
}
out[n] = values[i]
}
return map[string]any{"placement": out, "unreadable": unreadable}, nil
return map[string]any{"placement": out, "unreadable": unreadable, "candidates": candidates}, nil
}
// The states of a node's answer to a placement candidate, as the view and the assigner's check read it.
const (
// AnswerNone: the node's have holds no answer for the candidate's group (a node asleep, or not composing yet).
AnswerNone = "none"
// AnswerStale: the node answered an earlier head or revision of the candidate.
AnswerStale = "stale"
)
// CandidateRow is one placement candidate and the node's answer to it.
type CandidateRow struct {
Group string `json:"group"`
Node string `json:"node"`
Pull delivery.Head `json:"pull"`
Revision uint64 `json:"revision"`
Placement json.RawMessage `json:"placement,omitempty"` // the candidate value as stored
Removes bool `json:"removes,omitempty"` // the head removes the node's file
Answered string `json:"answered"` // same, differs, refused, stale or none
Answer *delivery.CandidateAnswer `json:"answer,omitempty"`
}
// PlacementCandidates is every placement candidate, narrowed by node and by a module its placement names, with each
// node's answer. A key that does not read is a row naming only its key's group and node, answered none, never dropped.
func (r Reader) PlacementCandidates(node, module string) ([]CandidateRow, error) {
st := r.PlacementCandidatesState
if st.Keys == nil || st.GetEntry == nil {
return []CandidateRow{}, nil
}
keys, err := st.Keys()
if err != nil {
return nil, fmt.Errorf("reading the placement candidates: %w", err)
}
sort.Strings(keys)
answers := map[string]map[string]delivery.CandidateAnswer{} // node, then group
rows := []CandidateRow{}
for _, key := range keys {
group, n, err := delivery.ParsePlacementCandidateKey(key)
if err != nil || (node != "" && n != node) {
continue
}
raw, rev, err := st.GetEntry(key)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", key, err)
}
if len(raw) == 0 {
continue
}
row := CandidateRow{Group: group, Node: n, Revision: rev, Placement: raw, Answered: AnswerNone}
c, err := delivery.DecodePlacementCandidate(raw)
if err == nil {
row.Pull, row.Removes = c.Pull, c.Placement == nil
if module != "" && (c.Placement == nil || !slices.Contains(c.Placement.Assigned(), module)) {
continue
}
} else if module != "" {
continue
}
if _, read := answers[n]; !read {
answers[n] = r.candidateAnswers(n)
}
if a, ok := answers[n][group]; ok && err == nil {
row.Answer = &a
row.Answered = AnswerOf(c, rev, a)
}
rows = append(rows, row)
}
return rows, nil
}
// AnswerOf is the state of a node's answer a to the candidate c at key revision rev: its verdict when it answers that
// head and revision, AnswerStale otherwise.
func AnswerOf(c delivery.PlacementCandidate, rev uint64, a delivery.CandidateAnswer) string {
if a.Head != c.Pull.Head || a.Revision != rev {
return AnswerStale
}
return a.Verdict
}
// candidateAnswers is a node's answers to its candidates, by group, from its `<node>.node` have; none when it has none.
func (r Reader) candidateAnswers(node string) map[string]delivery.CandidateAnswer {
if r.HavesState.Get == nil {
return nil
}
key, err := delivery.HaveNodeKey(node)
if err != nil {
return nil
}
raw, err := r.HavesState.Get(key)
if err != nil || len(raw) == 0 {
return nil
}
h, err := delivery.DecodeHaveNode(raw)
if err != nil {
return nil
}
return h.Candidates
}
+1 -1
View File
@@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
if m.Module != "mesh-delivery-view" {
t.Errorf("module = %q", m.Module)
}
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.placement-candidates", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // indexer_placement reads the placement candidates and every <node>.node (ADR 0559 §7); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
got := slices.Clone(m.Reads)
slices.Sort(got)
if !reflect.DeepEqual(got, want) {
+1
View File
@@ -49,6 +49,7 @@
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.placement",
"mesh-delivery-indexer.placement-candidates",
"mesh-delivery-indexer.taken",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",