Show placement candidates and each node's answer in indexer_placement, add indexer_placement_reread, and drop a group's candidates when its head changes no placement file, so a missed message or a revert is recoverable by tool (issue 1126, ADR 0559 §3, §7, §8)
This commit is contained in:
@@ -65,7 +65,9 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
|
||||
{Name: view.PlacementReadTool,
|
||||
Description: "Where modules run, as the indexer read it from the catalogue's placement files at the last merge (placement.<node>, novox/hq ADR 0549): " +
|
||||
"each node's modules in order with their host ports, bindings and settings placement, its pins and the seats its modules hold, and the catalogue revision. " +
|
||||
"With node, that node's; with module, every node placing it; with neither, every node's. Keys that do not read are listed as unreadable. Read-only.",
|
||||
"With node, that node's; with module, every node placing it; with neither, every node's. Keys that do not read are listed as unreadable. " +
|
||||
"Beside it, candidates: each open catalogue pull request's placement of a node that differs from the trunk's (placement-candidates <group>.<node>, issue 1126), " +
|
||||
"narrowed the same way, with the node's answer from its <node>.node have (answered: same, differs, refused, stale for an earlier head, none when it has not answered). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": nodeModule},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
node, module := nodeModuleOf(args)
|
||||
@@ -202,8 +204,9 @@ func read(state stdio.KeptState) view.StateRead {
|
||||
func main() {
|
||||
r := view.Reader{State: read(stdio.State("mesh-delivery-indexer.modules")), Builds: read(stdio.State("mesh-delivery-builder.builds")), CandidatesState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
WantsState: read(stdio.State("mesh-delivery-assigner.wants")), DifferencesState: read(stdio.State("mesh-delivery-assigner.differences")), HavesState: read(stdio.State("node-engine.have")),
|
||||
ReachState: read(stdio.State("reach")),
|
||||
PlacementState: read(stdio.State("mesh-delivery-indexer.placement"))}
|
||||
ReachState: read(stdio.State("reach")),
|
||||
PlacementState: read(stdio.State("mesh-delivery-indexer.placement")),
|
||||
PlacementCandidatesState: read(stdio.State("mesh-delivery-indexer.placement-candidates"))}
|
||||
u := view.UsageReader{Haves: r.HavesState, Resolved: read(stdio.State("mesh-config.resolved")), Tries: read(stdio.State("mesh-delivery-checker.tries")),
|
||||
Steps: map[string]view.StateRead{}, Pauses: map[string]view.StateRead{}, Backlog: backlogOf(socket())}
|
||||
for _, lang := range delivery.AskLanguages {
|
||||
|
||||
@@ -131,6 +131,50 @@ func TestModulesToolAnswersEveryKeyAsStored(t *testing.T) {
|
||||
if _, err := run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"}); err == nil || !strings.Contains(err.Error(), "no placement of novox") {
|
||||
t.Errorf("a node with no key: %v", err)
|
||||
}
|
||||
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); len(c) != 0 {
|
||||
t.Errorf("with no candidate open: %v", c)
|
||||
}
|
||||
|
||||
// ADR 0559 §7 (issue 1126): beside the trunk's, each open candidate key and the node's answer from its <node>.node:
|
||||
// answered at the key's head and revision, at an earlier head (stale), or not at all (none); a node the trunk does not
|
||||
// place yet is shown by its candidate alone.
|
||||
const h1, h2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222"
|
||||
cand := func(node, head string, placed bool) string {
|
||||
pl := `,"placement":{"v":1,"node":"` + node + `","modules":[{"module":"plex"}]}`
|
||||
if !placed {
|
||||
pl = ""
|
||||
}
|
||||
return `{"v":1,"group":"feat/p","pull":{"repository":"novox/mesh-catalog","number":9,"head":"` + head + `"}` + pl + `}`
|
||||
}
|
||||
cands := fake{"feat/p.ace": cand("ace", h2, true), "feat/p.g14": cand("g14", h2, false), "feat/p.novox": cand("novox", h2, true)}
|
||||
revs := map[string]uint64{"feat/p.ace": 4, "feat/p.g14": 5, "feat/p.novox": 6}
|
||||
p.PlacementCandidatesState = view.StateRead{Keys: cands.reader().State.Keys, GetEntry: func(key string) (json.RawMessage, uint64, error) {
|
||||
raw, _ := cands.reader().State.Get(key)
|
||||
return raw, revs[key], nil
|
||||
}}
|
||||
p.HavesState = fake{
|
||||
"ace.node": `{"v":1,"composed":"x","candidates":{"feat/p":{"head":"` + h2 + `","revision":4,"verdict":"same","same":["plex"]}}}`,
|
||||
"g14.node": `{"v":1,"composed":"x","candidates":{"feat/p":{"head":"` + h1 + `","revision":2,"verdict":"same"}}}`,
|
||||
}.reader().State
|
||||
out, err = run(t, p, view.PlacementReadTool, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answered := map[string]string{}
|
||||
for _, c := range out.(map[string]any)["candidates"].([]view.CandidateRow) {
|
||||
answered[c.Node] = c.Answered
|
||||
}
|
||||
if want := map[string]string{"ace": "same", "g14": view.AnswerStale, "novox": view.AnswerNone}; fmt.Sprint(answered) != fmt.Sprint(want) {
|
||||
t.Errorf("answers %v, want %v", answered, want)
|
||||
}
|
||||
out, err = run(t, p, view.PlacementReadTool, map[string]any{"node": "novox"})
|
||||
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); err != nil || len(c) != 1 || c[0].Pull.Head != h2 || len(out.(map[string]any)["placement"].(map[string]json.RawMessage)) != 0 {
|
||||
t.Errorf("a node placed by its candidate alone: %v, %v", out, err)
|
||||
}
|
||||
out, _ = run(t, p, view.PlacementReadTool, map[string]any{"module": "plex"})
|
||||
if c := out.(map[string]any)["candidates"].([]view.CandidateRow); len(c) != 2 {
|
||||
t.Errorf("the candidates placing plex: %v (g14's removes its file)", c)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61 h1:/c6ncICWjq55nBf8TlDM0ROe/PuH+qZIdnuVbtg8UuM=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026211913-68d65e6bbf61/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f h1:aWqn4HHTPUp/+28h7b8uFqQptAYvQFs05CALMHN7SkI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027010747-7e99c354ac7f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -43,6 +43,8 @@ type Reader struct {
|
||||
HavesState StateRead // the node-engine's haves
|
||||
ReachState StateRead // the view's own reach (ADR 0556)
|
||||
PlacementState StateRead // the indexer's placement
|
||||
// PlacementCandidatesState is the indexer's placement candidates (issue 1126); unset reads none.
|
||||
PlacementCandidatesState StateRead
|
||||
}
|
||||
|
||||
// ReachReadTool reads the view's reach: per repository, how far each module reached and the reached head (ADR 0556).
|
||||
@@ -652,11 +654,19 @@ func (r Reader) Haves(node, module string) (KeptAnswer, error) {
|
||||
// Placement is the indexer's placement as stored (ADR 0549 §2): one node's key with node, every node placing a module
|
||||
// with module (each node's key whole, so its pins, ports and bindings are beside the module), and every node's key with
|
||||
// neither. A key that does not read as a placement is listed as unreadable, never dropped.
|
||||
//
|
||||
// Beside it, candidates (issue 1126, ADR 0549 §4): each open catalogue pull request's placement of a node that differs from
|
||||
// the trunk's (the indexer's placement-candidates `<group>.<node>`), narrowed the same way, each with the node's answer from
|
||||
// its `<node>.node` have and whether that answer is for the candidate as it stands (its head and key revision).
|
||||
func (r Reader) Placement(node, module string) (map[string]any, error) {
|
||||
node, module = strings.TrimSpace(node), strings.TrimSpace(module)
|
||||
if r.PlacementState.Keys == nil || r.PlacementState.Get == nil {
|
||||
return nil, fmt.Errorf("the placement state cannot be read here")
|
||||
}
|
||||
candidates, err := r.PlacementCandidates(node, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if node != "" {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
@@ -666,10 +676,14 @@ func (r Reader) Placement(node, module string) (map[string]any, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
if len(raw) == 0 && len(candidates) == 0 {
|
||||
return nil, fmt.Errorf("no placement of %s: the catalogue holds no placement file for it, or the indexer has read none yet", node)
|
||||
}
|
||||
return map[string]any{"placement": map[string]json.RawMessage{node: raw}}, nil
|
||||
out := map[string]json.RawMessage{}
|
||||
if len(raw) > 0 {
|
||||
out[node] = raw
|
||||
}
|
||||
return map[string]any{"placement": out, "candidates": candidates}, nil
|
||||
}
|
||||
keys, err := r.PlacementState.Keys()
|
||||
if err != nil {
|
||||
@@ -698,5 +712,102 @@ func (r Reader) Placement(node, module string) (map[string]any, error) {
|
||||
}
|
||||
out[n] = values[i]
|
||||
}
|
||||
return map[string]any{"placement": out, "unreadable": unreadable}, nil
|
||||
return map[string]any{"placement": out, "unreadable": unreadable, "candidates": candidates}, nil
|
||||
}
|
||||
|
||||
// The states of a node's answer to a placement candidate, as the view and the assigner's check read it.
|
||||
const (
|
||||
// AnswerNone: the node's have holds no answer for the candidate's group (a node asleep, or not composing yet).
|
||||
AnswerNone = "none"
|
||||
// AnswerStale: the node answered an earlier head or revision of the candidate.
|
||||
AnswerStale = "stale"
|
||||
)
|
||||
|
||||
// CandidateRow is one placement candidate and the node's answer to it.
|
||||
type CandidateRow struct {
|
||||
Group string `json:"group"`
|
||||
Node string `json:"node"`
|
||||
Pull delivery.Head `json:"pull"`
|
||||
Revision uint64 `json:"revision"`
|
||||
Placement json.RawMessage `json:"placement,omitempty"` // the candidate value as stored
|
||||
Removes bool `json:"removes,omitempty"` // the head removes the node's file
|
||||
Answered string `json:"answered"` // same, differs, refused, stale or none
|
||||
Answer *delivery.CandidateAnswer `json:"answer,omitempty"`
|
||||
}
|
||||
|
||||
// PlacementCandidates is every placement candidate, narrowed by node and by a module its placement names, with each
|
||||
// node's answer. A key that does not read is a row naming only its key's group and node, answered none, never dropped.
|
||||
func (r Reader) PlacementCandidates(node, module string) ([]CandidateRow, error) {
|
||||
st := r.PlacementCandidatesState
|
||||
if st.Keys == nil || st.GetEntry == nil {
|
||||
return []CandidateRow{}, nil
|
||||
}
|
||||
keys, err := st.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the placement candidates: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
answers := map[string]map[string]delivery.CandidateAnswer{} // node, then group
|
||||
rows := []CandidateRow{}
|
||||
for _, key := range keys {
|
||||
group, n, err := delivery.ParsePlacementCandidateKey(key)
|
||||
if err != nil || (node != "" && n != node) {
|
||||
continue
|
||||
}
|
||||
raw, rev, err := st.GetEntry(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", key, err)
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
row := CandidateRow{Group: group, Node: n, Revision: rev, Placement: raw, Answered: AnswerNone}
|
||||
c, err := delivery.DecodePlacementCandidate(raw)
|
||||
if err == nil {
|
||||
row.Pull, row.Removes = c.Pull, c.Placement == nil
|
||||
if module != "" && (c.Placement == nil || !slices.Contains(c.Placement.Assigned(), module)) {
|
||||
continue
|
||||
}
|
||||
} else if module != "" {
|
||||
continue
|
||||
}
|
||||
if _, read := answers[n]; !read {
|
||||
answers[n] = r.candidateAnswers(n)
|
||||
}
|
||||
if a, ok := answers[n][group]; ok && err == nil {
|
||||
row.Answer = &a
|
||||
row.Answered = AnswerOf(c, rev, a)
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// AnswerOf is the state of a node's answer a to the candidate c at key revision rev: its verdict when it answers that
|
||||
// head and revision, AnswerStale otherwise.
|
||||
func AnswerOf(c delivery.PlacementCandidate, rev uint64, a delivery.CandidateAnswer) string {
|
||||
if a.Head != c.Pull.Head || a.Revision != rev {
|
||||
return AnswerStale
|
||||
}
|
||||
return a.Verdict
|
||||
}
|
||||
|
||||
// candidateAnswers is a node's answers to its candidates, by group, from its `<node>.node` have; none when it has none.
|
||||
func (r Reader) candidateAnswers(node string) map[string]delivery.CandidateAnswer {
|
||||
if r.HavesState.Get == nil {
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
raw, err := r.HavesState.Get(key)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
h, err := delivery.DecodeHaveNode(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return h.Candidates
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
if m.Module != "mesh-delivery-view" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.placement-candidates", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // indexer_placement reads the placement candidates and every <node>.node (ADR 0559 §7); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
||||
got := slices.Clone(m.Reads)
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
"mesh-delivery-indexer.candidates",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-indexer.placement",
|
||||
"mesh-delivery-indexer.placement-candidates",
|
||||
"mesh-delivery-indexer.taken",
|
||||
"mesh-delivery-teller.answers",
|
||||
"mesh-delivery-watchman.presence",
|
||||
|
||||
Reference in New Issue
Block a user