The delivery repository's contracts and SDK live under modules/ as mesh-delivery-contracts and mesh-delivery-sdk, their Go module paths renamed to match the folders (the TypeScript names already did), and every importer in mesh-delivery, mesh-catalog, mesh-host, mesh-tools and mesh-sdk follows in the same group: a design as it should be, one path per module, checked by the group's check building every importer. Builds: design 47 G §427, §88 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
507 lines
22 KiB
Go
507 lines
22 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
assignerclient "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
|
engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
|
placing "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
|
nodeseat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
|
watchdogsclient "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
|
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
|
|
"git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
|
|
"git.novox.be/novox/mesh-catalog/modules/restic/backupgood"
|
|
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
)
|
|
|
|
// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed.
|
|
type bus struct {
|
|
drop map[string]bool // stream or stream.consumer missing
|
|
change func(rows map[string]*streamRow)
|
|
closed string
|
|
signals error
|
|
asksFail error
|
|
// banned is each node's answer to node-intrusion-prevention.banned; a node absent does not answer.
|
|
banned map[string]string
|
|
// fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap.
|
|
fills []fillRow
|
|
}
|
|
|
|
func (b bus) rows() map[string]*streamRow {
|
|
rows := map[string]*streamRow{}
|
|
for _, s := range buscompose.MeshStreams() {
|
|
if b.drop[s.Name] {
|
|
continue
|
|
}
|
|
retention := "limits"
|
|
per := int64(s.MaxMsgsPerSubject)
|
|
switch s.Retention {
|
|
case buscompose.RetentionWorkQueue:
|
|
retention = "workqueue"
|
|
case buscompose.RetentionLastPerSubject:
|
|
per = 1
|
|
}
|
|
rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per}
|
|
}
|
|
if b.change != nil {
|
|
b.change(rows)
|
|
}
|
|
return rows
|
|
}
|
|
|
|
func (b bus) ask(key string, body any) (json.RawMessage, error) {
|
|
if b.asksFail != nil {
|
|
return nil, b.asksFail
|
|
}
|
|
switch key {
|
|
case StreamsTool:
|
|
stream, _ := body.(map[string]any)["stream"].(string)
|
|
var detail []streamRow
|
|
for name, r := range b.rows() {
|
|
if stream == "" {
|
|
detail = append(detail, *r)
|
|
} else if name == stream {
|
|
detail = append(detail, *r)
|
|
}
|
|
}
|
|
if stream == "" && len(b.fills) > 0 {
|
|
var rows []any
|
|
for _, r := range detail {
|
|
rows = append(rows, r)
|
|
}
|
|
for _, f := range b.fills {
|
|
rows = append(rows, f)
|
|
}
|
|
return json.Marshal(map[string]any{"detail": rows})
|
|
}
|
|
if stream != "" && len(detail) == 0 {
|
|
return nil, errors.New("no stream called " + stream)
|
|
}
|
|
return json.Marshal(map[string]any{"detail": detail})
|
|
case ClosedTool:
|
|
if b.closed == "" {
|
|
return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil
|
|
}
|
|
return json.RawMessage(b.closed), nil
|
|
case SignalsVerb:
|
|
if b.signals != nil {
|
|
return nil, b.signals
|
|
}
|
|
return json.RawMessage(`{"signals":[]}`), nil
|
|
}
|
|
if node, ok := strings.CutPrefix(key, BannedVerb+"@"); ok {
|
|
if a, ok := b.banned[node]; ok {
|
|
return json.RawMessage(a), nil
|
|
}
|
|
}
|
|
return nil, errors.New("nats: no responders available for request")
|
|
}
|
|
|
|
func keysOf(found []Finding) []string {
|
|
var out []string
|
|
for _, f := range found {
|
|
out = append(out, keyOf(f))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2).
|
|
func TestTheBusProbes(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
probe func(context.Context, *Checker) ([]Finding, error)
|
|
bus bus
|
|
want []string
|
|
err string
|
|
mesh *memStore // the bus states the machine probes read; nil reads none
|
|
}{
|
|
{"D7 all as defined", probeStreams, bus{}, nil, "", nil},
|
|
{"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) {
|
|
rows["EVENTS"].Retention = "workqueue"
|
|
}}, []string{"bus.EVENTS.stream-redefined.stream-wrong", "bus.NODES.stream-missing.stream-wrong"}, "", nil},
|
|
{"D15 quiet", probeReconnects, bus{}, nil, "", nil},
|
|
{"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[
|
|
{"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]},
|
|
{"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`},
|
|
[]string{"bus.g14=2enode-tools.bus-reconnects"}, "", nil},
|
|
{"DW answers", probeWatchdogs, bus{}, nil, "", nil},
|
|
{"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.watchdogs-silent"}, "", nil},
|
|
{"H-bus healthy", probeBusHealth, bus{}, nil, "", nil},
|
|
{"H-bus a stream missing", probeBusHealth,
|
|
bus{drop: map[string]bool{"NODES": true}}, []string{"bus.mesh.core-unhealthy"}, "", nil},
|
|
{"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed", nil},
|
|
// D8 (issue 238): the assignments' addresses and endpoints against each heard holder's ban list.
|
|
{"D8 nothing of the mesh banned", probeBans, bus{banned: map[string]string{"ace": `{"banned":["198.51.100.7"]}`,
|
|
"novox": `{"banned":[]}`}}, nil, "", machines()},
|
|
{"D8 a private address and an endpoint banned", probeBans, bus{banned: map[string]string{
|
|
"ace": `{"jails":{"sshd":["10.10.0.1","198.51.100.7"]}}`, "novox": `{"banned":["203.0.113.5"]}`}},
|
|
[]string{"machine.ace.own-address-banned", "machine.novox.own-address-banned"}, "", machines()},
|
|
{"D8 a machine not heard from is not asked", probeBans, bus{banned: map[string]string{"novox": `{"banned":[]}`,
|
|
"ace": `{"banned":[]}`}}, nil, "", machines()},
|
|
{"D8 a heard holder that does not answer is no pass", probeBans, bus{banned: map[string]string{}}, nil,
|
|
"a ban list could not be read", machines()},
|
|
// D13 (issue 273, ADR 0233): the readings disk-load keeps and the last good nights restic keeps, against the bounds
|
|
// the manifests declare; the kinds, keys and words are the controller's.
|
|
{"D13 all held", probeData, bus{}, nil, "", dataMesh(func(*dataRig) {})},
|
|
{"D13 shrank, gone, quiet, unprotected and stale", probeData, bus{}, []string{
|
|
"machine.ace=2epg=2edata.data-quiet", "machine.ace=2epg=2edata.backup-stale", "machine.ace=2epg=2edumps.data-missing",
|
|
"machine.ace=2eplex=2emovies.protection-missing", "machine.ace=2eplex=2ethumbs.data-shrank",
|
|
"machine.ace=2earray=2emd-md127.array-degraded"},
|
|
"", dataMesh(func(r *dataRig) {
|
|
r.size("ace", "pg", "data", 100<<20, 100<<20).write("ace", "pg", "data", -80*time.Hour)
|
|
r.gone("ace", "pg", "dumps")
|
|
r.size("ace", "plex", "thumbs", 10<<20, 600<<20)
|
|
r.size("ace", "plex", "movies", 1<<30, 1<<30).noArray("ace", "plex", "movies")
|
|
r.size("ace", "plex", "library", 1<<30, 1<<30).array("ace", "plex", "library", "md", "md127", false)
|
|
r.lastGood("ace", "pg", "data", -100*time.Hour)
|
|
})},
|
|
{"D13 a dataset shrank once for its items", probeData, bus{}, []string{"machine.ace=2edataset=2estorage-media.data-shrank"},
|
|
"", dataMesh(func(r *dataRig) {
|
|
r.size("ace", "plex", "thumbs", 10<<20, 600<<20).dataset("ace", "plex", "thumbs", "storage/media")
|
|
r.size("ace", "plex", "meta", 10<<20, 600<<20).dataset("ace", "plex", "meta", "storage/media")
|
|
})},
|
|
{"D13 a retired item is kept, not judged, and cleanup waits after thirty days", probeData, bus{}, []string{
|
|
"machine.ace=2epg=2edumps.cleanup-waiting"}, "", dataMesh(func(r *dataRig) {
|
|
r.size("ace", "pg", "dumps", 1<<20, 1<<30).retired("ace", "pg", "dumps", -31*24*time.Hour)
|
|
r.size("ace", "pg", "old", 1<<20, 1<<30).retired("ace", "pg", "old", -2*24*time.Hour)
|
|
})},
|
|
{"D13 a copy far smaller than a retired one elsewhere is an empty replacement", probeData, bus{},
|
|
[]string{"machine.ace=2eplex=2ethumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) {
|
|
r.size("ace", "plex", "thumbs", 10<<20, 10<<20)
|
|
r.size("g14", "plex", "thumbs", 600<<20, 600<<20).retired("g14", "plex", "thumbs", -24*time.Hour)
|
|
})},
|
|
{"D13 no good backup is counted from when the item was first measured", probeData, bus{},
|
|
[]string{"machine.ace=2epg=2edata.backup-stale"}, "", dataMesh(func(r *dataRig) {
|
|
delete(r.good, diskreadings.ReadingKey("ace", "pg", "data"))
|
|
delete(r.good, diskreadings.ReadingKey("ace", "plex", "meta"))
|
|
r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour)
|
|
})},
|
|
{"D13 a machine with a backup holder and no reading", probeData, bus{}, []string{"machine.g14.data-unmeasured"}, "",
|
|
dataMesh(func(r *dataRig) { r.holder("g14") })},
|
|
// D-fill (ADR 0297 §6): a module's bucket or log at three quarters of its cap, kept until under seven tenths.
|
|
{"D-fill all with room", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 74, MaxBytes: 100},
|
|
{Name: "LOG_mesh-delivery-builder_build-log", Bytes: 10, MaxBytes: 100}, {Name: "KV_nocap_x", Bytes: 999}}}, nil, "", nil},
|
|
{"D-fill a bucket and a log filling", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 75, MaxBytes: 100},
|
|
{Name: "LOG_mesh-delivery-builder_build-log", Bytes: 99, MaxBytes: 100}}},
|
|
[]string{"bus.KV_mesh-issues_issues.bucket-or-log-filling", "bus.LOG_mesh-delivery-builder_build-log.bucket-or-log-filling"}, "", nil},
|
|
{"D-fill between the thresholds raises nothing new", probeFill,
|
|
bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, nil, "", nil},
|
|
{"D-fill between the thresholds keeps what is said", probeFill,
|
|
bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}},
|
|
[]string{"bus.KV_mesh-issues_issues.bucket-or-log-filling"}, "", filling("KV_mesh-issues_issues")},
|
|
{"D-fill the bus module silent", probeFill, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil},
|
|
// DA (ADR 0266): the agent account each machine names, judged on its engine's report while it is heard from.
|
|
{"DA confined", probeAgentAccounts, bus{}, nil, "", agents(map[string]string{"novox": stateHealthy})},
|
|
{"DA a way to root found", probeAgentAccounts, bus{}, []string{"machine.novox.agent-can-become-root"}, "",
|
|
agents(map[string]string{"novox": stateUnhealthy})},
|
|
{"DA no verdict on it", probeAgentAccounts, bus{}, []string{"machine.novox.agent-can-become-root"}, "",
|
|
agents(map[string]string{"novox": ""})},
|
|
{"DA the search runs, within its quiet", probeAgentAccounts, bus{}, nil, "",
|
|
agents(map[string]string{"novox": "pending"})},
|
|
{"DA a machine not heard from has no verdict on now", probeAgentAccounts, bus{}, []string{"machine.g14.agent-can-become-root"},
|
|
"", agents(map[string]string{"novox": stateHealthy, "g14": stateHealthy})},
|
|
{"DA an engine older than the judging", probeAgentAccounts, bus{}, []string{"machine.novox.agent-can-become-root"}, "",
|
|
agents(map[string]string{"novox": "old"})},
|
|
}
|
|
for _, tc := range cases {
|
|
c := &Checker{Ask: tc.bus.ask, Store: tc.mesh, Catalog: tc.mesh, Now: func() time.Time { return dataNow }}
|
|
if tc.mesh == nil {
|
|
c.Store = newStore()
|
|
}
|
|
found, err := tc.probe(context.Background(), c)
|
|
// D13 judges the states it reads from memory, kept by watching them, never one key at a time through the runtime,
|
|
// which did not finish within its bound (issue 1952).
|
|
if strings.HasPrefix(tc.name, "D13") {
|
|
for _, s := range []string{ReadingsRead, GoodRead} {
|
|
if n := tc.mesh.gets[s]; n > 0 {
|
|
t.Errorf("%s: D13 read %s one key at a time %d times", tc.name, s, n)
|
|
}
|
|
}
|
|
}
|
|
if tc.err != "" {
|
|
if err == nil || !strings.Contains(err.Error(), tc.err) {
|
|
t.Errorf("%s: error %v, want %q", tc.name, err, tc.err)
|
|
}
|
|
continue
|
|
}
|
|
if err != nil {
|
|
t.Errorf("%s: %v", tc.name, err)
|
|
continue
|
|
}
|
|
if got := keysOf(found); !slices.Equal(got, tc.want) {
|
|
t.Errorf("%s: found %v, want %v", tc.name, got, tc.want)
|
|
}
|
|
}
|
|
// What changes after the watch began is judged at the next run, from memory: every reading gone is seen.
|
|
mesh := dataMesh(func(*dataRig) {})
|
|
c := &Checker{Store: mesh, Catalog: mesh, Now: func() time.Time { return dataNow }}
|
|
if found, err := probeData(context.Background(), c); err != nil || len(found) != 0 {
|
|
t.Fatalf("D13 on a held mesh: %v, %v", keysOf(found), err)
|
|
}
|
|
keys, _ := mesh.Keys(ReadingsRead)
|
|
for _, k := range keys {
|
|
_ = mesh.Delete(ReadingsRead, k)
|
|
}
|
|
if found, err := probeData(context.Background(), c); err != nil || !slices.Contains(keysOf(found), "machine.ace.data-unmeasured") {
|
|
t.Errorf("D13 after every reading went found %v, %v", keysOf(found), err)
|
|
}
|
|
}
|
|
|
|
// machines is a mesh of three: novox the hub with its endpoint, ace and g14 on the private network; ace and novox
|
|
// hold node-intrusion-prevention and are heard from, g14 holds it and is not.
|
|
func machines() *memStore {
|
|
s := newStore()
|
|
for _, p := range []struct {
|
|
node, address, endpoint string
|
|
online bool
|
|
}{{"novox", "10.10.0.1/16", "203.0.113.5:51820", true}, {"ace", "10.10.0.2/16", "", true}, {"g14", "10.10.0.3/16", "", false}} {
|
|
pl := placing.Assignments{V: 1, Node: p.node,
|
|
Decided: placing.NodeDecided{Address: p.address, Endpoint: p.endpoint, Hub: p.endpoint != ""}}
|
|
if p.node == "ace" {
|
|
pl.Modules = []placing.PlacedModule{{Module: "sshd"}} // a module key beside the node key
|
|
}
|
|
place(s, pl)
|
|
_ = s.Put(PresenceRead, p.node, watchdogsclient.NodePresence{V: 1, Online: p.online})
|
|
key, _ := assignerclient.SeatHoldersKey(BannedSeat, p.node)
|
|
_ = s.Put(SeatHoldersRead, key, placing.SeatHolders{V: 1, Seat: BannedSeat, Scope: "node", Node: p.node,
|
|
Holders: []placing.SeatHoldersHoldersItem{{Module: "fail2ban", Node: p.node}}})
|
|
}
|
|
return s
|
|
}
|
|
|
|
// place writes a node's assignments as the seat does, its node key and a key per module (research 072 row 37).
|
|
func place(s *memStore, p placing.Assignments) {
|
|
values, _ := assignerclient.AssignmentsValues(p)
|
|
for k, raw := range values {
|
|
_ = s.Put(AssignmentsRead, k, json.RawMessage(raw))
|
|
}
|
|
}
|
|
|
|
// placed is every node's assignments the store holds.
|
|
func placed(s *memStore) map[string]placing.Assignments {
|
|
keys, _ := s.Keys(AssignmentsRead)
|
|
values := map[string][]byte{}
|
|
for _, k := range keys {
|
|
values[k], _, _ = s.Get(AssignmentsRead, k)
|
|
}
|
|
out, _ := assignerclient.AssignmentsOf(values)
|
|
return out
|
|
}
|
|
|
|
// filling is a store holding the self-check's written fill finding of a stream: the condition open.
|
|
func filling(stream string) *memStore {
|
|
s := newStore()
|
|
o := finding(meshconditionsclient.ScopeBus, stream, "", "", Finding{Kind: "bucket-or-log-filling", Severity: meshconditionsclient.Warning,
|
|
Summary: "filling", Source: "D-fill", Observer: Module, Run: "run", At: time.Now(), Since: time.Now()})
|
|
raw, _ := json.Marshal(o)
|
|
_ = s.Put(FindingsState, keyOf(o), json.RawMessage(raw))
|
|
return s
|
|
}
|
|
|
|
// agents is machines() where each node named names the agent account "agent" and its report holds the verdict on it:
|
|
// a state, "" for none, "pending" for one waiting for its search, "old" for an engine older than the judging.
|
|
func agents(verdicts map[string]string) *memStore {
|
|
s := machines()
|
|
for node, v := range verdicts {
|
|
p := placed(s)[node]
|
|
p.Decided.AgentAccount = "agent"
|
|
place(s, p)
|
|
h := nodeseat.HealthByNodeValue{V: 1, Node: node, Contract: 13}
|
|
switch v {
|
|
case "":
|
|
case "old":
|
|
h.Contract = 2
|
|
h.Root = []nodeseat.RootAccountVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: stateHealthy}}
|
|
case "pending":
|
|
h.Root = []nodeseat.RootAccountVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent",
|
|
State: stateUnknown, Reason: rootPending + ": 3 of 9 trees"}}
|
|
default:
|
|
h.Root = []nodeseat.RootAccountVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: v,
|
|
Reason: "can become root without a person: a sudo rule"}}
|
|
}
|
|
raw, _ := engineclient.EncodeHealth(h)
|
|
_ = s.Put(HealthRead, node, json.RawMessage(raw))
|
|
}
|
|
return s
|
|
}
|
|
|
|
// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what
|
|
// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4).
|
|
func TestTheRegistryIsADR0564sTable(t *testing.T) {
|
|
moved := []string{"D2", "D3", "D4", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard",
|
|
"DW", "H-bus"}
|
|
var ids []string
|
|
for _, p := range Registry() {
|
|
ids = append(ids, p.ID)
|
|
if p.Kind == "" || p.Asserts == "" {
|
|
t.Errorf("%s has no kind or assertion", p.ID)
|
|
}
|
|
if (p.Run == nil) == (p.Deferred == "") {
|
|
t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID)
|
|
}
|
|
}
|
|
slices.Sort(ids)
|
|
slices.Sort(moved)
|
|
if !slices.Equal(ids, moved) {
|
|
t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved)
|
|
}
|
|
}
|
|
|
|
// ---- D13's rig ------------------------------------------------------------------------------------------------
|
|
|
|
var dataNow = time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)
|
|
|
|
// dataRig is the mesh D13 reads: a manifest per module with the items it declares, a reading per item, a last good night.
|
|
type dataRig struct {
|
|
s *memStore
|
|
readings map[string]*diskreadings.Reading
|
|
good map[string]time.Time
|
|
}
|
|
|
|
func (r *dataRig) key(node, module, item string) string {
|
|
return diskreadings.ReadingKey(node, module, item)
|
|
}
|
|
|
|
func (r *dataRig) reading(node, module, item string) *diskreadings.Reading {
|
|
k := r.key(node, module, item)
|
|
if r.readings[k] == nil {
|
|
at := dataNow
|
|
r.readings[k] = &diskreadings.Reading{V: diskreadings.ReadingVersion, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module,
|
|
ID: item, Class: catalogue.ClassValuable, Path: "/srv/" + module + "/" + item}, Measured: diskreadings.Measured{MeasuredAt: &at, Precision: "exact"}}
|
|
}
|
|
return r.readings[k]
|
|
}
|
|
|
|
func (r *dataRig) size(node, module, item string, size, peak int64) *dataRig {
|
|
x := r.reading(node, module, item)
|
|
x.SizeBytes, x.Peak = &size, &peak
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) write(node, module, item string, ago time.Duration) *dataRig {
|
|
t := dataNow.Add(ago)
|
|
r.reading(node, module, item).LastWrite = &t
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) gone(node, module, item string) *dataRig {
|
|
r.reading(node, module, item).Error = "/srv/" + module + "/" + item + " does not exist"
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) dataset(node, module, item, name string) *dataRig {
|
|
r.reading(node, module, item).Precision = "dataset " + name + ": its whole size"
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) array(node, module, item, kind, where string, healthy bool) *dataRig {
|
|
r.reading(node, module, item).Redundancy = &diskreadings.Redundancy{Kind: kind, Where: where, Healthy: &healthy, Said: "a member is missing"}
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) noArray(node, module, item string) *dataRig {
|
|
r.reading(node, module, item).Redundancy = nil
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) lastGood(node, module, item string, ago time.Duration) *dataRig {
|
|
r.good[r.key(node, module, item)] = dataNow.Add(ago)
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) retired(node, module, item string, ago time.Duration) *dataRig {
|
|
x := r.reading(node, module, item)
|
|
x.RetiredAt, x.RetiredWhy = dataNow.Add(ago), "no longer declared on "+node
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) firstSeen(node, module, item string, ago time.Duration) *dataRig {
|
|
r.reading(node, module, item).FirstSeen = dataNow.Add(ago)
|
|
return r
|
|
}
|
|
|
|
func (r *dataRig) holder(node string) {
|
|
key, _ := assignerclient.SeatHoldersKey("node-backup", node)
|
|
_ = r.s.Put(SeatHoldersRead, key, placing.SeatHolders{V: 1, Seat: "node-backup", Scope: "node", Node: node,
|
|
Holders: []placing.SeatHoldersHoldersItem{{Module: "restic", Node: node}}})
|
|
_ = r.s.Put(PresenceRead, node, watchdogsclient.NodePresence{V: 1, Online: true})
|
|
}
|
|
|
|
// dataMesh builds the mesh: pg declares data (backed up within 48 h, written all the time within 24 h) and dumps; plex
|
|
// declares movies (protected by redundancy), thumbs, meta and library (all valuable); every item is read as held, and what
|
|
// is given changes it.
|
|
func dataMesh(change func(*dataRig)) *memStore {
|
|
r := &dataRig{s: newStore(), readings: map[string]*diskreadings.Reading{}, good: map[string]time.Time{}}
|
|
r.holder("ace")
|
|
declare := func(module string, items map[string]catalogue.DataItem) {
|
|
var man catalogue.Manifest
|
|
man.Module = module
|
|
man.Data = &catalogue.Data{}
|
|
for id, it := range items {
|
|
it.ID = id
|
|
man.Data.Own = append(man.Data.Own, it)
|
|
}
|
|
raw, _ := json.Marshal(man)
|
|
var whole map[string]any
|
|
_ = json.Unmarshal(raw, &whole)
|
|
if r.s.modules == nil {
|
|
r.s.modules = map[string]catalog.CatalogEntry{}
|
|
}
|
|
r.s.modules[module] = catalog.CatalogEntry{Module: module, Manifest: whole}
|
|
}
|
|
declare("pg", map[string]catalogue.DataItem{
|
|
"data": {Class: catalogue.ClassValuable, Within: "48h", Active: "24h"},
|
|
"dumps": {Class: catalogue.ClassValuable}})
|
|
declare("plex", map[string]catalogue.DataItem{
|
|
"movies": {Class: catalogue.ClassValuable, Redundancy: "the array"},
|
|
"thumbs": {Class: catalogue.ClassValuable},
|
|
"meta": {Class: catalogue.ClassValuable},
|
|
"library": {Class: catalogue.ClassValuable, Redundancy: "the array"}})
|
|
for _, i := range [][2]string{{"pg", "data"}, {"pg", "dumps"}, {"plex", "movies"}, {"plex", "thumbs"}, {"plex", "meta"}, {"plex", "library"}} {
|
|
r.size("ace", i[0], i[1], 1<<30, 1<<30).lastGood("ace", i[0], i[1], -time.Hour).write("ace", i[0], i[1], -time.Minute)
|
|
if i[0] == "plex" && (i[1] == "movies" || i[1] == "library") {
|
|
r.array("ace", i[0], i[1], "md", "md127", true)
|
|
}
|
|
}
|
|
change(r)
|
|
for k, x := range r.readings {
|
|
_ = r.s.Put(ReadingsRead, k, x)
|
|
}
|
|
for k, t := range r.good {
|
|
node, rest, _ := strings.Cut(k, ".")
|
|
module, item, _ := strings.Cut(rest, ".")
|
|
_ = r.s.Put(GoodRead, k, backupgood.GoodRow{V: backupgood.GoodVersion, Node: node, Module: module, Item: item, CoveredBy: "/x", LastGood: t})
|
|
}
|
|
return r.s
|
|
}
|
|
|
|
// TestPresenceIsReadThroughTheWatchdogsClient holds the node's presence to the client's codec (ADR 0634 section 4): a
|
|
// part's key (`part.<part>.<instance>`) is no node, and a value of a version the client refuses is a refusal, not a
|
|
// node that is offline.
|
|
func TestPresenceIsReadThroughTheWatchdogsClient(t *testing.T) {
|
|
s := newStore()
|
|
_ = s.Put(PresenceRead, "ace", watchdogsclient.NodePresence{V: 1, Online: true})
|
|
_ = s.Put(PresenceRead, "part.watchman.1", watchdogsclient.PartPresence{V: 1, Online: true})
|
|
c := &Checker{Store: s}
|
|
heard, err := c.heard()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(heard) != 1 || !heard["ace"] {
|
|
t.Errorf("heard %v, want only ace: a part's presence is no node's", heard)
|
|
}
|
|
_ = s.Put(PresenceRead, "g14", watchdogsclient.NodePresence{V: 99, Online: true})
|
|
if _, err := c.heard(); err == nil {
|
|
t.Error("a presence of a version the client refuses was read as a node's")
|
|
}
|
|
}
|