diff --git a/modules/disk-load/.gitignore b/modules/disk-load/.gitignore index 96984eae..76f82dbe 100644 --- a/modules/disk-load/.gitignore +++ b/modules/disk-load/.gitignore @@ -1 +1,2 @@ -disk-load +/disk-load +/cmd/disk-load/disk-load diff --git a/modules/disk-load/cmd/disk-load/data.go b/modules/disk-load/cmd/disk-load/data.go new file mode 100644 index 00000000..b7561238 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/data.go @@ -0,0 +1,119 @@ +package main + +// Where disk-load learns the data items it measures (novox/hq ADR 0577 clause 2): the node-engine's state data +// (delivery.EngineDataBucket), one key `.` per module placed on this node that declares data items, as the +// engine composed them. disk-load watches its own node's keys and composes nothing: it reads no resolved values, no +// manifests and no placement, so a manifest its mesh-sdk cannot judge never silences it, and a round reads one watched +// map instead of several hundred keys over the tunnel (issue 1955). + +import ( + "context" + "errors" + "fmt" + "sync" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Watched is this node's data items as the engine's state data holds them, by module, kept from a watch of its keys. +type Watched struct { + Node string + + mu sync.Mutex + byKey map[string]string // key to its module, for a delete that carries no value + byModule map[string][]catalogue.PlacedDataItem + ready bool +} + +// ErrNotWatchedYet is a round asked before the watch handed over the engine's current keys: no item is known yet, and +// none is taken for gone. +var ErrNotWatchedYet = errors.New("the node-engine's state " + delivery.EngineDataBucket + " is not watched yet: " + + "no data item of this machine is known") + +// Take keeps one change of a key and answers the module it is of and whether its items changed. +func (w *Watched) Take(ch stdio.StateChange) (string, bool, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.byKey == nil { + w.byKey, w.byModule = map[string]string{}, map[string][]catalogue.PlacedDataItem{} + } + if ch.Op != stdio.OpPut { + module, known := w.byKey[ch.Key] + if !known { + return "", false, nil + } + delete(w.byKey, ch.Key) + delete(w.byModule, module) + return module, true, nil + } + d, err := delivery.DecodeEngineData(ch.Value) + if err != nil { + return "", false, fmt.Errorf("%s: %w", ch.Key, err) + } + if d.Node != w.Node { + return "", false, fmt.Errorf("%s names the machine %q, and this is %s", ch.Key, d.Node, w.Node) + } + for _, it := range d.Items { + if it.Module != d.Module { + return "", false, fmt.Errorf("%s holds an item of %q, and is %s's", ch.Key, it.Module, d.Module) + } + } + changed := !sameItems(w.byModule[d.Module], d.Items) + w.byKey[ch.Key] = d.Module + w.byModule[d.Module] = d.Items + return d.Module, changed, nil +} + +// Ready says the watch handed over every current key: from now on an item missing is an item gone. +func (w *Watched) Ready() { + w.mu.Lock() + w.ready = true + w.mu.Unlock() +} + +// All is every module's items, as the Keeper reads them; refused until the watch is ready. +func (w *Watched) All(context.Context) (map[string][]catalogue.PlacedDataItem, error) { + w.mu.Lock() + defer w.mu.Unlock() + if !w.ready { + return nil, ErrNotWatchedYet + } + out := make(map[string][]catalogue.PlacedDataItem, len(w.byModule)) + for m, its := range w.byModule { + out[m] = append([]catalogue.PlacedDataItem(nil), its...) + } + return out, nil +} + +func sameItems(a, b []catalogue.PlacedDataItem) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// Follow watches this node's keys of the engine's state data, and asks the keeper to measure a module whose items changed +// after the watch began; it returns once the current keys were handed over (stdio.KeptState.Follow). +func (w *Watched) Follow(k *Keeper) error { + pattern, err := delivery.EngineDataNodeKeys(w.Node) + if err != nil { + return err + } + stdio.State(delivery.EngineDataRead).Follow(pattern, func(ch stdio.StateChange) error { + module, changed, err := w.Take(ch) + if err != nil || !changed || ch.Current { + return err + } + k.Due(module) + return nil + }) + w.Ready() + return nil +} diff --git a/modules/disk-load/cmd/disk-load/data_test.go b/modules/disk-load/cmd/disk-load/data_test.go new file mode 100644 index 00000000..2306f159 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/data_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// ADR 0577 clauses 2, 3 (and the disk-load check of clause 5): the items measured come from the node-engine's keys of +// this machine as the watch hands them over, and none is known (none taken for gone) before the watch caught up; a key +// that changes after the watch began has its module measured again, and one deleted is measured no more; an item whose +// measurement fails is a failed item of the round, which measures the rest, and the round's key says so. +func TestTheItemsComeFromTheEnginesKeysAndARoundSaysHowFarItGot(t *testing.T) { + key := func(module string) string { k, _ := delivery.EngineDataKey("ace", module); return k } + value := func(module string, items ...catalogue.PlacedDataItem) json.RawMessage { + raw, err := delivery.EncodeEngineData(delivery.EngineData{Node: "ace", Module: module, Digest: "sha256:" + module, Items: items}) + if err != nil { + t.Fatal(err) + } + return raw + } + pg := catalogue.PlacedDataItem{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureShallow} + gone := catalogue.PlacedDataItem{Module: "pg", ID: "wal", Class: "valuable", Path: "/srv/missing", Measure: catalogue.MeasureShallow} + web := catalogue.PlacedDataItem{Module: "web", ID: "uploads", Class: "valuable", Path: "/srv/web", Measure: catalogue.MeasureShallow} + measured := map[string]int{} + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "test" && args[1] == "/srv/missing": + return "", errors.New("no such file") + case name == "test": + return "", nil + case name == "bash" && strings.Contains(args[1], "-maxdepth 1"): + measured[strings.Split(args[1], "'")[1]]++ + return "1760000000 2\n", nil + } + return "", os.ErrNotExist + } + now := time.Date(2026, 11, 1, 0, 30, 0, 0, time.UTC) + readings, rounds := &memStore{m: map[string]json.RawMessage{}}, &memStore{m: map[string]json.RawMessage{}} + w := &Watched{Node: "ace"} + var said []string + k := &Keeper{Node: "ace", Items: w.All, Store: readings, Rounds: rounds, + Say: func(f string, a ...any) { said = append(said, f) }, + Measurer: Measurer{Run: run, Now: func() time.Time { return now }}} + round := func() Round { + var r Round + raw, _ := rounds.Get("ace") + if err := json.Unmarshal(raw, &r); err != nil { + t.Fatalf("the round's key: %v", err) + } + return r + } + + // Before the watch caught up nothing is known, and a round says so in its key rather than retiring every reading. + if _, _, err := k.Round(context.Background(), false); !errors.Is(err, ErrNotWatchedYet) { + t.Fatalf("a round before the watch: %v", err) + } + if r := round(); r.Ended == nil || !strings.Contains(r.Error, "not watched yet") { + t.Fatalf("the round before the watch is kept as %+v", r) + } + for _, step := range []struct { + name string + change stdio.StateChange + module string + changed bool + err bool + }{ + {"a current key", stdio.StateChange{Key: key("pg"), Op: stdio.OpPut, Value: value("pg", pg, gone), Current: true}, "pg", true, false}, + {"another current key", stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", web), Current: true}, "web", true, false}, + {"the same items again", stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", web)}, "web", false, false}, + {"another machine's items", stdio.StateChange{Key: key("x"), Op: stdio.OpPut, Value: json.RawMessage( + `{"v":1,"node":"g14","module":"x","digest":"d","items":[]}`)}, "", false, true}, + {"a delete of a key never seen", stdio.StateChange{Key: key("y"), Op: stdio.OpDelete}, "", false, false}, + } { + module, changed, err := w.Take(step.change) + if module != step.module || changed != step.changed || (err != nil) != step.err { + t.Errorf("%s: taken as %q, changed %v, %v", step.name, module, changed, err) + } + } + w.Ready() + + // A round measures every item it was handed; the missing one fails, and the round goes on and says it. + if _, _, err := k.Round(context.Background(), false); err != nil { + t.Fatalf("the round: %v", err) + } + r := round() + if r.Ended == nil || r.Items != 3 || r.Measured != 3 || len(r.Failed) != 1 || r.Failed[0].Item != "pg.wal" || + !strings.Contains(r.Failed[0].Why, "does not exist") || measured["/srv/pg"] != 1 || measured["/srv/web"] != 1 { + t.Fatalf("the round's key says %+v; measured %v", r, measured) + } + if !strings.Contains(strings.Join(said, "\n"), "the round ended") || !strings.Contains(strings.Join(said, "\n"), "starts") { + t.Errorf("the journal said %v", said) + } + + // A key that changes after the watch began has its module measured again, at once, by Work. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go k.Work(ctx) + moved := web + moved.Path = "/srv/web2" + if module, changed, err := w.Take(stdio.StateChange{Key: key("web"), Op: stdio.OpPut, Value: value("web", moved)}); err != nil || !changed { + t.Fatalf("a changed key: %s %v %v", module, changed, err) + } + k.Due("web") + waitFor(t, func() bool { + raw := readingOf(readings, "ace.web.uploads") + return raw != nil && raw.Path == "/srv/web2" + }) + // A key deleted: its module's items are measured no more, and its reading is retired. + if module, changed, _ := w.Take(stdio.StateChange{Key: key("web"), Op: stdio.OpDelete}); module != "web" || !changed { + t.Fatalf("a deleted key was taken as %q, %v", module, changed) + } + k.Due("web") + waitFor(t, func() bool { r := readingOf(readings, "ace.web.uploads"); return r != nil && !r.RetiredAt.IsZero() }) + before := measured["/srv/web2"] + if _, _, err := k.Round(context.Background(), false); err != nil { + t.Fatal(err) + } + if measured["/srv/web2"] != before || round().Items != 2 { + t.Errorf("an item whose key was deleted was measured again: %d times, round %+v", measured["/srv/web2"]-before, round()) + } +} + +func readingOf(s *memStore, key string) *health.Reading { + raw, _ := s.Get(key) + if raw == nil { + return nil + } + r, err := health.DecodeReading(raw) + if err != nil { + return nil + } + return &r +} + +func waitFor(t *testing.T, done func() bool) { + t.Helper() + for end := time.Now().Add(5 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { + if done() { + return + } + } + t.Fatal("not done within 5 s") +} diff --git a/modules/disk-load/cmd/disk-load/whole_integration_test.go b/modules/disk-load/cmd/disk-load/whole_integration_test.go new file mode 100644 index 00000000..2cff6f38 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/whole_integration_test.go @@ -0,0 +1,267 @@ +//go:build integration + +package main + +// disk-load whole against a real bus (novox/hq ADR 0577 clause 5, the group's integration test): the built binary, played +// over its real stdin and stdout as the node's tool runtime plays it (mesh/state.* answered from a real nats-server's +// key-value buckets, a watch handed over from the bus's own watch, mesh/publish onto its subjects). A key of the +// node-engine's state data written after disk-load started has its module's item read within a minute, and disk-load +// composes nothing to learn it. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` +} + +// player is the node's tool runtime, as far as disk-load needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + asked []string // every state call, ` ` +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"jsonrpc": "2.0", "id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"jsonrpc": "2.0", "id": id, "result": result}) +} + +// bucket is a state's bucket: another module's `.`, or disk-load's own by its local name. +func bucket(state string) string { + if owner, name, other := strings.Cut(state, "."); other { + return owner + "_" + name + } + return "disk-load_" + state +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil || m.Method == "" { + continue // an answer to a change handed over + } + go p.handle(m) + } +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + p.mu.Lock() + p.asked = append(p.asked, m.Method+" "+a.State+" "+a.Key) + p.mu.Unlock() + if m.Method == "mesh/publish" { + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.disk-load.event."+e.Key, e.Body)) + return + } + if !strings.HasPrefix(m.Method, "mesh/state.") { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + kv, err := p.js.KeyValue(p.ctx, bucket(a.State)) + if err != nil { + p.answer(m.ID, nil, err) + return + } + switch m.Method { + case "mesh/state.get": + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + rev, err := kv.Put(p.ctx, a.Key, a.Value) + p.answer(m.ID, map[string]any{"revision": rev}, err) + case "mesh/state.keys": + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.watch": + p.watch(m.ID, a.State, kv, a.Key) + default: + p.answer(m.ID, nil, errors.New("not played here")) + } +} + +// watch hands over the bus's watch of the pattern: every current value, then the answer to the watch, then each change. +func (p *player) watch(id json.RawMessage, state string, kv jetstream.KeyValue, pattern string) { + var watchID string + _ = json.Unmarshal(id, &watchID) + if pattern == "" { + pattern = ">" + } + w, err := kv.Watch(p.ctx, strings.ReplaceAll(pattern, "**", ">")) + if err != nil { + p.answer(id, nil, err) + return + } + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(id, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + p.mu.Lock() + p.next++ + n := p.next + p.mu.Unlock() + p.send(map[string]any{"jsonrpc": "2.0", "id": fmt.Sprintf("rt-%d", n), "method": "mesh/state", "params": map[string]any{ + "watch": watchID, "change": map[string]any{"state": state, "key": e.Key(), "op": op, + "value": json.RawMessage(e.Value()), "revision": e.Revision(), "current": current}}}) + } +} + +// 0577/AC5 (the group's integration test): a key of the node-engine's state data written after disk-load started is +// read within a minute, its item's reading is on the bus, and disk-load read no composition input to learn it. +func TestDiskLoadWholeMeasuresAnItemWithinAMinuteOfItsKey(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), "disk-load") + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("disk-load does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{delivery.EngineDataBucket, health.ReadingsBucket, "disk-load_" + StateRounds} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE=disk-load") + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin} + go p.serve(stdout) + + // Once disk-load watches its machine's keys, the engine writes one. + watching := func() bool { + p.mu.Lock() + defer p.mu.Unlock() + for _, a := range p.asked { + if a == "mesh/state.watch "+delivery.EngineDataRead+" ace.*" { + return true + } + } + return false + } + for end := time.Now().Add(20 * time.Second); !watching(); time.Sleep(50 * time.Millisecond) { + if time.Now().After(end) { + t.Fatalf("disk-load did not watch its machine's keys of %s: asked %v", delivery.EngineDataBucket, p.asked) + } + } + data, _ := js.KeyValue(ctx, delivery.EngineDataBucket) + dir := t.TempDir() + item := catalogue.PlacedDataItem{Module: "pg", ID: "data", Class: "valuable", Path: dir, Measure: catalogue.MeasureShallow} + raw, err := delivery.EncodeEngineData(delivery.EngineData{Node: "ace", Module: "pg", Digest: "sha256:pg", Items: []catalogue.PlacedDataItem{item}}) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.EngineDataKey("ace", "pg") + written := time.Now() + if _, err := data.Put(ctx, key, raw); err != nil { + t.Fatal(err) + } + readings, _ := js.KeyValue(ctx, health.ReadingsBucket) + for { + e, err := readings.Get(ctx, health.ReadingKey("ace", "pg", "data")) + if err == nil { + r, err := health.DecodeReading(e.Value()) + if err != nil || r.Path != dir || r.MeasuredAt == nil { + t.Fatalf("the reading is %s (%v)", e.Value(), err) + } + break + } + if time.Since(written) > time.Minute { + t.Fatalf("no reading within a minute of the key's write: %v", err) + } + time.Sleep(200 * time.Millisecond) + } + // It composed nothing: every state it asked is the engine's data or its own. + p.mu.Lock() + defer p.mu.Unlock() + for _, a := range p.asked { + f := strings.Fields(a) + if len(f) > 1 && strings.HasPrefix(f[0], "mesh/state.") && f[1] != delivery.EngineDataRead && f[1] != StateReadings && f[1] != StateRounds { + t.Errorf("disk-load read %s, which is no state of its own nor the engine's data", a) + } + } +}