disk-load measures the items the node-engine keeps for its node, and composes nothing (ADR 0577)
It watches its node's keys of mesh-host_data instead of composing every module's items from about 370 resolved keys per round, so a round no longer takes 30 to 45 minutes across the tunnel and a manifest its mesh-sdk cannot judge no longer silences it. A changed key measures its module at once; each round writes how far it got to the per-node state rounds, said by disk_round. Pins mesh-sdk feat/data-items. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
@@ -25,47 +25,114 @@ type Store interface {
|
|||||||
Keys() ([]string, error)
|
Keys() ([]string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// busStore is the state through the runtime.
|
// busStore is one of disk-load's states through the runtime: readings, or rounds.
|
||||||
type busStore struct{}
|
type busStore struct{ state string }
|
||||||
|
|
||||||
func (busStore) Get(key string) (json.RawMessage, error) {
|
func (b busStore) Get(key string) (json.RawMessage, error) {
|
||||||
e, err := stdio.State(StateReadings).Get(key)
|
e, err := stdio.State(b.state).Get(key)
|
||||||
if err != nil || e == nil {
|
if err != nil || e == nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return e.Value, nil
|
return e.Value, nil
|
||||||
}
|
}
|
||||||
func (busStore) Put(key string, v any) error {
|
func (b busStore) Put(key string, v any) error {
|
||||||
_, err := stdio.State(StateReadings).Put(key, v)
|
_, err := stdio.State(b.state).Put(key, v)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
func (busStore) Keys() ([]string, error) { return stdio.State(StateReadings).Keys() }
|
func (b busStore) Keys() ([]string, error) { return stdio.State(b.state).Keys() }
|
||||||
|
|
||||||
// Keeper measures the items of its node and keeps the readings.
|
// Keeper measures the items of its node and keeps the readings.
|
||||||
type Keeper struct {
|
type Keeper struct {
|
||||||
Node string
|
Node string
|
||||||
// Items is every data item the modules placed on this node declare, composed from the bus (frombus.DataItems).
|
// Items is every data item the modules placed on this node declare, as the node-engine composed them and its state
|
||||||
|
// data holds them (Watched.All, novox/hq ADR 0577 clause 2).
|
||||||
Items func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error)
|
Items func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error)
|
||||||
Measurer Measurer
|
Measurer Measurer
|
||||||
Store Store
|
Store Store
|
||||||
Say func(string, ...any)
|
// Rounds is the state rounds, where each round says how far it got (ADR 0577 clause 3); nil keeps none.
|
||||||
|
Rounds Store
|
||||||
|
Say func(string, ...any)
|
||||||
|
|
||||||
mu sync.Mutex // one round at a time
|
mu sync.Mutex // one round at a time
|
||||||
|
|
||||||
|
// pending are the modules whose items changed since they were last measured (Due), measured by Work.
|
||||||
|
pmu sync.Mutex
|
||||||
|
pending map[string]bool
|
||||||
|
wake chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SayEvery is how many items a round measures between two lines saying how far it got.
|
||||||
|
const SayEvery = 50
|
||||||
|
|
||||||
|
// RoundVersion is the version of a round's layout.
|
||||||
|
const RoundVersion = 1
|
||||||
|
|
||||||
|
// Round is the value of a node's key of the state rounds: when its last round started and ended, how many items it
|
||||||
|
// had to measure, how many it measured, wrote and kept, and each item that failed and why. Ended is nil while it runs;
|
||||||
|
// Error says why a round measured nothing at all.
|
||||||
|
type Round struct {
|
||||||
|
V int `json:"v"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Walks bool `json:"walks,omitempty"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Ended *time.Time `json:"ended,omitempty"`
|
||||||
|
Items int `json:"items"`
|
||||||
|
Measured int `json:"measured"`
|
||||||
|
Written int `json:"written"`
|
||||||
|
Kept int `json:"kept"`
|
||||||
|
Failed []FailedItem `json:"failed,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailedItem is one item a round could not measure or keep, `<module>.<item>`, and why.
|
||||||
|
type FailedItem struct {
|
||||||
|
Item string `json:"item"`
|
||||||
|
Why string `json:"why"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// putRound writes the round to this node's key; a round that cannot be written is said, and measures on.
|
||||||
|
func (k *Keeper) putRound(r Round) {
|
||||||
|
if k.Rounds == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := k.Rounds.Put(k.Node, r); err != nil {
|
||||||
|
k.Say("the round's outcome cannot be kept in the state %s: %v", StateRounds, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Round measures every item that is not a cache, one at a time, and writes the readings that changed. An item
|
// Round measures every item that is not a cache, one at a time, and writes the readings that changed. An item
|
||||||
// measured by a walk is walked only when walks is true or its last walk is older than WalkEvery: the hourly round
|
// measured by a walk is walked only when walks is true or its last walk is older than WalkEvery: the hourly round
|
||||||
// reads counters, and a walk runs at most once a day (ADR 0233 §22).
|
// reads counters, and a walk runs at most once a day (ADR 0233 §22).
|
||||||
|
//
|
||||||
|
// A round says how far it got (novox/hq ADR 0577 clause 3): it writes its outcome to this node's key of the state rounds
|
||||||
|
// when it starts, every SayEvery items and when it ends, and says the same in the journal. An item that fails is a
|
||||||
|
// failed item of the round, never its end.
|
||||||
func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err error) {
|
func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err error) {
|
||||||
k.mu.Lock()
|
k.mu.Lock()
|
||||||
defer k.mu.Unlock()
|
defer k.mu.Unlock()
|
||||||
|
r := Round{V: RoundVersion, Node: k.Node, Walks: walks, Started: k.Measurer.Now().UTC()}
|
||||||
|
end := func() {
|
||||||
|
at := k.Measurer.Now().UTC()
|
||||||
|
r.Ended = &at
|
||||||
|
k.putRound(r)
|
||||||
|
}
|
||||||
|
k.putRound(r)
|
||||||
|
k.Say("a round of measuring the data declared here starts (walks: %v)", walks)
|
||||||
items, err := k.Items(ctx)
|
items, err := k.Items(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
r.Error = err.Error()
|
||||||
|
end()
|
||||||
|
k.Say("the round measured nothing: %v", err)
|
||||||
return 0, 0, err
|
return 0, 0, err
|
||||||
}
|
}
|
||||||
modules := make([]string, 0, len(items))
|
modules := make([]string, 0, len(items))
|
||||||
for m := range items {
|
for m := range items {
|
||||||
modules = append(modules, m)
|
modules = append(modules, m)
|
||||||
|
for _, it := range items[m] {
|
||||||
|
if it.Class != ClassCache {
|
||||||
|
r.Items++
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
sort.Strings(modules)
|
sort.Strings(modules)
|
||||||
var failed []string
|
var failed []string
|
||||||
@@ -74,29 +141,133 @@ func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err
|
|||||||
if it.Class == ClassCache {
|
if it.Class == ClassCache {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
w, err := k.measureOne(ctx, it, walks)
|
w, said, err := k.measureOne(ctx, it, walks)
|
||||||
if err != nil {
|
r.Measured++
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
failed = append(failed, fmt.Sprintf("%s.%s: %v", module, it.ID, err))
|
failed = append(failed, fmt.Sprintf("%s.%s: %v", module, it.ID, err))
|
||||||
continue
|
r.Failed = append(r.Failed, FailedItem{Item: module + "." + it.ID, Why: err.Error()})
|
||||||
|
case said != "":
|
||||||
|
// Measured, and the measurement failed: its reading says so, and the round counts it failed.
|
||||||
|
r.Failed = append(r.Failed, FailedItem{Item: module + "." + it.ID, Why: said})
|
||||||
}
|
}
|
||||||
if w {
|
if err == nil && w {
|
||||||
written++
|
written++
|
||||||
} else {
|
} else if err == nil {
|
||||||
kept++
|
kept++
|
||||||
}
|
}
|
||||||
|
r.Written, r.Kept = written, kept
|
||||||
|
if r.Measured%SayEvery == 0 {
|
||||||
|
k.Say("the round has measured %d of %d items: %d written, %d unchanged, %d failed", r.Measured, r.Items,
|
||||||
|
written, kept, len(r.Failed))
|
||||||
|
k.putRound(r)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
retired, err := k.retireGone(items)
|
retired, err := k.retireGone(items)
|
||||||
written += retired
|
written += retired
|
||||||
|
r.Written = written
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failed = append(failed, err.Error())
|
failed = append(failed, err.Error())
|
||||||
|
r.Failed = append(r.Failed, FailedItem{Item: "the readings no longer declared", Why: err.Error()})
|
||||||
}
|
}
|
||||||
|
end()
|
||||||
|
k.Say("the round ended: %d of %d items measured, %d readings written, %d unchanged, %d failed", r.Measured, r.Items,
|
||||||
|
written, kept, len(r.Failed))
|
||||||
if len(failed) > 0 {
|
if len(failed) > 0 {
|
||||||
return written, kept, errors.New("readings not kept: " + strings.Join(failed, "; "))
|
return written, kept, errors.New("readings not kept: " + strings.Join(failed, "; "))
|
||||||
}
|
}
|
||||||
return written, kept, nil
|
return written, kept, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Due asks Work to measure a module's items: its key of the engine's state data changed (novox/hq ADR 0577 clause 2).
|
||||||
|
func (k *Keeper) Due(module string) {
|
||||||
|
k.pmu.Lock()
|
||||||
|
if k.pending == nil {
|
||||||
|
k.pending, k.wake = map[string]bool{}, make(chan struct{}, 1)
|
||||||
|
}
|
||||||
|
k.pending[module] = true
|
||||||
|
wake := k.wake
|
||||||
|
k.pmu.Unlock()
|
||||||
|
select {
|
||||||
|
case wake <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Work measures each module Due names, one at a time, until ctx ends.
|
||||||
|
func (k *Keeper) Work(ctx context.Context) {
|
||||||
|
k.pmu.Lock()
|
||||||
|
if k.pending == nil {
|
||||||
|
k.pending, k.wake = map[string]bool{}, make(chan struct{}, 1)
|
||||||
|
}
|
||||||
|
wake := k.wake
|
||||||
|
k.pmu.Unlock()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-wake:
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
k.pmu.Lock()
|
||||||
|
var module string
|
||||||
|
for m := range k.pending {
|
||||||
|
module = m
|
||||||
|
break
|
||||||
|
}
|
||||||
|
delete(k.pending, module)
|
||||||
|
k.pmu.Unlock()
|
||||||
|
if module == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
k.MeasureModule(ctx, module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeasureModule measures one module's items now, as its key of the engine's state data changed, and retires the
|
||||||
|
// readings of a module that no longer declares any here. A walk is not repeated within a day for an item that did not
|
||||||
|
// change (measureOne).
|
||||||
|
func (k *Keeper) MeasureModule(ctx context.Context, module string) {
|
||||||
|
k.mu.Lock()
|
||||||
|
defer k.mu.Unlock()
|
||||||
|
items, err := k.Items(ctx)
|
||||||
|
if err != nil {
|
||||||
|
k.Say("%s's data items changed and are not measured: %v", module, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(items[module]) == 0 {
|
||||||
|
n, err := k.retireGone(items)
|
||||||
|
if err != nil {
|
||||||
|
k.Say("%s declares no data item here any more, and its readings are not retired: %v", module, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
k.Say("%s declares no data item here any more: %d reading(s) retired, measured no more", module, n)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
written, kept := 0, 0
|
||||||
|
var failed []string
|
||||||
|
for _, it := range items[module] {
|
||||||
|
if it.Class == ClassCache {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
w, said, err := k.measureOne(ctx, it, false)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
failed = append(failed, it.ID+": "+err.Error())
|
||||||
|
case said != "":
|
||||||
|
failed = append(failed, it.ID+": "+said)
|
||||||
|
case w:
|
||||||
|
written++
|
||||||
|
default:
|
||||||
|
kept++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
k.Say("%s's data items changed and were measured: %d readings written, %d unchanged, %d failed%s", module, written,
|
||||||
|
kept, len(failed), map[bool]string{true: " (" + strings.Join(failed, "; ") + ")", false: ""}[len(failed) > 0])
|
||||||
|
}
|
||||||
|
|
||||||
// retireGone marks retired every reading of this node whose item the node no longer declares — its module was unassigned
|
// retireGone marks retired every reading of this node whose item the node no longer declares — its module was unassigned
|
||||||
// there, or is no longer pulled in — with when and why, and keeps it: the record of what the module left behind, until a
|
// there, or is no longer pulled in — with when and why, and keeps it: the record of what the module left behind, until a
|
||||||
// person deletes it (ADR 0233). It is cleared when the item is declared again (measureOne).
|
// person deletes it (ADR 0233). It is cleared when the item is declared again (measureOne).
|
||||||
@@ -135,12 +306,15 @@ func (k *Keeper) retireGone(declared map[string][]catalogue.PlacedDataItem) (int
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, error) {
|
// measureOne measures one item, unless it is a walk done within WalkEvery of an unchanged item, and writes its reading
|
||||||
|
// when it changed. It answers whether it wrote the reading, what the measurement said failed ("" when it did not, or
|
||||||
|
// when the item was not measured now), and an error when the reading could not be read or kept.
|
||||||
|
func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, string, error) {
|
||||||
key := health.ReadingKey(k.Node, it.Module, it.ID)
|
key := health.ReadingKey(k.Node, it.Module, it.ID)
|
||||||
var prev *health.Reading
|
var prev *health.Reading
|
||||||
raw, err := k.Store.Get(key)
|
raw, err := k.Store.Get(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, "", err
|
||||||
}
|
}
|
||||||
if len(raw) > 0 {
|
if len(raw) > 0 {
|
||||||
if p, err := health.DecodeReading(raw); err == nil {
|
if p, err := health.DecodeReading(raw); err == nil {
|
||||||
@@ -148,8 +322,9 @@ func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, wa
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
now := k.Measurer.Now()
|
now := k.Measurer.Now()
|
||||||
if prev != nil && prev.RetiredAt.IsZero() && it.Measure == catalogue.MeasureWalk && !walks && prev.MeasuredAt != nil && now.Sub(*prev.MeasuredAt) < WalkEvery {
|
if prev != nil && prev.RetiredAt.IsZero() && prev.PlacedDataItem == it && it.Measure == catalogue.MeasureWalk && !walks &&
|
||||||
return false, nil
|
prev.MeasuredAt != nil && now.Sub(*prev.MeasuredAt) < WalkEvery {
|
||||||
|
return false, "", nil
|
||||||
}
|
}
|
||||||
m := k.Measurer.Measure(ctx, it)
|
m := k.Measurer.Measure(ctx, it)
|
||||||
next, changed := health.NextReading(prev, k.Node, it, m, now)
|
next, changed := health.NextReading(prev, k.Node, it, m, now)
|
||||||
@@ -158,9 +333,9 @@ func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, wa
|
|||||||
next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true
|
next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true
|
||||||
}
|
}
|
||||||
if !changed {
|
if !changed {
|
||||||
return false, nil
|
return false, m.Error, nil
|
||||||
}
|
}
|
||||||
return true, k.Store.Put(key, next)
|
return true, m.Error, k.Store.Put(key, next)
|
||||||
}
|
}
|
||||||
|
|
||||||
// MeasureNow measures one item now, walk or not, and writes its reading when it changed.
|
// MeasureNow measures one item now, walk or not, and writes its reading when it changed.
|
||||||
@@ -178,7 +353,7 @@ func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Re
|
|||||||
if it.Class == ClassCache {
|
if it.Class == ClassCache {
|
||||||
return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
|
return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item)
|
||||||
}
|
}
|
||||||
w, err := k.measureOne(ctx, it, true)
|
w, _, err := k.measureOne(ctx, it, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return health.Reading{}, false, err
|
return health.Reading{}, false, err
|
||||||
}
|
}
|
||||||
@@ -201,12 +376,7 @@ func (k *Keeper) Loop(ctx context.Context, every time.Duration, first time.Durat
|
|||||||
case <-time.After(first):
|
case <-time.After(first):
|
||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
w, kept, err := k.Round(ctx, false)
|
_, _, _ = k.Round(ctx, false) // it says its outcome itself, in the journal and its key of the state rounds
|
||||||
if err != nil {
|
|
||||||
k.Say("measuring the data declared here: %v", err)
|
|
||||||
} else {
|
|
||||||
k.Say("measured the data declared here: %d readings changed, %d unchanged", w, kept)
|
|
||||||
}
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -4,42 +4,52 @@
|
|||||||
// filesystems' space and the processes doing the I/O — and changes nothing of the machine.
|
// filesystems' space and the processes doing the I/O — and changes nothing of the machine.
|
||||||
//
|
//
|
||||||
// It also measures the data items the modules of its node declare (ADR 0233) and keeps each reading and the item's
|
// It also measures the data items the modules of its node declare (ADR 0233) and keeps each reading and the item's
|
||||||
// seven-day peak as its own state `readings` (ADR 0571 clause 3), composed from the bus (no file).
|
// seven-day peak as its own state `readings` (ADR 0571 clause 3). It learns the items from the node-engine's state data,
|
||||||
|
// watching its own node's keys, and composes nothing (ADR 0577 clause 2); each round says how far it got in its state
|
||||||
|
// `rounds` (clause 3).
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/compose/fromruntime"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) }
|
func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) }
|
||||||
|
|
||||||
|
// Kept is every state disk-load keeps, as it announces them: the readings, readable by restic, nfs-server, mounts and
|
||||||
|
// the self-check (their manifests' `reads`; checks/readable holds it), and the rounds, its own tools' alone.
|
||||||
|
var Kept = []announce.State{{Name: StateReadings, Readable: true}, {Name: StateRounds}}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
// Readable: restic, nfs-server, mounts and the self-check read it (their manifests' `reads`); checks/readable holds it.
|
if err := stdio.Keeps(Kept...); err != nil {
|
||||||
if err := stdio.Keeps(announce.State{Name: StateReadings, Readable: true}); err != nil {
|
|
||||||
logf("%v", err)
|
logf("%v", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
node := os.Getenv("MESH_NODE")
|
node := os.Getenv("MESH_NODE")
|
||||||
k := &Keeper{Node: node, Store: busStore{}, Say: logf, Measurer: Measurer{Run: RootRunner, Now: time.Now},
|
watched := &Watched{Node: node}
|
||||||
Items: func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error) {
|
k := &Keeper{Node: node, Store: busStore{StateReadings}, Rounds: busStore{StateRounds}, Say: logf,
|
||||||
return placedItems(ctx, node)
|
Measurer: Measurer{Run: RootRunner, Now: time.Now}, Items: watched.All}
|
||||||
}}
|
|
||||||
if node != "" {
|
if node != "" {
|
||||||
every := time.Hour
|
every := time.Hour
|
||||||
if d, err := time.ParseDuration(os.Getenv("MESH_DISK_MEASURE_EVERY")); err == nil && d >= time.Minute {
|
if d, err := time.ParseDuration(os.Getenv("MESH_DISK_MEASURE_EVERY")); err == nil && d >= time.Minute {
|
||||||
every = d
|
every = d
|
||||||
}
|
}
|
||||||
go k.Loop(context.Background(), every, 3*time.Minute)
|
ctx := context.Background()
|
||||||
|
go k.Work(ctx)
|
||||||
|
go func() {
|
||||||
|
<-stdio.Served() // the runtime's channel first: a watch is asked through it
|
||||||
|
if err := watched.Follow(k); err != nil {
|
||||||
|
logf("this machine's keys of the node-engine's data items cannot be watched, and nothing is measured: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logf("watching this machine's keys of the node-engine's data items")
|
||||||
|
}()
|
||||||
|
go k.Loop(ctx, every, 3*time.Minute)
|
||||||
}
|
}
|
||||||
tools := append(Tools(Here()), ReadingTools(k)...)
|
tools := append(Tools(Here()), ReadingTools(k)...)
|
||||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): disk-load.
|
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): disk-load.
|
||||||
@@ -48,15 +58,3 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// placedItems is every data item the modules placed on node declare, composed from the bus as the node composes itself.
|
|
||||||
func placedItems(ctx context.Context, node string) (map[string][]catalogue.PlacedDataItem, error) {
|
|
||||||
items, unplaced, err := frombus.DataItems(ctx, fromruntime.Reader(), frombus.Mesh{}, frombus.Scope{Node: node})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(unplaced) > 0 {
|
|
||||||
logf("data items left out as not placeable: %s", strings.Join(unplaced, "; "))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -8,5 +8,9 @@ import "git.novox.be/novox/mesh-sdk/go/health"
|
|||||||
// StateReadings is the state disk-load keeps.
|
// StateReadings is the state disk-load keeps.
|
||||||
const StateReadings = health.ReadingsState
|
const StateReadings = health.ReadingsState
|
||||||
|
|
||||||
|
// StateRounds is its state of each node's last round, key `<node>` (novox/hq ADR 0577 clause 3): apart from the readings,
|
||||||
|
// so a reader of the readings never meets a round.
|
||||||
|
const StateRounds = "rounds"
|
||||||
|
|
||||||
// ClassCache is the class of an item that is listed and never measured.
|
// ClassCache is the class of an item that is listed and never measured.
|
||||||
const ClassCache = "cache"
|
const ClassCache = "cache"
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -12,15 +13,26 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/health"
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
)
|
)
|
||||||
|
|
||||||
type memStore struct{ m map[string]json.RawMessage }
|
type memStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
m map[string]json.RawMessage
|
||||||
|
}
|
||||||
|
|
||||||
func (s *memStore) Get(k string) (json.RawMessage, error) { return s.m[k], nil }
|
func (s *memStore) Get(k string) (json.RawMessage, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.m[k], nil
|
||||||
|
}
|
||||||
func (s *memStore) Put(k string, v any) error {
|
func (s *memStore) Put(k string, v any) error {
|
||||||
raw, _ := json.Marshal(v)
|
raw, _ := json.Marshal(v)
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
s.m[k] = raw
|
s.m[k] = raw
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (s *memStore) Keys() ([]string, error) {
|
func (s *memStore) Keys() ([]string, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
var out []string
|
var out []string
|
||||||
for k := range s.m {
|
for k := range s.m {
|
||||||
out = append(out, k)
|
out = append(out, k)
|
||||||
|
|||||||
@@ -2,9 +2,11 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
"git.novox.be/novox/mesh-sdk/go/health"
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
@@ -68,6 +70,42 @@ func ReadingTools(k *Keeper) []stdio.Tool {
|
|||||||
"cap": "none declared: the bus's own limit on a state applies"}}, nil
|
"cap": "none declared: the bus's own limit on a state applies"}}, nil
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Name: "disk_round",
|
||||||
|
Description: "How far this machine's last round of measuring its data items got (novox/hq ADR 0577 clause 3), from " +
|
||||||
|
"disk-load's state `rounds`, key `<node>`: when it started and ended (no end while it runs), how many items it had, " +
|
||||||
|
"measured, wrote and kept unchanged, each item that failed and why, and how old the round is; `node` reads " +
|
||||||
|
"another machine's key. A round that never ends, or fails every item, shows here. (r)",
|
||||||
|
Input: map[string]any{
|
||||||
|
"node": map[string]any{"type": "string", "description": "the machine whose last round to read; this one by default"},
|
||||||
|
},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
node := text(args, "node")
|
||||||
|
if node == "" {
|
||||||
|
node = k.Node
|
||||||
|
}
|
||||||
|
if k.Rounds == nil {
|
||||||
|
return nil, fmt.Errorf("this disk-load keeps no rounds")
|
||||||
|
}
|
||||||
|
raw, err := k.Rounds.Get(node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return nil, fmt.Errorf("no round of %s is kept yet: its first round starts a few minutes after disk-load starts", node)
|
||||||
|
}
|
||||||
|
var r Round
|
||||||
|
if err := json.Unmarshal(raw, &r); err != nil {
|
||||||
|
return nil, fmt.Errorf("the round of %s is not readable: %w", node, err)
|
||||||
|
}
|
||||||
|
state := "ended"
|
||||||
|
if r.Ended == nil {
|
||||||
|
state = "running"
|
||||||
|
}
|
||||||
|
return map[string]any{"round": r, "state": state,
|
||||||
|
"age": k.Measurer.Now().UTC().Sub(r.Started).Round(time.Second).String()}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
Name: "disk_measure",
|
Name: "disk_measure",
|
||||||
Description: "Measure one data item of this machine now, whatever its method (a walk too), and write its reading when it " +
|
Description: "Measure one data item of this machine now, whatever its method (a walk too), and write its reading when it " +
|
||||||
|
|||||||
@@ -2,13 +2,21 @@ module diskload
|
|||||||
|
|
||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031213356-5ae69df8ccde
|
require (
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031230538-c28c645ca406
|
||||||
|
github.com/nats-io/nats.go v1.54.0
|
||||||
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||||
|
github.com/google/go-tpm v0.9.8 // indirect
|
||||||
github.com/klauspost/compress v1.20.0 // indirect
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
github.com/nats-io/nats.go v1.54.0 // indirect
|
github.com/minio/highwayhash v1.0.4 // indirect
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17 // indirect
|
||||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
github.com/nats-io/nuid v1.0.1 // indirect
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
golang.org/x/crypto v0.57.0 // indirect
|
golang.org/x/crypto v0.57.0 // indirect
|
||||||
golang.org/x/sys v0.48.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
|
golang.org/x/time v0.15.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,7 +1,17 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031213356-5ae69df8ccde h1:RyISctWEfdaykq69I3CYKOZ4N1wUx6PpiPbql4kpXkg=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031230538-c28c645ca406 h1:Z4RcH5Ft4OAAG3F538NXIHP9F3pkyk3j0LtP9mqu1qQ=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031213356-5ae69df8ccde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031230538-c28c645ca406/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||||
|
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||||
|
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||||
@@ -10,5 +20,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
|||||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
|
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
|
|||||||
@@ -9,6 +9,7 @@
|
|||||||
"disk_devices",
|
"disk_devices",
|
||||||
"cpu_top",
|
"cpu_top",
|
||||||
"disk_readings",
|
"disk_readings",
|
||||||
|
"disk_round",
|
||||||
"disk_measure"
|
"disk_measure"
|
||||||
],
|
],
|
||||||
"build": {
|
"build": {
|
||||||
@@ -27,14 +28,13 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"state": [
|
"state": [
|
||||||
"readings"
|
"readings",
|
||||||
|
{
|
||||||
|
"name": "rounds",
|
||||||
|
"per-machine": true
|
||||||
|
}
|
||||||
],
|
],
|
||||||
"reads": [
|
"reads": [
|
||||||
"mesh-config.resolved",
|
"mesh-host.data"
|
||||||
"mesh-delivery-assigner.placement",
|
|
||||||
"mesh-delivery-builder.builds",
|
|
||||||
"mesh-delivery-indexer.manifests",
|
|
||||||
"mesh-delivery-indexer.modules",
|
|
||||||
"node-engine.have"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user