A-112: the bus module's snapshots-cap and snapshot-keep-days are config values in a settings file the keeper reads at its start
Builds: design 47 G §633, §681, lead answer A-112 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
@@ -227,17 +227,10 @@ func percent(getenv func(string) string, name string, fallback int64) int64 {
|
|||||||
return fallback
|
return fallback
|
||||||
}
|
}
|
||||||
|
|
||||||
// number reads a positive whole number from the environment, or the fallback.
|
|
||||||
func number(getenv func(string) string, name string, fallback int64) int64 {
|
|
||||||
if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 {
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
return fallback
|
|
||||||
}
|
|
||||||
|
|
||||||
// runKeeper is the keeper's process: it returns when ctx ends, or when it has no credential.
|
// runKeeper is the keeper's process: it returns when ctx ends, or when it has no credential.
|
||||||
func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||||
logger := log.New(logOut, "nats-keeper: ", log.LstdFlags)
|
logger := log.New(logOut, "nats-keeper: ", log.LstdFlags)
|
||||||
|
settings := readSettings(getenv("MESH_NATS_SNAPSHOTS_DIR") + "/settings.json")
|
||||||
account, err := process.ReadBrokerAccount(getenv)
|
account, err := process.ReadBrokerAccount(getenv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -276,9 +269,10 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer
|
|||||||
Warrant: WarrantFrom(conn.Ask), Finding: writer,
|
Warrant: WarrantFrom(conn.Ask), Finding: writer,
|
||||||
Logf: logger.Printf,
|
Logf: logger.Printf,
|
||||||
Raise: percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70),
|
Raise: percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70),
|
||||||
// The settings snapshots-cap (20 GiB) and snapshot-keep-days (90) of the manifest (design 47 G §681).
|
// The settings snapshots-cap (20 GiB) and snapshot-keep-days (90) of the manifest, read from the file it declares
|
||||||
SnapshotsCap: number(getenv, "MESH_NATS_SNAPSHOTS_CAP", keeper.DefaultSnapshotsCap),
|
// at the start, its defaults when the file is missing (design 47 G §681, A-112).
|
||||||
SnapshotKeepDays: int(number(getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)),
|
SnapshotsCap: settings.SnapshotsCap,
|
||||||
|
SnapshotKeepDays: settings.SnapshotKeepDays,
|
||||||
}
|
}
|
||||||
// The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved.
|
// The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved.
|
||||||
stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf)))
|
stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf)))
|
||||||
@@ -454,3 +448,30 @@ func requestsOf(c *process.Conn) func(context.Context, func()) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// keeperSettings are the keeper's settings its manifest declares as a file (design 47 G A-112): the content of
|
||||||
|
// `${config:<key>}`, read at its start.
|
||||||
|
type keeperSettings struct {
|
||||||
|
SnapshotsCap int64 `json:"snapshots-cap"`
|
||||||
|
SnapshotKeepDays int `json:"snapshot-keep-days"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSettings reads the settings file at path; a file that is missing, or a value that is not above 0, gives its default.
|
||||||
|
func readSettings(path string) keeperSettings {
|
||||||
|
s := keeperSettings{SnapshotsCap: keeper.DefaultSnapshotsCap, SnapshotKeepDays: keeper.DefaultSnapshotKeepDays}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
var f keeperSettings
|
||||||
|
if json.Unmarshal(raw, &f) != nil {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
if f.SnapshotsCap > 0 {
|
||||||
|
s.SnapshotsCap = f.SnapshotsCap
|
||||||
|
}
|
||||||
|
if f.SnapshotKeepDays > 0 {
|
||||||
|
s.SnapshotKeepDays = f.SnapshotKeepDays
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
@@ -196,7 +197,19 @@ func allTools() []stdio.Tool {
|
|||||||
out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz))
|
out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz))
|
||||||
out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...)
|
out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...)
|
||||||
rs := RequestsInState()
|
rs := RequestsInState()
|
||||||
out = append(out, snapshotListTool(rs, int(number(os.Getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)))...)
|
out = append(out, snapshotListTool(rs, bundleKeepDays())...)
|
||||||
out = append(out, plansTool(PlansInState))
|
out = append(out, plansTool(PlansInState))
|
||||||
return append(out, requestTools(rs)...)
|
return append(out, requestTools(rs)...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bundleKeepDays is the setting snapshot-keep-days as this bundle's configuration holds it (design 47 G A-112), its
|
||||||
|
// default when the runtime gives none.
|
||||||
|
func bundleKeepDays() int {
|
||||||
|
if raw, err := stdio.Config("snapshot-keep-days"); err == nil {
|
||||||
|
var n int
|
||||||
|
if json.Unmarshal(raw, &n) == nil && n > 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return keeper.DefaultSnapshotKeepDays
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"nats-tools/internal/keeper"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The shape ADR 0480 clauses 1, 2, 3 and 5 give the bus's manifest: the upstream image by digest, the mesh's two
|
// The shape ADR 0480 clauses 1, 2, 3 and 5 give the bus's manifest: the upstream image by digest, the mesh's two
|
||||||
@@ -315,3 +317,50 @@ func TestTheKeepersProcessServesTheVerbsItAnswers(t *testing.T) {
|
|||||||
t.Errorf("the keeper process serves %v; its code answers %v", served, want)
|
t.Errorf("the keeper process serves %v; its code answers %v", served, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The keeper's snapshot settings are config values of the manifest, written into a file the keeper reads at its start,
|
||||||
|
// never environment (design 47 G A-112, §681): a missing file, or a value not above 0, gives the default.
|
||||||
|
func TestTheKeepersSnapshotSettingsAreAFileItsManifestDeclares(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Config map[string]struct{ Default float64 } `json:"config"`
|
||||||
|
Resources []struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Path string `json:"path"`
|
||||||
|
Owner string `json:"owner"`
|
||||||
|
TakesConfig []string `json:"takes-config"`
|
||||||
|
Env map[string]string `json:"env"`
|
||||||
|
} `json:"resources"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.Config["snapshots-cap"].Default != 21474836480 || m.Config["snapshot-keep-days"].Default != 90 {
|
||||||
|
t.Errorf("config %v: snapshots-cap 20 GiB and snapshot-keep-days 90 are the defaults", m.Config)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r.ID == "keeper-settings" {
|
||||||
|
found = r.Path == "${dir:keeper-state}/settings.json" && r.Owner == "nats-keeper" && slices.Equal(r.TakesConfig, []string{"snapshots-cap", "snapshot-keep-days"})
|
||||||
|
}
|
||||||
|
for k := range r.Env {
|
||||||
|
if strings.Contains(k, "SNAPSHOTS_CAP") || strings.Contains(k, "SNAPSHOT_KEEP_DAYS") {
|
||||||
|
t.Errorf("%s still sets %s in the environment", r.ID, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("the keeper has no settings file taking snapshots-cap and snapshot-keep-days")
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
if got := readSettings(dir + "/none.json"); got.SnapshotsCap != keeper.DefaultSnapshotsCap || got.SnapshotKeepDays != keeper.DefaultSnapshotKeepDays {
|
||||||
|
t.Errorf("a missing file gives %+v", got)
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(dir+"/s.json", []byte(`{"snapshots-cap": 1000, "snapshot-keep-days": 0}`), 0o600)
|
||||||
|
if got := readSettings(dir + "/s.json"); got.SnapshotsCap != 1000 || got.SnapshotKeepDays != keeper.DefaultSnapshotKeepDays {
|
||||||
|
t.Errorf("the file gives %+v: the cap it names, the day count's default for 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"snapshots-cap": 21474836480,
|
||||||
|
"snapshot-keep-days": 90
|
||||||
|
}
|
||||||
@@ -71,6 +71,20 @@
|
|||||||
"uses": [
|
"uses": [
|
||||||
"operator-channel"
|
"operator-channel"
|
||||||
],
|
],
|
||||||
|
"config": {
|
||||||
|
"snapshots-cap": {
|
||||||
|
"type": "number",
|
||||||
|
"description": "How many bytes the keeper's snapshots directory may hold (design 47 G §681): past 80 % of it the keeper tells a finding, and it never takes a snapshot past it. The keeper reads it at its start from its settings file.",
|
||||||
|
"minimum": 1,
|
||||||
|
"default": 21474836480
|
||||||
|
},
|
||||||
|
"snapshot-keep-days": {
|
||||||
|
"type": "number",
|
||||||
|
"description": "How many days a snapshot the keeper took before a drop or recreate, and its record, are kept (design 47 G §681, A-112). The keeper reads it at its start from its settings file; the tool nats_snapshots reads it as its configuration.",
|
||||||
|
"minimum": 1,
|
||||||
|
"default": 90
|
||||||
|
}
|
||||||
|
},
|
||||||
"streams": [
|
"streams": [
|
||||||
{
|
{
|
||||||
"name": "broker-events",
|
"name": "broker-events",
|
||||||
@@ -236,6 +250,20 @@
|
|||||||
"place": "mesh",
|
"place": "mesh",
|
||||||
"unknown": "program"
|
"unknown": "program"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "keeper-settings",
|
||||||
|
"trusted": false,
|
||||||
|
"type": "file",
|
||||||
|
"path": "${dir:keeper-state}/settings.json",
|
||||||
|
"mode": "0600",
|
||||||
|
"owner": "nats-keeper",
|
||||||
|
"takes-config": [
|
||||||
|
"snapshots-cap",
|
||||||
|
"snapshot-keep-days"
|
||||||
|
],
|
||||||
|
"merge": "json",
|
||||||
|
"template": "files/settings.json"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "server-conf",
|
"id": "server-conf",
|
||||||
"type": "file",
|
"type": "file",
|
||||||
@@ -278,8 +306,6 @@
|
|||||||
"MESH_BROKER_FILE": "${dir:keeper-state}/broker-keeper",
|
"MESH_BROKER_FILE": "${dir:keeper-state}/broker-keeper",
|
||||||
"MESH_NATS_FULL_RAISE": "75",
|
"MESH_NATS_FULL_RAISE": "75",
|
||||||
"MESH_NATS_FULL_CLEAR": "70",
|
"MESH_NATS_FULL_CLEAR": "70",
|
||||||
"MESH_NATS_SNAPSHOTS_CAP": "21474836480",
|
|
||||||
"MESH_NATS_SNAPSHOT_KEEP_DAYS": "90",
|
|
||||||
"MESH_NATS_SNAPSHOTS_DIR": "${dir:keeper-state}"
|
"MESH_NATS_SNAPSHOTS_DIR": "${dir:keeper-state}"
|
||||||
},
|
},
|
||||||
"serves": [
|
"serves": [
|
||||||
@@ -389,7 +415,6 @@
|
|||||||
"MESH_NATS_CONTAINER": "mesh-broker-nats",
|
"MESH_NATS_CONTAINER": "mesh-broker-nats",
|
||||||
"MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf",
|
"MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf",
|
||||||
"MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}",
|
"MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}",
|
||||||
"MESH_NATS_SNAPSHOT_KEEP_DAYS": "90",
|
|
||||||
"NATS_TOOLS_CREDENTIAL": "${dir:tools-secret}/broker-nats-tools"
|
"NATS_TOOLS_CREDENTIAL": "${dir:tools-secret}/broker-nats-tools"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in New Issue
Block a user