Merge branch 'feat/47-G-fix11-c3c' into feat/47-G-join11

# Conflicts:
#	modules/nats/cmd/nats-tools/keep.go
#	modules/nats/go.mod
This commit is contained in:
2026-11-12 10:26:43 +01:00
43 changed files with 1591 additions and 370 deletions
+4 -5
View File
@@ -60,13 +60,12 @@ func waitBlocksStop(session string, w Wait, now time.Time) (bool, string) {
"{\"op\":\"said\",\"session\":%q,\"by\":%q,\"id\":%q}.", w.ID, w.What, session, w.By, w.ID)
}
// watcherAdvice is what a session is told to do to follow a wait (ADR 0489 §3, review round 1 item 3): a
// family is a candidate for mesh_wait only when the tool runner is granted it (ADR 0367 §2), which this hook
// cannot know, so it is offered conditionally; a watcher that rereads the read address always works.
// watcherAdvice is what a session is told to do to follow a wait (ADR 0489 §3, design 47 E §20): a wait with an event is
// followed by mesh_wait on it, rereading what it reads after every answer, or by a watcher that does; with no event, by a
// watcher that rereads the read address. The advice puts no condition on a grant.
func watcherAdvice(w Wait) string {
if w.Family != "" {
return fmt.Sprintf("mesh_wait on %s if this session's tool runner is granted it, otherwise start a watcher "+
"that rereads %s", w.Family, w.Read.Address)
return fmt.Sprintf("mesh_wait on %s and reread %s after every answer, or start a watcher that does", w.Family, w.Read.Address)
}
return fmt.Sprintf("start a watcher that rereads %s", w.Read.Address)
}
@@ -300,15 +300,15 @@ func TestStopAllowsWhenTheRefusalCounterCannotBeWritten(t *testing.T) {
}
}
// Review round 1, item 3 (1743/AC1, §3 with ADR 0367 §2): a family is offered only conditionally (the tool
// runner may not be granted it); without one, the advice is only to start a watcher.
func TestWaitBlocksStopAdvisesMeshWaitOnlyConditionallyOtherwiseAWatcher(t *testing.T) {
// Design 47 E §20: a wait with an event is followed by mesh_wait on it and a reread after every answer, or a watcher
// that does, with no condition on a grant; without an event, the advice is only to start a watcher.
func TestWaitBlocksStopAdvisesMeshWaitOnItsEventOtherwiseAWatcher(t *testing.T) {
withFamily := openWait("w-f", 10)
withFamily.Read = WaitRead{Address: "mesh-delivery.recheck"}
withFamily.Family = "mesh-delivery.decided"
_, reason := waitBlocksStop("sess-f", withFamily, stopT0.Add(time.Minute))
if !strings.Contains(reason, "mesh_wait on mesh-delivery.decided if this session's tool runner is granted it") ||
!strings.Contains(reason, "otherwise start a watcher that rereads mesh-delivery.recheck") {
if !strings.Contains(reason, "mesh_wait on mesh-delivery.decided and reread mesh-delivery.recheck after every answer, or start a watcher that does") ||
strings.Contains(reason, "granted") {
t.Fatalf("%s", reason)
}
+14
View File
@@ -794,6 +794,12 @@ export class GiteaClient {
{ method: "PATCH", body: JSON.stringify(body) });
}
/** Remove the protection rule named rule (gitea_branch_protection_loosen, design 46 approval mark §21). As this
* client's account, which the forge refuses where it is no administrator. */
async deleteBranchProtection(owner: string, repo: string, rule: string): Promise<void> {
await this.request(`${repoPath(owner, repo)}/branch_protections/${pathSegments("branch", rule)}`, { method: "DELETE" });
}
async createPullRequest(
owner: string,
repo: string,
@@ -1097,6 +1103,14 @@ export class GiteaAdmin {
return res.body as BranchProtection;
}
/** Remove the protection rule named rule, as the forge's administrator (gitea_branch_protection_loosen, design 46
* approval mark §21): the forge answers 204. */
async deleteBranchProtection(owner: string, repo: string, rule: string): Promise<void> {
const path = `${repoPath(owner, repo)}/branch_protections/${pathSegments("branch", rule)}`;
const res = await this.request(path, { method: "DELETE" });
if (res.status !== 204 && res.status !== 200) GiteaAdmin.fail(path, res);
}
/** A repository as the forge answers it to its administrator: every setting (gitea_repo_settings_set_full). */
async repoSettings(owner: string, repo: string): Promise<Record<string, unknown>> {
const path = repoPath(owner, repo);
+24
View File
@@ -24,6 +24,10 @@ export const SEEN = "seen.";
/** The template state's cap (design 47 *The runner's kept data*): the fill is shown against it. */
export const CAP_BYTES = 64 * 1024 * 1024;
/** How long a `seen` key is kept after its `step-ended`, and how many go on one event (§581). */
export const SEEN_DAYS = 30;
export const SEEN_MOST_PER_EVENT = 100;
/** What a `seen` key holds: a commit's fingerprints at one module. */
export interface Seen {
v: 1;
@@ -79,6 +83,11 @@ export class FailedSteps {
* a kept failure's step clears it, and nothing else does.
*/
stepEnded(ev: StepEndedEvent): Promise<void> {
// The `seen` keys past their 30 days are judged on each step end, with no timer (§581).
return this.record(ev).then(() => this.expireSeen(new Date(Date.parse(String(ev?.at ?? "")) || Date.now())).then(() => {}));
}
private record(ev: StepEndedEvent): Promise<void> {
const r = ev?.result;
const fingerprint = r?.fingerprint || ev?.fp || "";
if (!r || !SHA.test(String(ev.commit ?? "")) || !fingerprint || !ev.module) return Promise.resolve();
@@ -163,6 +172,21 @@ export class FailedSteps {
});
}
/**
* Deletes the `seen` keys whose `at` is more than 30 days before `now`, the oldest first, at most `most` of them (§581): run on
* each `step-ended`, at the start, and by the tool `git_seen_expire`. A key with no readable `at` is kept. Answers the keys deleted.
*/
expireSeen(now: Date = new Date(), most = SEEN_MOST_PER_EVENT): Promise<string[]> {
return this.turn(async () => {
const { seen } = await this.all();
const limit = now.getTime() - SEEN_DAYS * 24 * 3600 * 1000;
const old = [...seen].filter(([, s]) => Date.parse(s.at) < limit).sort(([, a], [, b]) => Date.parse(a.at) - Date.parse(b.at)).slice(0, most);
for (const [k] of old) await this.st.delete(k);
if (old.length) this.log(`${old.length} seen key(s) past ${SEEN_DAYS} days deleted`);
return old.map(([k]) => k);
});
}
/** Drops one held key, a failure by its key or a `seen` key; false when it is not there. */
drop(key: string): Promise<boolean> {
return this.turn(async () => {
+3
View File
@@ -402,6 +402,9 @@ const heldHeads = new Heads();
let reloadHeads: () => Promise<void> = async () => {};
const failedSteps = new FailedSteps(state<unknown>("failed-steps"), (line) => console.log(`[gitea] ${line}`), (commit) => heldHeads.has(commit));
// The `seen` keys past their 30 days are deleted at the start, as on each step-ended (design 47 §581).
void failedSteps.expireSeen().catch((err) => console.log(`[gitea] the seen keys past 30 days could not be deleted at the start: ${err instanceof Error ? err.message : String(err)}`));
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). The seat git's verbs are
// served here, beside the one writer of a head's statuses: the seat's status takes its turn on the head's chain, after the
// announcement's pending and a trunk's move, so a recheck's pending never lands over a later verdict (issue 456).
+6
View File
@@ -162,6 +162,12 @@
]
}
],
"uses": [
"operator-channel"
],
"needs-approval": [
"gitea_branch_protection_loosen"
],
"data": {
"own": [
{
+12 -5
View File
@@ -218,16 +218,23 @@ export function trunksOf(): { names: string[]; from: string } {
return { names: [MAIN_BRANCH], from: TRUNK_FROM };
}
/** The tool that alone removes a trunk's protection or allows a force-push onto a trunk, marked in the manifest's
* `needs-approval`, so it runs only on the operator's Approve of that one call (design 46 approval mark §21). */
export const LOOSEN_TOOL = "gitea_branch_protection_loosen";
/** The two acts LOOSEN_TOOL performs (design 46 approval mark §21). */
export const LOOSEN_ACTS = ["remove-protection", "allow-force-push"] as const;
/** Why a call is the operator's, or null: it lets a force-push onto a rule covering a trunk (the operator, 2026-10-28
* 16:4x: a call waits for their Approve only if it removes branch protection or allows a force-push to a trunk). No
* tool of this module can ask for that Approve yet, so such a call is refused with nothing sent. */
* 16:4x: a call waits for their Approve only if it removes branch protection or allows a force-push to a trunk). Those
* two acts are LOOSEN_TOOL's alone (design 46 approval mark §21), so such a call is refused with nothing sent, naming it. */
export function forcePushToTrunk(rule: string, body: Record<string, unknown>, trunks: string[]): string | null {
const covered = trunks.filter((t) => ruleCovers(rule, t));
if (!covered.length) return null;
const opens = body.enable_force_push === true || body.force_push_allowlist_deploy_keys === true ||
(["force_push_allowlist_usernames", "force_push_allowlist_teams"] as const).some((k) => Array.isArray(body[k]) && (body[k] as unknown[]).length > 0);
if (!opens) return null;
return `the rule ${rule} covers ${covered.join(", ")}, and this call lets a force-push onto it: a force-push to a trunk is ` +
`the operator's Approve (the operator, 2026-10-28), and no tool of this module can ask it yet, so it is the operator's ` +
`at the forge. Nothing was sent`;
return `the rule ${rule} covers ${covered.join(", ")}, and this call lets a force-push onto it: removing a trunk's protection ` +
`or allowing a force-push onto a trunk is ${LOOSEN_TOOL}'s alone, which runs on the operator's Approve of that one call. ` +
`Nothing was sent`;
}
+22
View File
@@ -96,3 +96,25 @@ test("git_failed_steps shows what is held and its fill, and git_failed_steps_dro
assert.deepEqual([done.dropped, acts.length, acts[0].verb], [true, 1, "git_failed_steps_drop"]);
assert.equal(await f.judge(C2), null);
});
test("a seen key past 30 days is deleted at the next step-ended, a younger one kept, at most 100 oldest first, and git_seen_expire runs the same now (§581)", async () => {
const { failedStepsTools } = await import("../tools/failedsteps.js");
const { runAs } = await import("@novox/mesh-sdk/callers");
const { st, kv } = memory() as any;
const f = new FailedSteps(st as any);
// The step-ended is at 2026-11-11T12:00:00Z, so 30 days before it is 2026-10-12.
const now = new Date("2026-11-11T12:00:00Z");
const old = (n: number) => new Date(Date.parse("2026-10-01T00:00:00Z") - n * 60000).toISOString();
await st.put(seenKey(C1, "old"), { v: 1, commit: C1, module: "old", fingerprints: ["x"], at: old(0) });
await st.put(seenKey(C2, "young"), { v: 1, commit: C2, module: "young", fingerprints: ["x"], at: "2026-10-20T00:00:00Z" });
await f.stepEnded(ended(C3, "passed"));
assert.equal(kv.has(seenKey(C1, "old")), false, "past 30 days: deleted at the next step-ended");
assert.equal(kv.has(seenKey(C2, "young")), true, "younger: kept");
assert.equal(kv.has(seenKey(C3, "nats")), true, "the step's own key is new");
for (let i = 0; i < 101; i++) await st.put(seenKey(C1, `m${String(i).padStart(3, "0")}`), { v: 1, commit: C1, module: `m${i}`, fingerprints: ["x"], at: old(i) });
assert.equal((await f.expireSeen(now)).length, 100, "at most 100 for each event");
const acts: any[] = [];
const tool = failedStepsTools(f, async (a) => { acts.push(a); }).find((t) => t.name === "git_seen_expire")!;
const done: any = await runAs({ "mesh/caller": "operator" }, {}, () => tool.run({ why: "stale" }));
assert.deepEqual([done.deleted.length, acts[0].verb], [1, "git_seen_expire"]);
});
@@ -79,6 +79,12 @@ const server = createServer((req, res) => {
giteaEdit(r, body);
return send(200, r);
}
if (req.method === "DELETE" && m[1]) {
const i = rules.findIndex((x) => x.rule_name === decodeURIComponent(m[1]));
if (i < 0) return send(404, { message: "no rule" });
rules.splice(i, 1);
res.writeHead(204); return res.end();
}
if (req.method === "POST" && !m[1]) { const r = { rule_name: body.rule_name }; giteaEdit(r, body); rules.push(r); return send(201, r); }
send(405, { message: "no" });
});
@@ -132,6 +138,39 @@ test("an unknown field, a field of the wrong kind, no field, no reason or a logi
});
// judgedRule's cases (mesh-control cmd/mesh-controller/build_source.go), each in its own words.
// Design 46 approval mark §21: a trunk's protection is loosened only by gitea_branch_protection_loosen, marked in the
// manifest's needs-approval; gitea_branch_protection_set_full refuses a force-push onto a trunk, naming it.
const loosen = (args: Record<string, unknown>) =>
tools.find((t) => t.name === "gitea_branch_protection_loosen")!.run({ owner: "novox", repo: "de-spiegel", branch: "main", why: "L2", ...args });
test("set_full refuses a force-push onto a trunk, nothing sent, naming gitea_branch_protection_loosen", async () => {
for (const args of [{ enable_force_push: true }, { enable_force_push: true, enable_force_push_allowlist: true, force_push_allowlist_usernames: ["jschoubben"] }]) {
await assert.rejects(setFull(args), /gitea_branch_protection_loosen's alone/);
}
assert.deepEqual(sent, []);
});
test("gitea_branch_protection_loosen removes a trunk's protection, or lets a force-push onto it, and says the trunk rule no longer holds", async () => {
const forced: any = await loosen({ act: "allow-force-push" });
assert.deepEqual(sent, [{ method: "PATCH", path: "/repos/novox/de-spiegel/branch_protections/main", body: { enable_force_push: true } }]);
assert.equal(forced.before.fields.enable_force_push, false);
assert.equal(forced.after.fields.enable_force_push, true);
sent = [];
const removed: any = await loosen({ act: "remove-protection" });
assert.deepEqual(sent, [{ method: "DELETE", path: "/repos/novox/de-spiegel/branch_protections/main", body: undefined }]);
assert.equal(removed.after, null);
assert.deepEqual(rules, []);
assert.equal(removed.trunk_rule.holds, false);
assert.match(removed.trunk_rule.why, /no protection rule covers main/);
});
test("gitea_branch_protection_loosen refuses an unknown act, no reason or a rule the repository lacks, nothing sent", async () => {
await assert.rejects(loosen({ act: "unprotect" }), /one of remove-protection, allow-force-push/);
await assert.rejects(loosen({ act: "remove-protection", why: "" }), /say why/);
await assert.rejects(loosen({ act: "remove-protection", branch: "release/*" }), /no protection rule named release\/\*/);
assert.deepEqual(sent, []);
});
test("the trunk rule is judged as the controller judges it", () => {
const ok = { rule_name: "main", enable_push: false, enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"], block_admin_merge_override: true };
const cases: [any[], string, boolean, RegExp][] = [
+1 -1
View File
@@ -539,7 +539,7 @@ test("the tools register once there is a way to a token, and the first call mint
"gitea_get_file",
"gitea_list_branches",
"gitea_delete_branch",
"gitea_branch_protection_get", "gitea_team_rights", "gitea_branch_protection_set", "gitea_branch_protection_set_full", "gitea_repo_settings_set_full",
"gitea_branch_protection_get", "gitea_team_rights", "gitea_branch_protection_set", "gitea_branch_protection_set_full", "gitea_branch_protection_loosen", "gitea_repo_settings_set_full",
"gitea_default_branch_set", "gitea_pull_base_set",
"gitea_commit_statuses", "gitea_open_pulls", "gitea_commit_trees", "gitea_contains",
"gitea_list_labels", "gitea_create_label",
+18
View File
@@ -40,5 +40,23 @@ export function failedStepsTools(kept: FailedSteps, acts: (act: HandAct) => Prom
return { dropped: await kept.drop(key), key, why, by };
},
},
{
name: "git_seen_expire",
description: "Deletes the seen keys of the state failed-steps that are more than 30 days old, the oldest first, at most 100: the removal the forge module does on each step-ended, now, with why. " +
"Written in the hand-act log first (design 47 §581).",
input: { type: "object", required: ["why"], properties: { why: { type: "string", description: "why it is run now, in plain words" } } },
run: async (args: Record<string, unknown>) => {
const why = typeof args.why === "string" ? args.why.trim() : "";
if (!why) throw new Error("git_seen_expire says why");
const by = callerOf(args);
try {
await acts({ what: "deleted the seen keys of the forge module's failed-steps past 30 days", why: `${why} (${by})`, cause: "seen-expired", verb: "git_seen_expire" });
} catch (err) {
throw new Error(`the hand act was not recorded, so nothing was deleted: ${err instanceof Error ? err.message : String(err)}`);
}
const deleted = await kept.expireSeen();
return { deleted, why, by };
},
},
];
}
+50 -2
View File
@@ -12,7 +12,7 @@ import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"
import { emit } from "@novox/mesh-sdk/events";
import { GiteaAdmin, GiteaClient, type BranchProtection } from "../client.js";
import { readTeamRights } from "../teamrights.js";
import { PROTECTION_FIELDS, REPO_FIELDS, alsoChanged, forcePushToTrunk, fullProtectionBody, trunksOf, givenFields, trunkRule, unapplied } from "../protectfull.js";
import { LOOSEN_ACTS, LOOSEN_TOOL, PROTECTION_FIELDS, REPO_FIELDS, alsoChanged, forcePushToTrunk, fullProtectionBody, trunksOf, givenFields, trunkRule, unapplied } from "../protectfull.js";
import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js";
import {
MERGE_METHODS, Refused, checkName, judgeApiCall, judgeContentsWrite, judgeDelete, judgeProtectionChange, pathSegments, routedPath, sentQuery,
@@ -484,7 +484,7 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined
"enable_push_whitelist and push_whitelist_deploy_keys need enable_push: true in the same call, the force-push allowlist " +
"flags enable_force_push: true (the forge ignores them otherwise). The rule is read back after the write: a field the " +
"forge did not apply fails the call, naming it; also_changed names what the forge changed beside the call. Generic: it " +
"tightens or loosens as told (the operator, 2026-10-28), so a reason (why) is required and logged; but a call that lets a force-push onto a rule covering a trunk is refused, nothing sent, until the module can ask the operator's Approve for it. A repository's trunk, the branch the mesh installs from, is main, the mesh's one trunk of every repository (said in trunks.from). As the forge's " +
"tightens or loosens as told (the operator, 2026-10-28), so a reason (why) is required and logged; but a call that lets a force-push onto a rule covering a trunk is refused, nothing sent, naming gitea_branch_protection_loosen, which alone removes a trunk's protection or lets a force-push onto it, on the operator's Approve. A repository's trunk, the branch the mesh installs from, is main, the mesh's one trunk of every repository (said in trunks.from). As the forge's " +
"administrator. Answers the rule before and after, the repository's trunks, and trunk_rule: whether the branch is a trunk (is_trunk) and whether the controller now registers a module from " +
"this branch (judgedRule, mesh-control build_source.go: no push, a merge whitelist naming mesh-tools and no " +
"agent-<node>, no administrator override) and why not.",
@@ -530,6 +530,54 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined
},
},
{
name: LOOSEN_TOOL,
description: "Loosen a branch's protection rule in one of the two ways gitea_branch_protection_set_full refuses: remove the rule " +
"(act remove-protection), or let a force-push onto it (act allow-force-push, enable_force_push true). Marked in the " +
"module's needs-approval, so a call runs only on the operator's Approve of that one call (design 46 approval mark " +
"section 21); tightening stays gitea_branch_protection_set_full's, unasked. A reason (why) is required and logged. As " +
"the forge's administrator. Answers the rule before and after (null once removed), the repository's trunks, and " +
"trunk_rule: whether the controller still registers a module from the branch, and why not.",
input: {
owner: { type: "string", description: "the repository owner" },
repo: { type: "string", description: "the repository name" },
branch: { type: "string", description: "the branch (the rule's name, or a glob) whose rule is loosened" },
act: { type: "string", enum: [...LOOSEN_ACTS], description: "remove-protection removes the rule; allow-force-push lets a force-push onto it" },
why: { type: "string", description: "why, in a sentence: logged and answered" },
},
run: async (args) => {
const owner = String(args.owner), repo = String(args.repo), branch = String(args.branch ?? "").trim();
const act = String(args.act ?? ""), why = String(args.why ?? "").trim();
if (!branch) throw new Error("name the branch whose rule is loosened");
if (!(LOOSEN_ACTS as readonly string[]).includes(act)) throw new Error(`act is ${JSON.stringify(act)}: it is one of ${LOOSEN_ACTS.join(", ")}`);
if (!why) throw new Error("say why (why): a branch's protection is loosened for a reason, which is logged");
pathSegments("branch", branch);
const forge = admin ? admin() : gitea;
const rules = await forge.branchProtections(owner, repo);
const before = rules.find((p) => (p.rule_name ?? p.branch_name) === branch) ?? null;
if (!before) throw new Refused(`${owner}/${repo} has no protection rule named ${branch}: there is nothing to loosen. Nothing was sent`);
console.log(`[gitea] ${LOOSEN_TOOL} ${owner}/${repo} ${branch}: ${act} — why: ${why}`);
if (act === "remove-protection") await forge.deleteBranchProtection(owner, repo, branch);
else await forge.writeBranchProtection(owner, repo, branch, { enable_force_push: true }, false);
const after = await forge.branchProtections(owner, repo);
const read = after.find((p) => (p.rule_name ?? p.branch_name) === branch) ?? null;
if (act === "remove-protection" && read) {
throw new Error(`the forge answered the removal of ${owner}/${repo} ${branch}'s rule but still holds it: ${JSON.stringify(summarised(read))}`);
}
if (act === "allow-force-push") {
const missed = read ? unapplied({ enable_force_push: true }, read) : ["the rule itself (the forge no longer holds it)"];
if (missed.length) throw new Error(`the forge answered the write of ${owner}/${repo} ${branch}'s rule but did not apply ${missed.join("; ")}`);
}
const trunks = trunksOf();
return {
branch, act, why,
before: { ...summarised(before), fields: before },
after: read ? { ...summarised(read), fields: read } : null,
trunks, trunk_rule: trunkRule(after, branch, trunks.names),
};
},
},
{
name: "gitea_repo_settings_set_full",
description: "Set a repository's settings as the forge's administrator — its default branch (e.g. master to main), " +
@@ -7,7 +7,6 @@ package main
import (
"encoding/json"
"fmt"
"time"
broker "git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
)
@@ -48,10 +47,27 @@ func (k *Keeper) takeSource(module string) error {
return nil
}
// nudgeBound says, without waiting, that the bus module's entries may be read now (ListBound's retry).
func (k *Keeper) nudgeBound() {
k.mu.Lock()
k.init()
c := k.boundRetry
k.mu.Unlock()
select {
case c <- struct{}{}:
default:
}
}
// ListBound reads the entries the bus module holds at the keeper's start: every module with a `finding-events` entry is a
// source. A call that fails is asked again with a growing wait, the bus module being delivered after the keeper at times.
// source. A call that fails is asked again on the next event that says the bus module may answer, its `created` or a
// node-engine's `running` (the bus module being delivered after the keeper at times), never on a timer.
func ListBound(k *Keeper, call func(key string, body any) (json.RawMessage, error)) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
k.mu.Lock()
k.init()
retry := k.boundRetry
k.mu.Unlock()
for {
raw, err := call("seat:"+broker.Seat+"."+broker.VerbList, broker.ListInput{})
var answer broker.ListAnswer
if err == nil {
@@ -67,7 +83,7 @@ func ListBound(k *Keeper, call func(key string, body any) (json.RawMessage, erro
}
return
}
k.Logf("the entries of the bus module are not read yet (%v); asked again in %s", err, wait)
time.Sleep(wait)
k.Logf("the entries of the bus module are not read yet (%v); asked again when it says created or a node-engine says running", err)
<-retry
}
}
@@ -72,13 +72,17 @@ func (k *Keeper) FindingEvent(e stdio.Envelope) error {
if _, ok := emitterOf(e.Key, watchdogs.EventMachineBack); ok && strings.HasPrefix(e.Key, watchdogs.Seat+".") {
return k.MachinePresence(e.Body, false)
}
if _, _, _, ok := engineKey(e.Key); ok {
if event, _, _, ok := engineKey(e.Key); ok {
if event == engine.EventRunning {
k.nudgeBound()
}
return k.EngineEvent(e)
}
if strings.HasPrefix(e.Key, meshhealers.Seat+".attempt-changed.") {
return k.AttemptChanged(e.Key, e.Body)
}
if e.Key == broker.Seat+"."+broker.EventCreated {
k.nudgeBound()
return k.BrokerCreated(e.Body)
}
return nil
@@ -164,6 +164,9 @@ type Keeper struct {
stopped map[string]bool
// created are the modules whose `finding-events` entry the bus module made (bound.go).
created map[string]bool
// boundRetry is told, once, when the bus module says `created` or a node-engine says `running`: the events after which
// the entries of the bus module may be read (ListBound).
boundRetry chan struct{}
// followed are the finding streams the catalog half declared for each module, by module (catalog.go).
followed map[string]string
}
@@ -179,6 +182,9 @@ func (k *Keeper) init() {
k.halves, k.placed, k.stopped = map[string]map[string]catseat.CatalogEntry{}, map[string]bool{}, map[string]bool{}
k.views, k.keyNode, k.whys = map[string]*sourceView{}, map[string]map[string]string{}, map[string]string{}
}
if k.boundRetry == nil {
k.boundRetry = make(chan struct{}, 1)
}
if k.Now == nil {
k.Now = time.Now
}
@@ -723,3 +723,47 @@ func TestTheKeepersOwnFindingIsTheContractsFindingByItsWireForm(t *testing.T) {
t.Fatal("the condition stands after its finding was cleared")
}
}
// ListBound reads the bus module's entries once and, when the call fails, again on the event that says the bus module may
// answer (its `created`, or a node-engine's `running`), never on a timer (design 47 G §515, §663).
func TestListBoundIsAskedAgainOnTheEventsNotATimer(t *testing.T) {
for _, event := range []string{"mesh-broker.created", "node-engine.running.ace.mesh-conditions"} {
t.Run(event, func(t *testing.T) {
var asked []map[string]any
k := &Keeper{Publish: func(_ string, b any) error { asked = append(asked, b.(map[string]any)); return nil }}
calls := make(chan int)
ended := make(chan struct{})
go func() {
n := 0
ListBound(k, func(string, any) (json.RawMessage, error) {
n++
calls <- n
if n == 1 {
return nil, fmt.Errorf("the bus module is not delivered yet")
}
return mustJSON(t, map[string]any{"entries": []any{map[string]any{"module": "newpart", "template": FindingEventsTemplate}}}), nil
})
close(ended)
}()
<-calls
select {
case <-ended:
t.Fatal("ListBound ended with no answer")
default:
}
_ = k.FindingEvent(stdio.Envelope{Key: event, Body: []byte(`{}`)})
select {
case n := <-calls:
if n != 2 {
t.Fatalf("asked %d times", n)
}
case <-time.After(time.Second):
t.Fatal("not asked again on the event")
}
<-ended
if len(asked) != 1 || asked[0]["module"] != "newpart" {
t.Errorf("asked %v, want newpart once", asked)
}
})
}
}
+1
View File
@@ -78,6 +78,7 @@
"mesh-catalog modules/mesh-provisioner",
"mesh-catalog modules/mesh-provisioner-client",
"mesh-catalog modules/mesh-reads",
"mesh-catalog modules/mesh-recovery",
"mesh-catalog modules/mesh-self-check",
"mesh-catalog modules/mesh-vault",
"mesh-catalog modules/mesh-watcher",
+1
View File
@@ -190,6 +190,7 @@
"mesh-catalog modules/mesh-provisioner",
"mesh-catalog modules/mesh-provisioner-client",
"mesh-catalog modules/mesh-reads",
"mesh-catalog modules/mesh-recovery",
"mesh-catalog modules/mesh-self-check",
"mesh-catalog modules/mesh-vault",
"mesh-catalog modules/mesh-watcher",
@@ -21,7 +21,8 @@ type manifest struct {
Consumes []string `json:"consumes"`
Invokes []string `json:"invokes"`
Config map[string]struct {
Default float64 `json:"default"`
Default float64 `json:"default"`
Minimum *float64 `json:"minimum"`
} `json:"config"`
Data struct {
Own []struct{ ID string } `json:"own"`
@@ -75,6 +76,13 @@ func TestTheManifestSaysWhatTheModuleIs(t *testing.T) {
t.Errorf("setting %s defaults to %v, want %v", k, m.Config[k].Default, want)
}
}
// A bound of time is never 0, which the reader refuses, and may be a fraction (A-138), so a whole test sets
// seconds: its minimum is above 0 and below 1 (the manifest's minimum is inclusive).
for _, k := range []string{"point-keep-days", "dump-wait-minutes", "snapshot-restore-wait-minutes", "expire-every-hours"} {
if min := m.Config[k].Minimum; min == nil || *min <= 0 || *min >= 1 {
t.Errorf("setting %s has the minimum %v: a bound of 0 is refused by the reader and a fraction is allowed (A-138), so its minimum is above 0 and below 1", k, m.Config[k].Minimum)
}
}
// The keeper's snapshot verbs the recovery module asks, the cancel at its bound among them (§681).
for _, inv := range []string{"seat:mesh-broker.snapshot-get", "seat:mesh-broker.snapshot-restore", "seat:mesh-broker.snapshot-restore-cancel"} {
if !slices.Contains(m.Invokes, inv) {
+4 -4
View File
@@ -110,7 +110,7 @@
"point-keep-days": {
"type": "number",
"description": "days after which a restore point is removed, the newest of its kept data never (design 47 G sections 551, 557).",
"minimum": 0,
"minimum": 0.01,
"maximum": 3650,
"trusted": false,
"default": 90
@@ -118,7 +118,7 @@
"dump-wait-minutes": {
"type": "number",
"description": "minutes an act waits on the dump of the bus's item before recovery-waits is written (design 47 G sections 561, 557).",
"minimum": 0,
"minimum": 0.01,
"maximum": 10080,
"trusted": false,
"default": 60
@@ -126,7 +126,7 @@
"snapshot-restore-wait-minutes": {
"type": "number",
"description": "minutes a restore in place of a protected keeper's snapshot is waited on, and once more after its cancel (design 47 G section 681, A-138, A-140).",
"minimum": 0,
"minimum": 0.01,
"maximum": 10080,
"trusted": false,
"default": 90
@@ -134,7 +134,7 @@
"expire-every-hours": {
"type": "number",
"description": "hours between two removals of the points past their keep days, beside the one at the start and at each snapshot-kept; timed by a bound key of the state bounds (design 47 G sections 551, 557, A-138).",
"minimum": 0,
"minimum": 0.01,
"maximum": 8760,
"trusted": false,
"default": 24
+74 -2
View File
@@ -48,6 +48,20 @@ func HeldInState() (map[string]keeper.Asked, error) {
return out, nil
}
// capsOf is the sum of the bus's caps against its limits (design 47 G §547, §591, §661), in MiB: every entry made and
// every old section the catalog graph's entries declare, as the keeper's status last says them; a limit the keeper has
// not read, or that is none, is absent. The sum is over every module, whatever list is narrowed to.
func capsOf(all map[string]keeper.Asked, status func() (keeper.Status, error)) *meshbroker.ListAnswerCaps {
sum := keeper.CapSum(all, "", "")
c := &meshbroker.ListAnswerCaps{}
if s, err := status(); err == nil {
sum += s.OldBytes
c.AccountLimitMiB, c.ServerLimitMiB = s.Limits.AccountBytes/keeper.MiB, s.Limits.ServerBytes/keeper.MiB
}
c.SumMiB = sum / keeper.MiB
return c
}
// busStream is one stream as the monitoring endpoint answers it.
type busStream struct {
Config jetstream.StreamConfig `json:"config"`
@@ -157,7 +171,7 @@ func anyOf(m map[string]json.RawMessage) map[string]any {
}
// brokerTools are the four reading verbs and nats_history.
func brokerTools(read heldEntries, jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool {
func brokerTools(read heldEntries, status func() (keeper.Status, error), jsz func() ([]byte, error), history func(module, local string, limit int) ([]keeper.Event, error)) []stdio.Tool {
both := func() (map[string]keeper.Asked, map[string]busStream, error) {
entries, err := read()
if err != nil {
@@ -194,7 +208,11 @@ func brokerTools(read heldEntries, jsz func() ([]byte, error), history func(modu
if m := arg(a, "module"); m != "" {
entries = map[string]keeper.Asked{m: entries[m]}
}
return meshbroker.ListAnswer{Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil
all, err := read()
if err != nil {
return nil, err
}
return meshbroker.ListAnswer{Caps: capsOf(all, status), Entries: each(entries, held, func(keeper.Made) bool { return true })}, nil
}},
{Name: "mesh-broker.show",
Description: "One entry of a module's kept data on the bus: its bus name, template and options as asked, the settings the bus holds of each of its objects, and its fill against its cap. Whether it is refused, waiting, drifted, full or orphaned is what nats_history says of it.",
@@ -317,3 +335,57 @@ func sortedNames[V any](m map[string]V) []string {
sort.Strings(out)
return out
}
// PlansInState reads the keeper's state plans (design 47 G §538): each module's entries as last planned.
func PlansInState() (map[string]keeper.Planned, error) {
s := stdio.State(keeper.StatePlans)
keys, err := s.Keys()
if err != nil {
return nil, fmt.Errorf("the keeper's own %s cannot be read: %w", keeper.StatePlans, err)
}
out := map[string]keeper.Planned{}
for _, key := range keys {
e, err := s.Get(key)
if err != nil || e == nil {
continue
}
var p keeper.Planned
if json.Unmarshal(e.Value, &p) == nil {
out[key] = p
}
}
return out, nil
}
// plansTool is nats_plans: what the bus module planned for each module, from its own state plans (design 47 G §538).
func plansTool(read func() (map[string]keeper.Planned, error)) stdio.Tool {
return stdio.Tool{Name: "nats_plans",
Description: "What the bus module planned for each module: the bus-state entries it last planned, their digest (a verdict whose entries digest the same plans nothing), the revision of the verdict and when. Narrowed by a module.",
Input: map[string]any{"module": str("one module")},
Run: func(a map[string]any) (any, error) {
plans, err := read()
if err != nil {
return nil, err
}
type line struct {
Module string `json:"module"`
Digest string `json:"digest"`
Revision uint64 `json:"revision"`
Entries []string `json:"entries"`
At string `json:"at"`
}
out := []line{}
for _, module := range sortedNames(plans) {
if m := arg(a, "module"); m != "" && m != module {
continue
}
p := plans[module]
names := []string{}
for _, e := range p.Entries {
names = append(names, e.Name)
}
out = append(out, line{Module: module, Digest: p.Digest, Revision: p.Revision, Entries: names, At: p.At.UTC().Format(time.RFC3339)})
}
return map[string]any{"plans": out}, nil
}}
}
+37 -3
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"encoding/json"
"fmt"
"reflect"
@@ -37,9 +38,13 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) {
}}}
jsz := `{"account_details":[{"stream_detail":[
{"name":"KV_m_drafts","config":{"name":"KV_m_drafts","retention":"limits","discard":"new","max_bytes":67108864,"max_msgs_per_subject":1,"max_msg_size":262144},"state":{"messages":3,"bytes":1048576}},
{"name":"OBJ_m_drafts-objects","config":{"name":"OBJ_m_drafts-objects","retention":"limits","discard":"new","max_bytes":16777216},"state":{"messages":0,"bytes":0}},
{"name":"m_log","config":{"name":"m_log","retention":"limits","discard":"new","max_bytes":33554432,"max_msg_size":262144},"state":{"messages":9,"bytes":9000}}]}]}`
tools := map[string]stdio.Tool{}
for _, tl := range brokerTools(func() (map[string]keeper.Asked, error) { return held, nil },
func() (keeper.Status, error) {
return keeper.Status{Limits: keeper.Limits{AccountBytes: 2048 << 20, ServerBytes: 4096 << 20}, OldBytes: 256 << 20}, nil
},
func() ([]byte, error) { return []byte(jsz), nil },
func(module, local string, limit int) ([]keeper.Event, error) {
return []keeper.Event{{Module: module, Local: local, Why: "first"}, {Module: module, Local: local, Why: "second"}}[:limit], nil
@@ -53,10 +58,13 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) {
err string
}{
// Each answers the seat's contract (mesh-contracts meshbroker ListAnswer, Held, CheckAnswer, OrphansAnswer).
{"mesh-broker.list", nil, []string{`{"entries":[`, `"bus-name":"m_drafts"`, `"local-name":"drafts"`, `"storage-type":"key-value"`,
`"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":67108864,"count":3}`, `"options":{"max-mib":16}`,
{"mesh-broker.list", nil, []string{`"entries":[`, `"bus-name":"m_drafts"`, `"local-name":"drafts"`, `"storage-type":"key-value"`,
`"KV_m_drafts":{"max-bytes":67108864`, `"fill":{"bytes":1048576,"cap":83886080,"count":3}`, `"options":{"max-mib":16}`,
// The binding of an entry the bus holds is the one its created carries (design 25 bindings-first-view §2).
`"binding":{"bucket":"m_drafts","local":"drafts","storage":"key-value","template":"state"}`}, ""},
`"binding":{"bucket":"m_drafts","local":"drafts","objects":"m_drafts-objects","storage":"key-value","template":"state"}`}, ""},
// The sum of the caps against both limits (design 47 G §547, §591): drafts 64+16, log 16, old 1024 (orphaned, still
// held), and the old sections the catalog graph's entries declare, 256; later is not made.
{"mesh-broker.list", nil, []string{`"caps":{"account-limit-mib":2048,"server-limit-mib":4096,"sum-mib":1376`}, ""},
{"mesh-broker.show", map[string]any{"module": "m", "local-name": "log"}, []string{`"bus-name":"m_log"`, `"template":"log"`,
`"m_log":{"max-bytes":33554432`}, ""},
{"mesh-broker.show", map[string]any{"module": "m", "local-name": "later"}, []string{`"local-name":"later"`}, ""},
@@ -114,3 +122,29 @@ func TestTheSeatsReadingVerbsAnswerFromTheKeeperAndTheBus(t *testing.T) {
})
}
}
// The server's storage limit is read from the monitoring document's config (design 47 G §547); none is 0.
func TestTheServersStorageLimitIsReadFromItsMonitor(t *testing.T) {
for doc, want := range map[string]int64{`{"config":{"max_storage":5368709120}}`: 5368709120, `{"config":{"max_storage":-1}}`: 0, `{}`: 0} {
got, err := serverStorageLimit(func(context.Context, string) ([]byte, error) { return []byte(doc), nil })(context.Background())
if err != nil || got != want {
t.Errorf("%s: %d (%v), want %d", doc, got, err, want)
}
}
}
// nats_plans answers each module's entries as last planned, narrowed by a module (design 47 G §538).
func TestNatsPlansReadsTheKeepersPlans(t *testing.T) {
read := func() (map[string]keeper.Planned, error) {
return map[string]keeper.Planned{
"a": {Module: "a", Digest: "sha256:1", Revision: 3, Entries: []catalogue.BusStateEntry{{Name: "drafts"}}},
"b": {Module: "b", Digest: "sha256:2", Revision: 4},
}, nil
}
tool := plansTool(read)
out, err := tool.Run(map[string]any{"module": "a"})
raw, _ := json.Marshal(out)
if err != nil || !strings.Contains(string(raw), `"entries":["drafts"]`) || !strings.Contains(string(raw), `"digest":"sha256:1"`) || strings.Contains(string(raw), `"module":"b"`) {
t.Errorf("nats_plans answered %s (%v)", raw, err)
}
}
+98 -6
View File
@@ -1,10 +1,13 @@
package main
import (
"cmp"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"maps"
"slices"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -12,15 +15,18 @@ import (
"nats-tools/internal/keeper"
)
// The keeper reads a module's asks from the module's entry in the catalog half of its revision view, not from the
// indexer's event (design 47 G §344, §433).
// The keeper reads a module's asks from the module's entry in the catalog half of each watched branch, not from the
// indexer's event (design 47 G §344, §433, §532).
// Asks is what one catalog graph entry says of a module for the keeper: what its manifest asks of the bus as an accepted
// verdict, the seats it claims, and whether it declares an event of its own (the facts keeper.Sends reads).
// verdict, the seats it claims, whether it declares an event of its own (the facts keeper.Sends reads), whether it is a
// foundation part (an old `objects` section `sent-objects`, §533), and the caps of its old sections in bytes (§547, §591).
type Asks struct {
Accepted keeper.Accepted
Claims []string
EmitsOwn bool
Accepted keeper.Accepted
Claims []string
EmitsOwn bool
Foundation bool
OldBytes int64
}
// AsksOf reads one entry. An entry whose manifest is not accepted asks nothing of the bus now, as an accepted verdict of
@@ -48,9 +54,95 @@ func AsksOf(e catalog.CatalogEntry) (Asks, error) {
for _, c := range m.Claims {
out.Claims = append(out.Claims, c.Name)
}
// The old sections' caps (design 47 G §547, §591): an objects section its max-mib, a log or a stream its max-mib or
// the catalogue's default.
for _, o := range m.Objects {
out.Foundation = out.Foundation || o.Name == keeper.SentLocal
out.OldBytes += int64(o.MaxMiB) * keeper.MiB
}
for _, l := range m.Logs {
out.OldBytes += int64(cmp.Or(l.MaxMiB, catalogue.LogDefaultMiB)) * keeper.MiB
}
for _, st := range m.Streams {
out.OldBytes += int64(cmp.Or(st.MaxMiB, catalogue.StreamDefaultMiB)) * keeper.MiB
}
return out, nil
}
// Branch is one watched branch's newest revision of the catalog half: its name, its revision and its entries by module.
type Branch struct {
Name string
Revision int64
Entries map[string]catalog.CatalogEntry
}
// ViewOf is the catalog half as the keeper plans from it (design 47 G §532): every module some branch's newest revision
// lists, its kept data planned from each such branch, each local name once, from the first of the module's watched
// branches in its registration's order (order: the registration's branches; a branch it does not name comes after, in
// the order given) whose entry asks it; another branch asking that local name with other settings is a drift naming
// the branch, and plans nothing. The module sends when a branch's entry says so; it is a foundation part, and its old
// sections' caps are read, from the first branch.
func ViewOf(branches []Branch, order func(module string) []string) (keeper.CatalogView, error) {
modules := map[string]bool{}
for _, b := range branches {
for m := range b.Entries {
modules[m] = true
}
}
var view keeper.CatalogView
for _, module := range slices.Sorted(maps.Keys(modules)) {
var listing []Branch
var named []string
if order != nil {
named = order(module)
}
for _, name := range named {
for _, b := range branches {
if _, ok := b.Entries[module]; ok && b.Name == name {
listing = append(listing, b)
}
}
}
for _, b := range branches {
if _, ok := b.Entries[module]; ok && !slices.Contains(named, b.Name) {
listing = append(listing, b)
}
}
var cm keeper.CatalogModule
from := map[string]string{}
asked := map[string]string{}
for i, b := range listing {
a, err := AsksOf(b.Entries[module])
if err != nil {
return keeper.CatalogView{}, fmt.Errorf("the branch %s: %w", b.Name, err)
}
if i == 0 {
cm = keeper.CatalogModule{Accepted: keeper.Accepted{Module: module, Digest: b.Entries[module].ManifestDigest, Revision: uint64(max(b.Revision, 0))},
Foundation: a.Foundation, OldBytes: a.OldBytes}
}
cm.EmitsOwn = cm.EmitsOwn || a.EmitsOwn
for _, c := range a.Claims {
if !slices.Contains(cm.Claims, c) {
cm.Claims = append(cm.Claims, c)
}
}
for _, e := range a.Accepted.BusState {
settings := string(mustJSON(e))
if f, ok := from[e.Name]; ok {
if asked[e.Name] != settings {
view.Drifts = append(view.Drifts, keeper.Drift{Module: module, Local: e.Name, Branch: b.Name, From: f})
}
continue
}
from[e.Name], asked[e.Name] = b.Name, settings
cm.BusState = append(cm.BusState, e)
}
}
view.Modules = append(view.Modules, cm)
}
return view, nil
}
func mustJSON(v any) []byte {
b, _ := json.Marshal(v)
return b
+209 -115
View File
@@ -14,6 +14,8 @@ import (
"syscall"
"time"
"sync"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -22,11 +24,14 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/sourcebranches"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/bussnap"
@@ -34,33 +39,10 @@ import (
)
// The keeper (`nats-tools keep`): the bus module's process on its own account (own secret broker-keeper, ADR 0436 §1),
// started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, asks the
// indexer's first view once, then hears manifest-accepted and the operator's decided on its durable consumer, makes what
// each verdict asks for, and acts on the persons' requests its tools put in its own state requests (requests.go) as they
// are written or answered. It answers the seat mesh-broker's snapshot verbs itself (snapshots.go; design 47 G A-107).
// AcceptedOf reads a manifest-accepted body: the seat mesh-catalog's typed event, with the indexer's revision of the
// verdict when the event carries it (0 from an indexer that does not say it).
func AcceptedOf(body []byte) (keeper.Accepted, error) {
var a catalog.ManifestAcceptedEvent
if err := json.Unmarshal(body, &a); err != nil {
return keeper.Accepted{}, err
}
if a.Module == "" {
return keeper.Accepted{}, fmt.Errorf("a manifest-accepted names no module")
}
var state []catalogue.BusStateEntry // the seat types an entry as the manifest wrote it; the keeper reads the catalogue's form
if raw, err := json.Marshal(a.BusState); err != nil || json.Unmarshal(raw, &state) != nil {
return keeper.Accepted{}, fmt.Errorf("the bus-state of the manifest-accepted of %s is not readable", a.Module)
}
if len(a.BusState) == 0 {
state = nil
}
return keeper.Accepted{Module: a.Module, Digest: a.Digest, Revision: uint64(max(a.Revision, 0)), BusState: state}, nil
}
// AcceptedSubject is the seat mesh-catalog's manifest-accepted on the shared events stream.
var AcceptedSubject = catalog.SubjectManifestAccepted
// started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, follows the
// catalog half of every watched branch through meshcatalogclient's branch follower and the seat list (design 47 G §344,
// §433, §532, §614), plans again at each of their changes, hears the operator's decided on its durable consumer, and acts
// on the persons' requests its tools put in its own state requests (requests.go) as they are written or answered. It answers the seat mesh-broker's snapshot verbs itself (snapshots.go; design 47 G A-107).
// The router's records of asks, read under this module's own name (ADR 0259 §3), as the mail sender reads them.
const asksBucket = "messenger_asks"
@@ -190,36 +172,6 @@ func sayer(js jetstream.JetStream) func(oc.Condition) error {
}
}
// firstView asks the indexer's indexer_bus_state as the keeper's own account.
func firstView(c *process.Conn) func() (keeper.FirstView, error) {
return func() (keeper.FirstView, error) {
raw, err := c.Call(keeper.FirstViewOwner+"."+keeper.FirstViewVerb, map[string]any{})
if err != nil {
return keeper.FirstView{}, err
}
return FirstViewOf(raw)
}
}
// FirstViewOf reads indexer_bus_state's answer, as the tool runner hands it back ({result, error} or bare).
func FirstViewOf(raw []byte) (keeper.FirstView, error) {
var wrapped struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
if json.Unmarshal(raw, &wrapped) == nil && (wrapped.Result != nil || wrapped.Error != "") {
if wrapped.Error != "" {
return keeper.FirstView{}, fmt.Errorf("%s", wrapped.Error)
}
raw = wrapped.Result
}
var v keeper.FirstView
if err := json.Unmarshal(raw, &v); err != nil {
return keeper.FirstView{}, fmt.Errorf("the answer is not a first view: %w", err)
}
return v, nil
}
func percent(getenv func(string) string, name string, fallback int64) int64 {
if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 && n < 100 {
return n
@@ -227,17 +179,10 @@ func percent(getenv func(string) string, name string, fallback int64) int64 {
return fallback
}
// number reads a positive whole number from the environment, or the fallback.
func number(getenv func(string) string, name string, fallback int64) int64 {
if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 {
return n
}
return fallback
}
// runKeeper is the keeper's process: it returns when ctx ends, or when it has no credential.
func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "nats-keeper: ", log.LstdFlags)
settings := readSettings(getenv("MESH_NATS_SNAPSHOTS_DIR") + "/settings.json")
account, err := process.ReadBrokerAccount(getenv)
if err != nil {
return err
@@ -267,8 +212,8 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer
defer writer.Close()
k := &keeper.Keeper{
Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR")), NC: nc,
Dir: getenv("MESH_NATS_SNAPSHOTS_DIR"), Log: logger.Printf},
Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn),
Dir: getenv("MESH_NATS_SNAPSHOTS_DIR"), Log: logger.Printf, Server: serverStorageLimit(LiveMonitor())},
Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js),
Act: func(what, why, cause string) error {
_, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause})
return err
@@ -276,29 +221,173 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer
Warrant: WarrantFrom(conn.Ask), Finding: writer,
Logf: logger.Printf,
Raise: percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70),
// The settings snapshots-cap (20 GiB) and snapshot-keep-days (90) of the manifest (design 47 G §681).
SnapshotsCap: number(getenv, "MESH_NATS_SNAPSHOTS_CAP", keeper.DefaultSnapshotsCap),
SnapshotKeepDays: int(number(getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)),
// The settings snapshots-cap (20 GiB) and snapshot-keep-days (90) of the manifest, read from the file it declares
// at the start, its defaults when the file is missing (design 47 G §681, A-112).
SnapshotsCap: settings.SnapshotsCap,
SnapshotKeepDays: settings.SnapshotKeepDays,
}
// The catalog half (design 47 G §433, §532, §614): meshcatalogclient's branch follower on the keeper's views reader, which
// holds its finding writer; and the seat list, each definition as the seat answers that one seat, resolved (§537). Each
// change of a branch's view, of the registrations or of the seat list plans again.
moved := make(chan struct{}, 1)
wakeMoved := func() {
select {
case moved <- struct{}{}:
default:
}
}
follow, err := followCatalog(ctx, conn, writer, wakeMoved, logger.Printf)
if err != nil {
return err
}
k.Catalog, k.Seats, k.Again = follow.view, follow.seats, follow.again
// The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved.
stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf)))
if err != nil {
return fmt.Errorf("the seat %s's snapshot verbs cannot be answered: %w", meshbroker.Seat, err)
}
defer stopAnswers()
logger.Printf("started on the account %s: asking %s.%s for the first view", account.User, keeper.FirstViewOwner, keeper.FirstViewVerb)
return Keep(ctx, k, hearer(js, account, logger.Printf), requestsOf(conn), logger.Printf)
logger.Printf("started on the account %s: following the catalog half of every watched branch through the seat %s", account.User, catalog.Seat)
return Keep(ctx, k, hearer(js, account, follow.take, follow.bound, logger.Printf), requestsOf(conn), moved, logger.Printf)
}
// hearer binds the keeper's durable consumer on EVENTS (<node>_nats_keeper, filtered on manifest-accepted) and hands
// each message to handle, which acknowledges it once applied.
func hearer(js jetstream.JetStream, a process.Account, logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error {
// catalogFollow is the keeper's reading of the catalog half and the seat list (design 47 G §532, §614): the reader, the
// branch follower over it, the seat list, and the events of both, routed from the keeper's one durable consumer.
type catalogFollow struct {
ctx context.Context
reader *views.ProcessReader
branches *meshcatalogclient.Branches
seatList *seatlist.Reader
routes *eventRoutes
}
// followCatalog wires the branch follower and the seat list, and asks their first views; moved is told of each change.
func followCatalog(ctx context.Context, conn *process.Conn, writer views.WireWriter, moved func(), logf func(string, ...any)) (*catalogFollow, error) {
f := &catalogFollow{ctx: ctx, routes: &eventRoutes{by: map[string]func(jetstream.Msg){}}}
f.reader = &views.ProcessReader{Module: keeper.Module, Call: conn.Call, Logf: logf, Findings: func() views.WireWriter { return writer }}
branches, err := meshcatalogclient.FollowBranchesProcess(f.reader, conn.Call)
if err != nil {
return nil, fmt.Errorf("the catalog half cannot be followed: %w", err)
}
branches.Logf = logf
branches.Changed = func(string, string, json.RawMessage, bool, time.Time) { moved() }
// A registration that changes its watched branches' order moves what is planned from which branch (§532).
reconcile := branches.Registrations.Follower.Changed
branches.Registrations.Follower.Changed = func(key string, value json.RawMessage, deleted bool, at time.Time) {
reconcile(key, value, deleted, at)
moved()
}
f.branches = branches
f.seatList, err = seatlist.Start(seatlist.Config{Bus: seatlist.ProcessBus(conn.Call, f.routes), Bootstrap: seatlist.Bootstrapped(),
OnChange: func([]catalogue.Seat) { moved() }})
if err != nil {
logf("the seat list cannot be followed (%v): a module's sent store waits for it", err)
}
f.reader.Ask(ctx, nil)
return f, nil
}
// view is the catalog half as the keeper plans from it: an error while a watched branch has no first view yet.
func (f *catalogFollow) view() (keeper.CatalogView, error) {
if err := f.branches.Ready(); err != nil {
return keeper.CatalogView{}, err
}
var bs []Branch
for _, name := range f.branches.Followed() {
if c, ok := f.branches.Catalog(name); ok {
if half, ok := c.View(); ok {
bs = append(bs, Branch{Name: name, Revision: half.Revision.Revision, Entries: half.Entries})
}
}
}
return ViewOf(bs, func(module string) []string {
reg, ok := f.branches.Registrations.Registration(module)
if !ok {
return nil
}
return reg.Branches
})
}
// seats are the seat list's definitions once the seat has answered; before, none, so no sent store is planned yet
// (ADR 0636 §1: a reader that has not heard acts on nothing new).
func (f *catalogFollow) seats() map[string]seatdef.Definition {
if f.seatList == nil || !f.seatList.Status().Heard {
return nil
}
return f.seatList.Definitions()
}
// again asks every first view of the catalog half and the seat list again, by hand.
func (f *catalogFollow) again() error {
var errs []error
for _, fl := range f.reader.Followers() {
errs = append(errs, fl.Again(f.ctx, "asked again by hand (nats_first_view_again)"))
}
if f.seatList != nil {
errs = append(errs, f.seatList.Reload())
}
return errors.Join(errs...)
}
// take hands a message of the catalog half or the seat list to its reader, which acknowledges it; false for any other.
func (f *catalogFollow) take(m jetstream.Msg) bool {
if h := f.routes.get(m.Subject()); h != nil {
h(m)
return true
}
if strings.HasPrefix(m.Subject(), "mesh.seat."+catalog.Seat+".event.") || strings.HasPrefix(m.Subject(), seatdef.EventSubject(sourcebranches.Seat, sourcebranches.EventSourceMoved)) {
f.reader.Hear(f.ctx)(m)
return true
}
return false
}
// bound tells each view the bus made the keeper's consumer then, so a first view taken before is asked again.
func (f *catalogFollow) bound(created time.Time) {
for _, fl := range f.reader.Followers() {
fl.ConsumerMade(f.ctx, created)
}
}
// eventRoutes are the seat list's handlers of its events by subject, on the keeper's one durable consumer.
type eventRoutes struct {
mu sync.Mutex
by map[string]func(jetstream.Msg)
}
func (r *eventRoutes) Handle(subject string, handle func(jetstream.Msg)) error {
r.mu.Lock()
defer r.mu.Unlock()
if r.by[subject] != nil {
return fmt.Errorf("the events %q are handled already", subject)
}
r.by[subject] = handle
return nil
}
func (r *eventRoutes) get(subject string) func(jetstream.Msg) {
r.mu.Lock()
defer r.mu.Unlock()
return r.by[subject]
}
// hearer binds the keeper's durable consumer on EVENTS (<node>_nats_keeper, filtered on its consumes): a message of the
// catalog half or the seat list goes to take, which acknowledges it; every other to handle, acknowledged once applied.
// bound is told when the bus made the consumer.
func hearer(js jetstream.JetStream, a process.Account, take func(jetstream.Msg) bool, bound func(time.Time), logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error {
return func(ctx context.Context, handle func(string, []byte) error) error {
cons, err := js.Consumer(ctx, process.EventsStream, a.QueueConsumerName())
if err != nil {
return fmt.Errorf("the keeper's consumer %s on %s: %w", a.QueueConsumerName(), process.EventsStream, err)
}
if info, err := cons.Info(ctx); err == nil && bound != nil {
bound(info.Created.UTC())
}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
if take != nil && take(m) {
return
}
if err := handle(m.Subject(), m.Data()); err != nil {
logf("%v: delivered again", err)
_ = m.NakWithDelay(30 * time.Second)
@@ -322,39 +411,25 @@ var FillEvery = time.Minute
var DecidedSubject = oc.SubjectDecided(keeper.Module)
// Keep is the keeper's loop, apart from the bus so a test drives it: a fill pass at once and every minute, from its
// start, so the store's one guard runs while the first view is still asked (design 25 § 12.3); the first view asked
// until it answers, on its backoff; then hear, which applies each manifest-accepted and wakes the persons' requests on
// the operator channel's decided for the bus module's own asks; the waiting entries made again on the same backoff; and
// the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx ends.
// start, so the store's one guard runs while the catalog half's first views are still asked (design 25 § 12.3); a plan
// from the catalog half at once and at each move of it (moved: a branch's view, the registrations or the seat list
// changed), nothing planned until every watched branch has its first view (design 47 G §532, §614); hear, which wakes the
// persons' requests on the operator channel's decided for the bus module's own asks; the waiting entries made again on
// a backoff; and the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx
// ends.
func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func(string, []byte) error) error,
requests func(context.Context, func()) error, logf func(string, ...any)) error {
requests func(context.Context, func()) error, moved <-chan struct{}, logf func(string, ...any)) error {
tick := time.NewTicker(FillEvery)
defer tick.Stop()
k.FillPass()
k.SnapshotsAtStart()
k.EndCutOffRestores()
var wait time.Duration
for {
next, err := k.AskFirstView(wait)
if err == nil {
break
}
logf("the first view did not answer (%v): nothing is created or orphaned until it does; asked again in %s", err, next)
wait = next
again := time.After(next)
waiting:
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-tick.C:
k.FillPass()
case <-again:
break waiting
}
plan := func() {
if err := k.PlanCatalog(); err != nil {
logf("nothing is planned yet: %v", err)
}
}
logf("the first view answered at revision %d", k.Status().FirstView.Revision)
plan()
wake := func(c chan struct{}) func() {
return func() {
select {
@@ -363,31 +438,20 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func
}
}
}
heard, asked := make(chan struct{}, 1), make(chan struct{}, 1)
asked := make(chan struct{}, 1)
for backoff := 10 * time.Second; ; backoff = min(backoff*2, 5*time.Minute) {
err := hear(ctx, func(subject string, body []byte) error {
if subject == DecidedSubject {
// The operator answered one of the bus module's asks: the waiting requests are read again now.
wake(asked)()
return nil
}
if subject != AcceptedSubject && !strings.HasSuffix(subject, ".event."+catalog.EventManifestAccepted) {
return nil
}
a, err := AcceptedOf(body)
if err != nil {
logf("a manifest-accepted cannot be read (%v): taken, and nothing made of it", err)
return nil
}
k.Heard(a)
wake(heard)()
return nil
})
if err == nil {
break
}
logf("manifest-accepted cannot be heard yet (%v): asked again in %s", err, backoff)
// The fill pass runs on while the consumer is bound again (review round 2, running F1).
logf("the keeper's consumer cannot be heard yet (%v): asked again in %s", err, backoff)
// The fill pass and the plans run on while the consumer is bound again (review round 2, running F1).
bound := time.After(backoff)
binding:
for {
@@ -396,6 +460,8 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func
return ctx.Err()
case <-tick.C:
k.FillPass()
case <-moved:
plan()
case <-bound:
break binding
}
@@ -421,7 +487,8 @@ func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func
k.FillPass()
case <-asked:
k.ActOnRequests()
case <-heard:
case <-moved:
plan()
case <-again:
again = nil
k.CheckAll()
@@ -454,3 +521,30 @@ func requestsOf(c *process.Conn) func(context.Context, func()) error {
})
}
}
// keeperSettings are the keeper's settings its manifest declares as a file (design 47 G A-112): the content of
// `${config:<key>}`, read at its start.
type keeperSettings struct {
SnapshotsCap int64 `json:"snapshots-cap"`
SnapshotKeepDays int `json:"snapshot-keep-days"`
}
// readSettings reads the settings file at path; a file that is missing, or a value that is not above 0, gives its default.
func readSettings(path string) keeperSettings {
s := keeperSettings{SnapshotsCap: keeper.DefaultSnapshotsCap, SnapshotKeepDays: keeper.DefaultSnapshotKeepDays}
raw, err := os.ReadFile(path)
if err != nil {
return s
}
var f keeperSettings
if json.Unmarshal(raw, &f) != nil {
return s
}
if f.SnapshotsCap > 0 {
s.SnapshotsCap = f.SnapshotsCap
}
if f.SnapshotKeepDays > 0 {
s.SnapshotKeepDays = f.SnapshotKeepDays
}
return s
}
+50 -44
View File
@@ -6,67 +6,73 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"nats-tools/internal/keeper"
)
// The keeper's inputs (design 25 § 12.2; CAT-N2): a manifest-accepted read with the indexer's revision when it carries
// one (0 when not yet), its entries whole; the first view of indexer_bus_state read bare or as the tool runner wraps it,
// an error answer an error; anything else refused, naming why.
func TestTheKeepersInputsAreRead(t *testing.T) {
// The catalog half as the keeper plans from it (design 47 G §532, §533, §547, §591): every module some watched branch's
// newest revision lists, each local name from the first of its watched branches in its registration's order whose entry
// asks it, another branch asking it with other settings a drift naming that branch; a branch the registration does not
// name after; a verdict not accepted asks nothing; the claims of every branch; a foundation part and its old sections'
// caps from the first branch.
func TestTheCatalogHalfIsReadPerWatchedBranch(t *testing.T) {
entryOf := func(verdict string, manifest map[string]any) catalog.CatalogEntry {
manifest["module"] = "m"
return catalog.CatalogEntry{Module: "m", Verdict: verdict, ManifestDigest: "sha256:" + verdict, Manifest: manifest}
}
state := func(opts string) map[string]any {
e := map[string]any{"name": "drafts", "storage": "key-value", "template": "state"}
if opts != "" {
e["options"] = map[string]any{"history": opts}
}
return e
}
log := map[string]any{"name": "log", "storage": "stream", "template": "log"}
main := entryOf("accepted", map[string]any{"bus-state": []any{state("")}, "claims": []any{map[string]any{"name": "a"}}})
feature := entryOf("accepted", map[string]any{"bus-state": []any{state("5"), log}, "claims": []any{map[string]any{"name": "b"}}})
for _, row := range []struct {
name, body, want, refused string
view bool
name string
branches []Branch
order []string
want string
}{
{name: "accepted with its revision", body: `{"v":1,"module":"m","digest":"sha256:a","judge":"j","revision":7,"bus-state":[{"name":"drafts","storage":"key-value","template":"state"}]}`,
want: "m sha256:a 7 [drafts/key-value/state]"},
{name: "accepted before the event carries a revision", body: `{"v":1,"module":"m","digest":"sha256:b","judge":"j","bus-state":[]}`,
want: "m sha256:b 0 []"},
{name: "accepted naming no module", body: `{"v":1,"digest":"x"}`, refused: "names no module"},
{name: "first view as the tool runner answers it", view: true,
body: `{"result":{"revision":12,"modules":[{"module":"m","digest":"d","revision":11,"bus-state":[{"name":"log","storage":"stream","template":"log"}]}]}}`,
want: "12: m d 11 [log/stream/log]"},
{name: "first view bare", view: true, body: `{"revision":3,"modules":[]}`, want: "3:"},
{name: "first view answering an error", view: true, body: `{"error":"not granted"}`, refused: "not granted"},
{name: "first view that is none", view: true, body: `[1,2]`, refused: "is not a first view"},
{"one branch", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}}, []string{"main"},
"m 3 [drafts/key-value/state] [a] false 0;"},
{"a feature branch asking a local name with other settings drifts, its new one planned",
[]Branch{{Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}, {Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}},
[]string{"main", "feature"}, "m 3 [drafts/key-value/state log/stream/log] [a b] false 0; drifted m drafts feature from main"},
{"the registration's order decides", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": main}}, {Name: "feature", Revision: 9, Entries: map[string]catalog.CatalogEntry{"m": feature}}},
[]string{"feature"}, "m 9 [drafts/key-value/state log/stream/log] [b a] false 0; drifted m drafts main from feature"},
{"a verdict not accepted asks nothing", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("refused", map[string]any{"bus-state": []any{state("")}})}}}, []string{"main"},
"m 3 [] [] false 0;"},
{"a foundation part, its old sections counted", []Branch{{Name: "main", Revision: 3, Entries: map[string]catalog.CatalogEntry{"m": entryOf("accepted", map[string]any{
"objects": []any{map[string]any{"name": "sent-objects", "max-mib": 256}}, "logs": []any{map[string]any{"name": "l", "max-mib": 4}}, "streams": []any{map[string]any{"name": "s"}}})}}}, []string{"main"},
"m 3 [] [] true 1284;"},
} {
t.Run(row.name, func(t *testing.T) {
v, err := ViewOf(row.branches, func(string) []string { return row.order })
if err != nil {
t.Fatal(err)
}
var got string
var err error
if row.view {
v, e := FirstViewOf([]byte(row.body))
err = e
got = fmt.Sprintf("%d:", v.Revision)
for _, m := range v.Modules {
got += " " + line(m.Module, m.Digest, m.Revision, m.BusState)
for _, m := range v.Modules {
es := []string{}
for _, e := range m.BusState {
es = append(es, e.Name+"/"+e.Storage+"/"+e.Template)
}
} else {
a, e := AcceptedOf([]byte(row.body))
err = e
got = line(a.Module, a.Digest, a.Revision, a.BusState)
got += fmt.Sprintf("%s %d [%s] %v %v %d;", m.Module, m.Revision, strings.Join(es, " "), m.Claims, m.Foundation, m.OldBytes/keeper.MiB)
}
if row.refused != "" {
if err == nil || !strings.Contains(err.Error(), row.refused) {
t.Fatalf("read %s (%v): refused, naming %q", got, err, row.refused)
}
return
for _, d := range v.Drifts {
got += fmt.Sprintf(" drifted %s %s %s from %s", d.Module, d.Local, d.Branch, d.From)
}
if err != nil || got != row.want {
t.Errorf("read %q (%v), want %q", got, err, row.want)
if got != row.want {
t.Errorf("read %q, want %q", got, row.want)
}
})
}
}
func line(module, digest string, revision uint64, entries []catalogue.BusStateEntry) string {
es := []string{}
for _, e := range entries {
es = append(es, e.Name+"/"+e.Storage+"/"+e.Template)
}
return fmt.Sprintf("%s %s %d [%s]", module, digest, revision, strings.Join(es, " "))
}
// The keeper's health tool (CAT-N6): healthy naming its pass; unhealthy, naming why, when its pass is old or verdicts
// wait on its consumer `<node>_nats_keeper` past the bound.
func TestTheKeepersHealthToolReadsItsStatusAndItsConsumer(t *testing.T) {
+1 -1
View File
@@ -82,7 +82,7 @@ func keeperHealth(status func() (keeper.Status, error), read heldEntries, jsz fu
func keeperHealthTool(status func() (keeper.Status, error), read heldEntries, jsz func() ([]byte, error)) stdio.Tool {
return stdio.Tool{Name: "nats_keeper_health",
Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, its first view (the indexer's indexer_bus_state) has not answered, or accepted verdicts wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).",
Description: "The bus module's keeper's health, as the node-engine asks it: unhealthy, naming why, when its fill pass is more than three minutes old, the catalog half's first views of its watched branches (the seat mesh-catalog's revision) have not answered, or events wait on its consumer unanswered past three minutes; it names beside the entries it could not create, the ones waiting, and those past the raise mark (another module's full entry raises that module's condition and leaves the keeper healthy).",
Run: func(map[string]any) (any, error) {
return keeperHealth(status, read, jsz, func() time.Time { return time.Now().UTC() }), nil
}}
@@ -9,6 +9,7 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"slices"
"strings"
@@ -24,8 +25,10 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views"
"git.novox.be/novox/mesh-sdk/go/bussnap"
@@ -108,7 +111,8 @@ func eventually(t *testing.T, what string, ok func() bool) {
}
// The bus module runs whole (design 25 § 12.7, "The bus module runs whole" and "The bus module's first view"; CAT-N10):
// its keeper against the real bus server, hearing manifest-accepted on its durable consumer, creating on the bus,
// its keeper against the real bus server, planning from the catalog half of every watched branch and the seat list
// (design 47 G §344, §532, §533, §537, §538) at each move of them, creating on the bus,
// saying its events on its own stream BROKER_EVENTS, keeping its state in its own buckets, and acting on a person's
// requests. One server for every case, in order, as the mesh runs them.
func TestTheBusModuleRunsWhole(t *testing.T) {
@@ -144,7 +148,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
// The keeper's consumer hears the indexer's verdicts and the operator channel's decided for the bus module's own
// asks, as the grants part composes it from its consumes and the seats it uses.
if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: account.QueueConsumerName(),
FilterSubjects: []string{AcceptedSubject, DecidedSubject}, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 5}); err != nil {
FilterSubjects: []string{DecidedSubject}, AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 5}); err != nil {
t.Fatal(err)
}
// The router's own record of the asks, which the keeper reads under its own process account (process.Conn.Ask), never
@@ -158,7 +162,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
t.Fatal(err)
}
defer pconn.Close()
for _, st := range []string{keeper.StateAsked, keeper.StateRequests, keeper.StateSpent, "findings"} {
for _, st := range []string{keeper.StateAsked, keeper.StateRequests, keeper.StateSpent, keeper.StatePlans, "findings"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "nats_" + st, History: 5}); err != nil {
t.Fatal(err)
}
@@ -166,7 +170,18 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
store := kvStore{js}
snapshots := t.TempDir()
var mu sync.Mutex
view := keeper.FirstView{Revision: 1}
// The catalog half as meshcatalogclient's branch follower holds it, branch by branch in the registrations' order, and
// the seat list's definitions; each change moves the keeper, as the follower's and the seat list's changes do.
branches := []Branch{{Name: "main", Revision: 1, Entries: map[string]catalog.CatalogEntry{}}}
order := []string{"main"}
defs := map[string]seatdef.Definition{}
moved := make(chan struct{}, 1)
move := func() {
select {
case moved <- struct{}{}:
default:
}
}
var acts []string
// The bus module's findings go through a held writer over its state findings, started as the keeper's process starts it.
writer, err := views.NewHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{
@@ -182,7 +197,13 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
newKeeper := func() *keeper.Keeper {
return &keeper.Keeper{Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, snapshots), NC: nc, Dir: snapshots, Log: t.Logf}, Store: store,
Finding: writer, Emit: streamEmitter(js, account), Say: sayer(js),
View: func() (keeper.FirstView, error) { mu.Lock(); defer mu.Unlock(); return view, nil },
Catalog: func() (keeper.CatalogView, error) {
mu.Lock()
defer mu.Unlock()
return ViewOf(branches, func(string) []string { return order })
},
Seats: func() map[string]seatdef.Definition { mu.Lock(); defer mu.Unlock(); return maps.Clone(defs) },
Again: func() error { return nil },
Act: func(what, why, cause string) error {
mu.Lock()
defer mu.Unlock()
@@ -208,15 +229,28 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
}()
return nil
}
go func() { _ = Keep(kctx, k, hearer(js, account, t.Logf), watch, t.Logf); close(done) }()
go func() { _ = Keep(kctx, k, hearer(js, account, nil, nil, t.Logf), watch, moved, t.Logf); close(done) }()
return func() { stop(); <-done }
}
accepted := func(module string, revision uint64, entries ...catalogue.BusStateEntry) {
body, _ := json.Marshal(map[string]any{"v": 1, "module": module, "digest": fmt.Sprintf("sha256:%d", revision),
"judge": "j", "revision": revision, "bus-state": entries})
if _, err := js.Publish(ctx, AcceptedSubject, body); err != nil {
t.Fatal(err)
// onBranch puts a module's entry in a branch's newest revision (none: the module leaves it), and moves the keeper.
onBranch := func(branch, module string, revision int64, manifest map[string]any) {
mu.Lock()
for i := range branches {
if branches[i].Name == branch {
branches[i].Revision = revision
if manifest == nil {
delete(branches[i].Entries, module)
} else {
manifest["module"] = module
branches[i].Entries[module] = catalog.CatalogEntry{Module: module, Verdict: "accepted", ManifestDigest: fmt.Sprintf("sha256:%d", revision), Manifest: manifest}
}
}
}
mu.Unlock()
move()
}
accepted := func(module string, revision uint64, entries ...catalogue.BusStateEntry) {
onBranch("main", module, int64(revision), map[string]any{"bus-state": entries})
}
// saidOn reads what one stream holds of the bus module's events, by its subjects' prefix: its own stream (its
// history), or EVENTS on the seat mesh-broker's subjects (what every consumer of mesh-broker.<event> hears).
@@ -264,11 +298,13 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
// one guard never waits on the indexer.
t.Run("the fill pass runs while the first view is asked", func(t *testing.T) {
down := newKeeper()
down.View = func() (keeper.FirstView, error) { return keeper.FirstView{}, errors.New("no responders") }
down.Catalog = func() (keeper.CatalogView, error) {
return keeper.CatalogView{}, errors.New("the catalog half has no first view of main yet")
}
stopDown := run(down)
eventually(t, "a fill pass with no first view", func() bool { return !down.Status().Pass.IsZero() && down.Status().FirstView.Answered.IsZero() })
stopDown()
// And while its consumer of manifest-accepted is bound again on its backoff (review round 2, running F1).
// And while its consumer is bound again on its backoff (review round 2, running F1).
was := FillEvery
FillEvery = 20 * time.Millisecond
defer func() { FillEvery = was }()
@@ -276,7 +312,7 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
kctx, stopDeaf := context.WithCancel(ctx)
done := make(chan struct{})
go func() {
_ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, t.Logf)
_ = Keep(kctx, deaf, func(context.Context, func(string, []byte) error) error { return errors.New("no consumer yet") }, nil, nil, t.Logf)
close(done)
}()
eventually(t, "the first view answered", func() bool { return !deaf.Status().FirstView.Answered.IsZero() })
@@ -309,27 +345,6 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
stop = run(k)
eventually(t, "created n jobs", saidHas("created n jobs"))
})
t.Run("a replay older than the first view is not applied; a set-aside module keeps its key", func(t *testing.T) {
stop()
mu.Lock()
view = keeper.FirstView{Revision: 9, Modules: []keeper.Accepted{{Module: "p", Revision: 9, BusState: []catalogue.BusStateEntry{state}}}}
mu.Unlock()
accepted("p", 4)
k = newKeeper()
stop = run(k)
eventually(t, "created p drafts", saidHas("created p drafts"))
eventually(t, "the replay heard", func() bool {
info, _ := js.Consumer(ctx, "EVENTS", account.QueueConsumerName())
i, _ := info.Info(ctx)
return i.NumPending == 0 && i.NumAckPending == 0
})
if saidHas("orphaned p drafts")() {
t.Error("the replay at revision 4 orphaned what the first view at 9 asked")
}
if _, _, found, _ := store.Get(keeper.StateAsked, "m"); !found {
t.Error("m, missing from the first view, lost its key")
}
})
t.Run("drift is said, a growth applied and the drift cleared", func(t *testing.T) {
accepted("m", 5, state, entry("log", "stream", "log", map[string]string{"max-mib": "1", "max-age-days": "2"}))
eventually(t, "drifted m log", saidHas("drifted m log"))
@@ -521,8 +536,11 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
if a := restore("live"); !strings.Contains(a.Error, "names live") {
t.Errorf("a restore over a live stream answered %+v", a)
}
if err := js.DeleteStream(ctx, "KV_m_drafts"); err != nil {
t.Fatal(err)
// The state's snapshot holds its objects store beside it (design 47 G §58): a restore goes over neither.
for _, name := range []string{"KV_m_drafts", "OBJ_m_drafts-objects"} {
if err := js.DeleteStream(ctx, name); err != nil {
t.Fatal(err)
}
}
a := restore("it was removed by mistake")
if a.Accepted == "" {
@@ -551,6 +569,63 @@ func TestTheBusModuleRunsWhole(t *testing.T) {
t.Error("the snapshot's record outlived its file")
}
})
// §532: a branch registered while the keeper runs is followed with no restart; a feature branch asking a local name with
// other settings than the default branch's is said drifted, naming it, while the default branch's are planned, and its
// new local name is planned from it.
t.Run("a branch registered while it runs is planned from, a local name with other settings drifted", func(t *testing.T) {
mu.Lock()
branches = append(branches, Branch{Name: "feature", Revision: 4, Entries: map[string]catalog.CatalogEntry{"m": {Module: "m", Verdict: "accepted",
Manifest: map[string]any{"module": "m", "bus-state": []any{state, entry("log", "stream", "log", map[string]string{"max-mib": "2"}), entry("extra", "key-value", "state", nil)}}}}})
order = []string{"main", "feature"}
mu.Unlock()
move()
eventually(t, "created m extra", saidHas("created m extra"))
eventually(t, "drifted m log", saidHas("drifted m log"))
if info, _ := held("m_log"); info.Config.MaxBytes != 4<<20 {
t.Errorf("m_log's cap is %d: the default branch's 4 MiB planned", info.Config.MaxBytes)
}
mu.Lock()
branches, order = branches[:1], []string{"main"}
mu.Unlock()
move()
eventually(t, "orphaned m extra", saidHas("orphaned m extra"))
})
// §344, §533, §537: a module of the new form claiming a seat with a verb gets a sent store, one with neither none, a seat
// whose verbs come only from an included part one, a seat including only bodies none, a foundation part none; a
// definition gaining a verb gives its holder one. §538: a module in no watched branch's newest revision loses its plans key.
t.Run("the sent stores follow the catalog half and the seat list", func(t *testing.T) {
mu.Lock()
defs["verbed"] = seatdef.Definition{Seat: "verbed", Verbs: []seatdef.Verb{{Name: "go"}}}
defs["bare"] = seatdef.Definition{Seat: "bare"}
defs["resolved"] = seatdef.Definition{Seat: "resolved", Includes: []string{"part"}, Verbs: []seatdef.Verb{{Name: "from-the-part"}}}
defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}}
mu.Unlock()
claims := func(seat string) map[string]any { return map[string]any{"claims": []any{map[string]any{"name": seat}}} }
onBranch("main", "sa", 5, claims("verbed"))
onBranch("main", "sb", 5, claims("bare"))
onBranch("main", "sc", 5, claims("resolved"))
onBranch("main", "sd", 5, claims("bodies"))
f := claims("verbed")
f["objects"] = []any{map[string]any{"name": "sent-objects", "max-mib": 256}}
onBranch("main", "se", 5, f)
eventually(t, "sa and sc's sent stores", func() bool {
_, a := held("OBJ_sa_sent-objects")
_, c := held("OBJ_sc_sent-objects")
return a && c
})
for _, none := range []string{"sb", "sd", "se"} {
if _, ok := held("OBJ_" + none + "_sent-objects"); ok {
t.Errorf("%s has a sent store", none)
}
}
mu.Lock()
defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}, Verbs: []seatdef.Verb{{Name: "new"}}}
mu.Unlock()
move()
eventually(t, "sd's sent store once its seat gains a verb", func() bool { _, ok := held("OBJ_sd_sent-objects"); return ok })
onBranch("main", "sb", 6, nil)
eventually(t, "sb's plan deleted", func() bool { _, _, found, _ := store.Get(keeper.StatePlans, "sb"); return !found })
})
t.Run("nats_first_view_again asks at once and records the hand act", func(t *testing.T) {
raw, _ := json.Marshal(keeper.Request{ID: "r3", Verb: keeper.VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
_, _ = store.Put(keeper.StateRequests, "r3", raw)
+16 -2
View File
@@ -6,6 +6,7 @@ package main
import (
"context"
"encoding/json"
"fmt"
"os"
"time"
@@ -192,10 +193,23 @@ func allTools() []stdio.Tool {
}
out := append(tools(monitor, ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime, HeldInState)...)
out = append(out, expiryTools(LiveBus(), SaidByTheRuntime, RecordedBySeat)...)
out = append(out, brokerTools(HeldInState, jsz, StreamHistory(LiveBus()))...)
out = append(out, brokerTools(HeldInState, StatusInState, jsz, StreamHistory(LiveBus()))...)
out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz))
out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...)
rs := RequestsInState()
out = append(out, snapshotListTool(rs, int(number(os.Getenv, "MESH_NATS_SNAPSHOT_KEEP_DAYS", keeper.DefaultSnapshotKeepDays)))...)
out = append(out, snapshotListTool(rs, bundleKeepDays())...)
out = append(out, plansTool(PlansInState))
return append(out, requestTools(rs)...)
}
// bundleKeepDays is the setting snapshot-keep-days as this bundle's configuration holds it (design 47 G A-112), its
// default when the runtime gives none.
func bundleKeepDays() int {
if raw, err := stdio.Config("snapshot-keep-days"); err == nil {
var n int
if json.Unmarshal(raw, &n) == nil && n > 0 {
return n
}
}
return keeper.DefaultSnapshotKeepDays
}
+66 -12
View File
@@ -8,6 +8,8 @@ import (
"slices"
"strings"
"testing"
"nats-tools/internal/keeper"
)
// The shape ADR 0480 clauses 1, 2, 3 and 5 give the bus's manifest: the upstream image by digest, the mesh's two
@@ -159,8 +161,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
}
}
// The bus module provides bus state (design 25 § 12.2-12.3, ADR 0595 §4-§6; CAT-N1): its keeper is a process of
// its own on its own account (own secret broker-keeper, ADR 0436 §1), creating only bus state; it hears the indexer's
// manifest-accepted, keeps what it heard in its own old state `asked` (ADR 0595 §10) and the persons' requests in
// its own on its own account (own secret broker-keeper, ADR 0436 §1), creating only bus state; it reads the catalog half
// of every watched branch and the seat list (design 47 G §433, §532, §614), keeps what it planned in its own old state `asked` (ADR 0595 §10) and the persons' requests in
// `requests`; its nine events are its own (ADR 0620 §2), on its own stream that keeps its history (ADR 0592 §6, the manifest
// validator's predefined form) at 256 MiB; the seat's four
// reading verbs are served, its four acting verbs are the keeper's own work and no other account's; its health is
@@ -192,17 +194,22 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, append(slices.Clone(ReadingVerbs), SnapshotVerbs...)) {
t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v and its snapshot verbs %v alone", nm.Claims, ReadingVerbs, SnapshotVerbs)
}
if !slices.Equal(nm.Consumes, []string{"mesh-catalog.manifest-accepted", "mesh-conditions.findings-asked"}) {
t.Errorf("consumes %v: the seat mesh-catalog's manifest-accepted (ADR 0595 §4) and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes)
// The catalog half replaces manifest-accepted and indexer_bus_state (design 47 G §433, §614), and the seat list is read
// through the seat mesh-catalog (§537).
catalogHalf := []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved",
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"}
if !slices.Equal(nm.Consumes, append(slices.Clone(catalogHalf), "mesh-conditions.findings-asked")) {
t.Errorf("consumes %v: the catalog half's and the seat list's events %v, and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes, catalogHalf)
}
if AcceptedSubject != "mesh.seat.mesh-catalog.event.manifest-accepted" {
t.Errorf("the keeper's durable consumer filters on %s, not on the seat mesh-catalog's event", AcceptedSubject)
}
for _, want := range []string{"mesh-delivery-indexer.indexer_bus_state", "seat:mesh-hand-acts.hand-act"} {
for _, want := range []string{"seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source",
"seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act"} {
if !slices.Contains(nm.Invokes, want) {
t.Errorf("invokes %v lack %s", nm.Invokes, want)
}
}
if strings.Contains(string(raw), "indexer_bus_state") || strings.Contains(string(raw), "manifest-accepted") {
t.Error("the manifest still names indexer_bus_state or manifest-accepted (design 47 G §433)")
}
if !slices.Contains(nm.Uses, "operator-channel") {
t.Errorf("uses %v: the operator channel, for the warrant of recreate and drop", nm.Uses)
}
@@ -228,7 +235,7 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
}
// spent, the warrants the keeper spent, is written by the keeper alone and only read by the tools (review round 2,
// security F2: the tool runner's account, which writes requests, may never delete a spend record).
if !slices.Equal(states, []string{"asked", "requests", "spent", "findings"}) || !slices.Equal(readOnly, []string{"spent"}) {
if !slices.Equal(states, []string{"asked", "requests", "spent", "plans", "findings"}) || !slices.Equal(readOnly, []string{"spent", "plans"}) {
t.Errorf("state is %v, read-only %v: asked and requests, old sections (ADR 0595 §10), and spent, read-only", states, readOnly)
}
if _, ok := nm.OwnSecrets["broker-keeper"]; !ok {
@@ -238,7 +245,7 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
bus, _ := keeper["bus"].(map[string]any)
health, _ := keeper["health"].(map[string]any)
if keeper["type"] != "process" || keeper["artifact"] != "tools" || fmt.Sprint(keeper["run"]) != "[./nats-tools keep]" ||
fmt.Sprint(bus["state"]) != "[asked requests spent findings]" ||
fmt.Sprint(bus["state"]) != "[asked requests spent plans findings]" ||
health["kind"] != "tool" || health["tool"] != "nats_keeper_health" {
t.Errorf("the keeper is %v: a process of the tools bundle, run as keep, keeping asked, requests and spent, judged by nats_keeper_health", keeper)
}
@@ -248,8 +255,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) {
t.Fatal(err)
}
for _, st := range whole.State {
if (st.Name == "spent") != st.ReadOnly {
t.Errorf("the SDK reads %s read-only %v: spent alone is read-only", st.Name, st.ReadOnly)
if (st.Name == "spent" || st.Name == "plans") != st.ReadOnly {
t.Errorf("the SDK reads %s read-only %v: spent and plans alone are read-only", st.Name, st.ReadOnly)
}
}
if b := whole.ProcessBuses()["keeper"]; !b.Keeps {
@@ -315,3 +322,50 @@ func TestTheKeepersProcessServesTheVerbsItAnswers(t *testing.T) {
t.Errorf("the keeper process serves %v; its code answers %v", served, want)
}
}
// The keeper's snapshot settings are config values of the manifest, written into a file the keeper reads at its start,
// never environment (design 47 G A-112, §681): a missing file, or a value not above 0, gives the default.
func TestTheKeepersSnapshotSettingsAreAFileItsManifestDeclares(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Config map[string]struct{ Default float64 } `json:"config"`
Resources []struct {
ID string `json:"id"`
Path string `json:"path"`
Owner string `json:"owner"`
TakesConfig []string `json:"takes-config"`
Env map[string]string `json:"env"`
} `json:"resources"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
if m.Config["snapshots-cap"].Default != 21474836480 || m.Config["snapshot-keep-days"].Default != 90 {
t.Errorf("config %v: snapshots-cap 20 GiB and snapshot-keep-days 90 are the defaults", m.Config)
}
found := false
for _, r := range m.Resources {
if r.ID == "keeper-settings" {
found = r.Path == "${dir:keeper-state}/settings.json" && r.Owner == "nats-keeper" && slices.Equal(r.TakesConfig, []string{"snapshots-cap", "snapshot-keep-days"})
}
for k := range r.Env {
if strings.Contains(k, "SNAPSHOTS_CAP") || strings.Contains(k, "SNAPSHOT_KEEP_DAYS") {
t.Errorf("%s still sets %s in the environment", r.ID, k)
}
}
}
if !found {
t.Errorf("the keeper has no settings file taking snapshots-cap and snapshot-keep-days")
}
dir := t.TempDir()
if got := readSettings(dir + "/none.json"); got.SnapshotsCap != keeper.DefaultSnapshotsCap || got.SnapshotKeepDays != keeper.DefaultSnapshotKeepDays {
t.Errorf("a missing file gives %+v", got)
}
_ = os.WriteFile(dir+"/s.json", []byte(`{"snapshots-cap": 1000, "snapshot-keep-days": 0}`), 0o600)
if got := readSettings(dir + "/s.json"); got.SnapshotsCap != 1000 || got.SnapshotKeepDays != keeper.DefaultSnapshotKeepDays {
t.Errorf("the file gives %+v: the cap it names, the day count's default for 0", got)
}
}
+21
View File
@@ -11,6 +11,7 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
@@ -86,3 +87,23 @@ func ParseWritten(s string) (time.Time, error) {
}
return t, nil
}
// serverStorageLimit reads the server's own JetStream storage limit in bytes from its monitoring endpoint (`config`'s
// max_storage; 0 or less is none), shown beside the account's (design 47 G §547).
func serverStorageLimit(monitor Monitor) func(ctx context.Context) (int64, error) {
return func(ctx context.Context) (int64, error) {
raw, err := monitor(ctx, "/jsz?config=1")
if err != nil {
return 0, err
}
var doc struct {
Config struct {
MaxStorage int64 `json:"max_storage"`
} `json:"config"`
}
if err := json.Unmarshal(raw, &doc); err != nil {
return 0, err
}
return max(doc.Config.MaxStorage, 0), nil
}
}
+1 -1
View File
@@ -209,7 +209,7 @@ func requestTools(rs requestStore) []stdio.Tool {
}}
}
firstView := stdio.Tool{Name: "nats_first_view_again",
Description: "Ask the bus module's keeper to ask the indexer's indexer_bus_state, its first view of every accepted manifest's bus state, at once (design 25 § 12.2): by hand, beside its own backoff, recorded as a hand act; answers what the first view answered and its revision.",
Description: "Ask the bus module's keeper to ask the catalog half's first views of every watched branch and the seat list again at once, and plan from them (design 47 G §614): by hand, recorded as a hand act; answers whether they answered and the newest revision planned from.",
Input: map[string]any{"why": str("why, in plain words: kept with the act"), "request": str("a request made before: its end is read")},
Run: func(a map[string]any) (any, error) {
if id := arg(a, "request"); id != "" {
@@ -145,3 +145,21 @@ func TestAPersonsRequestIsHandedToTheKeeperAsTheProvedCaller(t *testing.T) {
t.Errorf("an end not written: %+v on %d watch(es)", r, watches)
}
}
// The ask for a recreate or a drop lasts what its highest option level allows (design 47 G §570), so the asker client
// takes it: an expiry at the bound passes occ.Check, and the fixed 12 hours it was is refused.
func TestTheAskLastsWhatItsLevelAllows(t *testing.T) {
now := time.Now()
r := keeper.Request{ID: "r1", Verb: keeper.VerbDrop, Module: "m", Local: "drafts", Why: "unused", Caller: "person.jo"}
a, err := askFor(r, now)
if err != nil {
t.Fatal(err)
}
if err := occ.Check(a, now); err != nil {
t.Errorf("the ask is refused by the asker client: %v", err)
}
a.Expires = occ.FormatTime(now.Add(12 * time.Hour))
if err := occ.Check(a, now); err == nil || !strings.Contains(err.Error(), "at most") {
t.Errorf("12 hours is held to the level's bound: %v", err)
}
}
+4
View File
@@ -0,0 +1,4 @@
{
"snapshots-cap": 21474836480,
"snapshot-keep-days": 90
}
+2
View File
@@ -8,7 +8,9 @@ require (
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
github.com/klauspost/compress v1.20.1
+5
View File
@@ -21,6 +21,11 @@ func (k *Keeper) FillPass() {
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if lim, err := k.Bus.Limits(ctx); err == nil {
k.setStatus(func(s *Status) { s.Limits = lim })
} else {
k.logf("the bus's storage limits cannot be read: %v", err)
}
for _, module := range keys {
held, _, found := k.asked(module)
if !found {
+21
View File
@@ -23,6 +23,27 @@ type JetStream struct {
NC *nats.Conn
Dir string
Log func(string, ...any)
// Server reads the server's own storage limit in bytes (0 none), from its monitoring endpoint.
Server func(ctx context.Context) (int64, error)
}
// Limits reads the account's storage limit (-1 is none) and the server's (design 47 G §547).
func (b JetStream) Limits(ctx context.Context) (Limits, error) {
info, err := b.JS.AccountInfo(ctx)
if err != nil {
return Limits{}, err
}
var l Limits
if info.Limits.MaxStore > 0 {
l.AccountBytes = info.Limits.MaxStore
}
if b.Server != nil {
// The server's limit is only shown, never held against (§547): one that cannot be read is shown as none.
if n, err := b.Server(ctx); err == nil {
l.ServerBytes = n
}
}
return l, nil
}
// Held reads the object's stream.
+286 -60
View File
@@ -3,6 +3,7 @@ package keeper
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
@@ -16,6 +17,7 @@ import (
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/membership"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// Module is the bus module's own name: the publisher of its events and its conditions.
@@ -26,6 +28,9 @@ const Module = "nats"
const (
StateAsked = "asked"
StateRequests = "requests"
// StatePlans holds, per module, the bus-state entries as last planned and their digest (design 47 G §538): what a
// verdict's entries are compared with, never the manifest's digest, which changes with any field.
StatePlans = "plans"
// KeyStatus is the keeper's own status in StateAsked, read by its health tool: a key no module can have, since a
// module's name holds no underscore.
KeyStatus = "_keeper"
@@ -77,8 +82,8 @@ type EventFill struct {
Cap int64 `json:"cap"`
}
// Accepted is a manifest-accepted the keeper heard, or a module of its first view: the module, its manifest's digest,
// the indexer's revision of the verdict (0 when the event does not say it yet), and its bus-state entries whole.
// Accepted is one module's entry as the catalog half's watched branches give it (design 47 G §344, §433, §532): the
// module, its manifest's digest, the catalog revision of the branch it was read from, and its bus-state entries whole.
type Accepted struct {
Module string `json:"module"`
Digest string `json:"digest"`
@@ -86,10 +91,51 @@ type Accepted struct {
BusState []catalogue.BusStateEntry `json:"bus-state"`
}
// FirstView is the indexer's answer of indexer_bus_state: every accepted manifest's bus-state entries, by module.
type FirstView struct {
Revision uint64 `json:"revision"`
Modules []Accepted `json:"modules"`
// CatalogModule is one module of the catalog half for the bus module (design 47 G §344, §532, §533, §547): what its
// watched branches' newest revisions ask of the bus, the seats they claim, whether one declares an event of its own,
// whether it is a foundation part (it declares an old `objects` section `sent-objects`), and the caps of the old
// sections it declares, in bytes.
type CatalogModule struct {
Accepted
Claims []string `json:"claims,omitempty"`
EmitsOwn bool `json:"emits-own,omitempty"`
Foundation bool `json:"foundation,omitempty"`
OldBytes int64 `json:"old-bytes,omitempty"`
}
// Drift is a local name a later watched branch asks with other settings than the branch it is planned from (design 47
// G §532): said as the bus module's drifted, naming the branch, and planned from nothing of that branch.
type Drift struct {
Module string `json:"module"`
Local string `json:"local-name"`
Branch string `json:"branch"`
// From is the branch whose settings are planned.
From string `json:"from"`
}
// CatalogView is the catalog half as the bus module plans from it: every module in a watched branch's newest revision,
// and the drifts between branches.
type CatalogView struct {
Modules []CatalogModule
Drifts []Drift
}
// Planned is the value of a module's key in StatePlans: the entries as planned, their digest and the revision of the verdict.
type Planned struct {
V int `json:"v"`
Module string `json:"module"`
Digest string `json:"digest"`
Revision uint64 `json:"revision"`
Entries []catalogue.BusStateEntry `json:"entries"`
At time.Time `json:"at"`
}
// EntriesDigest is the digest of bus-state entries alone (design 47 G §538): equal entries have an equal digest whatever
// else a manifest changed.
func EntriesDigest(entries []catalogue.BusStateEntry) string {
raw, _ := json.Marshal(entries)
sum := sha256.Sum256(raw)
return "sha256:" + hex.EncodeToString(sum[:])
}
// Asked is what the keeper holds of one module, the value of its key in StateAsked: what its newest accepted manifest
@@ -101,7 +147,9 @@ type Asked struct {
Revision uint64 `json:"revision"`
Entries []catalogue.BusStateEntry `json:"entries"`
Made map[string]Made `json:"made,omitempty"`
At time.Time `json:"at"`
// Drifts are the local names a later watched branch asks with other settings, as last said (design 47 G §532).
Drifts map[string]Drift `json:"drifts,omitempty"`
At time.Time `json:"at"`
}
// Made is one entry as the keeper made it: its plan and what stands of it now.
@@ -127,9 +175,15 @@ type Status struct {
Started time.Time `json:"started"`
// Pass is when the last fill pass ended.
Pass time.Time `json:"pass"`
// FirstView is the indexer's first view: when asked, when answered and at which revision, or why not.
// Limits are the bus's storage limits as last read (design 47 G §547): what `list` shows against the sum of the caps.
Limits Limits `json:"limits,omitempty"`
// OldBytes is the sum of the caps of every old section the catalog graph's entries declare, in bytes, as last planned
// (design 47 G §547, §591): counted in the sum of the bus's caps beside the entries made.
OldBytes int64 `json:"old-sections-bytes,omitempty"`
// FirstView is the catalog half's first views (design 47 G §614): when last planned from, when answered and at which
// revision, or what it waits for.
FirstView FirstViewStatus `json:"first-view"`
// Heard is when the last manifest-accepted was heard, and Unanswered since when one has waited unhandled.
// Heard is when the catalog half last moved, and Unanswered since when an event has waited unhandled.
Heard time.Time `json:"heard,omitempty"`
Unanswered time.Time `json:"unanswered,omitempty"`
// Unsaid are the events the bus did not take, kept with their id and the subjects still to say them on, and
@@ -167,10 +221,11 @@ type FirstViewStatus struct {
Next time.Time `json:"next,omitempty"`
}
// The first view's owner and verb (design 25 § 12.2).
// The first view's seat and verb: the catalog half of each watched branch, read through the seat mesh-catalog's
// `revision` (design 47 G §433, §614).
const (
FirstViewOwner = "mesh-delivery-indexer"
FirstViewVerb = "indexer_bus_state"
FirstViewOwner = "mesh-catalog"
FirstViewVerb = "revision"
)
// Bus is the bus as the keeper's own account reaches it.
@@ -193,6 +248,54 @@ type Bus interface {
// keeper's own partial stream of a restore that did not end).
Restore(ctx context.Context, path string) error
DeleteStream(ctx context.Context, name string) error
// Limits reads the bus's storage limits: the keeper's account's and the server's (design 47 G §547).
Limits(ctx context.Context) (Limits, error)
}
// Limits are the bus's storage limits in bytes; 0 is no limit (an unlimited account, or a server that sets none).
type Limits struct {
AccountBytes int64 `json:"account-bytes,omitempty"`
ServerBytes int64 `json:"server-bytes,omitempty"`
}
// CapSum is the sum, in bytes, of the caps of every object the entries made hold, but for the entry of skipModule and
// skipLocal (design 47 G §547): what the bus's limits are read against. An entry never made, or refused, holds none.
func CapSum(held map[string]Asked, skipModule, skipLocal string) int64 {
var sum int64
for module, a := range held {
for local, m := range a.Made {
if (module == skipModule && local == skipLocal) || !(m.Created || m.Orphaned) {
continue
}
for _, o := range m.Plan.Objects {
sum += o.Settings.MaxBytes
}
}
}
return sum
}
// capRoom refuses a plan that adds or raises a cap when the sum of the bus's caps would pass the account's limit,
// naming both (design 47 G §547); caps already made stay as they are. With no account limit nothing is refused.
func (k *Keeper) capRoom(ctx context.Context, held *Asked, m *Made) error {
lim, err := k.Bus.Limits(ctx)
if err != nil {
return fmt.Errorf("%w: the bus's storage limits cannot be read: %v", ErrRetry, err)
}
k.setStatus(func(s *Status) { s.Limits = lim })
if lim.AccountBytes <= 0 {
return nil
}
all := k.Held()
all[held.Module] = *held
sum := CapSum(all, held.Module, m.Plan.Local) + k.Status().OldBytes
for _, o := range m.Plan.Objects {
sum += o.Settings.MaxBytes
}
if sum > lim.AccountBytes {
return fmt.Errorf("the plan takes the sum of the bus's caps to %d MiB, past the account's storage limit of %d MiB", sum/MiB, lim.AccountBytes/MiB)
}
return nil
}
// Fill is how much one object holds: its bytes and messages.
@@ -225,8 +328,12 @@ type Keeper struct {
// Emit says one of the bus module's events (Saying); Say raises or clears a condition through the condition keeper.
Emit func(s *Saying) error
Say func(oc.Condition) error
// View asks the indexer's indexer_bus_state.
View func() (FirstView, error)
// Catalog reads the catalog half of every watched branch as it stands (design 47 G §433, §532): an error while a
// branch has no first view yet, when the keeper plans nothing. Seats are the seat list's definitions, each resolved
// with the parts it includes (§537). Again asks every first view of the catalog half again, by hand.
Catalog func() (CatalogView, error)
Seats func() map[string]seatdef.Definition
Again func() error
// Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask.
Act func(what, why, cause string) error
Warrant func(askID string) (*oc.Ask, *oc.Warrant, error)
@@ -345,70 +452,149 @@ func (k *Keeper) statusWritten(s Status, err error) {
}
}
// AskFirstView asks the indexer's first view once and applies it (design 25 § 12.2): every module it answers is
// applied as an accepted verdict, a module the keeper holds and the view leaves out is kept (a set-aside manifest is
// missing from it, never removed), and then check runs over what is held. It answers the error and when to ask again,
// on the backoff from 10 s doubling to 5 min; the keeper's health is unhealthy meanwhile, naming the verb.
func (k *Keeper) AskFirstView(wait time.Duration) (time.Duration, error) {
// PlanCatalog plans from the catalog half as it stands (design 47 G §344, §433, §532, §537, §538): each module in a
// watched branch's newest revision is applied (Heard), a module of the new form that sends gets its sent store, a drift
// between branches is said once, and a module in no watched branch's newest revision has its plans key deleted, its
// entries then orphans. Before every watched branch has its first view it plans nothing, and says what it waits for in
// its status; the first plan checks what is held.
func (k *Keeper) PlanCatalog() error {
at := k.now()
k.setStatus(func(s *Status) { s.FirstView.Asked = at })
view, err := k.View()
view, err := k.Catalog()
if err != nil {
next := wait * 2
if wait <= 0 {
next = 10 * time.Second
}
if next > 5*time.Minute {
next = 5 * time.Minute
}
k.setStatus(func(s *Status) {
s.FirstView.Error = fmt.Sprintf("%s.%s did not answer: %v", FirstViewOwner, FirstViewVerb, err)
s.FirstView.Next = at.Add(next)
})
return next, err
k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, err.Error() })
return err
}
k.mu.Lock()
first := !k.viewed
k.viewed = true
k.mu.Unlock()
k.setStatus(func(s *Status) {
s.FirstView.Answered, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = k.now(), view.Revision, "", time.Time{}
})
for _, a := range view.Modules {
k.Heard(a)
var revision uint64
var old int64
for _, m := range view.Modules {
revision = max(revision, m.Revision)
old += m.OldBytes
}
k.CheckAll()
return 0, nil
k.setStatus(func(s *Status) {
s.FirstView.Asked, s.FirstView.Revision, s.FirstView.Error, s.FirstView.Next = at, revision, "", time.Time{}
if s.FirstView.Answered.IsZero() {
s.FirstView.Answered = at
}
s.OldBytes = old
})
var defs map[string]seatdef.Definition
if k.Seats != nil {
defs = k.Seats()
}
listed := map[string]bool{}
for _, m := range view.Modules {
listed[m.Module] = true
k.Heard(m.Accepted)
if !m.Foundation && Sends(m.Claims, m.EmitsOwn, defs) {
k.Sends(m.Module)
}
}
k.drifts(view.Drifts)
gone := map[string]bool{}
for module := range k.Held() {
gone[module] = !listed[module]
}
if keys, err := k.Store.Keys(StatePlans); err == nil {
for _, module := range keys {
gone[module] = !listed[module]
}
}
for _, module := range sortedKeys(gone) {
if !gone[module] {
continue
}
if _, ok := k.planned(module); ok {
k.Heard(Accepted{Module: module, Revision: revision})
}
if err := k.Store.Delete(StatePlans, module); err != nil {
k.logf("the plan of %s cannot be removed from its own %s: %v", module, StatePlans, err)
}
}
if first {
k.CheckAll()
}
return nil
}
// Heard applies one accepted verdict (manifest-accepted, or a module of the first view): applied only when newer than
// what the keeper holds for that module by the indexer's revision alone, so a replay never takes an entry back (a
// verdict that says no revision is applied when its digest differs). Each entry asked is made where the bus lacks it;
// each entry made that is no longer asked is an orphan. An empty verdict ("this module asks nothing now") ends the
// module's key once nothing it made is held.
// drifts says each drift once (design 47 G §532): kept in the module's key, said again only when the branch or the
// settings planned change.
func (k *Keeper) drifts(ds []Drift) {
by := map[string]map[string]Drift{}
for _, d := range ds {
if by[d.Module] == nil {
by[d.Module] = map[string]Drift{}
}
by[d.Module][d.Local] = d
}
for module, a := range k.Held() {
if by[module] == nil && len(a.Drifts) > 0 {
by[module] = map[string]Drift{}
}
}
for _, module := range sortedKeys(by) {
held, _, found := k.asked(module)
if !found {
held = Asked{V: 1, Module: module, At: k.now()}
}
now := by[module]
changed := len(now) != len(held.Drifts)
for local, d := range now {
if held.Drifts[local] != d {
changed = true
var p Plan
for _, e := range held.Entries {
if e.Name == local {
p, _ = PlanOf(module, e)
}
}
p.Module, p.Local = module, local
k.emit(EventDrifted, p, held.Revision, fmt.Sprintf("the branch %s asks %s of %s with other settings than the branch %s, whose are planned: nothing of %s is planned", d.Branch, local, module, d.From, d.Branch), nil, nil)
}
}
if !changed {
continue
}
held.Drifts = nil
if len(now) > 0 {
held.Drifts = now
}
if found || len(now) > 0 {
k.write(held)
}
}
}
// Heard applies one module's entry of the catalog half: it plans again only when its bus-state entries differ from those
// it last planned (design 47 G §538), never on the manifest's digest, which changes with any field. Each entry asked is
// made where the bus lacks it; each entry made that is no longer asked is an orphan. An entry that asks nothing ("this
// module asks nothing now") ends the module's key once nothing it made is held.
func (k *Keeper) Heard(a Accepted) {
k.setStatus(func(s *Status) { s.Heard = k.now() })
if !k.Viewed() {
k.logf("manifest-accepted of %s heard before the first view: kept on the consumer until it is answered", a.Module)
k.logf("the entry of %s is heard before the catalog half's first views: nothing is planned until they are in", a.Module)
return
}
held, _, found := k.asked(a.Module)
if found && a.Revision != 0 && a.Revision <= held.Revision {
k.logf("manifest-accepted of %s at revision %d is not newer than the %d held: not applied", a.Module, a.Revision, held.Revision)
return
}
if found && a.Revision == 0 && a.Digest != "" && a.Digest == held.Digest {
return
}
if !found {
held = Asked{V: 1, Module: a.Module}
}
if held.Made == nil {
held.Made = map[string]Made{}
}
held.Digest, held.Entries, held.At = a.Digest, a.BusState, k.now()
if a.Revision > held.Revision {
held.Revision = a.Revision
digest := EntriesDigest(a.BusState)
if p, ok := k.planned(a.Module); ok && found && p.Digest == digest {
if held.Digest != a.Digest || held.Revision != a.Revision {
held.Digest, held.Revision, held.At = a.Digest, a.Revision, k.now()
k.write(held)
}
return
}
held.Digest, held.Entries, held.Revision, held.At = a.Digest, a.BusState, a.Revision, k.now()
defer k.writePlan(a.Module, digest, held.Revision, a.BusState)
asked := map[string]bool{}
for _, e := range a.BusState {
asked[e.Name] = true
@@ -428,10 +614,10 @@ func (k *Keeper) Heard(a Accepted) {
}
m.Orphaned = true
held.Made[local] = m
k.emit(EventOrphaned, m.Plan, held.Revision, "no accepted manifest of "+a.Module+" asks for it now", nil, nil)
k.emit(EventOrphaned, m.Plan, held.Revision, "no watched branch's newest revision of "+a.Module+" asks for it now", nil, nil)
k.raise(conditionOrphaned, a.Module, local, occ.SeverityWarning, "Kept data no manifest asks for",
"the operator's drop of it, or a manifest asking for it again",
fmt.Sprintf("%s of %s is on the bus and no accepted manifest asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module))
fmt.Sprintf("%s of %s is on the bus and no watched branch's newest revision asks for it: it goes only by the bus module's drop, on your warrant (nats_drop), or a manifest asking for it again keeps it", BusName(a.Module, local), a.Module))
}
k.write(held)
}
@@ -488,6 +674,9 @@ func (k *Keeper) ensure(held *Asked, m *Made) {
defer cancel()
asked, holds := map[string]Settings{}, map[string]Settings{}
var drifts []string
// madeLater is an object newly made for an entry that was made whole before (a state's `<state>-objects`, design 47 G
// §58): its binding gained a name, so `created` is said again.
madeLater := false
for _, o := range m.Plan.Objects {
asked[o.Name] = o.Settings
s, _, found, err := k.Bus.Held(ctx, o)
@@ -496,6 +685,18 @@ func (k *Keeper) ensure(held *Asked, m *Made) {
return
}
if !found {
if err := k.capRoom(ctx, held, m); err != nil {
if errors.Is(err, ErrRetry) {
m.Waiting = err.Error()
return
}
m.Refused, m.Waiting = fmt.Sprintf("%s: %v", o.Stream(), err), ""
k.emit(EventRefused, m.Plan, held.Revision, m.Refused, asked, nil)
k.raise(conditionRefused, held.Module, m.Plan.Local, occ.SeverityWarning, "The bus module refused to create kept data",
"a smaller cap, or more storage for the bus's account",
fmt.Sprintf("%s of %s was not created: %s. The module gets no binding for it and fails at its start, naming it", m.Plan.Local, held.Module, m.Refused))
return
}
if err := k.Bus.Create(ctx, o); err != nil {
if errors.Is(err, ErrRetry) {
m.Waiting = fmt.Sprintf("%s cannot be created yet: %v", o.Stream(), err)
@@ -509,6 +710,7 @@ func (k *Keeper) ensure(held *Asked, m *Made) {
return
}
holds[o.Name] = o.Settings
madeLater = madeLater || m.Created
continue
}
holds[o.Name] = s
@@ -516,6 +718,12 @@ func (k *Keeper) ensure(held *Asked, m *Made) {
continue
}
if Grows(o.Settings, s) {
if o.Settings.MaxBytes > s.MaxBytes {
if err := k.capRoom(ctx, held, m); err != nil {
m.Waiting = err.Error()
return
}
}
if err := k.Bus.Update(ctx, o); err != nil {
m.Waiting = fmt.Sprintf("%s's larger settings cannot be applied yet: %v", o.Stream(), err)
return
@@ -530,7 +738,7 @@ func (k *Keeper) ensure(held *Asked, m *Made) {
if wasRefused {
k.clear(conditionRefused, held.Module, m.Plan.Local)
}
if !m.Created {
if !m.Created || madeLater {
// Marked created only once `created` is said whole: the grants part binds on it, so one the bus did not take is
// said again on the backoff (Waiting), never marked made and forgotten.
if err := k.sayNow(EventCreated, m.Plan, held.Revision, "", asked, holds); err != nil {
@@ -757,6 +965,23 @@ func EventID() string {
return hex.EncodeToString(b[:])
}
func (k *Keeper) planned(module string) (Planned, bool) {
raw, _, found, err := k.Store.Get(StatePlans, module)
if err != nil || !found {
return Planned{}, false
}
var p Planned
return p, json.Unmarshal(raw, &p) == nil
}
// writePlan keeps the entries a module was last planned for and their digest (§538).
func (k *Keeper) writePlan(module, digest string, revision uint64, entries []catalogue.BusStateEntry) {
raw, _ := json.Marshal(Planned{V: 1, Module: module, Digest: digest, Revision: revision, Entries: entries, At: k.now()})
if _, err := k.Store.Put(StatePlans, module, raw); err != nil {
k.logf("the plan of %s cannot be written to its own %s: %v", module, StatePlans, err)
}
}
func (k *Keeper) asked(module string) (Asked, uint64, bool) {
raw, rev, found, err := k.Store.Get(StateAsked, module)
if err != nil || !found {
@@ -772,10 +997,11 @@ func (k *Keeper) asked(module string) (Asked, uint64, bool) {
// write keeps a module's key, or removes it when its manifest asks for nothing and nothing it made is held: a key goes
// only so, never because a module is missing from a first view.
func (k *Keeper) write(a Asked) {
if len(a.Entries) == 0 && len(a.Made) == 0 {
if len(a.Entries) == 0 && len(a.Made) == 0 && len(a.Drifts) == 0 {
if err := k.Store.Delete(StateAsked, a.Module); err != nil {
k.logf("the key of %s cannot be removed from its own %s: %v", a.Module, StateAsked, err)
}
_ = k.Store.Delete(StatePlans, a.Module)
return
}
raw, _ := json.Marshal(a)
+183 -36
View File
@@ -38,6 +38,10 @@ type fakeBus struct {
restored []string
gate chan struct{}
deletedStreams []string
// limits are the bus's storage limits the fake answers.
limits Limits
// reads counts the reads of an object's stream.
reads int
}
func (b *fakeBus) SnapshotStreams(path string) ([]string, error) {
@@ -73,6 +77,12 @@ func (b *fakeBus) DeleteStream(_ context.Context, name string) error {
return nil
}
func (b *fakeBus) Limits(context.Context) (Limits, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.limits, nil
}
func newFakeBus() *fakeBus {
return &fakeBus{held: map[string]Settings{}, fill: map[string]uint64{}, refuse: map[string]error{}}
}
@@ -80,6 +90,7 @@ func newFakeBus() *fakeBus {
func (b *fakeBus) Held(_ context.Context, o Object) (Settings, Fill, bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.reads++
s, ok := b.held[o.Stream()]
return s, Fill{Bytes: b.fill[o.Stream()]}, ok, nil
}
@@ -291,12 +302,12 @@ func (f *findingLog) Clear(_ context.Context, scope, id, kind, _ string) error {
return nil
}
func newKeeper(view FirstView) (*Keeper, *fakeBus, *memStore, *said) {
func newKeeper(view CatalogView) (*Keeper, *fakeBus, *memStore, *said) {
bus, store, out := newFakeBus(), newMemStore(), &said{}
at := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC)
k := &Keeper{Bus: bus, Store: store, Finding: &findingLog{}, Emit: out.emit, Say: out.say, Files: &memFiles{size: map[string]int64{}},
View: func() (FirstView, error) { return view, nil },
Now: func() time.Time { return at }}
Catalog: func() (CatalogView, error) { return view, nil },
Now: func() time.Time { return at }}
return k, bus, store, out
}
@@ -328,8 +339,13 @@ func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) {
refused string
}{
{name: "state", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "5"}),
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}",
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 5, new, size 262144}; object-store mesh-knowledge_drafts-objects {16 MiB, age 0, history 0, new, size 0}",
binding: "bucket mesh-knowledge_drafts"},
{name: "state naming objects-max-mib (§591)", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"objects-max-mib": "512"}),
objects: "key-value mesh-knowledge_drafts {64 MiB, age 0, history 1, new, size 262144}; object-store mesh-knowledge_drafts-objects {512 MiB, age 0, history 0, new, size 0}",
binding: "bucket mesh-knowledge_drafts"},
{name: "state naming objects-max-mib 0", entry: entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"objects-max-mib": "0"}),
refused: "not a size above 0"},
{name: "events, default age and cap", entry: entry("events", catalogue.StorageStream, catalogue.TemplateEvents, nil),
objects: "stream mesh-knowledge_events [mesh.bus.mesh-knowledge.events.>] {1024 MiB, age 604800, history 0, old, size 0}",
binding: "stream mesh-knowledge_events on mesh.bus.mesh-knowledge.events"},
@@ -429,12 +445,12 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
if len(bus.created) != 0 {
t.Fatalf("created %v before the first view", bus.created)
}
if _, err := k.AskFirstView(0); err != nil {
if err := k.PlanCatalog(); err != nil {
t.Fatal(err)
}
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
if want := "[KV_m_drafts m_log]"; fmt.Sprint(bus.created) != want {
if want := "[KV_m_drafts OBJ_m_drafts-objects m_log]"; fmt.Sprint(bus.created) != want {
t.Errorf("created %v, want %s", bus.created, want)
}
if want := "[created m drafts created m log]"; fmt.Sprint(out.events) != want {
@@ -455,7 +471,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
k.Sends("m")
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}})
k.Sends("m")
if want := "[OBJ_m_sent-objects KV_m_drafts]"; fmt.Sprint(bus.created) != want {
if want := "[OBJ_m_sent-objects KV_m_drafts OBJ_m_drafts-objects]"; fmt.Sprint(bus.created) != want {
t.Fatalf("created %v, want %s", bus.created, want)
}
if s := bus.held["OBJ_m_sent-objects"]; s.MaxBytes != 256*MiB || s.MaxAgeSeconds != 7*24*3600 || s.Discard != DiscardNew {
@@ -476,11 +492,66 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
t.Errorf("the sent store was orphaned or removed: %+v %v", held.Made[SentLocal], bus.deleted)
}
}},
{"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}})
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil})
if a := k.Held()["m"]; a.Revision != 9 || len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
t.Errorf("held %+v: the replay at 4 took back what 9 asked", a)
// §344, §533, §537: a module of the new form claiming a seat whose resolved definition has a verb or an event, or
// declaring an event of its own, gets a sent store; a seat including only bodies, and a foundation part, none; a
// definition gaining a verb gives its holder one at the next plan.
{"the catalog half and the seat list plan the sent stores", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
defs := map[string]seatdef.Definition{
"verbed": {Seat: "verbed", Verbs: []seatdef.Verb{{Name: "go"}}},
"resolved": {Seat: "resolved", Includes: []string{"part"}, Verbs: []seatdef.Verb{{Name: "from-the-part"}}},
"bodies": {Seat: "bodies", Includes: []string{"shared-bodies"}},
}
k.Seats = func() map[string]seatdef.Definition { return defs }
k.Catalog = func() (CatalogView, error) {
return CatalogView{Modules: []CatalogModule{
{Accepted: Accepted{Module: "a"}, Claims: []string{"verbed"}},
{Accepted: Accepted{Module: "b"}, Claims: []string{"resolved"}},
{Accepted: Accepted{Module: "c"}, Claims: []string{"bodies"}},
{Accepted: Accepted{Module: "d"}, Claims: []string{"verbed"}, Foundation: true},
{Accepted: Accepted{Module: "e"}, EmitsOwn: true},
}}, nil
}
_ = k.PlanCatalog()
if want := "[OBJ_a_sent-objects OBJ_b_sent-objects OBJ_e_sent-objects]"; fmt.Sprint(bus.created) != want {
t.Fatalf("created %v, want %s", bus.created, want)
}
defs["bodies"] = seatdef.Definition{Seat: "bodies", Includes: []string{"shared-bodies"}, Verbs: []seatdef.Verb{{Name: "new"}}}
_ = k.PlanCatalog()
if want := "[OBJ_a_sent-objects OBJ_b_sent-objects OBJ_e_sent-objects OBJ_c_sent-objects]"; fmt.Sprint(bus.created) != want {
t.Errorf("created %v, want %s: the holder of a definition gaining a verb gets its store", bus.created, want)
}
}},
// §532: a later branch asking a local name with other settings is said drifted once, naming the branch; §547: the
// old sections' caps count in the sum.
{"a drift between branches is said once, naming the branch", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
view := CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{drafts}}, OldBytes: 8 * MiB}},
Drifts: []Drift{{Module: "m", Local: "drafts", Branch: "feature", From: "main"}}}
k.Catalog = func() (CatalogView, error) { return view, nil }
_ = k.PlanCatalog()
_ = k.PlanCatalog()
if fmt.Sprint(out.events) != "[created m drafts drifted m drafts]" || !strings.Contains(out.bodies[1].Why, "the branch feature asks drafts") {
t.Errorf("said %v (%+v): created from main, drifted once naming feature", out.events, out.bodies)
}
if k.Status().OldBytes != 8*MiB {
t.Errorf("old sections %d: 8 MiB counted", k.Status().OldBytes)
}
}},
// §538: a module in no watched branch's newest revision has its plans key deleted, its entries then orphans.
{"a module no watched branch lists loses its plan, its entries orphans", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Catalog = func() (CatalogView, error) {
return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil
}
_ = k.PlanCatalog()
if _, ok := k.planned("m"); !ok {
t.Fatal("m's plan is not kept")
}
k.Catalog = func() (CatalogView, error) { return CatalogView{}, nil }
_ = k.PlanCatalog()
if _, ok := k.planned("m"); ok {
t.Error("m's plan is kept while no watched branch lists it")
}
if a := k.Held()["m"]; !a.Made["drafts"].Orphaned || len(bus.deleted) != 0 {
t.Errorf("held %+v, deleted %v: drafts an orphan, kept on the bus", a, bus.deleted)
}
}},
{"a grant not composed yet waits, then is made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
@@ -543,10 +614,64 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
t.Errorf("said %v %v: cleared once asked as held", out.events, out.conds)
}
}},
{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
{"a plan whose caps pass the account's storage limit is refused, naming the sum and the limit, and the caps made stay (§547)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
// drafts: 64 MiB state + 16 MiB objects = 80 MiB; log: 1024 MiB.
bus.limits = Limits{AccountBytes: 100 * MiB}
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
k.Heard(Accepted{Module: "m", Revision: 2, BusState: []catalogue.BusStateEntry{drafts, log}})
m := k.Held()["m"].Made
if fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" || !m["drafts"].Created || m["log"].Created ||
!strings.Contains(m["log"].Refused, "1104 MiB, past the account's storage limit of 100 MiB") || fmt.Sprint(out.events) != "[created m drafts refused m log]" {
t.Errorf("created %v, said %v, log %+v: the caps made stay, the plan that passes the limit refused naming both", bus.created, out.events, m["log"])
}
if CapSum(k.Held(), "", "") != 80*MiB {
t.Errorf("the sum is %d, want 80 MiB: a refused entry holds none", CapSum(k.Held(), "", ""))
}
bus.limits = Limits{}
k.CheckAll()
if !k.Held()["m"].Made["log"].Created {
t.Errorf("with no account limit nothing is refused: %+v", k.Held()["m"].Made["log"])
}
}},
{"a verdict plans again only when its entries differ, never on the manifest's digest (§538)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
reads := bus.reads
k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 2, BusState: []catalogue.BusStateEntry{drafts}})
if bus.reads != reads {
t.Errorf("the bus was read %d times more for a manifest that changed outside its entries", bus.reads-reads)
}
raw, _, found, _ := k.Store.Get(StatePlans, "m")
var p Planned
_ = json.Unmarshal(raw, &p)
if !found || p.Digest != EntriesDigest([]catalogue.BusStateEntry{drafts}) || len(p.Entries) != 1 || k.Held()["m"].Digest != "d2" || k.Held()["m"].Revision != 2 {
t.Errorf("the plan kept %+v (%v), the digest and revision follow the verdict: %+v", p, found, k.Held()["m"])
}
bigger := entry("drafts", catalogue.StorageKeyValue, catalogue.TemplateState, map[string]string{"history": "7"})
k.Heard(Accepted{Module: "m", Digest: "d3", Revision: 3, BusState: []catalogue.BusStateEntry{bigger}})
if bus.reads == reads || bus.updated == nil {
t.Errorf("a changed option plans the difference: reads %d (was %d), updated %v", bus.reads, reads, bus.updated)
}
k.Heard(Accepted{Module: "m", Digest: "d4", Revision: 4, BusState: nil})
if _, _, found, _ := k.Store.Get(StatePlans, "m"); !found {
t.Errorf("an entry made and now unasked is an orphan the keeper still holds, with its plan")
}
}},
{"a state made before its objects store existed gets the store and its binding said again (§58)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
p, _ := PlanOf("m", drafts)
bus.held["KV_m_drafts"] = p.Objects[0].Settings
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
out.reset()
bus.created = nil
delete(bus.held, "OBJ_m_drafts-objects")
k.CheckAll()
if fmt.Sprint(bus.created) != "[OBJ_m_drafts-objects]" || fmt.Sprint(out.events) != "[created m drafts]" || out.bodies[0].Binding.Objects != "m_drafts-objects" {
t.Errorf("created %v, said %v: the missing store made, its binding said again", bus.created, out.events)
}
}},
{"a bus that holds an entry the keeper never made is taken as made", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
p, _ := PlanOf("m", drafts)
bus.held["KV_m_drafts"], bus.held["OBJ_m_drafts-objects"] = p.Objects[0].Settings, p.Objects[1].Settings
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
if len(bus.created) != 0 || fmt.Sprint(out.events) != "[created m drafts]" {
t.Errorf("created %v, said %v: its binding said, nothing made twice", bus.created, out.events)
}
@@ -571,12 +696,14 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
}},
{"a module missing from the first view keeps its key; one never made leaves none", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
k.View = func() (FirstView, error) { return FirstView{Revision: 5}, nil }
if _, err := k.AskFirstView(0); err != nil {
t.Fatal(err)
k.Catalog = func() (CatalogView, error) {
return CatalogView{}, errors.New("the catalog half has no first view of main yet")
}
if err := k.PlanCatalog(); err == nil {
t.Fatal("planned while a watched branch has no first view")
}
if a := k.Held()["m"]; len(a.Entries) != 1 || a.Made["drafts"].Orphaned {
t.Errorf("held %+v: kept as asked, never orphaned by a first view", a)
t.Errorf("held %+v: kept as asked, never orphaned while a first view waits", a)
}
bus.refuse["KV_n_x"] = errors.New("no")
k.Heard(Accepted{Module: "n", Revision: 1, BusState: []catalogue.BusStateEntry{entry("x", catalogue.StorageKeyValue, catalogue.TemplateState, nil)}})
@@ -617,19 +744,21 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
if h := HealthOf(k.Status(), nil, 2, later); !strings.Contains(h.Why, "2 accepted verdict(s) wait") {
t.Errorf("health %+v: names the verdicts waiting", h)
}
k.View = func() (FirstView, error) { return FirstView{}, errors.New("no responders") }
k.Catalog = func() (CatalogView, error) {
return CatalogView{}, errors.New("the catalog half has no first view of main yet")
}
k.mu.Lock()
k.status.FirstView.Answered = time.Time{}
k.mu.Unlock()
if next, err := k.AskFirstView(10 * time.Second); err == nil || next != 20*time.Second {
t.Fatalf("asked again in %s (%v): on the backoff", next, err)
if err := k.PlanCatalog(); err == nil {
t.Fatal("planned with no first view")
}
// The first view unanswered is unhealthy only past the bound since the start (review round 1: health.go:41).
if h := HealthOf(k.Status(), nil, 0, at); !h.Healthy {
t.Errorf("health %+v: healthy within the bound while the first view is asked", h)
}
k.FillPass()
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-delivery-indexer.indexer_bus_state") {
if h := HealthOf(k.Status(), nil, 0, later); h.Healthy || !strings.Contains(h.Why, "mesh-catalog.revision, has not answered: the catalog half has no first view of main yet") {
t.Errorf("health %+v: unhealthy past the bound, naming the verb", h)
}
// What waits on its consumer unread is never skipped silently (review round 1: health.go:47).
@@ -639,6 +768,22 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
}},
// CAT-N8: a person's requests: a drop removes an orphan alone, only on the operator's warrant binding exactly
// that drop, after a snapshot, said dropped and recorded; a recreate rebuilds a drifted entry the same way.
{"an ask the router refused ends its request with the router's words (§570)", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
store := k.Store.(*memStore)
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}})
k.Heard(Accepted{Module: "m", Revision: 2, BusState: nil})
w := &oc.Warrant{Ask: "bus-1", Asker: Module, Outcome: oc.WarrantOutcomeRefused, Words: "an ask that destroys lasts at most 10m0s"}
k.Warrant = func(string) (*oc.Ask, *oc.Warrant, error) { return nil, w, nil }
raw, _ := json.Marshal(Request{ID: "rr", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", At: k.now()})
_, _ = store.Put(StateRequests, "rr", raw)
k.ActOnRequests()
raw, _, _, _ = store.Get(StateRequests, "rr")
var got Request
_ = json.Unmarshal(raw, &got)
if got.Done.IsZero() || got.Waiting != "" || !strings.Contains(got.Error, "refused") || !strings.Contains(got.Error, w.Words) || len(bus.deleted) != 0 {
t.Errorf("ended %+v, deleted %v: the router's refusal and its words, nothing removed", got, bus.deleted)
}
}},
{"a drop waits for the warrant, refuses another act's, then acts after a snapshot", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
store := k.Store.(*memStore)
k.Heard(Accepted{Module: "m", Revision: 1, BusState: []catalogue.BusStateEntry{drafts, log}})
@@ -696,7 +841,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
bus.snaps = nil
out.reset()
got := put(Request{ID: "r2", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-1", Caller: "person.jo", Why: "unused"})
if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts]" ||
if got.Error != "" || fmt.Sprint(bus.snaps) != "[[KV_m_drafts OBJ_m_drafts-objects]]" || fmt.Sprint(bus.deleted) != "[KV_m_drafts OBJ_m_drafts-objects]" ||
fmt.Sprint(out.events) != "[dropped m drafts]" || len(acts) != 1 || !strings.Contains(acts[0], "nats_drop: dropped drafts of m") ||
!strings.Contains(acts[0], "person.jo") {
t.Errorf("ended %+v, snapshots %v, deleted %v, said %v, recorded %v", got, bus.snaps, bus.deleted, out.events, acts)
@@ -733,7 +878,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
out.reset()
k.ActOnRequests()
got := get("r10")
if got.Path != "/snapshots/KV_m_drafts.tar" || got.Size != 100 || !strings.HasPrefix(got.Digest, "sha256:") || !got.Protected || got.Error != "" {
if got.Path != "/snapshots/KV_m_drafts+OBJ_m_drafts-objects.tar" || got.Size != 100 || !strings.HasPrefix(got.Digest, "sha256:") || !got.Protected || got.Error != "" {
t.Fatalf("the drop's record: %+v", got)
}
findings := k.Finding.(*findingLog)
@@ -753,7 +898,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
k.Now = func() time.Time { return base.AddDate(0, 0, 91) }
findings.calls = nil
k.ExpireSnapshots()
if _, _, found, _ := store.Get(StateRequests, "r10"); found || fmt.Sprint(files.removed) != "[/snapshots/m_log.tar /snapshots/KV_m_drafts.tar]" ||
if _, _, found, _ := store.Get(StateRequests, "r10"); found || fmt.Sprint(files.removed) != "[/snapshots/m_log.tar /snapshots/KV_m_drafts+OBJ_m_drafts-objects.tar]" ||
len(journal) != 1 || !strings.Contains(journal[0], "past its 90 days") || fmt.Sprint(findings.calls) != "[clear module.nats.snapshots.snapshots-filling]" {
t.Errorf("after 91 days: removed %v, journal %v, finding calls %v", files.removed, journal, findings.calls)
}
@@ -767,7 +912,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
{"a snapshot is restored in place, verified, refused while its names are live, cancelled, and its failure removes the partial stream", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) {
store, files := k.Store.(*memStore), k.Files.(*memFiles)
out.sig = make(chan struct{}, 8)
path := "/snapshots/KV_m_drafts.tar"
path := "/snapshots/KV_m_drafts+OBJ_m_drafts-objects.tar"
bus.snapStreams = map[string][]string{path: {"KV_m_drafts"}}
files.missing = map[string]bool{"/snapshots/gone.tar": true}
digest := "sha256:" + hex.EncodeToString([]byte(path))
@@ -942,13 +1087,13 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
_ = json.Unmarshal(raw, &got)
return got
}
if got := put("r5"); got.Error != "" || len(bus.deleted) != 1 {
if got := put("r5"); got.Error != "" || len(bus.deleted) != 2 {
t.Fatalf("the first drop: %+v, deleted %v", got, bus.deleted)
}
// The entry is asked for and orphaned again, so a second drop would remove it: the spent warrant refuses it.
k.Heard(Accepted{Module: "m", Revision: 3, BusState: []catalogue.BusStateEntry{drafts, log}})
k.Heard(Accepted{Module: "m", Revision: 4, BusState: []catalogue.BusStateEntry{log}})
if got := put("r6"); !strings.Contains(got.Error, "already acted (drop of m's drafts, request r5") || len(bus.deleted) != 1 {
if got := put("r6"); !strings.Contains(got.Error, "already acted (drop of m's drafts, request r5") || len(bus.deleted) != 2 {
t.Errorf("the second request on the same warrant: %+v, deleted %v", got, bus.deleted)
}
if raw, _, found, _ := store.Get(StateSpent, "bus-5"); !found || !strings.Contains(string(raw), `"request":"r5"`) {
@@ -960,7 +1105,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
for _, key := range keys {
_ = store.Delete(StateRequests, key)
}
if got := put("r6b"); !strings.Contains(got.Error, "already acted") || len(bus.deleted) != 1 {
if got := put("r6b"); !strings.Contains(got.Error, "already acted") || len(bus.deleted) != 2 {
t.Errorf("with requests wiped, the spent warrant: %+v, deleted %v", got, bus.deleted)
}
}},
@@ -977,7 +1122,7 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
_ = json.Unmarshal(raw, &got)
return got
}
raw, _ := json.Marshal(Request{ID: "r7", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-7", Caller: "person.jo", At: k.now().Add(-time.Hour)})
raw, _ := json.Marshal(Request{ID: "r7", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-7", Caller: "person.jo", At: k.now().Add(-AskLasts / 2)})
_, _ = store.Put(StateRequests, "r7", raw)
raw, _ = json.Marshal(Request{ID: "r8", Verb: VerbDrop, Module: "m", Local: "drafts", Ask: "bus-8", Caller: "person.jo", At: k.now().Add(-AskLasts - time.Minute)})
_, _ = store.Put(StateRequests, "r8", raw)
@@ -1071,24 +1216,26 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) {
store := k.Store.(*memStore)
var acts []string
k.Act = func(what, why, cause string) error { acts = append(acts, cause+": "+what); return nil }
k.View = func() (FirstView, error) {
return FirstView{Revision: 8, Modules: []Accepted{{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}, nil
k.Catalog = func() (CatalogView, error) {
return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 8, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil
}
again := 0
k.Again = func() error { again++; return nil }
raw, _ := json.Marshal(Request{ID: "r9", Verb: VerbFirstViewAgain, Why: "a verdict thought missed", Caller: "person.jo"})
_, _ = store.Put(StateRequests, "r9", raw)
k.ActOnRequests()
raw, _, _, _ = store.Get(StateRequests, "r9")
var got Request
_ = json.Unmarshal(raw, &got)
if got.Outcome != "the first view answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts]" ||
len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked mesh-delivery-indexer.indexer_bus_state again by hand") {
if got.Outcome != "the first views answered at revision 8" || fmt.Sprint(bus.created) != "[KV_m_drafts OBJ_m_drafts-objects]" || again != 1 ||
len(acts) != 1 || !strings.Contains(acts[0], "nats_first_view_again: asked the catalog half's first views (mesh-catalog.revision) again by hand") {
t.Errorf("ended %+v, created %v, recorded %v", got, bus.created, acts)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
k, bus, _, out := newKeeper(FirstView{Revision: 1})
if _, err := k.AskFirstView(0); err != nil {
k, bus, _, out := newKeeper(CatalogView{})
if err := k.PlanCatalog(); err != nil {
t.Fatal(err)
}
row.run(t, k, bus, out)
@@ -1111,7 +1258,7 @@ func TestSends(t *testing.T) {
}{
{"a seat with a verb", []string{"verbed"}, false, true},
{"a seat with an event", []string{"evented"}, false, true},
{"a seat that includes parts", []string{"included"}, false, true},
{"a seat that only includes parts (§537: resolved by the seat list, including alone counts for nothing)", []string{"included"}, false, false},
{"a seat with neither", []string{"bare"}, false, false},
{"a seat the list does not hold", []string{"unknown"}, false, false},
{"an event of its own", nil, true, true},
+21 -7
View File
@@ -1,8 +1,7 @@
// Package keeper is the bus module's keeper (novox/hq design 25 § 12.2-12.3, ADR 0595 §4-§6): it creates the bus state
// every accepted manifest asks for in `requires`, compares what the bus holds with what was asked, finds orphans and
// reads the fill of what refuses when full, and says each on the bus module's own events. It reads nothing of another
// part's state: what to create comes on the indexer's manifest-accepted, and its first view from the indexer's tool
// indexer_bus_state. Everything it touches it reaches through the interfaces in keeper.go; this file is pure.
// part's state: what to create comes from the catalog half of each watched branch (design 47 G §433, §532). Everything it touches it reaches through the interfaces in keeper.go; this file is pure.
package keeper
import (
@@ -49,16 +48,20 @@ const (
SentAgeDays = 7
)
// StateObjectsCapMiB is the cap of a state's object store `<state>-objects` unless the state names `objects-max-mib`
// (design 47 G §591, §547).
const StateObjectsCapMiB = 16
// Sends says whether a module sends (design 47 G §344): it claims a seat whose definition has a verb or an event, or it
// declares an event of its own (emitsOwn). claims are the seats its catalog graph entry claims, defs the seat list's
// definitions by seat; a claimed seat the list does not hold counts for nothing. A definition that includes parts takes
// their verbs and events whole (seatdef.Resolve), so one counts as having some.
// definitions by seat, each as the seat list answers that one seat, resolved with the parts it includes; a claimed seat
// the list does not hold counts for nothing, and including parts alone counts for nothing (design 47 G §537).
func Sends(claims []string, emitsOwn bool, defs map[string]seatdef.Definition) bool {
if emitsOwn {
return true
}
for _, c := range claims {
if d, ok := defs[c]; ok && (len(d.Verbs) > 0 || len(d.Events) > 0 || len(d.Includes) > 0) {
if d, ok := defs[c]; ok && (len(d.Verbs) > 0 || len(d.Events) > 0) {
return true
}
}
@@ -156,8 +159,19 @@ func PlanOf(module string, e catalogue.BusStateEntry) (Plan, error) {
if err != nil {
return Plan{}, err
}
p.Binding.Bucket = name
p.Objects = []Object{{Kind: KindBucket, Name: name, Settings: s}}
// Beside each state its object store `<state>-objects`, which the grants part grants to the state's owner alone (design
// 47 G §58, §591): 16 MiB unless the state's requires names `objects-max-mib`.
objCap, err := opts.int(catalogue.OptionObjectsCapMiB, StateObjectsCapMiB)
if err != nil {
return Plan{}, err
}
if objCap <= 0 {
return Plan{}, fmt.Errorf("the option %s of %q is %d, not a size above 0", catalogue.OptionObjectsCapMiB, e.Name, objCap)
}
objects := BusName(module, catalogue.StateObjectsName(e.Name))
p.Binding.Bucket, p.Binding.Objects = name, objects
p.Objects = []Object{{Kind: KindBucket, Name: name, Settings: s},
{Kind: KindObjects, Name: objects, Settings: Settings{MaxBytes: objCap * MiB, Discard: DiscardNew}}}
case e.Storage == catalogue.StorageStream && e.Template == catalogue.TemplateEvents:
age, err := opts.int(catalogue.OptionAgeDays, EventsDays)
if err != nil {
+21 -8
View File
@@ -9,6 +9,7 @@ import (
"time"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
)
// A person's request (design 25 § 12.3; CAT-N8): the bus module's tools hold no grant of the seat's acting verbs, so
@@ -23,7 +24,7 @@ const (
VerbApply = "apply"
VerbRecreate = "recreate"
VerbDrop = "drop"
// VerbFirstViewAgain asks the indexer's first view at once, by hand (design 25 § 12.2), beside the backoff.
// VerbFirstViewAgain asks the catalog half's first views again at once, by hand (design 47 G §614).
VerbFirstViewAgain = "first-view-again"
)
@@ -67,10 +68,11 @@ type Request struct {
Unrecorded string `json:"unrecorded,omitempty"`
}
// AskLasts is how long the operator's ask for a recreate or a drop waits for an answer: shorter than the day the state
// requests keeps a key, so a request whose ask ended unanswered is ended by the keeper, saying so, before its key ages
// out, and the warrant's acted key (ActedKey) outlives the ask it spends.
const AskLasts = 12 * time.Hour
// AskLasts is how long the operator's ask for a recreate or a drop waits for an answer: the bound of its highest option
// level, destroy (design 47 G §570: no asker asks for longer than its level allows; the asker client refuses more). It is
// shorter than the day the state requests keeps a key, so a request whose ask ended unanswered is ended by the keeper,
// saying so, before its key ages out, and the warrant's acted key (ActedKey) outlives the ask it spends.
const AskLasts = occ.DestroyLasts
// ActedPrefix started the spend record's key while it was kept in StateRequests (`acted.<ask id>`); a key of that form
// still met there is no request and is skipped.
@@ -173,13 +175,19 @@ func (k *Keeper) ActOnRequests() {
func (k *Keeper) act(r *Request) (string, error) {
if r.Verb == VerbFirstViewAgain {
_, err := k.AskFirstView(0)
err := fmt.Errorf("the keeper follows no catalog half")
if k.Again != nil {
err = k.Again()
}
if err == nil {
err = k.PlanCatalog()
}
s := k.Status().FirstView
k.record(r, fmt.Sprintf("asked %s.%s again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision))
k.record(r, fmt.Sprintf("asked the catalog half's first views (%s.%s) again by hand: answered %v at revision %d", s.Owner, s.Verb, err == nil, s.Revision))
if err != nil {
return "asked, not answered", err
}
return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil
return fmt.Sprintf("the first views answered at revision %d", s.Revision), nil
}
if !k.Viewed() {
return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant)
@@ -288,6 +296,11 @@ func (k *Keeper) warranted(r *Request) error {
if err != nil {
return fmt.Errorf("%w: the router's record of the ask %s cannot be read now: %v", ErrNoWarrant, r.Ask, err)
}
// An ask the router refused at take-in ends its request with the router's words, never "not answered yet" (design 47 G
// §570).
if w != nil && w.Outcome == oc.WarrantOutcomeRefused {
return fmt.Errorf("the router refused the operator's ask %s, so nothing is removed: %s", r.Ask, w.Words)
}
// An ask that ended unanswered ends its request, said: never a request that waits until its key ages out. And a
// warrant is not acted on once its ask ended, so a request handed after its acted key aged out acts on nothing.
var expires time.Time
+49 -7
View File
@@ -62,15 +62,36 @@
],
"invokes": [
"seat:mesh-hand-acts.hand-act",
"mesh-delivery-indexer.indexer_bus_state"
"seat:mesh-catalog.revision",
"seat:mesh-catalog.registrations",
"seat:mesh-delivery.source",
"seat:mesh-catalog.seat-definitions"
],
"consumes": [
"mesh-catalog.manifest-accepted",
"mesh-catalog.catalog-revision",
"mesh-catalog.registration-changed",
"mesh-delivery.source-moved",
"mesh-catalog.seat-definition-changed",
"mesh-catalog.seat-definition-removed",
"mesh-conditions.findings-asked"
],
"uses": [
"operator-channel"
],
"config": {
"snapshots-cap": {
"type": "number",
"description": "How many bytes the keeper's snapshots directory may hold (design 47 G §681): past 80 % of it the keeper tells a finding, and it never takes a snapshot past it. The keeper reads it at its start from its settings file.",
"minimum": 1,
"default": 21474836480
},
"snapshot-keep-days": {
"type": "number",
"description": "How many days a snapshot the keeper took before a drop or recreate, and its record, are kept (design 47 G §681, A-112). The keeper reads it at its start from its settings file; the tool nats_snapshots reads it as its configuration.",
"minimum": 1,
"default": 90
}
},
"streams": [
{
"name": "broker-events",
@@ -115,6 +136,11 @@
"ttl-seconds": 86400,
"read-only": true
},
{
"name": "plans",
"history": 1,
"read-only": true
},
{
"name": "findings",
"history": 1
@@ -154,7 +180,8 @@
"nats_keeper_health",
"nats_first_view_again",
"nats_requests",
"nats_snapshots"
"nats_snapshots",
"nats_plans"
],
"guards": [
8222
@@ -230,6 +257,20 @@
"place": "mesh",
"unknown": "program"
},
{
"id": "keeper-settings",
"trusted": false,
"type": "file",
"path": "${dir:keeper-state}/settings.json",
"mode": "0600",
"owner": "nats-keeper",
"takes-config": [
"snapshots-cap",
"snapshot-keep-days"
],
"merge": "json",
"template": "files/settings.json"
},
{
"id": "server-conf",
"type": "file",
@@ -272,8 +313,6 @@
"MESH_BROKER_FILE": "${dir:keeper-state}/broker-keeper",
"MESH_NATS_FULL_RAISE": "75",
"MESH_NATS_FULL_CLEAR": "70",
"MESH_NATS_SNAPSHOTS_CAP": "21474836480",
"MESH_NATS_SNAPSHOT_KEEP_DAYS": "90",
"MESH_NATS_SNAPSHOTS_DIR": "${dir:keeper-state}"
},
"serves": [
@@ -287,12 +326,16 @@
"asked",
"requests",
"spent",
"plans",
"findings"
],
"asks": true,
"keeps": true,
"invokes": [
"mesh-delivery-indexer.indexer_bus_state",
"seat:mesh-catalog.revision",
"seat:mesh-catalog.registrations",
"seat:mesh-delivery.source",
"seat:mesh-catalog.seat-definitions",
"seat:mesh-hand-acts.hand-act"
]
},
@@ -382,7 +425,6 @@
"MESH_NATS_CONTAINER": "mesh-broker-nats",
"MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf",
"MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}",
"MESH_NATS_SNAPSHOT_KEEP_DAYS": "90",
"NATS_TOOLS_CREDENTIAL": "${dir:tools-secret}/broker-nats-tools"
}
},
+3
View File
@@ -14,6 +14,9 @@
]
}
],
"uses": [
"operator-channel"
],
"tools": [
"zsh_config"
],
+4
View File
@@ -28,6 +28,10 @@ test("it claims node-login-shell serving execute, and declares no seat", () => {
assert.deepEqual(m.claims, [{ name: "node-login-shell", scope: "node", serves: ["execute"] }]);
});
test("it uses operator-channel, since its seat's verb execute is marked needs-approval (design 46/7 section 21)", () => {
assert.deepEqual(m.uses, ["operator-channel"]);
});
test("the .zshrc block is at the start and holds the three slots in order, after the defaults' header", () => {
const rc = resource("rc");
assert.equal(rc.path, "${machine:account-home}/.zshrc");