Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d8cbe7b81 | ||
|
|
e74c32ed50 | ||
|
|
146c48fd96 | ||
|
|
1f3abd3e0e | ||
|
|
6d620f77c3 | ||
|
|
f8286c063d | ||
|
|
e096b4595a | ||
|
|
09c0c6b367 | ||
|
|
d1fc25f682 | ||
|
|
eda457f415 | ||
|
|
59f4d486b1 | ||
|
|
801552c0eb | ||
|
|
ede9bce6ef | ||
|
|
0f0028785c | ||
|
|
6fdcfad8d3 | ||
|
|
8d9d33ae85 | ||
|
|
cc25baa563 | ||
|
|
68a2ebdcc3 | ||
|
|
69b99eec68 | ||
|
|
09bd0eec4f | ||
|
|
222a38e050 | ||
|
|
ee99a24f77 | ||
|
|
f68521da28 | ||
|
|
296064c799 | ||
|
|
df9231c734 | ||
|
|
843b709b59 | ||
|
|
f506fb34ec | ||
|
|
c34b937dd3 | ||
|
|
d84c9699b3 | ||
|
|
002d5e578c | ||
|
|
2421b82ad2 | ||
|
|
0ebd48a6a8 | ||
|
|
67e291c02a | ||
|
|
8a400d165e | ||
|
|
53d3cd7ce9 | ||
|
|
6648e4a5c8 | ||
|
|
7fa2568ce7 | ||
|
|
4b382ceffd | ||
|
|
ce86d09d22 | ||
|
|
853be00ebe |
@@ -111,6 +111,14 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
|
||||
@@ -85,7 +85,7 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||
|
||||
@@ -676,6 +676,17 @@ type answers struct {
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
|
||||
// journal was the only place that said so for a day (04-ISSUES/179).
|
||||
failing []inventory.ProviderStanding
|
||||
// overflowing is every module whose identity overflows the bound of a provision it requires
|
||||
// (novox/hq ADR 0225): its provider leaves it out of the grants and composes everything else, so
|
||||
// this is the one place it is said across the mesh. Not well while there is any.
|
||||
overflowing []catalogue.Overflow
|
||||
// handActs is how many acts were done by hand in the last seven days (novox/hq to-be 45 §7), nil
|
||||
// where the log is not on hand; handActsUnread why it could not be read when it could not.
|
||||
handActs *int
|
||||
handActsUnread string
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
|
||||
// RequestTimeout) — the shortest wait of a caller the mesh ships.
|
||||
const consoleWaits = 30 * time.Second
|
||||
|
||||
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
|
||||
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
|
||||
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
|
||||
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
||||
if link.AnswerWithin >= consoleWaits/2 {
|
||||
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
|
||||
}
|
||||
if link.AnswerWithin >= broker.ResponseTTL {
|
||||
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
|
||||
}
|
||||
}
|
||||
|
||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "w"}, true},
|
||||
{"push", map[string]any{"why": "w"}, true},
|
||||
{"command", map[string]any{"command": "push anchor --why w"}, true},
|
||||
{"command", map[string]any{"command": "push --behind --why=w"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
||||
}
|
||||
if got := answersFirst(argv); got != c.want {
|
||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,17 @@ import (
|
||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||
// refused what it could not see would teach people to ignore it.
|
||||
//
|
||||
// **And every identity against every bound it meets** (novox/hq ADR 0225, issue 263): each module's
|
||||
// identity, on a machine whose name is `longestMachine` characters, against the bound of every
|
||||
// provision it wants that a manifest given here offers. An overflow is refused in the pull request
|
||||
// that introduces it — a new requirement, a lowered bound, a longer slug — instead of on the
|
||||
// provider's machine when a real machine's name first meets the module's.
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
return moduleCheckFor(paths, catalogue.DefaultLongestMachine, out)
|
||||
}
|
||||
|
||||
func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
|
||||
if len(paths) == 0 {
|
||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||
"manifest of a repository together so the rules between them are checked too")
|
||||
@@ -74,6 +84,15 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
// Between the manifests too: an identity against the bounds of the provisions it wants, which
|
||||
// only the provider's manifest states.
|
||||
identities := catalogue.IdentityProblems(shelf, longestMachine)
|
||||
sort.Strings(identities)
|
||||
for _, p := range identities {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(identities)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
@@ -109,7 +128,8 @@ func moduleCheck(paths []string, out io.Writer) error {
|
||||
}
|
||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||
"module not given here reads as unknown\n", len(paths))
|
||||
"module not given here reads as unknown. Identities judged on a %d-character machine name, "+
|
||||
"against the bounds of the providers given here\n", len(paths), longestMachine)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// recordBuildDuration measures one build from its ask to its outcome heard.
|
||||
func recordBuildDuration(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, asked time.Time) {
|
||||
if asked.IsZero() || result.ID == "" {
|
||||
return
|
||||
}
|
||||
subject := result.Module
|
||||
if subject == "" {
|
||||
subject = result.Repository
|
||||
}
|
||||
detail := "built"
|
||||
if result.Failed != "" {
|
||||
detail = "failed: " + firstLine(result.Failed)
|
||||
}
|
||||
if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationBuild, Subject: subject,
|
||||
Node: result.On, Ref: result.ID, Started: asked, Took: time.Since(asked), Detail: detail}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: how long it took could not be recorded: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
//
|
||||
// **Which verbs ask why.** `plans close` and `plans stop`, `broker consumer-reset` and `hand-act
|
||||
// record` refuse without it everywhere: nothing automated runs them, so a call without a reason is a
|
||||
// person who has not given one. A named `push` asks for it through the mesh-controller seat, which is
|
||||
// how a person or an agent acts by hand on the mesh; at a shell `--why` is recorded when given and not
|
||||
// required, because the installer and the lab push by command line as a step of what they do, and a
|
||||
// step of a procedure is not a repair. `conditions silence` joins them when the condition store does
|
||||
// (Phase 1).
|
||||
|
||||
// handActFlags are the flags every repairing verb takes.
|
||||
type handActFlags struct {
|
||||
why, cause, condition *string
|
||||
}
|
||||
|
||||
func addHandActFlags(set *flag.FlagSet) handActFlags {
|
||||
return handActFlags{
|
||||
why: set.String("why", "", "why this is done by hand — recorded in the hand-act log (novox/hq to-be 45 §7)"),
|
||||
cause: set.String("cause", "", "the cause, in a word or a condition's kind; the verb's own name when not given"),
|
||||
condition: set.String("condition", "", "the key of the condition this act addresses, if any"),
|
||||
}
|
||||
}
|
||||
|
||||
// given is whether a reason was given.
|
||||
func (f handActFlags) given() bool { return strings.TrimSpace(*f.why) != "" }
|
||||
|
||||
// require refuses an act without a reason, before anything is done.
|
||||
func (f handActFlags) require(verb string) error {
|
||||
if f.given() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s is a repair done by hand, and says why: --why <text> (recorded in the hand-act "+
|
||||
"log, novox/hq to-be 45 §7). Nothing was done", verb)
|
||||
}
|
||||
|
||||
// handActConn is the serving controller's connection, for what it reads of the log itself; a
|
||||
// command dials its own.
|
||||
var handActConn *nats.Conn
|
||||
|
||||
// onTheBus runs f with a connection to the bus: the serving controller's, or one of its own.
|
||||
func onTheBus(f func(*nats.Conn) error) error {
|
||||
if handActConn != nil {
|
||||
return f(handActConn)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
return f(js.Conn())
|
||||
}
|
||||
|
||||
// record writes the entry for an act about to be done. **Before the act, and never instead of it**:
|
||||
// a log that cannot be written is said loudly, and the repair it was about still happens — a mesh
|
||||
// whose bus is down is exactly the mesh somebody is repairing by hand.
|
||||
func (f handActFlags) record(ctx context.Context, verb string, args []string) {
|
||||
if !f.given() {
|
||||
return
|
||||
}
|
||||
act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
act = written
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "this act by hand could NOT be recorded in the hand-act log, and is done anyway: %v\n", err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("recorded as %s in the hand-act log: %s, because %q (cause: %s)\n", act.ID, act.By, act.Why, act.Cause)
|
||||
}
|
||||
|
||||
// handActCommand is `hand-act record` and `hand-acts`.
|
||||
func handActCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "record" {
|
||||
set := flag.NewFlagSet("hand-act record", flag.ContinueOnError)
|
||||
f := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
what := strings.TrimSpace(strings.Join(positionals, " "))
|
||||
if what == "" {
|
||||
return errors.New("hand-act record <what was done> --why <text> [--cause <word>] [--condition <key>]")
|
||||
}
|
||||
if err := f.require("hand-act record"); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*f.cause) == "" {
|
||||
return errors.New("hand-act record says the cause too: --cause <word>, the word a second " +
|
||||
"act for the same reason will use — it is how a repair done twice is found")
|
||||
}
|
||||
act := link.HandAct{Verb: "hand-act record", Args: []string{what}, Why: strings.TrimSpace(*f.why),
|
||||
Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
written, err := link.RecordHandAct(ctx, conn, act)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the act could not be recorded: %w", err)
|
||||
}
|
||||
fmt.Printf("recorded as %s: %s did %q, because %q (cause: %s)\n", written.ID, written.By, what,
|
||||
written.Why, written.Cause)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(args) > 0 && args[0] != "list" && !strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("hand-act record <what> --why <text> --cause <word> | hand-acts [--days N] [--json]")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "list" {
|
||||
args = args[1:]
|
||||
}
|
||||
set := flag.NewFlagSet("hand-acts", flag.ContinueOnError)
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "as data")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
return onTheBus(func(conn *nats.Conn) error {
|
||||
now := time.Now()
|
||||
acts, err := link.HandActs(ctx, conn, now.Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeated := link.RepeatedCauses(acts, now)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"acts": acts, "repeated": repeated}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(acts) == 0 {
|
||||
fmt.Printf("nothing was done by hand in the last %d day(s)\n", *days)
|
||||
return nil
|
||||
}
|
||||
for i := len(acts) - 1; i >= 0; i-- {
|
||||
a := acts[i]
|
||||
fmt.Printf("%s %s %s %s\n by %s — %s (cause: %s", a.At.Local().Format("2006-01-02 15:04"), a.ID,
|
||||
a.Verb, strings.Join(a.Args, " "), a.By, a.Why, a.Cause)
|
||||
if a.Condition != "" {
|
||||
fmt.Printf(", condition %s", a.Condition)
|
||||
}
|
||||
fmt.Println(")")
|
||||
}
|
||||
if len(repeated) > 0 {
|
||||
causes := make([]string, 0, len(repeated))
|
||||
for c, n := range repeated {
|
||||
causes = append(causes, fmt.Sprintf("%s ×%d", c, n))
|
||||
}
|
||||
sort.Strings(causes)
|
||||
fmt.Printf("\ndone by hand more than once in a fortnight — a healer is wanted (to-be 45 S15): %s\n",
|
||||
strings.Join(causes, ", "))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// handActsThisWeek is how many acts were done by hand in the last seven days, for `status`; -1 when
|
||||
// the log could not be read, which status says rather than reading as none.
|
||||
func handActsThisWeek(ctx context.Context) (int, string) {
|
||||
n := -1
|
||||
err := onTheBus(func(conn *nats.Conn) error {
|
||||
reading, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
acts, err := link.HandActs(reading, conn, time.Now().Add(-7*24*time.Hour))
|
||||
n = len(acts)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return -1, err.Error()
|
||||
}
|
||||
return n, ""
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before
|
||||
// anything is done, through the seat, through `command`, and at a shell where nothing automated runs
|
||||
// the verb.
|
||||
func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}},
|
||||
{"push", map[string]any{}},
|
||||
{"plans", map[string]any{"close": "plan-1"}},
|
||||
{"plans", map[string]any{"stop": "plan-1"}},
|
||||
{"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}},
|
||||
{"command", map[string]any{"command": "push anchor"}},
|
||||
{"command", map[string]any{"command": "plans close plan-1"}},
|
||||
{"command", map[string]any{"command": "broker consumer-reset EVENTS controller"}},
|
||||
{"command", map[string]any{"command": "hand-act record restarted --cause x"}},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if c.verb == "plans" && err == nil {
|
||||
// The seat composes the command line; the command refuses it, before opening anything.
|
||||
err = plansCommand(context.Background(), argv[1:])
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), "why") {
|
||||
t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{{"EVENTS", "controller"}} {
|
||||
if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("consumer-reset without why: %v", err)
|
||||
}
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--why") {
|
||||
t.Errorf("hand-act record without why: %v", err)
|
||||
}
|
||||
if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "--cause") {
|
||||
t.Errorf("hand-act record without a cause: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"},
|
||||
"push anchor --wait 0 --why stuck --cause sent-not-reported"},
|
||||
{"plans", map[string]any{"close": "plan-1", "why": "the report will not come"},
|
||||
"plans close plan-1 --why the report will not come"},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"},
|
||||
{"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"},
|
||||
"hand-act record restarted --why hung --cause proxy --condition machine.a.silent"},
|
||||
{"command", map[string]any{"command": "push anchor --why stuck"}, "push anchor --why stuck"},
|
||||
{"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil || strings.Join(argv, " ") != c.want {
|
||||
t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
||||
// issue 259, ADR 0221).
|
||||
//
|
||||
// A named push ends by sending every other machine whose declaration differs from what it was last
|
||||
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
||||
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
||||
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
||||
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
||||
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
||||
// everywhere at once.
|
||||
//
|
||||
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
||||
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
||||
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
||||
|
||||
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
||||
//
|
||||
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
||||
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
||||
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
||||
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
||||
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
||||
// is not known, so a held upgrade cannot be told from anything else.
|
||||
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
||||
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
||||
if !known {
|
||||
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
||||
"mesh kept them, or sent a declaration by hand"}
|
||||
}
|
||||
var why []string
|
||||
for _, m := range modules {
|
||||
now := current[m]
|
||||
was, carried := sent[m]
|
||||
if carried && was == now.Commit {
|
||||
continue
|
||||
}
|
||||
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
||||
if !now.RollOut {
|
||||
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
||||
continue
|
||||
}
|
||||
if id := planStillToSend(plans, m, node); id != "" {
|
||||
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
||||
}
|
||||
}
|
||||
sort.Strings(why)
|
||||
return why
|
||||
}
|
||||
|
||||
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
||||
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
||||
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
||||
// per machine.
|
||||
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
||||
for _, p := range plans {
|
||||
s, holds := p.Modules[module]
|
||||
if !p.Open() || !holds {
|
||||
continue
|
||||
}
|
||||
if s == nil {
|
||||
return p.ID
|
||||
}
|
||||
if s.SentAt != nil || s.State == "failed" {
|
||||
continue
|
||||
}
|
||||
first := false
|
||||
for _, n := range s.First {
|
||||
if n == node {
|
||||
first = true
|
||||
}
|
||||
}
|
||||
if !first {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func fromBuild(was string, carried bool) string {
|
||||
if !carried {
|
||||
return "(never sent it) "
|
||||
}
|
||||
return "from " + buildName(was) + " "
|
||||
}
|
||||
|
||||
func buildName(commit string) string {
|
||||
if commit == "" {
|
||||
return "a build with no source"
|
||||
}
|
||||
return shortCommit(commit)
|
||||
}
|
||||
|
||||
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
||||
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
||||
// to the send, which says why.
|
||||
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
||||
out := map[string][]string{}
|
||||
if len(names) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
inv := open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
modules := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
|
||||
out[node] = why
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
||||
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
||||
func sayHeld(w io.Writer, node string, why []string) {
|
||||
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
||||
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
||||
"grant from this push among it — waits with it. `push %s` sends it\n",
|
||||
node, strings.Join(why, "; "), node)
|
||||
}
|
||||
|
||||
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
||||
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
||||
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
||||
//
|
||||
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
||||
// machines that could not be composed, as refusals.
|
||||
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||
w io.Writer) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var refusals []string
|
||||
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
||||
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
||||
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
||||
rounds := 0
|
||||
for {
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
behind, err := inv.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var also []string
|
||||
for _, m := range behind {
|
||||
if !handled[m.Node] {
|
||||
also = append(also, m.Node)
|
||||
}
|
||||
}
|
||||
if len(also) == 0 {
|
||||
return refusals, nil
|
||||
}
|
||||
if rounds++; rounds > len(nodes) {
|
||||
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
||||
"should not happen; run `push --behind` to finish\n",
|
||||
rounds-1, strings.Join(also, ", "))
|
||||
return refusals, nil
|
||||
}
|
||||
sort.Strings(also)
|
||||
held, err := heldMachines(ctx, open, also)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var sending []string
|
||||
for _, name := range also {
|
||||
// Every candidate this round is marked handled — the sent ones so they are not
|
||||
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
||||
// that cannot be composed does not make the loop spin on it for ever.
|
||||
handled[name] = true
|
||||
if why, isHeld := held[name]; isHeld {
|
||||
sayHeld(w, name, why)
|
||||
continue
|
||||
}
|
||||
sending = append(sending, name)
|
||||
}
|
||||
if len(sending) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
||||
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
||||
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
||||
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
||||
// is left out of it said, and its declaration allocated.
|
||||
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
||||
return func(held context.Context, node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
if err == nil {
|
||||
reportLeftOut(node, declared)
|
||||
}
|
||||
return declared, err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
|
||||
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
|
||||
// else that moved is still a consequence the push sends (ADR 0083).
|
||||
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{
|
||||
"resolver": {Commit: "c2c2c2c2c2"},
|
||||
"agent": {Commit: "a2", RollOut: true},
|
||||
"network": {},
|
||||
}
|
||||
modules := []string{"network", "resolver", "agent"}
|
||||
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
|
||||
|
||||
// A module whose policy records moved: held, naming it, both builds and why.
|
||||
why := heldBack("laptop", modules, sent, true, current, nil)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
|
||||
!strings.Contains(why[0], "upgrade policy records") {
|
||||
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
|
||||
}
|
||||
|
||||
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
|
||||
sent["resolver"] = "c2c2c2c2c2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a machine whose builds are all current was held: %v", why)
|
||||
}
|
||||
|
||||
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
|
||||
sent["agent"] = "a1"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
|
||||
}
|
||||
|
||||
// The last send's builds are not known: held whole.
|
||||
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "not known") {
|
||||
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
|
||||
}
|
||||
|
||||
// A module the machine was never sent, under a recording policy: held, and said so.
|
||||
delete(sent, "resolver")
|
||||
sent["agent"] = "a2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
|
||||
t.Fatalf("a module never sent under a recording policy: %v", why)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
|
||||
// naming some other machine must not send them for it.
|
||||
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
|
||||
at := time.Now()
|
||||
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
|
||||
sent := map[string]string{"agent": "a1"}
|
||||
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
|
||||
|
||||
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
|
||||
t.Fatalf("a machine the plan has not reached was not held: %v", why)
|
||||
}
|
||||
// The first machine itself was sent by the plan: not held by it.
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
|
||||
t.Fatalf("the plan's first machine was held: %v", why)
|
||||
}
|
||||
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
|
||||
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
|
||||
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
|
||||
Modules: map[string]*inventory.PlanModule{"agent": s}}}
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
|
||||
t.Errorf("%s: not held: %v", what, why)
|
||||
}
|
||||
}
|
||||
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
|
||||
for what, plans := range map[string][]inventory.Plan{
|
||||
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
|
||||
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "failed"}}}},
|
||||
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built"}}}},
|
||||
} {
|
||||
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
|
||||
t.Errorf("%s: held: %v", what, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A send records the build of each module it carried; a module left out of it keeps the build it
|
||||
// was last sent, since the machine keeps that one.
|
||||
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
|
||||
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
|
||||
current, map[string]string{"a": "a1", "b": "b1"})
|
||||
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
|
||||
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
|
||||
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
|
||||
// reads the machine as current — and writes down which machines it declared.
|
||||
type recordedDelivery struct {
|
||||
inv *inventory.Inventory
|
||||
declared []string
|
||||
}
|
||||
|
||||
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
// says which build it is.
|
||||
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
|
||||
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
|
||||
}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A third machine on the private network: once it is sent, every other machine's peers change with
|
||||
// it, which is a consequence a push must still send — no build moved.
|
||||
func aThirdMachine(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.AddNode(ctx, "spare")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
|
||||
{"name": "systemd", "present": true}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var profile map[string]any
|
||||
if err := json.Unmarshal(reported, &profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
|
||||
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
|
||||
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
aResolver(t, open, "c1c1c1c1c1", asked)
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
||||
}
|
||||
digestOfLaptop := func() string {
|
||||
sent, err := inv.Outstanding(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sent
|
||||
}
|
||||
before := digestOfLaptop()
|
||||
|
||||
// The change merges; the policy is the default, record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
if len(d.declared) != 0 {
|
||||
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
||||
}
|
||||
if digestOfLaptop() != before {
|
||||
t.Fatal("the laptop's last send moved: it was sent the held build")
|
||||
}
|
||||
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
||||
"upgrade policy records", "`push laptop` sends it"} {
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
||||
}
|
||||
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
||||
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A policy that rolls out: the laptop is a consequence like any other, and sent.
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
||||
t.Fatalf("the new send did not record the new build: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
|
||||
// for another reason is sent by a push that names someone else.
|
||||
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
||||
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if strings.Contains(said.String(), "was not sent") {
|
||||
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
// question.
|
||||
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
||||
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||
@@ -90,3 +92,72 @@ func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||
//
|
||||
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||
// answers is a lookup that waits out its timeout on every name.
|
||||
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||
var replicated []catalogue.Seat
|
||||
for _, s := range catalogue.Seats() {
|
||||
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||
replicated = append(replicated, s)
|
||||
}
|
||||
}
|
||||
if len(replicated) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for _, seat := range replicated {
|
||||
var nodes []string
|
||||
for _, h := range recorded {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||
nodes = append(nodes, h.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
var derived []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
derived = append(derived, e.On...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(derived) == 1 {
|
||||
nodes = derived
|
||||
}
|
||||
}
|
||||
at := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
if address[n] != "" {
|
||||
at[overlay.InternalName(n)] = address[n]
|
||||
}
|
||||
}
|
||||
out[seat.Name] = at
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an
|
||||
// overflow is refused before merge by `module check`, and a provider's machine is never refused for
|
||||
// one consumer's identity.
|
||||
|
||||
// `module check` refuses the pull request that introduces an overflow, naming the module.
|
||||
func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) string {
|
||||
p := filepath.Join(dir, name+".json")
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
objects := write("objects", `{"module":"objects","version":"1",
|
||||
"provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}],
|
||||
"receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`)
|
||||
resolver := write("resolver", `{"module":"resolver","version":"1",
|
||||
"provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`)
|
||||
album := write("photoalbum", `{"module":"photoalbum","version":"1","requires":["s3-bucket"]}`)
|
||||
nm := write("networkmanager", `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheckFor([]string{resolver, nm}, 6, &out); err != nil {
|
||||
t.Fatalf("a long name requiring a keyless provision was refused (issue 263): %v\n%s", err, out.String())
|
||||
}
|
||||
out.Reset()
|
||||
err := moduleCheckFor([]string{objects, album, resolver, nm}, 6, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("an identity overflowing an S3 access key passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(out.String(), "`slug` of at most 8 characters") ||
|
||||
strings.Contains(out.String(), "networkmanager wants") {
|
||||
t.Fatalf("the refusal does not name the one overflowing module and its remedy:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case, through the commands: networkmanager on a six-character machine requires the
|
||||
// resolver provision, and a second consumer there overflows an object store's access key. The
|
||||
// provider's machine still composes; the overflowing consumer is left out of its grants and named,
|
||||
// by push and by `status`, and the keyless consumer is granted with its long name.
|
||||
func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "objects", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh,
|
||||
Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}})
|
||||
register(t, open, catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}})
|
||||
register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1",
|
||||
Requires: []string{"wildcard-resolution"}})
|
||||
register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}})
|
||||
for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"},
|
||||
{"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} {
|
||||
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
|
||||
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's machine resolves, and says which of its modules no provider will grant.
|
||||
consumer, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
over := consumer.Overflowing()
|
||||
if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" {
|
||||
t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over)
|
||||
}
|
||||
|
||||
// The provider's machine composes. Under ADR 0049's one bound this was a refusal naming
|
||||
// networkmanager, and no push to the provider could go through.
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "anchor", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err)
|
||||
}
|
||||
if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" {
|
||||
t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld)
|
||||
}
|
||||
grants, _, err := grantsFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keyless bool
|
||||
for _, g := range grants {
|
||||
keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager"
|
||||
}
|
||||
if !keyless {
|
||||
t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants)
|
||||
}
|
||||
var granted []string
|
||||
for _, c := range declared.Received["objects"]["s3-bucket"] {
|
||||
granted = append(granted, c.From)
|
||||
}
|
||||
if strings.Join(granted, ",") != "files" {
|
||||
t.Fatalf("the object store grants %v; files and only files fit", granted)
|
||||
}
|
||||
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
|
||||
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
|
||||
!strings.Contains(said, "left out of anchor's grants") {
|
||||
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
|
||||
}
|
||||
|
||||
// And `status` names it, and does not call the mesh well while it stands.
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" {
|
||||
t.Fatalf("status does not carry the overflow: %+v", asked.overflowing)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh with a consumer left out of its grants reads as well")
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
if !strings.Contains(shown, "identified too long for a provision they require") ||
|
||||
!strings.Contains(shown, "mesh_laptop_photoalbum") {
|
||||
t.Fatalf("status does not say it:\n%s", shown)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Overflowing []catalogue.Overflow `json:"overflowing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 ||
|
||||
doc.Overflowing[0].Bound.Max != 20 {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
|
||||
@@ -145,6 +145,14 @@ func run() error {
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
// Acts done by hand, and why (novox/hq to-be 45 §7).
|
||||
case "hand-act":
|
||||
return handActCommand(ctx, args[1:])
|
||||
case "hand-acts":
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
@@ -194,6 +202,7 @@ func usage() {
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||
@@ -225,19 +234,26 @@ func usage() {
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
plans stop|close <id> --why <text> end a plan by hand; recorded in the hand-act log
|
||||
hand-act record <what> --why <text> --cause <word> [--condition <key>]
|
||||
record an act done by hand outside the mesh (to-be 45 §7)
|
||||
hand-acts [--days N] [--json] what was done by hand lately, why, and which causes repeat
|
||||
durations [--kind K] [--days N] [--json]
|
||||
apply, heartbeat, plan-tier and build durations, per machine or module
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||
rotate <provision> [--consumer <n>] [--module <m>] a new credential for every holder, both ends at once
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||
push [<node>] [--behind] [--why <text>] send a node everything it should be, or only those
|
||||
that need it; --why records it in the hand-act log
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
@@ -290,6 +306,9 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
// And how long it took, asked to heard, which a build's bound will be set from (novox/hq to-be 45
|
||||
// Phase 0). Said if lost; never a reason not to take the build in.
|
||||
recordBuildDuration(ctx, b.inv, result, asked)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
@@ -316,5 +335,7 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
// A module registered may be one a machine is now behind: `status` is composed again.
|
||||
statusFrom.nudge()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
||||
//
|
||||
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
||||
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
||||
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
||||
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
||||
// built from that repository stayed behind and were built by hand.
|
||||
//
|
||||
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
||||
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
||||
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
||||
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
||||
// would still move something was never acted on — it is said, and acted on now.
|
||||
const (
|
||||
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
||||
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
||||
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
||||
mergeGrace = 10 * time.Minute
|
||||
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
||||
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
||||
mergeLookBack = 24 * time.Hour
|
||||
// mergeCatchUpEvery is how often the stream is read back.
|
||||
mergeCatchUpEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
||||
type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := open.inventory.ReadRepositories(ctx)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
tick := time.NewTicker(mergeCatchUpEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("merges the bus may not have handed over are looked for again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
||||
// move something is said and acted on, oldest first.
|
||||
//
|
||||
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
||||
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
||||
// would have.
|
||||
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
||||
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
||||
|
||||
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, read, err := catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||
isAre(len(names)))
|
||||
if err := act(ctx, a.SourceMoved); err != nil {
|
||||
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
||||
a.Owner, a.Repo, a.Commit, err)
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// announcedList is the events stream's merges as a test gives them.
|
||||
type announcedList []link.AnnouncedMerge
|
||||
|
||||
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
var out []link.AnnouncedMerge
|
||||
for _, m := range a {
|
||||
if !m.At.Before(since) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
||||
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
||||
type aCatalogue struct {
|
||||
entries []inventory.Entry
|
||||
acted []string
|
||||
fail error
|
||||
}
|
||||
|
||||
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
||||
}
|
||||
|
||||
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
||||
return func(_ context.Context, m link.SourceMoved) error {
|
||||
if c.fail != nil {
|
||||
return c.fail
|
||||
}
|
||||
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
||||
for _, moved := range wouldMove(m, c.entries, nil) {
|
||||
for i := range c.entries {
|
||||
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
||||
c.entries[i].Source.Head = m.Commit
|
||||
c.entries[i].Source.Seen = now()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func at(s string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
||||
return link.AnnouncedMerge{
|
||||
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
MergedAt: mergedAt, Paths: paths,
|
||||
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
||||
At: at(onTheBus),
|
||||
}
|
||||
}
|
||||
|
||||
func built(module, repo, path, commit, seen string) inventory.Entry {
|
||||
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
||||
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
||||
return e
|
||||
}
|
||||
|
||||
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
||||
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
||||
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
||||
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
||||
// has had its time with it.
|
||||
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{
|
||||
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
// Acted on when it was announced: looked at after it was merged.
|
||||
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
||||
}}
|
||||
stream := announcedList{
|
||||
// Nothing the mesh holds is built from the records repository.
|
||||
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
||||
// Acted on: its module was looked at since.
|
||||
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
||||
// Never handed over.
|
||||
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
||||
"node-tools/internal/console/console.go"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:50:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
pass()
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T22:58:00Z")
|
||||
pass()
|
||||
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
||||
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
||||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
||||
t.Fatalf("the missed merge was not said as one: %q", said)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T23:03:00Z")
|
||||
pass()
|
||||
if len(cat.acted) != 1 || len(said) != 1 {
|
||||
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
||||
// that could not be acted on is said and tried again on the next pass.
|
||||
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
||||
cat := &aCatalogue{
|
||||
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
||||
fail: errors.New("the store is restarting"),
|
||||
}
|
||||
stream := announcedList{
|
||||
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
||||
"modules/plex/module.json", "modules/plex/x.ts"),
|
||||
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
||||
t.Fatalf("a failed catch-up was not said: %q", said)
|
||||
}
|
||||
cat.fail = nil
|
||||
clock = at("2026-10-05T22:05:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
||||
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge older than the look-back is left to the operator: a controller that did not run this
|
||||
// missed it, and acting on it days later would be a surprise rebuild.
|
||||
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
||||
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
||||
func(string, ...any) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
||||
}
|
||||
}
|
||||
@@ -59,8 +59,19 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||
if args[0] == "check" {
|
||||
set := flag.NewFlagSet("module check", flag.ContinueOnError)
|
||||
// The longest machine name an identity must fit on (novox/hq ADR 0225): a mesh passes its own.
|
||||
longest := set.Int("longest-machine-name", catalogue.DefaultLongestMachine,
|
||||
"judge each module's identity on a machine name this many characters long")
|
||||
given, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *longest < 1 {
|
||||
return errors.New("--longest-machine-name is a length, at least 1")
|
||||
}
|
||||
var paths []string
|
||||
for _, a := range args[1:] {
|
||||
for _, a := range given {
|
||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||
under, err := manifestsUnder(a)
|
||||
if err != nil {
|
||||
@@ -71,7 +82,7 @@ func moduleCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
paths = append(paths, a)
|
||||
}
|
||||
return moduleCheck(paths, os.Stdout)
|
||||
return moduleCheckFor(paths, *longest, os.Stdout)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -661,7 +672,7 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife
|
||||
// durable subscription nobody reads.
|
||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
|
||||
@@ -275,25 +275,9 @@ func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// No registry trust is composed here any more: the container runtime's module states it, told
|
||||
// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
|
||||
g, err := overlay.From(nodes, cidr, "")
|
||||
if g != nil {
|
||||
// The artifact store, as this network reaches it. Found rather than configured: the
|
||||
// provider is whichever module offers it, on whichever machine holds that module — and if
|
||||
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
|
||||
// no trust to write and nothing is written (novox/hq ADR 0082).
|
||||
//
|
||||
// Refused rather than composed without it when the question could not be answered: a
|
||||
// declaration missing the trust because a lookup failed is a machine that cannot pull,
|
||||
// delivered by a push that reported success — and nothing recomposes it until the next
|
||||
// push (the shape of novox/hq issues 042/048, reappearing as a race).
|
||||
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
|
||||
if storeErr != nil {
|
||||
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
|
||||
}
|
||||
if found {
|
||||
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
|
||||
}
|
||||
}
|
||||
if err != nil && len(refused) > 0 {
|
||||
// The network is missing something, and some machines could not be resolved at all. Those
|
||||
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
||||
|
||||
@@ -387,7 +387,7 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(args[1:])
|
||||
return consumerReset(ctx, args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
@@ -414,10 +414,21 @@ func brokerCommand(ctx context.Context, args []string) error {
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
||||
func consumerReset(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("broker consumer-reset", flag.ContinueOnError)
|
||||
// A repair by hand, which says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
args, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer> --why <text>, e.g. broker consumer-reset EVENTS controller --why ...")
|
||||
}
|
||||
if err := why.require("broker consumer-reset"); err != nil {
|
||||
return err
|
||||
}
|
||||
why.record(ctx, "broker consumer-reset", args)
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -505,6 +516,19 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
||||
fmt.Printf(" public domain %s\n", domain)
|
||||
}
|
||||
|
||||
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
|
||||
// capabilities, because it is something not working now and they are a description.
|
||||
failing, err := failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if here := failingOn(failing, name); len(here) > 0 {
|
||||
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
|
||||
for _, line := range failingLines(here, time.Now()) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+107
-19
@@ -397,9 +397,49 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
return sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
out := sendable{Resources: composed.Resources, Adoption: adoption,
|
||||
Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
|
||||
LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld}
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
if !known {
|
||||
before = nil
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// carriedBuilds is the build of each module a declaration carries, as a send records it (novox/hq
|
||||
// issue 259): the module's current build for each module in it, and for a module left out of it
|
||||
// (ADR 0163, rule 6) the build it was last sent, since the machine keeps that one — or nothing, when
|
||||
// that is not known. Never nil, so a send through here always records what it knows.
|
||||
func carriedBuilds(modules []string, leftOut map[string]string, current map[string]inventory.CurrentBuild,
|
||||
before map[string]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, m := range modules {
|
||||
if _, left := leftOut[m]; left {
|
||||
if was, kept := before[m]; kept {
|
||||
out[m] = was
|
||||
}
|
||||
continue
|
||||
}
|
||||
out[m] = current[m].Commit
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedKeysOf is a map's keys, sorted — so what a declaration says it left out does not move
|
||||
@@ -424,6 +464,11 @@ func reportLeftOut(node string, declared sendable) {
|
||||
"what the machine holds for it is kept and its containers are untouched. %s\n",
|
||||
node, m, declared.leftOutWhy[m])
|
||||
}
|
||||
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
|
||||
// 0225): the machine is sent everything else, and the consumer is named.
|
||||
for _, o := range declared.withheld {
|
||||
fmt.Printf("%s: %s\n", node, o)
|
||||
}
|
||||
}
|
||||
|
||||
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||
@@ -431,7 +476,7 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||
inv := open.inventory
|
||||
grants, err := grantsFor(ctx, open, node)
|
||||
grants, withheld, err := grantsFor(ctx, open, node)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
@@ -667,6 +712,13 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// And who holds each replicated seat, where (novox/hq ADR 0223): every machine's resolver file
|
||||
// lists every holder of the mesh's resolver.
|
||||
replicas, err := replicatedHolders(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
|
||||
// **The bus is never public** (novox/hq ADR 0169). It was a foundation port — widened from the
|
||||
// broker's own `from: mesh` to from-anywhere on the broker's host, so a machine could enrol
|
||||
// before it had an address on the private network. A machine joins through the tunnel now, and
|
||||
@@ -732,15 +784,22 @@ func renderingFor(ctx context.Context, open *stores, node string,
|
||||
if err != nil {
|
||||
return catalogue.Rendering{}, inventory.Node{}, err
|
||||
}
|
||||
// Where this machine reaches each mesh seat's holder, for ${seat:<seat>:reach} (novox/hq ADR
|
||||
// 0222): the artifact store as this network reaches it, which the container runtime is told to
|
||||
// trust (ADR 0082). The same address composed into every reference the mesh built.
|
||||
var reach map[string]string
|
||||
if artifactStore != "" {
|
||||
reach = map[string]string{"mesh-artifact-store": artifactStore}
|
||||
}
|
||||
return catalogue.Rendering{
|
||||
BusMembership: memberships[node],
|
||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||
Machines: machines, Zones: zones,
|
||||
Machines: machines, Zones: zones, Holders: replicas,
|
||||
Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation,
|
||||
Kept: kept, Adopted: record.Adopted, OutwardLinks: outwardLinks,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, Built: built,
|
||||
BusUsers: busUsers,
|
||||
Given: given, Taken: taken, Seats: seats, ArtifactStore: artifactStore, SeatReach: reach, Built: built,
|
||||
BusUsers: busUsers, Withheld: withheld,
|
||||
}, record, nil
|
||||
}
|
||||
|
||||
@@ -876,28 +935,34 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
|
||||
// The mirror of what a consumer is given, and the half that makes the credential real: a password
|
||||
// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so
|
||||
// the mesh hands over something it cannot itself use.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, error) {
|
||||
//
|
||||
// **One consumer's identity never refuses the provider's machine** (novox/hq ADR 0225, issue 263).
|
||||
// A consumer whose identity overflows the provision's bound is left out of the grants and returned
|
||||
// beside them, for push, plan and `status` to say; every other consumer is granted and the provider's
|
||||
// declaration composes. Refusing here once made a whole machine unpushable for one module elsewhere.
|
||||
func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Grant, []catalogue.Overflow, error) {
|
||||
inv := open.inventory
|
||||
issued, err := inv.SecretsFrom(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Where each consumer is, so a provider that must reach back to one does not have to know how
|
||||
// the mesh names machines.
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// What each consumer actually asked for, taken from that machine's own resolution rather than
|
||||
// from a record beside it. A provider told to create a password and not what to create it for
|
||||
// can do nothing with it, and the name a consumer wants is the consumer's to say.
|
||||
out := make([]catalogue.Grant, 0, len(issued))
|
||||
var withheld []catalogue.Overflow
|
||||
for _, s := range issued {
|
||||
plan, settings, err := planFor(ctx, open, s.Consumer)
|
||||
switch {
|
||||
@@ -910,11 +975,11 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// The mesh could not be asked what they wanted, which is not the same as their wanting
|
||||
// nothing — and withholding a grant on that reading takes a consumer's access away
|
||||
// (novox/hq 04-ISSUES/152).
|
||||
return nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
return nil, nil, fmt.Errorf("what %s asked of %s cannot be read: %w", s.Consumer, s.Name, err)
|
||||
}
|
||||
values, asks, err := plan.ContributionsFrom(s.Name, s.ConsumerModule, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
// A port in there is the consumer's software port until this. The consumer is on another
|
||||
// machine, so the assignment that moved it is that machine's — fetched here rather than
|
||||
@@ -922,7 +987,7 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
// this case (novox/hq 04-ISSUES/038, the cross-node half).
|
||||
published, err := portsOn(ctx, inv, s.Consumer, s.ConsumerModule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
values = catalogue.AtPublishedPort(values, s.ConsumerModule, published)
|
||||
from := s.ConsumerModule
|
||||
@@ -933,9 +998,10 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
from = ""
|
||||
}
|
||||
// The consumer's identity slug, from its own manifest, carried on the grant so the provider
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Refused here if it still would
|
||||
// not fit the tightest backend — the mesh chose the name, so the mesh refuses it, with the
|
||||
// remedy a short slug rather than a login a provider silently shortened.
|
||||
// derives the same login the consumer does (novox/hq ADR 0049). Judged against the bound of
|
||||
// this provision, as the consumer's resolution states it from the provider's offer (ADR
|
||||
// 0225): a consumer it would not fit is left out of the grants and said, rather than a login a
|
||||
// provider silently shortened — and rather than this whole machine refused for it.
|
||||
slug := ""
|
||||
for _, mm := range plan.Modules {
|
||||
if mm.Module == s.ConsumerModule {
|
||||
@@ -944,15 +1010,32 @@ func grantsFor(ctx context.Context, open *stores, node string) ([]catalogue.Gran
|
||||
}
|
||||
}
|
||||
if from != "" {
|
||||
if err := catalogue.CheckIdentity(s.Consumer, catalogue.IdentitySource(slug, s.ConsumerModule)); err != nil {
|
||||
return nil, err
|
||||
bound := boundOfGrant(plan, s, node)
|
||||
source := catalogue.IdentitySource(slug, s.ConsumerModule)
|
||||
if catalogue.CheckIdentityWithin(s.Consumer, source, bound) != nil {
|
||||
withheld = append(withheld, catalogue.Overflow{Provision: s.Name, Provider: node,
|
||||
Consumer: s.Consumer, Module: s.ConsumerModule,
|
||||
Identity: catalogue.ConsumerIdentity(s.Consumer, source), Bound: bound})
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, catalogue.Grant{
|
||||
Provision: s.Name, Consumer: s.Consumer, At: onNetwork[s.Consumer],
|
||||
From: from, Values: values, Slug: slug, Sealed: s.ForProvider, Local: s.Local})
|
||||
}
|
||||
return out, nil
|
||||
return out, withheld, nil
|
||||
}
|
||||
|
||||
// boundOfGrant is the identity bound the consumer's own resolution states for the requirement this
|
||||
// grant answers. A requirement not found there is held to the tightest bound the mesh knows rather
|
||||
// than to none: what the provider keeps of it is not known here.
|
||||
func boundOfGrant(consumer catalogue.Resolution, s inventory.Secret, provider string) catalogue.IdentityBound {
|
||||
for _, n := range consumer.Needs {
|
||||
if n.Name == s.Name && n.For == s.ConsumerModule && n.From == provider {
|
||||
return n.Identity
|
||||
}
|
||||
}
|
||||
return catalogue.DefaultIdentityBound
|
||||
}
|
||||
|
||||
// listensLines is what a person is told about what this module would open, and why — the same
|
||||
@@ -1028,6 +1111,11 @@ func planCommand(ctx context.Context, args []string) error {
|
||||
left := plan.LeftOut(settings, record.Adopted)
|
||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||
}
|
||||
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||
for _, o := range plan.Overflowing() {
|
||||
fmt.Printf("%s: %s\n", args[0], o)
|
||||
}
|
||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||
// stored before its definition moved from under it.
|
||||
for _, m := range plan.Modules {
|
||||
|
||||
+128
-79
@@ -105,7 +105,10 @@ func serve(ctx context.Context) error {
|
||||
|
||||
work := link.Enrolment{Inventory: inv, Identity: ident, Broker: known,
|
||||
OnNATS: true}
|
||||
server, err := connectLink(ctx, inv, work, work)
|
||||
// `status` from a summary kept current here (novox/hq to-be 45 Phase 0): a machine saying
|
||||
// something new is one thing that moves it, so the listener nudges it.
|
||||
statusFrom = newStatusSummary(composeStatus(open))
|
||||
server, err := connectLink(ctx, inv, work, nudgingListener{work, statusFrom})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -121,17 +124,27 @@ func serve(ctx context.Context) error {
|
||||
// Open plans move on a timer as well as on outcomes (novox/hq ADR 0162): a tier waiting for
|
||||
// machines to report moves when they have, and a plan left by a replaced controller resumes.
|
||||
go planTicker(ctx, open)
|
||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||
go forgettingOldDurations(ctx, inv)
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
// above; this is the other half — the control plane is the only one of the three that knows
|
||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||
if err := server.Follows(following{open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the merges the bus announced and never handed over, read back on a timer and acted on late
|
||||
// rather than never (novox/hq issue 266).
|
||||
go catchingUpOnMerges(ctx, open, server)
|
||||
// And a catalogue that has just started, asking for what it missed. The same type answers
|
||||
// both: what a build meant and what the builds were are two questions about one record.
|
||||
if err := server.Answers(following{open}); err != nil {
|
||||
return err
|
||||
}
|
||||
// And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR
|
||||
// 0224): a provider's journal must not be the only place that says so.
|
||||
if err := server.Watches(standings{inv}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And the mesh's own verbs, as the seat this control plane holds (novox/hq ADR 0154). Served
|
||||
// from the store's row, so what the seat declares is what is answered.
|
||||
@@ -150,6 +163,21 @@ func serve(ctx context.Context) error {
|
||||
if !isNATS {
|
||||
return errors.New("the mesh's verbs are served over the bus, and this control plane is not on it")
|
||||
}
|
||||
// The hand-act log is counted for `status` on this connection rather than a new one a minute.
|
||||
handActConn = bus.Conn
|
||||
// Composed now and kept current, before the verb that answers from it is served.
|
||||
go statusFrom.keep(ctx)
|
||||
// A call that outlasts its caller's patience is followed by `calls` (novox/hq issue 265).
|
||||
link.Calls.Follow = catalogue.ControllerSeatName + ".calls"
|
||||
// And every call is kept on the bus, so a restart of this process keeps what came of each
|
||||
// (novox/hq to-be 45 §6). A bus without the bucket is said and served from memory, as before:
|
||||
// answering no calls at all would be worse than answering them without the record.
|
||||
said := log.New(os.Stdout, "", log.LstdFlags)
|
||||
if keeper, err := link.CallsOnTheBus(ctx, bus.Conn); err != nil {
|
||||
fmt.Printf("calls are kept in memory only, and lost when this controller stops: %v\n", err)
|
||||
} else if err := link.Calls.Durably(ctx, keeper, controllerProcess(), said); err != nil {
|
||||
fmt.Printf("calls are kept on the bus from now on; the ones kept before could not be read: %v\n", err)
|
||||
}
|
||||
stopServing, err := bus.ServeSeatTools(catalogue.ControllerSeatName, handlers, log.New(os.Stdout, "", log.LstdFlags))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -217,8 +245,9 @@ func declare(ctx context.Context, args []string) error {
|
||||
}
|
||||
// Written down like every other send (novox/hq issue 204): a declaration a person sent by hand
|
||||
// is still what the machine was last told, and status must not read it as current for the one
|
||||
// the mesh would compose.
|
||||
if _, err := recordSent(ctx, inv, node, raw); err != nil {
|
||||
// the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259):
|
||||
// the mesh did not compose it, so a push that does not name this machine treats it as held.
|
||||
if _, err := recordSent(ctx, inv, node, raw, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw))
|
||||
@@ -249,6 +278,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
// (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget.
|
||||
wait := set.Duration("wait", 0,
|
||||
"for a named node, how long to wait for it to report applying what it was sent (0: do not wait)")
|
||||
// A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat,
|
||||
// recorded when given at a shell — see handacts.go for why a shell is not refused.
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -263,6 +295,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return errors.New("push <node> or push --behind, not both: one names a machine and the " +
|
||||
"other asks which machines need one")
|
||||
}
|
||||
recorded := append([]string(nil), args...)
|
||||
if *behind {
|
||||
recorded = append(recorded, "--behind")
|
||||
}
|
||||
why.record(ctx, "push", recorded)
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -316,7 +353,7 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
if len(needsOne) == 0 {
|
||||
// Said rather than doing nothing quietly. "Nothing needed one" and "this did not run"
|
||||
// must never look the same.
|
||||
fmt.Println("every machine is doing what it was told")
|
||||
fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -357,6 +394,18 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
asked = append(asked, n.Name)
|
||||
}
|
||||
// **A push that named no machine says so, first.** Through the console a machine the caller
|
||||
// meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as
|
||||
// `push --behind`, and every machine behind was pushed by someone who thought they had pushed one.
|
||||
// Its whole-mesh reach is the first line of the answer, with the machines it is about to send.
|
||||
if len(args) == 0 {
|
||||
which := "every machine"
|
||||
if *behind {
|
||||
which = "every machine that is behind"
|
||||
}
|
||||
fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n",
|
||||
which, len(asked), strings.Join(asked, ", "))
|
||||
}
|
||||
|
||||
// **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's
|
||||
// user list, and a module's new grants are refused by the bus until that list says them. Among
|
||||
@@ -367,6 +416,23 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **Not when its own modules are held back** (novox/hq issue 259, ADR 0221): added rather than
|
||||
// named, it is sent its whole declaration, and a build its policy records or a plan has not sent
|
||||
// it yet would go with the user list. Named and left, with what that costs.
|
||||
saidHeld := map[string]bool{}
|
||||
if holderBehind && len(args) == 1 {
|
||||
held, err := heldMachines(ctx, open, []string{holder})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if why, isHeld := held[holder]; isHeld {
|
||||
sayHeld(os.Stdout, holder, why)
|
||||
fmt.Printf("%s holds the bus, and the user list it would carry has changed: until it is "+
|
||||
"sent, the bus may refuse what this push's machines were newly granted\n", holder)
|
||||
holderBehind = false
|
||||
saidHeld[holder] = true
|
||||
}
|
||||
}
|
||||
asked = brokerFirst(asked, holder, holderBehind)
|
||||
|
||||
// Held from composing to sending, so a converge on one of them cannot send between the two
|
||||
@@ -407,7 +473,11 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
release()
|
||||
fmt.Printf("\n%d node(s) told\n", len(sending))
|
||||
told := make([]string, 0, len(sending))
|
||||
for _, r := range sending {
|
||||
told = append(told, r.node)
|
||||
}
|
||||
fmt.Printf("\n%d node(s) told: %s\n", len(sending), strings.Join(told, ", "))
|
||||
reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld)
|
||||
|
||||
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
||||
@@ -418,76 +488,21 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
//
|
||||
// Compared against what each machine was last SENT, not against a before/after of this push:
|
||||
// the mint usually happened at `assign` or `module issue`, before this command ran, so the
|
||||
// only durable signal is "what it should be" versus "what it last received". A machine behind
|
||||
// for an unrelated reason is caught here too, which is not a cost — a named push that knew a
|
||||
// machine was behind and left it so would be the very silence this removes. Bounded: a
|
||||
// only durable signal is "what it should be" versus "what it last received". Bounded: a
|
||||
// flushed send may itself mint, so this converges over a few rounds.
|
||||
//
|
||||
// **Except a machine a policy or a plan holds back** (novox/hq issue 259, ADR 0221): one whose
|
||||
// modules would move to a build their upgrade policy records rather than rolls out, or that an
|
||||
// open plan has not sent it yet. It is named, with why, and left for a push that names it.
|
||||
if len(args) == 1 {
|
||||
flushed := map[string]bool{args[0]: true}
|
||||
// Bounded by the node count: a node is marked flushed the round it is handled and is
|
||||
// never handled twice, so the loop cannot run more than len(nodes) rounds. The bound is
|
||||
// a guard against a logic error, not a real limit — if it were ever hit, that is a bug
|
||||
// rather than a cascade legitimately still converging, so it is said rather than passed
|
||||
// over in silence, unlike the earlier fixed cap that could stop a real cascade short.
|
||||
rounds := 0
|
||||
for {
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
behind, err := inv.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var also []string
|
||||
for _, m := range behind {
|
||||
if !flushed[m.Node] {
|
||||
also = append(also, m.Node)
|
||||
}
|
||||
}
|
||||
if len(also) == 0 {
|
||||
break
|
||||
}
|
||||
if rounds++; rounds > len(nodes) {
|
||||
fmt.Printf("\nstopped cascading after %d rounds with %s still behind — this "+
|
||||
"should not happen; run `push --behind` to finish\n",
|
||||
rounds-1, strings.Join(also, ", "))
|
||||
break
|
||||
}
|
||||
sort.Strings(also)
|
||||
fmt.Printf("\nthis push left %s behind — a provision granted from there, or a "+
|
||||
"declaration since changed; sending it too\n", strings.Join(also, ", "))
|
||||
// Tolerantly, exactly as the named send above: a machine that cannot be composed is
|
||||
// collected as a refusal and reported at the end, and the others are still sent
|
||||
// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
|
||||
// all-or-nothing — so one swept machine's compose error failed the operator's named
|
||||
// push and skipped its --wait, the very intolerance the main path exists to avoid.
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, also,
|
||||
func(held context.Context, node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
if err == nil {
|
||||
reportLeftOut(node, declared)
|
||||
}
|
||||
return declared, err
|
||||
},
|
||||
bus, holder)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Every candidate this round is marked handled — the sent ones so they are not
|
||||
// re-listed, and the refused ones so a machine that cannot be composed does not make
|
||||
// the loop spin on it for ever. Its refusal is already in the report.
|
||||
for _, name := range also {
|
||||
flushed[name] = true
|
||||
}
|
||||
handled := map[string]bool{args[0]: true}
|
||||
for n := range saidHeld {
|
||||
handled[n] = true
|
||||
}
|
||||
refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open, gens), bus, holder, os.Stdout)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -762,7 +777,7 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body)
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -1050,6 +1065,20 @@ func digestOf(body []byte) string {
|
||||
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
|
||||
func wouldSend(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node) (map[string]string, error) {
|
||||
return wouldSendFrom(ctx, open, nodes, nil)
|
||||
}
|
||||
|
||||
// planned is one machine's plan as planFor answered it, for a caller that already asked.
|
||||
type planned struct {
|
||||
plan catalogue.Resolution
|
||||
settings catalogue.SettingsBy
|
||||
}
|
||||
|
||||
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
|
||||
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
|
||||
// to-be 45 Phase 0). A machine absent from plans is worked out here.
|
||||
func wouldSendFrom(ctx context.Context, open *stores,
|
||||
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
|
||||
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
@@ -1057,9 +1086,12 @@ func wouldSend(ctx context.Context, open *stores,
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
known, have := plans[n.Name]
|
||||
plan, settings := known.plan, known.settings
|
||||
if !have {
|
||||
if plan, settings, err = planFor(ctx, open, n.Name); err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
@@ -1147,6 +1179,11 @@ func raiseTheBus(ctx context.Context, inv *inventory.Inventory, address string)
|
||||
if err := broker.RaiseCancelledSets(js, inventory.MeshSeats()); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the controller's own buckets (novox/hq to-be 45 §1): the calls it serves and the acts done
|
||||
// by hand, kept where a restart of this process does not take them.
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Every module's state (novox/hq ADR 0201), from the catalogue: a bucket exists from
|
||||
// registration, so a module reading one may watch it before its owner runs anywhere. One that
|
||||
// nothing declares any more is said and kept — what it holds is data.
|
||||
@@ -1244,7 +1281,11 @@ func allot(ctx context.Context, inv *inventory.Inventory, node string) (int64, e
|
||||
// never wrote it down: status read "applied, current" over a machine that had just been sent
|
||||
// something else. What was sent was sent; the record of it must not depend on the sender living
|
||||
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte) (string, error) {
|
||||
//
|
||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||
builds map[string]string) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
@@ -1252,7 +1293,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body
|
||||
return "", err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := inv.RecordSent(kept, record.ID, digest); err != nil {
|
||||
if err := inv.RecordSent(kept, record.ID, digest, builds); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return digest, nil
|
||||
@@ -1278,3 +1319,11 @@ func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[stri
|
||||
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
|
||||
}
|
||||
}
|
||||
|
||||
// controllerProcess names this serving process among controllers: the machine, the process and when
|
||||
// it started — what a call kept on the bus carries, so the next controller can tell a call this one
|
||||
// left running from one it is running itself (novox/hq to-be 45 §6).
|
||||
func controllerProcess() string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("controller@%s pid %d since %s", host, os.Getpid(), time.Now().UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
@@ -78,6 +78,19 @@ type meshStatus struct {
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
// HandActsThisWeek is how many acts were done by hand in the last seven days (novox/hq to-be 45
|
||||
// §7): every one is a repair a healer could have made. Absent where the log is not on hand;
|
||||
// HandActsUnread says why when it could not be read, rather than reading as none.
|
||||
HandActsThisWeek *int `json:"handActsThisWeek,omitempty"`
|
||||
HandActsUnread string `json:"handActsUnread,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing, with the class of error, since
|
||||
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
|
||||
// document without this called the mesh well while the identity provider refused every consumer
|
||||
// for a day (04-ISSUES/179).
|
||||
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
|
||||
// Overflowing is every module whose identity overflows the bound of a provision it requires, and
|
||||
// so is left out of its provider's grants (novox/hq ADR 0225). Absent when every identity fits.
|
||||
Overflowing []catalogue.Overflow `json:"overflowing,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
@@ -210,6 +223,9 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.HandActsThisWeek, out.HandActsUnread = asked.handActs, asked.handActsUnread
|
||||
out.Failing = asked.failing
|
||||
out.Overflowing = asked.overflowing
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
|
||||
@@ -986,10 +986,18 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules")
|
||||
paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root")
|
||||
modules := set.String("modules", "", "or the modules it would change, comma-separated")
|
||||
// Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7).
|
||||
why := addHandActFlags(set)
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") {
|
||||
// Refused before anything is opened: a repair by hand says why.
|
||||
if err := why.require("plans " + positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1061,8 +1069,9 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
if !p.Open() {
|
||||
return fmt.Errorf("%s is already %s", p.ID, p.State)
|
||||
}
|
||||
why.record(ctx, "plans "+positionals[0], positionals[1:])
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier)
|
||||
p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + strings.TrimSpace(*why.why)
|
||||
sayUnsent(&p, func(m string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, m)
|
||||
return err == nil && u.RollOut
|
||||
|
||||
@@ -189,7 +189,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
|
||||
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||
// would bury the ones that matter under a list nobody can act on.
|
||||
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// rotateCommand replaces a credential and moves both ends together.
|
||||
@@ -33,12 +35,16 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||
// One consuming module rather than every module on the machine. A machine runs many consumers
|
||||
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
||||
// issue 268) is no reason to restart every other one on the machine.
|
||||
module := set.String("module", "", "only this consuming module's credential")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
||||
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
|
||||
}
|
||||
provision := positionals[0]
|
||||
|
||||
@@ -53,6 +59,12 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
holders = ofModule(holders, *module)
|
||||
if len(holders) == 0 && *module != "" {
|
||||
return fmt.Errorf(
|
||||
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
|
||||
"says what a machine holds", *module, onMachine(*only), provision)
|
||||
}
|
||||
if len(holders) == 0 {
|
||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||
// and a rotation somebody believes happened is worse than one they know did not.
|
||||
@@ -116,6 +128,27 @@ func rotateCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
||||
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
||||
if module == "" {
|
||||
return holders
|
||||
}
|
||||
var out []inventory.Holder
|
||||
for _, h := range holders {
|
||||
if h.ConsumerModule == module {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func onMachine(machine string) string {
|
||||
if machine == "" {
|
||||
return ""
|
||||
}
|
||||
return " on " + machine
|
||||
}
|
||||
|
||||
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||
func asLocal(local string) string {
|
||||
if local == "" {
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// One consuming module's credential, and not its neighbours' on the same machine (novox/hq issue
|
||||
// 268): a module that leaked its database password is no reason to restart every other consumer.
|
||||
func TestARotationNarrowedToAModuleTouchesOnlyThatModulesCredential(t *testing.T) {
|
||||
holders := []inventory.Holder{
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Provider: "ace"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"},
|
||||
{Provision: "postgres-database", Consumer: "ace", ConsumerModule: "letta", Local: "reader", Provider: "ace"},
|
||||
}
|
||||
got := ofModule(holders, "letta")
|
||||
if len(got) != 2 || got[0].ConsumerModule != "letta" || got[1].Local != "reader" {
|
||||
t.Fatalf("narrowed to letta: %+v", got)
|
||||
}
|
||||
if len(ofModule(holders, "")) != 3 {
|
||||
t.Fatal("no module named narrowed anyway")
|
||||
}
|
||||
if len(ofModule(holders, "absent")) != 0 {
|
||||
t.Fatal("a module holding nothing matched")
|
||||
}
|
||||
}
|
||||
@@ -29,11 +29,13 @@ type seatHolder struct {
|
||||
|
||||
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
||||
type seatRow struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
@@ -47,7 +49,7 @@ func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []cata
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Holders: []seatHolder{}}
|
||||
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
@@ -104,9 +106,13 @@ func seatCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
if len(args) == 3 && args[1] == "--to" {
|
||||
return handOver(ctx, args[0], args[2])
|
||||
return handOver(ctx, args[0], args[2], false)
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module>")
|
||||
if len(args) == 3 && args[1] == "--add" {
|
||||
return handOver(ctx, args[0], args[2], true)
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
|
||||
"seat <name> --add <node>/<module>")
|
||||
}
|
||||
|
||||
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
||||
@@ -119,7 +125,12 @@ func seatCommand(ctx context.Context, args []string) error {
|
||||
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
||||
// and refusing to record a handover to a module the node has not started would make the handover
|
||||
// impossible to do before the switch instead of as the switch.
|
||||
func handOver(ctx context.Context, seatName, to string) error {
|
||||
//
|
||||
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
|
||||
// records the named assignment as a further holder and leaves every holder on record as it is. A
|
||||
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
|
||||
// the one named, as it always did.
|
||||
func handOver(ctx context.Context, seatName, to string, adding bool) error {
|
||||
nodeName, module, ok := strings.Cut(to, "/")
|
||||
if !ok || nodeName == "" || module == "" {
|
||||
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||
@@ -135,6 +146,10 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
if !known {
|
||||
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
||||
}
|
||||
if adding && !seat.Replicated {
|
||||
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
|
||||
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -157,6 +172,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
||||
}
|
||||
var was string
|
||||
var held []string
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -164,6 +180,7 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
for _, h := range holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
||||
was = h.Node
|
||||
held = append(held, h.Node)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,6 +204,15 @@ func handOver(ctx context.Context, seatName, to string) error {
|
||||
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
||||
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
||||
}
|
||||
if adding {
|
||||
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
|
||||
strings.Join(held, ", "))
|
||||
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
|
||||
return nil
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -9,9 +9,11 @@ import (
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -36,19 +38,194 @@ type verbAnswer struct {
|
||||
|
||||
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
|
||||
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
|
||||
//
|
||||
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
|
||||
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
|
||||
// the verb declares but did not use for the command line it composed — given beside another that
|
||||
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
|
||||
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
|
||||
// not take that" would have stopped it before anything was sent.
|
||||
func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
str := func(key string) string {
|
||||
v, _ := args[key].(string)
|
||||
return strings.TrimSpace(v)
|
||||
a, err := readArguments(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
need := func(keys ...string) error {
|
||||
for _, k := range keys {
|
||||
if str(k) == "" {
|
||||
return fmt.Errorf("%s needs %q", verb, k)
|
||||
argv, err := a.commandLine()
|
||||
if len(a.misread) > 0 {
|
||||
// The table and the command line disagree: the verb reads an argument no caller can see
|
||||
// in its schema, so no caller could ever pass it.
|
||||
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
|
||||
"table and its command lines disagree", verb, quoteAll(a.misread))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unused := a.unused(); len(unused) > 0 {
|
||||
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
|
||||
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
|
||||
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
|
||||
// command line was composed from.
|
||||
type verbArguments struct {
|
||||
verb string
|
||||
given map[string]string
|
||||
used map[string]bool
|
||||
declared map[string]bool
|
||||
misread []string // arguments the command line read that the schema does not declare: a bug here
|
||||
}
|
||||
|
||||
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
||||
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
||||
// wrote.
|
||||
func controllerVerb(name string) (catalogue.Verb, bool) {
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == name {
|
||||
return v, true
|
||||
}
|
||||
}
|
||||
return catalogue.Verb{}, false
|
||||
}
|
||||
|
||||
// declaredArguments are a schema's properties, and which of them are switches.
|
||||
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
|
||||
switches = map[string]bool{}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
for name, p := range props {
|
||||
names = append(names, name)
|
||||
desc, _ := p.(map[string]any)
|
||||
switch enum := desc["enum"].(type) {
|
||||
case []string:
|
||||
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||
case []any:
|
||||
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, switches
|
||||
}
|
||||
|
||||
// readArguments refuses what the verb does not take, before anything is composed.
|
||||
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
|
||||
v, known := controllerVerb(verb)
|
||||
if !known {
|
||||
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
declared := map[string]bool{}
|
||||
for _, n := range names {
|
||||
declared[n] = true
|
||||
}
|
||||
takes := "none"
|
||||
if len(names) > 0 {
|
||||
takes = quoteAll(names)
|
||||
}
|
||||
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if !declared[k] {
|
||||
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
|
||||
}
|
||||
var value string
|
||||
switch x := args[k].(type) {
|
||||
case nil:
|
||||
continue
|
||||
case string:
|
||||
value = strings.TrimSpace(x)
|
||||
case bool:
|
||||
if !switches[k] {
|
||||
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
|
||||
}
|
||||
value = fmt.Sprint(x)
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
|
||||
}
|
||||
if switches[k] {
|
||||
switch value {
|
||||
case "true":
|
||||
case "false", "":
|
||||
continue // said and off: the same as not given, and nothing passed over
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
if value != "" {
|
||||
a.given[k] = value
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// str is one argument's value, marked as used.
|
||||
func (a *verbArguments) str(key string) string {
|
||||
if !a.declared[key] {
|
||||
a.misread = append(a.misread, key)
|
||||
}
|
||||
a.used[key] = true
|
||||
return a.given[key]
|
||||
}
|
||||
|
||||
// on is a switch, marked as used.
|
||||
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
|
||||
|
||||
// need refuses a call missing a required argument, in the verb's own words.
|
||||
func (a *verbArguments) need(keys ...string) error {
|
||||
for _, k := range keys {
|
||||
if a.str(k) == "" {
|
||||
return fmt.Errorf("%s needs %q", a.verb, k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unused are the arguments given that the command line was not composed from.
|
||||
func (a *verbArguments) unused() []string {
|
||||
var out []string
|
||||
for k := range a.given {
|
||||
if !a.used[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *verbArguments) usedGiven() []string {
|
||||
var out []string
|
||||
for k := range a.given {
|
||||
if a.used[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
return []string{"nothing"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func quoteAll(xs []string) string {
|
||||
q := make([]string, len(xs))
|
||||
for i, x := range xs {
|
||||
if x == "nothing" {
|
||||
q[i] = x
|
||||
continue
|
||||
}
|
||||
q[i] = fmt.Sprintf("%q", x)
|
||||
}
|
||||
return strings.Join(q, ", ")
|
||||
}
|
||||
|
||||
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
|
||||
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
|
||||
func (a *verbArguments) commandLine() ([]string, error) {
|
||||
verb, str, on, need := a.verb, a.str, a.on, a.need
|
||||
switch verb {
|
||||
case "command":
|
||||
// The generic verb: the command line as given, split as a shell would split it, with
|
||||
@@ -64,7 +241,15 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("command names no command")
|
||||
}
|
||||
// The generic verb is no way round the hand-act log (novox/hq to-be 45 §7): a repair through
|
||||
// it says why, as it would through its own verb.
|
||||
if repair := repairingCommand(argv); repair != "" && !slices.ContainsFunc(argv, isWhyFlag) {
|
||||
return nil, fmt.Errorf("%s is a repair done by hand, and says why: add --why <text> to the command "+
|
||||
"line (recorded in the hand-act log). Nothing was done", repair)
|
||||
}
|
||||
return argv, nil
|
||||
case "tools":
|
||||
return nil, errors.New("tools is answered from the records, not by a command")
|
||||
case "status":
|
||||
return []string{"status", "--json"}, nil
|
||||
case "nodes":
|
||||
@@ -82,10 +267,14 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if id := str("log"); id != "" {
|
||||
return []string{"builds", "--log", id}, nil
|
||||
}
|
||||
if m := str("module"); m != "" {
|
||||
return []string{"builds", m}, nil
|
||||
argv := []string{"builds"}
|
||||
if n := str("limit"); n != "" {
|
||||
argv = append(argv, "-n", n)
|
||||
}
|
||||
return []string{"builds"}, nil
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, m)
|
||||
}
|
||||
return argv, nil
|
||||
case "plans":
|
||||
if r := str("repository"); r != "" {
|
||||
argv := []string{"plans", "--what-if", r}
|
||||
@@ -97,19 +286,30 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if id := str("stop"); id != "" {
|
||||
return []string{"plans", "stop", id}, nil
|
||||
}
|
||||
if id := str("close"); id != "" {
|
||||
return []string{"plans", "close", id}, nil
|
||||
}
|
||||
if id := str("retry"); id != "" {
|
||||
return []string{"plans", "retry", id}, nil
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if id := str(act); id != "" {
|
||||
argv := []string{"plans", act, id}
|
||||
if act == "retry" {
|
||||
return argv, nil
|
||||
}
|
||||
// Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without.
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
}
|
||||
if c := str("cause"); c != "" {
|
||||
argv = append(argv, "--cause", c)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"plans", id}, nil
|
||||
}
|
||||
return []string{"plans"}, nil
|
||||
argv := []string{"plans"}
|
||||
if n := str("limit"); n != "" {
|
||||
argv = append(argv, "-n", n)
|
||||
}
|
||||
return argv, nil
|
||||
// The build queue (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return []string{"queue"}, nil
|
||||
@@ -119,7 +319,7 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
}
|
||||
return []string{verb, str("id")}, nil
|
||||
case "clear":
|
||||
if str("dead") == "true" {
|
||||
if on("dead") {
|
||||
return []string{"clear", "--dead"}, nil
|
||||
}
|
||||
return []string{"clear"}, nil
|
||||
@@ -133,10 +333,10 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"replay", str("id")}
|
||||
if str("register") == "true" {
|
||||
if on("register") {
|
||||
argv = append(argv, "--register")
|
||||
}
|
||||
if str("older") == "true" {
|
||||
if on("older") {
|
||||
argv = append(argv, "--older")
|
||||
}
|
||||
return argv, nil
|
||||
@@ -149,6 +349,9 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if on("files") {
|
||||
return []string{"plan", str("node"), "--files"}, nil
|
||||
}
|
||||
return []string{"plan", str("node"), "--json"}, nil
|
||||
case "assign", "unassign":
|
||||
if err := need("node", "module"); err != nil {
|
||||
@@ -171,23 +374,72 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||
if n := str("node"); n != "" {
|
||||
return []string{"push", n, "--wait", "0"}, nil
|
||||
// A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7).
|
||||
if err := need("why"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err)
|
||||
}
|
||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
||||
why := []string{"--why", str("why")}
|
||||
if c := str("cause"); c != "" {
|
||||
why = append(why, "--cause", c)
|
||||
}
|
||||
if n := str("node"); n != "" {
|
||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||
// a machine and asking for every machine behind are two requests, and guessing one
|
||||
// would push a machine nobody named, or not push one somebody did.
|
||||
return append([]string{"push", n, "--wait", "0"}, why...), nil
|
||||
}
|
||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||
// first, so a caller who meant one machine reads that it was not one.
|
||||
on("behind")
|
||||
return append([]string{"push", "--behind", "--wait", "0"}, why...), nil
|
||||
case "hand-act":
|
||||
if err := need("what", "why", "cause"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"hand-act", "record", str("what"), "--why", str("why"), "--cause", str("cause")}
|
||||
if c := str("condition"); c != "" {
|
||||
argv = append(argv, "--condition", c)
|
||||
}
|
||||
return argv, nil
|
||||
case "hand-acts":
|
||||
argv := []string{"hand-acts", "--json"}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "durations":
|
||||
argv := []string{"durations", "--json"}
|
||||
if k := str("kind"); k != "" {
|
||||
argv = append(argv, "--kind", k)
|
||||
}
|
||||
if d := str("days"); d != "" {
|
||||
argv = append(argv, "--days", d)
|
||||
}
|
||||
return argv, nil
|
||||
case "rotate":
|
||||
if p := str("provision"); p != "" {
|
||||
argv := []string{"rotate", p}
|
||||
if c := str("consumer"); c != "" {
|
||||
argv = append(argv, "--consumer", c)
|
||||
}
|
||||
// With a provision, module narrows to one consuming module (novox/hq issue 268); node
|
||||
// and secret stay the other shape's, and are refused as passed over.
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, "--module", m)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
if str("node") != "" && str("module") != "" && str("secret") != "" {
|
||||
_, node := a.given["node"]
|
||||
_, module := a.given["module"]
|
||||
_, secret := a.given["secret"]
|
||||
if node || module || secret {
|
||||
if err := need("node", "module", "secret"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
|
||||
}
|
||||
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
||||
}
|
||||
// Half of either shape: the command says its usage, which names both shapes, and that is
|
||||
// the answer the caller needs.
|
||||
// Neither shape: the command says its usage, which names both, and that is the answer the
|
||||
// caller needs.
|
||||
return []string{"rotate"}, nil
|
||||
case "settings":
|
||||
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||
@@ -195,15 +447,16 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
if err := need("module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"settings", "set", str("module")}
|
||||
switch {
|
||||
case str("clear") == "true":
|
||||
var argv []string
|
||||
if on("clear") {
|
||||
argv = []string{"settings", "clear", str("module")}
|
||||
case str("values") != "":
|
||||
argv = append(argv, str("values"))
|
||||
} else {
|
||||
argv = []string{"settings", "set", str("module")}
|
||||
// Neither values nor clear: the command says its usage, which names both.
|
||||
if v := str("values"); v != "" {
|
||||
argv = append(argv, v)
|
||||
}
|
||||
}
|
||||
// Neither values nor clear: the command says its usage, which names both, and that is the
|
||||
// answer the caller needs — the same as `rotate` given half of either shape.
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
@@ -235,11 +488,33 @@ func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
||||
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
|
||||
verb, catalogue.ControllerSeatName)
|
||||
}
|
||||
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true,
|
||||
"hand-acts": true, "durations": true}
|
||||
|
||||
// repairingCommand names a command line that repairs by hand, and so says why: a push, a plan stopped
|
||||
// or closed, a consumer re-made (novox/hq to-be 45 §7). Empty for any other.
|
||||
func repairingCommand(argv []string) string {
|
||||
switch {
|
||||
case argv[0] == "push":
|
||||
return "push"
|
||||
case argv[0] == "plans" && len(argv) > 1 && (argv[1] == "stop" || argv[1] == "close"):
|
||||
return "plans " + argv[1]
|
||||
case argv[0] == "broker" && len(argv) > 1 && argv[1] == "consumer-reset":
|
||||
return "broker consumer-reset"
|
||||
case argv[0] == "hand-act":
|
||||
return "hand-act record"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isWhyFlag(word string) bool {
|
||||
return word == "--why" || word == "-why" || strings.HasPrefix(word, "--why=") || strings.HasPrefix(word, "-why=")
|
||||
}
|
||||
|
||||
// runVerb runs this binary with the given command line and gathers what it said.
|
||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
@@ -251,6 +526,12 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||
// from a shell in this container would have, because it is that.
|
||||
cmd.Env = os.Environ()
|
||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||
caller := link.CallerIn(ctx)
|
||||
if caller == "" {
|
||||
caller = "a seat call whose caller the bus did not name"
|
||||
}
|
||||
cmd.Env = append(cmd.Env, link.CallerVar+"="+caller+", through the "+catalogue.ControllerSeatName+" seat")
|
||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||
@@ -287,8 +568,22 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
handlers := map[string]link.ToolHandler{}
|
||||
for _, v := range seat.Serves {
|
||||
verb := v.Name
|
||||
if verb == "tools" {
|
||||
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||
if inProcess[verb] {
|
||||
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||
args := map[string]any{}
|
||||
if len(bytes.TrimSpace(raw)) > 0 {
|
||||
if err := json.Unmarshal(raw, &args); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
||||
}
|
||||
}
|
||||
// Refused like any verb's: what a verb does not take is not ignored.
|
||||
a, err := readArguments(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verb == "calls" {
|
||||
return callsAnswer(link.Calls, a.given["call"])
|
||||
}
|
||||
return seatTools(), nil
|
||||
}
|
||||
continue
|
||||
@@ -327,12 +622,85 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verb == "status" && statusFrom != nil {
|
||||
// At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0).
|
||||
return statusFrom.answer(ctx)
|
||||
}
|
||||
if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) {
|
||||
// Whatever it did, `status` is composed again once it has.
|
||||
defer statusFrom.nudge()
|
||||
}
|
||||
if answersFirst(argv) {
|
||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
link.Acknowledge(ctx)
|
||||
}
|
||||
return runVerb(ctx, argv)
|
||||
}
|
||||
}
|
||||
return handlers, behind, nil
|
||||
}
|
||||
|
||||
// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them.
|
||||
func actsOnAPlan(args map[string]any) bool {
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if v, _ := args[act].(string); strings.TrimSpace(v) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||
// the records, `calls` from what this process served.
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true}
|
||||
|
||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
|
||||
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
|
||||
func answersFirst(argv []string) bool {
|
||||
return len(argv) > 0 && argv[0] == "push"
|
||||
}
|
||||
|
||||
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||
// one call whole. Kept on the bus, so a call a controller before this one served is answered too
|
||||
// (novox/hq to-be 45 §6); where the bus cannot be read, what this process served is answered and
|
||||
// the reason said beside it.
|
||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||
if id != "" {
|
||||
c, ok, err := log.Get(id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("call %s is not in this controller's memory, and the calls kept on the "+
|
||||
"bus could not be read: %w", id, err)
|
||||
}
|
||||
if !ok {
|
||||
if log.IsDurable() {
|
||||
return nil, fmt.Errorf("no call %s is kept: the bus keeps the last %d calls, or %s, and this "+
|
||||
"is not among them — `calls` lists them", id, broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
}
|
||||
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
recent, err := log.Recent()
|
||||
for i := range recent {
|
||||
recent[i].Answer = nil
|
||||
}
|
||||
answer := map[string]any{"calls": recent, "note": "newest first; `calls` with a call's id gives its whole answer"}
|
||||
if log.IsDurable() {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls, or %s, on the bus — across a restart of the controller",
|
||||
broker.KeptCallsDurably, broker.CallsKeptFor)
|
||||
} else {
|
||||
answer["kept"] = fmt.Sprintf("the last %d calls this controller served, in its memory only", link.KeptCalls)
|
||||
}
|
||||
if err != nil {
|
||||
answer["unread"] = err.Error()
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
|
||||
func seatTools() map[string]any {
|
||||
@@ -353,9 +721,10 @@ func seatTools() map[string]any {
|
||||
}
|
||||
|
||||
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
|
||||
// verb runnable rather than that it happens to want the arguments the check guessed.
|
||||
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
|
||||
// more, since an argument a verb does not declare is refused.
|
||||
func sampleArguments(v catalogue.Verb) map[string]any {
|
||||
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
|
||||
sample := map[string]any{}
|
||||
switch required := v.Input["required"].(type) {
|
||||
case []string:
|
||||
for _, k := range required {
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The schemas the controller serves, verb by verb, as the console receives them: from the
|
||||
// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
|
||||
func servedSchemas(t *testing.T) map[string]catalogue.Verb {
|
||||
t.Helper()
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(behind) != 0 {
|
||||
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||
}
|
||||
served := map[string]catalogue.Verb{}
|
||||
for _, e := range seatAnnouncement(handlers).Endpoints {
|
||||
var input map[string]any
|
||||
if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
|
||||
t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
|
||||
}
|
||||
served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
|
||||
}
|
||||
if len(served) != len(catalogue.ControllerVerbs) {
|
||||
t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// subsetsOf is every subset of the names, the empty one included.
|
||||
func subsetsOf(names []string) [][]string {
|
||||
var out [][]string
|
||||
for mask := 0; mask < 1<<len(names); mask++ {
|
||||
var s []string
|
||||
for i, n := range names {
|
||||
if mask&(1<<i) != 0 {
|
||||
s = append(s, n)
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func argumentsFor(subset []string, switches map[string]bool) map[string]any {
|
||||
args := map[string]any{}
|
||||
for _, n := range subset {
|
||||
if switches[n] {
|
||||
args[n] = "true"
|
||||
} else {
|
||||
args[n] = "x-" + n
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// saidOutright are the switches that say the default aloud, so the line is the same without them —
|
||||
// on purpose, and only these.
|
||||
var saidOutright = map[string]string{
|
||||
"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
|
||||
}
|
||||
|
||||
// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
|
||||
// every combination of the arguments its schema declares, the verb either refuses the call, or
|
||||
// composes a command line that each given argument changed: taking any one away changes the line
|
||||
// or makes it refused. An argument the line is the same without is one the verb ignored — the
|
||||
// shape of the push that named a machine and pushed every machine behind.
|
||||
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
if inProcess[name] {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
for _, subset := range subsetsOf(names) {
|
||||
args := argumentsFor(subset, switches)
|
||||
argv, err := argvFor(name, args)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "does not declare") {
|
||||
t.Errorf("%s %v: %v", name, args, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, dropped := range subset {
|
||||
fewer := map[string]any{}
|
||||
for k, val := range args {
|
||||
if k != dropped {
|
||||
fewer[k] = val
|
||||
}
|
||||
}
|
||||
without, err := argvFor(name, fewer)
|
||||
if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
|
||||
t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
|
||||
// what it requires and one argument more; and `node` in particular, for every verb whose schema
|
||||
// does not take a machine.
|
||||
func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
names, _ := declaredArguments(v)
|
||||
strangers := []string{"no-such-argument"}
|
||||
if !contains(names, "node") {
|
||||
strangers = append(strangers, "node")
|
||||
}
|
||||
for _, stranger := range strangers {
|
||||
args := sampleArguments(v)
|
||||
args[stranger] = "x"
|
||||
_, err := argvFor(name, args)
|
||||
if inProcess[name] {
|
||||
_, err = readArguments(name, args)
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
|
||||
t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
|
||||
t.Error("a switch took a word that is neither true nor false, and would have read it as false")
|
||||
}
|
||||
if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
|
||||
t.Error("a number was taken as a machine's name, or as no machine")
|
||||
}
|
||||
if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
|
||||
t.Errorf("a switch given as JSON true: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||
// naming one beside behind is refused rather than one of the two guessed.
|
||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||
argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"})
|
||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0 --why w" {
|
||||
t.Fatalf("a named push: %v %v", argv, err)
|
||||
}
|
||||
for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} {
|
||||
argv, err := argvFor("push", args)
|
||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" {
|
||||
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil ||
|
||||
!strings.Contains(err.Error(), `"behind"`) {
|
||||
t.Fatalf("a named push with behind was taken: %v", err)
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
|
||||
t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// commandFlags is every flag set this package's commands parse, by the name the set is made with,
|
||||
// and the flags defined on it — read from the source, so a flag added to a command is seen here
|
||||
// without anyone remembering to.
|
||||
func commandFlags(t *testing.T) map[string][]string {
|
||||
t.Helper()
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fset := token.NewFileSet()
|
||||
out := map[string][]string{}
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
file, err := parser.ParseFile(fset, f, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, decl := range file.Decls {
|
||||
fn, ok := decl.(*ast.FuncDecl)
|
||||
if !ok || fn.Body == nil {
|
||||
continue
|
||||
}
|
||||
sets := map[string]string{} // variable → the set's name
|
||||
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
||||
if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
|
||||
if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
|
||||
if id, ok := assign.Lhs[0].(*ast.Ident); ok {
|
||||
if name, ok := stringLit(call.Args[0]); ok {
|
||||
sets[id.Name] = name
|
||||
out[name] = append(out[name], []string{}...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
recv, ok := sel.X.(*ast.Ident)
|
||||
if !ok || sets[recv.Name] == "" {
|
||||
return true
|
||||
}
|
||||
arg := 0
|
||||
switch sel.Sel.Name {
|
||||
case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
|
||||
case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
|
||||
arg = 1
|
||||
default:
|
||||
return true
|
||||
}
|
||||
if arg < len(call.Args) {
|
||||
if flagName, ok := stringLit(call.Args[arg]); ok {
|
||||
out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isSelector(e ast.Expr, pkg, name string) bool {
|
||||
sel, ok := e.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
id, ok := sel.X.(*ast.Ident)
|
||||
return ok && id.Name == pkg && sel.Sel.Name == name
|
||||
}
|
||||
|
||||
func stringLit(e ast.Expr) (string, bool) {
|
||||
lit, ok := e.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
return "", false
|
||||
}
|
||||
s, err := strconv.Unquote(lit.Value)
|
||||
return s, err == nil
|
||||
}
|
||||
|
||||
// accountedFlags are the flags of a verb's command that are not an argument of the same name:
|
||||
// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
|
||||
// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
|
||||
var accountedFlags = map[string]map[string]string{
|
||||
"status": {"json": "set by the verb: the answer is data"},
|
||||
"seats": {"json": "set by the verb: the answer is data"},
|
||||
"queue": {"json": "not set: the verb answers the table a person reads"},
|
||||
"plan": {"json": "set by the verb unless files is asked"},
|
||||
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
|
||||
"build": {
|
||||
"wait": "set by the verb to 0: the id follows the build (issue 176)",
|
||||
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
|
||||
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
|
||||
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
},
|
||||
"builds": {"n": "=limit"},
|
||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||
"durations": {
|
||||
"json": "set by the verb: the answer is data",
|
||||
"all": "withheld: every measurement of a fortnight is more than a call should carry; `command` reaches it",
|
||||
},
|
||||
"hand-acts": {"json": "set by the verb: the answer is data"},
|
||||
}
|
||||
|
||||
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||
// from the source, so a flag added to a command — or a verb added whose command takes flags —
|
||||
// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
|
||||
// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
|
||||
// reads.
|
||||
func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
|
||||
flags := commandFlags(t)
|
||||
if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
|
||||
t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
|
||||
}
|
||||
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
|
||||
served := servedSchemas(t)
|
||||
for name, v := range served {
|
||||
if inProcess[name] || name == "command" {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
for _, subset := range subsetsOf(names) {
|
||||
argv, err := argvFor(name, argumentsFor(subset, switches))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// The flag set is named by the longest run of leading words that names one.
|
||||
var words []string
|
||||
for _, w := range argv {
|
||||
if strings.HasPrefix(w, "-") {
|
||||
break
|
||||
}
|
||||
words = append(words, w)
|
||||
}
|
||||
for n := len(words); n > 0; n-- {
|
||||
if _, has := flags[strings.Join(words[:n], " ")]; has {
|
||||
if reached[name] == nil {
|
||||
reached[name] = map[string]bool{}
|
||||
}
|
||||
reached[name][strings.Join(words[:n], " ")] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
used := map[string]map[string]bool{}
|
||||
verbs := make([]string, 0, len(reached))
|
||||
for verb := range reached {
|
||||
verbs = append(verbs, verb)
|
||||
}
|
||||
sort.Strings(verbs)
|
||||
for _, verb := range verbs {
|
||||
declared, _ := declaredArguments(served[verb])
|
||||
for set := range reached[verb] {
|
||||
if used[set] == nil {
|
||||
used[set] = map[string]bool{}
|
||||
}
|
||||
for _, flagName := range flags[set] {
|
||||
why, accounted := accountedFlags[set][flagName]
|
||||
switch {
|
||||
case accounted && strings.HasPrefix(why, "="):
|
||||
used[set][flagName] = true
|
||||
if !contains(declared, strings.TrimPrefix(why, "=")) {
|
||||
t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
|
||||
verb, flagName, set, strings.TrimPrefix(why, "="))
|
||||
}
|
||||
case accounted:
|
||||
used[set][flagName] = true
|
||||
case contains(declared, flagName):
|
||||
default:
|
||||
t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
|
||||
"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for set, fs := range accountedFlags {
|
||||
for flagName := range fs {
|
||||
if !used[set][flagName] {
|
||||
t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every verb's required arguments are properties of its schema: a schema that requires what it
|
||||
// does not describe is the uncallable verb of issue 244 from the other side.
|
||||
func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
names, _ := declaredArguments(v)
|
||||
for k := range sampleArguments(v) {
|
||||
if !contains(names, k) {
|
||||
t.Errorf("%s requires %q and does not describe it", name, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,29 +10,6 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
|
||||
// schema names and no other (novox/hq ADR 0154, ADR 0035).
|
||||
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == "tools" {
|
||||
continue
|
||||
}
|
||||
args := map[string]any{}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
for name := range props {
|
||||
args[name] = "x"
|
||||
}
|
||||
argv, err := argvFor(v.Name, args)
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", v.Name, err)
|
||||
continue
|
||||
}
|
||||
if argv[0] == "" {
|
||||
t.Errorf("%s: empty command", v.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `builds` given a build's id reads that build's log from the bus rather than listing builds
|
||||
// (novox/hq ADR 0157).
|
||||
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
|
||||
@@ -66,13 +43,22 @@ func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||
t.Fatalf("a pair credential: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "module": "letta"})
|
||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace --module letta" {
|
||||
t.Fatalf("one consuming module's pair credential: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||
t.Fatalf("an own secret: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
|
||||
if strings.Join(argv, " ") != "rotate" {
|
||||
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
|
||||
if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
|
||||
t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
|
||||
}
|
||||
if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
|
||||
t.Fatalf("neither shape falls to the command's usage: %v", argv)
|
||||
}
|
||||
if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
|
||||
t.Fatal("both shapes at once were taken, and one of them passed over")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,8 +107,8 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0 --why w" {
|
||||
t.Fatalf("push waits: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||
|
||||
@@ -43,6 +43,13 @@ type sendable struct {
|
||||
LeftOut []string
|
||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||
leftOutWhy map[string]string
|
||||
// withheld is every consumer this machine's grants leave out, because its identity overflows the
|
||||
// provision's bound (novox/hq ADR 0225); for push and plan to say, never on the wire.
|
||||
withheld []catalogue.Overflow
|
||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||
// Composed only on the send path; nil records that it is not known.
|
||||
Builds map[string]string
|
||||
}
|
||||
|
||||
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
|
||||
//
|
||||
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
|
||||
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
|
||||
|
||||
// standings keeps what providers say, in the inventory.
|
||||
type standings struct{ inv *inventory.Inventory }
|
||||
|
||||
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
|
||||
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
|
||||
Consumer: st.Consumer, ConsumerNode: st.Node,
|
||||
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
|
||||
})
|
||||
}
|
||||
|
||||
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
|
||||
//
|
||||
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
|
||||
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
|
||||
// consumer clears it.
|
||||
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
|
||||
all, err := inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
|
||||
}
|
||||
assigned := map[string]map[string]bool{}
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range all {
|
||||
on, asked := assigned[s.ProviderNode]
|
||||
if !asked {
|
||||
modules, err := inv.Assigned(ctx, s.ProviderNode)
|
||||
if err != nil {
|
||||
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
|
||||
modules = nil
|
||||
}
|
||||
on = map[string]bool{}
|
||||
for _, m := range modules {
|
||||
on[m] = true
|
||||
}
|
||||
assigned[s.ProviderNode] = on
|
||||
}
|
||||
if on[s.Module] {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
|
||||
// that stopped repeating itself said so.
|
||||
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
|
||||
var out []string
|
||||
for _, s := range list {
|
||||
where := s.Module
|
||||
if s.ProviderNode != "" {
|
||||
where += " on " + s.ProviderNode
|
||||
}
|
||||
whom := s.Consumer
|
||||
if s.ConsumerNode != "" {
|
||||
whom += " (" + s.ConsumerNode + ")"
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
|
||||
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
|
||||
s.Since.Local().Format("2006-01-02 15:04")))
|
||||
if e := strings.TrimSpace(s.Error); e != "" {
|
||||
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
|
||||
}
|
||||
if s.Quiet(now) {
|
||||
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
|
||||
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orUnclassed(class string) string {
|
||||
if class == "" {
|
||||
return "failing"
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// printFailing is the status section, said when there is anything to say.
|
||||
func printFailing(list []inventory.ProviderStanding, now time.Time) {
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
|
||||
for _, line := range failingLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
|
||||
}
|
||||
|
||||
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
|
||||
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range list {
|
||||
if s.ProviderNode == node || s.ConsumerNode == node {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
|
||||
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
|
||||
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
|
||||
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
|
||||
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
|
||||
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept := standings{open.inventory}
|
||||
since := time.Now().Add(-23 * time.Hour)
|
||||
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
|
||||
if _, err := kept.Stood(ctx, failing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
|
||||
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
if strings.Contains(said, "all doing what they were told") {
|
||||
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Failing []inventory.ProviderStanding `json:"failing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||
}
|
||||
}
|
||||
|
||||
// Recovered: gone, and the mesh may be well again as far as this is concerned.
|
||||
failing.Failing = false
|
||||
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
|
||||
t.Fatalf("%v %v", cleared, err)
|
||||
}
|
||||
asked, err = theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||
// not a problem.
|
||||
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("%+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
|
||||
now := time.Now()
|
||||
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
|
||||
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
|
||||
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
|
||||
}}, now), "\n")
|
||||
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Fatalf("%q not in:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lines, "more") {
|
||||
t.Fatalf("more than the first line of an error:\n%s", lines)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
@@ -129,6 +130,10 @@ func printStatus(asked answers) error {
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// A provider failing a consumer, beside machines failing what they were told: both are something
|
||||
// not working now (novox/hq ADR 0224).
|
||||
printFailing(asked.failing, time.Now())
|
||||
|
||||
if len(quiet) > 0 {
|
||||
var said []string
|
||||
for _, n := range quiet {
|
||||
@@ -298,6 +303,29 @@ func printStatus(asked answers) error {
|
||||
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||
}
|
||||
|
||||
if len(asked.overflowing) > 0 {
|
||||
// **Reported, and the provider still pushed** (novox/hq ADR 0225, issue 263). Each is left
|
||||
// out of its provider's grants, so the module holds a login nothing created; the provider's
|
||||
// machine is sent everything else rather than refused for one consumer elsewhere.
|
||||
fmt.Printf("%d module(s) identified too long for a provision they require, and not granted it:\n",
|
||||
len(asked.overflowing))
|
||||
for _, o := range asked.overflowing {
|
||||
fmt.Printf(" %-12s %-20s %-22s %q is %d, %s keeps %d\n", o.Consumer, o.Module, o.Provision,
|
||||
o.Identity, len(o.Identity), o.Bound.In, o.Bound.Max)
|
||||
}
|
||||
fmt.Printf("\n a shorter `slug` in the module's definition fits it; `module check` refuses one before merge\n\n")
|
||||
}
|
||||
|
||||
// Repairs done by hand this week (novox/hq to-be 45 §7). Not a fault, so it does not break "all
|
||||
// well"; each is a healer the mesh does not have yet, and the count is how that is watched.
|
||||
switch {
|
||||
case asked.handActsUnread != "":
|
||||
fmt.Printf("the hand-act log could not be read, so how much was done by hand this week is not known: %s\n\n",
|
||||
asked.handActsUnread)
|
||||
case asked.handActs != nil && *asked.handActs > 0:
|
||||
fmt.Printf("%d act(s) done by hand in the last seven days — `hand-acts` lists them, and why\n\n", *asked.handActs)
|
||||
}
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
@@ -406,15 +434,27 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||
// the private network is built from and should say nothing else; a machine that does not
|
||||
// resolve is already in refused, and is passed over here.
|
||||
plans := map[string]planned{}
|
||||
for _, n := range out.nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
plan, settings, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
continue
|
||||
}
|
||||
return answers{}, err
|
||||
}
|
||||
plans[n.Name] = planned{plan, settings}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
// And which of its modules a provider leaves out of its grants, for an identity too long
|
||||
// for what the provision keeps (novox/hq ADR 0225) — judged from the consumer's own
|
||||
// resolution, as the provider's composition judges it.
|
||||
out.overflowing = append(out.overflowing, plan.Overflowing()...)
|
||||
}
|
||||
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
|
||||
// providers announced: nothing else in the mesh knows whether a provision is being made.
|
||||
out.failing, err = failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
@@ -422,6 +462,16 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
}
|
||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||
// to read the log from; a process with none has no log to count.
|
||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||
n, unread := handActsThisWeek(ctx)
|
||||
if unread != "" {
|
||||
out.handActsUnread = unread
|
||||
} else {
|
||||
out.handActs = &n
|
||||
}
|
||||
}
|
||||
|
||||
// And which machines are not running what the mesh would send them. The same question as a
|
||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||
@@ -433,7 +483,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||
// reason is kept and reported as data; every question that does not depend on it is still
|
||||
// answered.
|
||||
would, err := wouldSend(ctx, open, out.nodes)
|
||||
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
|
||||
if err != nil {
|
||||
out.network = err.Error()
|
||||
would = map[string]string{}
|
||||
@@ -528,7 +578,7 @@ func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inven
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0 && len(a.overflowing) == 0
|
||||
}
|
||||
|
||||
// hostSplit is which machines report which host version, for every version more than one machine
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// `status` answered from a summary the serving controller keeps current (novox/hq to-be 45 Phase 0,
|
||||
// §4's D9 and §8's health of the controller).
|
||||
//
|
||||
// **Asked, it was composed: every machine resolved, twice, while its caller waited.** On 2026-10-06
|
||||
// the verb took eighteen seconds on a mesh of four machines, so its caller read "still running" and
|
||||
// had to ask `calls` for the answer to "is the mesh alright" — the one question that must answer at
|
||||
// once, and the one a self-check and a rollout gate will ask every few minutes. So the serving
|
||||
// controller composes it in the background — at its start, after anything that changes what it says
|
||||
// (a machine's report, a build, a verb that acts), and every minute regardless — and the verb answers
|
||||
// the last composition at once, saying when it was composed and how long that took. A caller who
|
||||
// needs it newer than that reads the time and asks again; nothing is answered as current that is not.
|
||||
|
||||
// statusEvery is how often the summary is composed with nothing having nudged it; statusSettle how
|
||||
// long a nudge waits for the next, so a push answered by four machines is composed once.
|
||||
var (
|
||||
statusEvery = time.Minute
|
||||
statusSettle = 2 * time.Second
|
||||
// statusComposeWithin bounds one composition, so a store that hangs cannot stop the summary for
|
||||
// good; the attempt is said as failed, and the last summary stands with its age.
|
||||
statusComposeWithin = 2 * time.Minute
|
||||
)
|
||||
|
||||
// statusSummary is the last composed `status --json`, and when.
|
||||
type statusSummary struct {
|
||||
compose func(context.Context) ([]byte, error)
|
||||
// every, settle and within are the clocks above, read once when it is made.
|
||||
every, settle, within time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
body []byte
|
||||
composedAt time.Time
|
||||
took time.Duration
|
||||
failed string
|
||||
failedAt time.Time
|
||||
started time.Time
|
||||
first chan struct{} // closed when the first attempt ends, either way
|
||||
|
||||
nudged chan struct{}
|
||||
}
|
||||
|
||||
func newStatusSummary(compose func(context.Context) ([]byte, error)) *statusSummary {
|
||||
return &statusSummary{compose: compose, started: time.Now(), first: make(chan struct{}),
|
||||
nudged: make(chan struct{}, 1), every: statusEvery, settle: statusSettle, within: statusComposeWithin}
|
||||
}
|
||||
|
||||
// statusFrom is the serving controller's summary; nil in any other process, where `status` is
|
||||
// composed when asked, as at a shell.
|
||||
var statusFrom *statusSummary
|
||||
|
||||
// nudge asks for a composition soon. Never blocks: one pending is as good as many.
|
||||
func (s *statusSummary) nudge() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.nudged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// keep composes until ctx ends: now, on a nudge once things settle, and every statusEvery.
|
||||
func (s *statusSummary) keep(ctx context.Context) {
|
||||
once := sync.Once{}
|
||||
for {
|
||||
s.composeOnce(ctx)
|
||||
once.Do(func() { close(s.first) })
|
||||
timer := time.NewTimer(s.every)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return
|
||||
case <-timer.C:
|
||||
case <-s.nudged:
|
||||
timer.Stop()
|
||||
// Let what else is arriving arrive, then compose once for all of it.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(s.settle):
|
||||
}
|
||||
select {
|
||||
case <-s.nudged:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statusSummary) composeOnce(ctx context.Context) {
|
||||
start := time.Now()
|
||||
asking, cancel := context.WithTimeout(ctx, s.within)
|
||||
body, err := s.compose(asking)
|
||||
cancel()
|
||||
took := time.Since(start)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err != nil {
|
||||
s.failed, s.failedAt = err.Error(), time.Now()
|
||||
fmt.Printf("status could not be composed (after %s): %v — `status` answers the last summary, "+
|
||||
"with its age\n", took.Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
s.body, s.composedAt, s.took, s.failed = body, start, took, ""
|
||||
}
|
||||
|
||||
// answer is what the `status` verb answers: the last summary at once, the same document `status
|
||||
// --json` prints, with when it was composed. Before the first composition has ended it waits for it,
|
||||
// but never past the caller's window; a controller that has none says so and why, rather than
|
||||
// answering an empty mesh as a well one.
|
||||
func (s *statusSummary) answer(ctx context.Context) (any, error) {
|
||||
wait := time.NewTimer(link.AnswerWithin - time.Second)
|
||||
defer wait.Stop()
|
||||
select {
|
||||
case <-s.first:
|
||||
case <-wait.C:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.body == nil {
|
||||
why := "its first composition has not finished"
|
||||
if s.failed != "" {
|
||||
why = "it could not be composed: " + s.failed
|
||||
}
|
||||
return nil, fmt.Errorf("this controller started %s ago and has no status to answer yet — %s. "+
|
||||
"Ask again shortly", time.Since(s.started).Round(time.Second), why)
|
||||
}
|
||||
var parsed any
|
||||
_ = json.Unmarshal(s.body, &parsed)
|
||||
out := map[string]any{
|
||||
"output": string(s.body), "ok": true, "answer": parsed,
|
||||
"composed": s.composedAt.UTC().Format(time.RFC3339),
|
||||
"age": time.Since(s.composedAt).Round(time.Second).String(),
|
||||
"composedIn": s.took.Round(time.Millisecond).String(),
|
||||
"note": "composed by the serving controller at its start, after each report, build or act, and " +
|
||||
"every minute; answered at once from the last composition",
|
||||
}
|
||||
if s.failed != "" && s.failedAt.After(s.composedAt) {
|
||||
out["lastAttemptFailed"] = fmt.Sprintf("%s: %s — this summary is the last that could be composed",
|
||||
s.failedAt.UTC().Format(time.RFC3339), s.failed)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeStatus is `status --json`, composed in this process against its stores.
|
||||
func composeStatus(open *stores) func(context.Context) ([]byte, error) {
|
||||
return func(ctx context.Context) ([]byte, error) {
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return statusAsJSON(asked)
|
||||
}
|
||||
}
|
||||
|
||||
// readingVerbs are the verbs that only read; after any other, what `status` says may have changed, so the summary is
|
||||
// composed again; a verb that only reads leaves it alone, or a console polling `nodes` would keep the
|
||||
// controller composing for ever.
|
||||
var readingVerbs = map[string]bool{
|
||||
"tools": true, "calls": true, "status": true, "nodes": true, "node": true, "modules": true,
|
||||
"seats": true, "builds": true, "plan": true, "queue": true, "durations": true, "hand-acts": true,
|
||||
}
|
||||
|
||||
// nudgingListener is the enrolment, nudging the summary when a machine said something new.
|
||||
type nudgingListener struct {
|
||||
link.Enrolment
|
||||
summary *statusSummary
|
||||
}
|
||||
|
||||
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
|
||||
news, err := l.Enrolment.Heard(ctx, report)
|
||||
if news {
|
||||
l.summary.nudge()
|
||||
}
|
||||
return news, err
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// quickly shortens the summary's clocks for one test.
|
||||
func quickly(t *testing.T) {
|
||||
t.Helper()
|
||||
every, settle := statusEvery, statusSettle
|
||||
statusEvery, statusSettle = time.Hour, 10*time.Millisecond
|
||||
t.Cleanup(func() { statusEvery, statusSettle = every, settle })
|
||||
}
|
||||
|
||||
// **`status` answers in full within ten seconds, five times in a row** (novox/hq to-be 45 Phase 0,
|
||||
// D9) — however long composing it takes. On 2026-10-06 composing took eighteen seconds and the
|
||||
// verb answered "still running"; from the summary it answers at once, in full, saying when.
|
||||
func TestStatusAnswersAtOnceHoweverLongComposingTakes(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
slow := make(chan struct{})
|
||||
s := newStatusSummary(func(ctx context.Context) ([]byte, error) {
|
||||
if composed.Add(1) > 1 {
|
||||
<-slow // every composition after the first outlasts any caller
|
||||
}
|
||||
return []byte(`{"wrong":[],"machines":4}`), nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
defer close(slow)
|
||||
go s.keep(ctx)
|
||||
for i := 0; i < 5; i++ {
|
||||
s.nudge() // a composition is under way and does not finish
|
||||
start := time.Now()
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if took := time.Since(start); took > time.Second {
|
||||
t.Fatalf("answer %d took %s", i+1, took)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
if m["answer"].(map[string]any)["machines"] != float64(4) || m["composed"] == "" || m["ok"] != true {
|
||||
t.Fatalf("answer %d was not in full: %v", i+1, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The first composition is waited for, never past the caller's window; a controller with none yet
|
||||
// says so rather than answering an empty mesh as a well one.
|
||||
func TestStatusBeforeItsFirstCompositionSaysSo(t *testing.T) {
|
||||
quickly(t)
|
||||
was := link.AnswerWithin
|
||||
link.AnswerWithin = 1100 * time.Millisecond
|
||||
t.Cleanup(func() { link.AnswerWithin = was })
|
||||
never := make(chan struct{})
|
||||
defer close(never)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { <-never; return nil, nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
start := time.Now()
|
||||
_, err := s.answer(ctx)
|
||||
if err == nil || !strings.Contains(err.Error(), "has no status to answer yet") {
|
||||
t.Fatalf("answered %v", err)
|
||||
}
|
||||
if took := time.Since(start); took > link.AnswerWithin {
|
||||
t.Fatalf("waited %s, past the caller's window", took)
|
||||
}
|
||||
}
|
||||
|
||||
// A nudge composes it again, once for several close together; a failed composition leaves the last
|
||||
// summary standing and says it is the last that could be composed.
|
||||
func TestANudgeComposesAgainAndAFailureKeepsTheLastSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
var composed atomic.Int32
|
||||
fail := atomic.Bool{}
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) {
|
||||
n := composed.Add(1)
|
||||
if fail.Load() {
|
||||
return nil, errors.New("the store did not answer")
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"n": n})
|
||||
return body, nil
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
if _, err := s.answer(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 2 })
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if n := composed.Load(); n != 2 {
|
||||
t.Fatalf("three nudges together composed %d times after the first", n-1)
|
||||
}
|
||||
fail.Store(true)
|
||||
s.nudge()
|
||||
waitFor(t, func() bool { return composed.Load() == 3 })
|
||||
waitFor(t, func() bool {
|
||||
got, err := s.answer(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := got.(map[string]any)
|
||||
return m["lastAttemptFailed"] != nil && m["answer"].(map[string]any)["n"] == float64(2)
|
||||
})
|
||||
}
|
||||
|
||||
// The seat's `status` answers from the summary when this process keeps one.
|
||||
func TestTheStatusVerbAnswersFromTheSummary(t *testing.T) {
|
||||
quickly(t)
|
||||
s := newStatusSummary(func(context.Context) ([]byte, error) { return []byte(`{"from":"summary"}`), nil })
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go s.keep(ctx)
|
||||
statusFrom = s
|
||||
t.Cleanup(func() { statusFrom = nil })
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := handlers["status"](ctx, json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.(map[string]any)["answer"].(map[string]any)["from"] != "summary" {
|
||||
t.Fatalf("answered %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for !ok() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("never happened")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -75,21 +75,25 @@ func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
||||
t.Fatalf("a plan was closed by hand without saying why: %v", err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "its report will not come"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") ||
|
||||
!strings.Contains(closed.Note, "its report will not come") {
|
||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID, "--why", "again"}); err == nil {
|
||||
t.Fatal("a plan already closed was closed again")
|
||||
}
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID, "why": "w"}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID, "--why", "w"}) {
|
||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -266,6 +267,11 @@ func notNow(err error) error {
|
||||
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
||||
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
||||
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
@@ -276,34 +282,7 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
return notNow(err)
|
||||
}
|
||||
|
||||
// Two kinds of module are affected by one merge, and they are affected differently.
|
||||
//
|
||||
// A module **built from** this repository and branch has moved: the mesh records the new commit
|
||||
// as what its source now has, and only what the merge actually changed is rebuilt. A module that
|
||||
// only **packages source from** it has not moved — its own source is somewhere else, at the
|
||||
// commit it already records — so it is rebuilt and its record left alone. Writing this commit as
|
||||
// its source would make it permanently behind a repository its manifest does not come from.
|
||||
var from, packaging []inventory.Entry
|
||||
already := 0
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case sourceIs(e.Source, m):
|
||||
if e.Source.BuiltFrom == m.Commit {
|
||||
already++
|
||||
continue
|
||||
}
|
||||
// **A merge older than the last look at the source is history, not a move.** The forge
|
||||
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
||||
// the recorded head backwards and rebuild everything built from that repository, once
|
||||
// per old merge (2026-09-28).
|
||||
if isHistory(m.MergedAt, e.Source.Seen) {
|
||||
continue
|
||||
}
|
||||
from = append(from, e)
|
||||
case readsFrom(read[e.Manifest.Module], m):
|
||||
packaging = append(packaging, e)
|
||||
}
|
||||
}
|
||||
from, packaging, already := mergeCandidates(m, entries, read)
|
||||
if len(from) == 0 && len(packaging) == 0 {
|
||||
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
||||
@@ -438,6 +417,57 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||
var actingOnMerges sync.Mutex
|
||||
|
||||
// mergeCandidates is what one merge could move, judged against what the catalogue holds.
|
||||
//
|
||||
// Two kinds of module are affected by one merge, and they are affected differently.
|
||||
//
|
||||
// A module **built from** this repository and branch has moved: the mesh records the new commit as
|
||||
// what its source now has, and only what the merge actually changed is rebuilt. A module that only
|
||||
// **packages source from** it has not moved — its own source is somewhere else, at the commit it
|
||||
// already records — so it is rebuilt and its record left alone. Writing this commit as its source
|
||||
// would make it permanently behind a repository its manifest does not come from. `already` counts
|
||||
// the modules built from this repository that are already built from this very commit.
|
||||
func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) (from, packaging []inventory.Entry, already int) {
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case sourceIs(e.Source, m):
|
||||
if e.Source.BuiltFrom == m.Commit {
|
||||
already++
|
||||
continue
|
||||
}
|
||||
// **A merge older than the last look at the source is history, not a move.** The forge
|
||||
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
||||
// the recorded head backwards and rebuild everything built from that repository, once
|
||||
// per old merge (2026-09-28).
|
||||
if isHistory(m.MergedAt, e.Source.Seen) {
|
||||
continue
|
||||
}
|
||||
from = append(from, e)
|
||||
case readsFrom(read[e.Manifest.Module], m):
|
||||
packaging = append(packaging, e)
|
||||
}
|
||||
}
|
||||
return from, packaging, already
|
||||
}
|
||||
|
||||
// wouldMove is the modules built from the merged repository that acting on this merge would mark as
|
||||
// moved and rebuild — SourceMoved's judgement, made without acting (novox/hq issue 266). Empty for a
|
||||
// merge already acted on: acting marks each of them as looked at, so the merge then reads as history.
|
||||
//
|
||||
// **Only the modules built from it, never the ones that merely package source from it.** Acting
|
||||
// records nothing about those, so a merge acted on would go on reading as unacted for them, and be
|
||||
// acted on again on every look. A merge that moves both is caught by the first kind, and acting on it
|
||||
// rebuilds the second as well.
|
||||
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||
from, _, _ := mergeCandidates(m, entries, read)
|
||||
return whatTheMergeTouched(from, entries, m)
|
||||
}
|
||||
|
||||
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
||||
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
||||
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// The controller's own key-value buckets (novox/hq to-be 45 §1, §6, §7).
|
||||
//
|
||||
// **What the controller must remember across its own restart, it keeps on the bus.** A call's
|
||||
// outcome lived in the memory of the process that served it (novox/hq issue 265), so a controller
|
||||
// replaced while a push ran answered "no such call" for the one thing its caller had been told to
|
||||
// ask about. The bus already outlives the controller and is the shape ADR 0201 gives a module's
|
||||
// current state: one value per key, written by one owner, read by anybody granted it. These are the
|
||||
// controller's, written by it alone — the writers table of to-be 45 §1 — and asserted on every start
|
||||
// like the streams, so a bus raised from nothing has them before the first call is served.
|
||||
|
||||
// CallsBucket keeps every call of the mesh's own verbs and what came of it; HandActsBucket every act
|
||||
// a person did by hand, with why.
|
||||
var (
|
||||
CallsBucket = BucketName(ControllerSeat, "calls")
|
||||
HandActsBucket = BucketName(ControllerSeat, "hand-acts")
|
||||
)
|
||||
|
||||
// The bounds to-be 45 §6 sets for calls: the last thousand, or fourteen days, whichever is fewer.
|
||||
// A call is two keys — its record, and its answer apart so a listing does not read every answer —
|
||||
// so the stream holds twice as many messages as it keeps calls.
|
||||
const (
|
||||
KeptCallsDurably = 1000
|
||||
CallsKeptFor = 14 * 24 * time.Hour
|
||||
// CallAnswerBytes is the most of one answer kept: a whole declaration is far smaller, and an
|
||||
// answer larger is cut and says so.
|
||||
CallAnswerBytes = 64 << 10
|
||||
// HandActsKeptFor is as long as a condition's history (to-be 45 §2): an act by hand is read
|
||||
// back beside what it addressed.
|
||||
HandActsKeptFor = 90 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// IsControllerBucket says a bucket is the controller's own, not a module's state nothing declares.
|
||||
func IsControllerBucket(bucket string) bool {
|
||||
return bucket == CallsBucket || bucket == HandActsBucket
|
||||
}
|
||||
|
||||
// ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
|
||||
type ControllerBucketsAsserter interface {
|
||||
EnsureControllerBuckets() error
|
||||
}
|
||||
|
||||
// EnsureControllerBuckets creates the controller's buckets if absent and brings their options to
|
||||
// match. An update, never a delete: what they hold is the record of what the mesh was asked.
|
||||
func (j *JetStream) EnsureControllerBuckets() error {
|
||||
js, err := jetstream.New(j.conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: CallsBucket,
|
||||
Description: "the calls of the mesh's own verbs and what came of each (novox/hq to-be 45 §6, issue " +
|
||||
"265): written by the controller alone, read through `calls`; the last thousand, or fourteen days",
|
||||
History: 1,
|
||||
TTL: CallsKeptFor,
|
||||
MaxValueSize: CallAnswerBytes + 4<<10,
|
||||
MaxBytes: 2 * KeptCallsDurably * (CallAnswerBytes + 4<<10),
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", CallsBucket, err)
|
||||
}
|
||||
// **The count, on the stream under the bucket.** A bucket has an age and a size and no count;
|
||||
// the stream it is made of does, and with one value per key the oldest message is the oldest
|
||||
// call. Asserted after the bucket, every time, because asserting the bucket writes the stream's
|
||||
// configuration whole and puts the count back to none.
|
||||
stream, err := js.Stream(ctx, "KV_"+CallsBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the stream under %s: %w", CallsBucket, err)
|
||||
}
|
||||
cfg := stream.CachedInfo().Config
|
||||
if cfg.MaxMsgs != 2*KeptCallsDurably {
|
||||
cfg.MaxMsgs = 2 * KeptCallsDurably
|
||||
cfg.Discard = jetstream.DiscardOld
|
||||
if _, err := js.UpdateStream(ctx, cfg); err != nil {
|
||||
return fmt.Errorf("bounding %s to the last %d calls: %w", CallsBucket, KeptCallsDurably, err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
|
||||
Bucket: HandActsBucket,
|
||||
Description: "every act a person did by hand, with why (novox/hq to-be 45 §7): written by the " +
|
||||
"controller's repairing verbs and `hand-act record`, read through `hand-acts`",
|
||||
History: 1,
|
||||
TTL: HandActsKeptFor,
|
||||
MaxValueSize: 16 << 10,
|
||||
MaxBytes: 64 << 20,
|
||||
Storage: jetstream.FileStorage,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("asserting bucket %s: %w", HandActsBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a
|
||||
// key is a publish to the bucket's own subject, which the management interface's grant does not
|
||||
// cover: the cancelled sets' writes timed out for want of this, and the controller's own buckets
|
||||
// would have.
|
||||
func TestTheControllerMayWriteEveryBucketItWrites(t *testing.T) {
|
||||
p, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"$KV." + CallsBucket + ".>", "$KV." + HandActsBucket + ".>"}
|
||||
for _, seat := range seatsTheControllerAsks {
|
||||
if hasCancelledSet(seat) {
|
||||
want = append(want, "$KV."+CancelledSetName(seat)+".>")
|
||||
}
|
||||
}
|
||||
for _, subject := range want {
|
||||
if !slices.Contains(p.Publish, subject) {
|
||||
t.Errorf("the controller may not publish %s, so it cannot write that bucket", subject)
|
||||
}
|
||||
}
|
||||
if slices.Contains(p.Publish, "$KV.>") {
|
||||
t.Error("the controller may write any bucket, a module's state included")
|
||||
}
|
||||
}
|
||||
+23
-3
@@ -18,6 +18,7 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A Kind is what a principal is, which decides the shape of its authority rather than its
|
||||
@@ -189,6 +190,13 @@ type Permissions struct {
|
||||
AllowResponses bool
|
||||
}
|
||||
|
||||
// ResponseTTL is how long the bus lets a principal answer a request it received. Its one answer has
|
||||
// to come inside this, and a seat's holder answers within link.AnswerWithin — inside it by design.
|
||||
// **A broker reloading its user list forgets every answer it was about to permit**, whatever this
|
||||
// says (novox/hq issue 265): a call that is still running when the list reloads has its answer
|
||||
// refused, which is why a holder answers before it does what can reload it.
|
||||
const ResponseTTL = time.Minute
|
||||
|
||||
// PermissionsFor derives a principal's authority. Pure, and the only place authority is decided:
|
||||
// a permission that cannot be derived from a declaration is a permission nobody can explain.
|
||||
func PermissionsFor(p Principal) (Permissions, error) {
|
||||
@@ -229,6 +237,12 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// building, kill it, pause it, resume it. The queue is the controller's to show and to
|
||||
// change, and what one machine is doing with an ask it took only that machine can say.
|
||||
pub = append(pub, "mesh.seat."+seat+".tool.>")
|
||||
// **And its cancelled set** (novox/hq ADR 0219, issue 269): a cancel writes the ask's id
|
||||
// there before it deletes the ask, and a write is a publish to the bucket's subject, which
|
||||
// `$JS.API.>` does not cover — so every cancel timed out, refused by this list.
|
||||
if hasCancelledSet(seat) {
|
||||
pub = append(pub, "$KV."+CancelledSetName(seat)+".>")
|
||||
}
|
||||
}
|
||||
// **And what the mesh says it did** (novox/hq ADR 0134). The control plane states its own
|
||||
// facts under the seat it holds, because a role's events belong to the role and keep their
|
||||
@@ -253,10 +267,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// And says so (novox/hq ADR 0197): it answers discovery for the seat it serves.
|
||||
sub = append(sub, announcing(ControllerSeat)...)
|
||||
|
||||
// The two events it reacts to, and its ack subject on the stream they arrive from
|
||||
// The events it reacts to, and its ack subject on the stream they arrive from
|
||||
// (streams.go). **Each named, not a pattern**: `mesh.mod.*.event.>` would make the
|
||||
// controller a subscriber to every event in the mesh, and its permission list would stop
|
||||
// saying what it is for. The ack grant below is scoped per stream because the controller's
|
||||
// saying what it is for. The one wildcard is the emitter of a provider's standing (ADR
|
||||
// 0224) — still two named events, from whichever module provides. The ack grant below is scoped per stream because the controller's
|
||||
// consumer name is the same on both and `$JS.ACK.CONTROL.controller.>` does not cover a
|
||||
// delivery from EVENTS — a consumer that cannot ack has every message redelivered for
|
||||
// ever, refused by the list it already has.
|
||||
@@ -279,6 +294,11 @@ func PermissionsFor(p Principal) (Permissions, error) {
|
||||
// enrolments and can reach nothing else.
|
||||
pub = append(pub, "_INBOX."+enrolmentPrefix+".>")
|
||||
|
||||
// **And its own buckets** (novox/hq to-be 45 §1): the calls it served and the acts done by
|
||||
// hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>`
|
||||
// does not cover; each bucket named, not `$KV.>`, which would let it write any module's state.
|
||||
pub = append(pub, "$KV."+CallsBucket+".>", "$KV."+HandActsBucket+".>")
|
||||
|
||||
case KindPerson:
|
||||
// Tools, and nothing else. Every subject a person may publish is a tool call; a person
|
||||
// who could publish an event would be able to claim a module said something.
|
||||
@@ -777,7 +797,7 @@ func ComposeAccounts(principals []Principal) (string, error) {
|
||||
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quoted(perms.Publish))
|
||||
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quoted(perms.Subscribe))
|
||||
if perms.AllowResponses {
|
||||
b.WriteString(" allow_responses: { max: 1, ttl: \"1m\" }\n")
|
||||
fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(ResponseTTL/time.Minute))
|
||||
}
|
||||
b.WriteString(" } }\n")
|
||||
}
|
||||
|
||||
@@ -5,16 +5,16 @@ import "testing"
|
||||
// A node-scoped seat's tool carries the node (novox/hq ADR 0132, design 33 §4): two nodes holding one
|
||||
// node-scoped seat derive two addresses, and a user of the seat may publish any node's.
|
||||
func TestTwoNodesHoldingOneNodeSeatDeriveTwoToolAddresses(t *testing.T) {
|
||||
seat := Seat{Name: "node-dns-resolver", Scope: "node", Serves: []string{"lookup"}}
|
||||
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "dnsmasq", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.one")
|
||||
has(t, two.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-dns-resolver.tool.lookup.two")
|
||||
seat := Seat{Name: "node-hostname", Scope: "node", Serves: []string{"entries"}}
|
||||
one, _ := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
two, _ := PermissionsFor(Principal{Kind: KindModule, Node: "two", Module: "hostname", Holds: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.one")
|
||||
has(t, two.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries")
|
||||
hasNot(t, one.Subscribe, "mesh.seat.node-hostname.tool.entries.two")
|
||||
|
||||
user, _ := PermissionsFor(Principal{Kind: KindModule, Node: "three", Module: "asker", Uses: []Seat{seat}, PasswordHash: "x"})
|
||||
has(t, user.Publish, "mesh.seat.node-dns-resolver.tool.lookup.*")
|
||||
has(t, user.Publish, "mesh.seat.node-hostname.tool.entries.*")
|
||||
}
|
||||
|
||||
// A mesh-scoped seat's tool stays flat: nothing about it changes.
|
||||
|
||||
@@ -160,8 +160,9 @@ func RaiseBuckets(a BucketAsserter, buckets []Bucket) (undeclared []string, err
|
||||
return nil, fmt.Errorf("listing the bus's state: %w", err)
|
||||
}
|
||||
for _, n := range names {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state.
|
||||
if !declared[n] && !IsCancelledSet(n) {
|
||||
// A seat's cancelled set is the mesh's own (novox/hq ADR 0219), not a module's state; so are
|
||||
// the controller's own buckets (novox/hq to-be 45 §1).
|
||||
if !declared[n] && !IsCancelledSet(n) && !IsControllerBucket(n) {
|
||||
undeclared = append(undeclared, n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,8 +226,23 @@ var ControllerFollows = []string{
|
||||
// registers build-agent itself comes from there. Appended, for the same reason as above; goes
|
||||
// with the retired seat row.
|
||||
seatEventSubject("mesh-build-machine", "built"),
|
||||
// **Every provider's standing** (novox/hq ADR 0224): a consumer it has failed for minutes, and
|
||||
// that consumer recovered. The one pattern on this list, and a narrow one — two named events,
|
||||
// from whichever module provides — because the rule is about every provider, and a list of
|
||||
// providers here would be a list somebody forgets to extend. On 2026-10-05 the identity provider
|
||||
// failed every consumer for a day and only its journal said so (issue 179). Appended, because
|
||||
// the index is a name.
|
||||
moduleEventSubject("*", ProvisionerFailing),
|
||||
moduleEventSubject("*", ProvisionerRecovered),
|
||||
}
|
||||
|
||||
// The provider standing events, by their local names. Written here as well as in the catalogue
|
||||
// (catalogue.ProvisionerEvents), which this package cannot import; a test keeps them agreeing.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// moduleEventSubject is where one module's event lands. The same derivation PermissionsFor uses, so
|
||||
// what the controller subscribes and what the emitter is permitted to publish cannot drift apart.
|
||||
func moduleEventSubject(module, event string) string {
|
||||
@@ -271,7 +286,7 @@ func MeshConsumers() []Consumer {
|
||||
// client and come back to be acted on again.
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
|
||||
Why: "the events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||
"become actionable",
|
||||
},
|
||||
|
||||
+2
-2
@@ -24,8 +24,8 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_hand-acts.>", "_INBOX.enrol.>", "mesh.assignment.>", "mesh.control.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.refused", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.tool.>"] }
|
||||
subscribe: { allow: ["$JS.API.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.gitea.event.pull.merged", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built"] }
|
||||
allow_responses: { max: 1, ttl: "1m" }
|
||||
} }
|
||||
{ user: "enrol.one", password: "$2a$11$eeeeeeeeeeeeeeeeeeeeee", permissions: {
|
||||
|
||||
@@ -119,3 +119,41 @@ func TestEveryCatalogueStoreSaysHowItIsBackedUp(t *testing.T) {
|
||||
t.Fatal("no module in the catalogue provides a store, so this proved nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryCatalogueIdentityFitsWhatItRequires is ADR 0225's check over the real catalogue: every
|
||||
// module's identity, on the longest machine name, fits the bound of every provision it wants. An
|
||||
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
|
||||
// the first time a real machine's name meets the module's (issue 263).
|
||||
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
|
||||
root := catalogueRoot(t)
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil || len(found) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
shelf := Shelf{}
|
||||
for _, p := range found {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", p, err)
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, p := range IdentityProblems(shelf, DefaultLongestMachine) {
|
||||
t.Error(p)
|
||||
}
|
||||
// The night it was found: the resolver provision bounds nothing, and the object store still 20.
|
||||
if dns, ok := shelf["dnsmasq"]; ok {
|
||||
if b := dns.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||
t.Errorf("the resolver provision bounds its consumers' identities: %+v", b)
|
||||
}
|
||||
}
|
||||
if store, ok := shelf["minio"]; ok {
|
||||
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 {
|
||||
t.Errorf("the object store's access key is not bounded at 20: %+v", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,7 +159,8 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// reloading answers the runtime's trust as the networking module does (novox/hq ADR 0102): a file
|
||||
// reloading answers the runtime's trust as the networking module once did (novox/hq ADR 0102; it no
|
||||
// longer does, ADR 0222 — and beside the runtime's module it is refused, generated_collision_test): a file
|
||||
// written into, and the runtime reloaded on it.
|
||||
type reloading struct{}
|
||||
|
||||
|
||||
@@ -114,7 +114,9 @@ func consumerInto(value, as string) (string, error) {
|
||||
// asDNSLabel writes a minted identity as a DNS label.
|
||||
//
|
||||
// The mesh's identities are already lower-case letters, digits and `_` (ConsumerIdentity), and
|
||||
// already short enough for the tightest backend they reach (CheckIdentity, twenty characters). So
|
||||
// already inside the bound of the provision serving them: a provider that serves one as a DNS label
|
||||
// bounds it at 63 or less (CheckServes, novox/hq ADR 0225), and one that serves it at all without
|
||||
// saying is held to twenty (IdentityBoundOf). So
|
||||
// this is the separator and nothing else — no lower-casing of what is already lower case, no
|
||||
// truncation to a limit the identity is already inside, no padding of a name that is already long
|
||||
// enough. Each of those would be the mesh guessing at a rule it has not been given.
|
||||
@@ -141,11 +143,31 @@ func CheckServes(m Manifest) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
|
||||
}
|
||||
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
|
||||
// bound has to keep the identity inside one (ADR 0225).
|
||||
if strings.Contains(text, "${consumer:as:dns}") {
|
||||
if b := m.IdentityBoundOf(provision); !b.Bounded() || b.Max > dnsLabelLimit {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s serves %s's consumers their identity as a DNS label in %q, and bounds "+
|
||||
"that identity at %s: a label keeps %d — state an `identity` of at most %d",
|
||||
m.Module, provision, key, boundWords(b), dnsLabelLimit, dnsLabelLimit))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// dnsLabelLimit is the longest DNS label (RFC 1035 §2.3.4).
|
||||
const dnsLabelLimit = 63
|
||||
|
||||
func boundWords(b IdentityBound) string {
|
||||
if !b.Bounded() {
|
||||
return "nothing"
|
||||
}
|
||||
return fmt.Sprintf("%d", b.Max)
|
||||
}
|
||||
|
||||
func sortedServes(serves map[string]map[string]any) []string {
|
||||
out := make([]string, 0, len(serves))
|
||||
for k := range serves {
|
||||
|
||||
@@ -159,12 +159,21 @@ type Rendering struct {
|
||||
// 0199): the mesh's resolver forwards each one there.
|
||||
Zones []ZoneAt
|
||||
|
||||
// Holders is, for each replicated mesh seat, every machine holding it, by internal name and
|
||||
// private address (novox/hq ADR 0223) — the same shape as Machines. What a machine's resolver
|
||||
// file lists: every holder of the mesh's resolver, this machine first if it is one.
|
||||
Holders map[string]map[string]string
|
||||
|
||||
Settings SettingsBy
|
||||
Generators map[string]Generator
|
||||
// Grants are the credentials this node must create, for the provisions it offers. Passed in
|
||||
// rather than resolved, because who consumes a node is a fact about the rest of the mesh and
|
||||
// resolution answers questions about one machine.
|
||||
Grants []Grant
|
||||
// Withheld is every consumer left out of Grants because its identity overflows the provision's
|
||||
// bound (novox/hq ADR 0225). Composed into nothing; carried so the machine's declaration can say
|
||||
// whom it does not serve, and why, beside what it does.
|
||||
Withheld []Overflow
|
||||
|
||||
// Ports is where this machine puts what each module needs reachable, by module and by the
|
||||
// port the software itself uses (novox/hq ADR 0038).
|
||||
@@ -201,6 +210,11 @@ type Rendering struct {
|
||||
// stored (novox/hq 04-ISSUES/102).
|
||||
ArtifactStore string
|
||||
|
||||
// SeatReach is where this machine reaches the holder of each mesh-scoped seat it may be asked
|
||||
// about (host:port), by seat: what ${seat:<seat>:reach} answers with (novox/hq ADR 0222). Absent
|
||||
// when no holder is reachable yet; see seat_into.go for which seats are answered.
|
||||
SeatReach map[string]string
|
||||
|
||||
// Built is every `<module>/<artifact>` the mesh has built. What tells a reference recorded
|
||||
// with an address — before references were kept without one — from an image a module runs
|
||||
// straight from a public registry.
|
||||
@@ -426,6 +440,11 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
generated, err := r.generatedHere(with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var out []map[string]any
|
||||
for _, m := range r.Modules {
|
||||
if with.Adopted && m.Filtering != nil {
|
||||
@@ -692,23 +711,14 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
first = append(first, file)
|
||||
}
|
||||
if m.Computed != "" {
|
||||
generator, known := with.Generators[m.Computed]
|
||||
if !known {
|
||||
return nil, fmt.Errorf(
|
||||
"%s says its resources are computed by %q, and this control plane has no %q",
|
||||
m.Module, m.Computed, m.Computed)
|
||||
}
|
||||
generated, part, err := generator.Resources(r.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mine, part := generated[m.Module]
|
||||
if !part {
|
||||
// Assigned, and not yet part of what this generates. Nothing to put on the
|
||||
// machine, which is different from an error: a node given the network module
|
||||
// before it has an address is in exactly that state, briefly.
|
||||
continue
|
||||
}
|
||||
resources = generated
|
||||
resources = mine
|
||||
}
|
||||
|
||||
// Now, and not before: a module whose resources are computed replaces them wholesale, and
|
||||
@@ -979,7 +989,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
|
||||
// plane's; making a name resolve is the module's software. Emitted as ordinary files under
|
||||
// this module's name, so they are applied, reported and removed exactly as anything else
|
||||
// it declares.
|
||||
given, err := FactsWithZonesInto(m, r, with.Names, with.Machines, with.Accounts, with.Suffix, with.Zones)
|
||||
given, err := FactsFrom(m, r, with)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -2348,3 +2358,49 @@ func accountsFirst(resources []map[string]any) (accounts, rest []map[string]any)
|
||||
}
|
||||
return accounts, rest
|
||||
}
|
||||
|
||||
// generatedHere is what each computed module's generator answers for this machine, by module —
|
||||
// absent for a module whose machine is not yet part of what it generates — held to the rule every
|
||||
// module is held to: no two modules on a machine declare one path, unit, name or package (novox/hq
|
||||
// issue 190, step 5; ADR 0222).
|
||||
//
|
||||
// Resolution checks the catalogue's manifests, and a computed module's manifest has none of the
|
||||
// resources it will declare — they exist only once its generator has answered for this machine,
|
||||
// here — so a generated resource writing into another module's file was never seen. That is how
|
||||
// the private network came to declare the container runtime's daemon file and service beside the
|
||||
// runtime's own module. Asked once, so what is checked is exactly what is declared.
|
||||
func (r Resolution) generatedHere(with Rendering) (map[string][]map[string]any, error) {
|
||||
generated := map[string][]map[string]any{}
|
||||
placed := make([]Manifest, 0, len(r.Modules))
|
||||
for _, m := range r.Modules {
|
||||
if m.Computed == "" {
|
||||
placed = append(placed, m)
|
||||
continue
|
||||
}
|
||||
generator, known := with.Generators[m.Computed]
|
||||
if !known {
|
||||
return nil, fmt.Errorf(
|
||||
"%s says its resources are computed by %q, and this control plane has no %q",
|
||||
m.Module, m.Computed, m.Computed)
|
||||
}
|
||||
resources, part, err := generator.Resources(r.Node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
computed := m
|
||||
computed.Resources = nil
|
||||
if part {
|
||||
generated[m.Module] = resources
|
||||
computed.Resources = resources
|
||||
}
|
||||
placed = append(placed, computed)
|
||||
}
|
||||
if len(generated) == 0 {
|
||||
return generated, nil // nothing resolution has not already judged
|
||||
}
|
||||
if problems := checkResources(placed); len(problems) > 0 {
|
||||
return nil, fmt.Errorf("what the mesh computes for this machine collides with a module's own: %s",
|
||||
strings.Join(problems, "; "))
|
||||
}
|
||||
return generated, nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package catalogue
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -159,3 +160,34 @@ func consumePattern(pattern string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The events a provider says about its consumers (novox/hq ADR 0224): a consumer it has failed
|
||||
// without one success for minutes, and that consumer succeeding again or being withdrawn. The
|
||||
// controller follows them from every module and `status` names a consumer failing until it recovers.
|
||||
const (
|
||||
ProvisionerFailing = "provisioner.failing"
|
||||
ProvisionerRecovered = "provisioner.recovered"
|
||||
)
|
||||
|
||||
// ProvisionerEvents are both, in the order they are said.
|
||||
var ProvisionerEvents = []string{ProvisionerFailing, ProvisionerRecovered}
|
||||
|
||||
// EmitsAll is every event a module may publish: what it declares and, for a module that receives
|
||||
// contributions — a provider, running a provisioner over them — the provider's standing events.
|
||||
//
|
||||
// **Derived, not declared**, because they are the mesh's rule about every provider rather than
|
||||
// anything one module chose to say: a provider whose manifest forgot them would fail its consumers
|
||||
// as silently as on 2026-10-05, with its announcement refused by the bus (novox/hq issue 179). Every
|
||||
// grant of a module's publishing reads this, never the declared list alone.
|
||||
func (m Manifest) EmitsAll() []string {
|
||||
out := append([]string(nil), m.Emits...)
|
||||
if len(m.Receives) == 0 {
|
||||
return out
|
||||
}
|
||||
for _, e := range ProvisionerEvents {
|
||||
if !slices.Contains(out, e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq issue 190, step 5 (ADR 0222): what the mesh computes for a module is held to the rule
|
||||
// every module is — no two modules on a machine declare one path, unit, name or package. The
|
||||
// private network once declared the container runtime's file and service beside the runtime's own
|
||||
// module, and nothing refused it, because the collision check only ever saw catalogue manifests.
|
||||
|
||||
func runtimesOwn() Manifest {
|
||||
return Manifest{Module: "docker", Resources: []map[string]any{
|
||||
{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", "into": "json",
|
||||
"content": `{"live-restore": true}`},
|
||||
{"id": "runtime", "type": "service", "unit": "docker.service", "state": "running",
|
||||
"reload-on": []any{"daemon"}},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAGeneratedResourceCollidingWithAModulesIsRefused(t *testing.T) {
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{
|
||||
{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
|
||||
runtimesOwn(),
|
||||
}}
|
||||
_, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": reloading{}}})
|
||||
if err == nil {
|
||||
t.Fatal("a generated resource declaring the runtime's file beside the runtime's module was accepted")
|
||||
}
|
||||
for _, want := range []string{"mesh-network", "docker", "/etc/docker/daemon.json", "docker.service"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGeneratedResourceBesideAModulesOwnIsComposed(t *testing.T) {
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{
|
||||
{Module: "mesh-network", Computed: "mesh-network", Provides: Offers("private-network")},
|
||||
runtimesOwn(),
|
||||
}}
|
||||
gen := &fake{on: map[string]bool{"workstation": true}}
|
||||
out, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": gen}})
|
||||
if err != nil {
|
||||
t.Fatalf("disjoint resources were refused: %v", err)
|
||||
}
|
||||
if fileNamed(out, "docker.daemon") == nil {
|
||||
t.Fatalf("the runtime's own file is missing: %v", out)
|
||||
}
|
||||
// And a machine not on the network yet generates nothing, which collides with nothing.
|
||||
if _, err := r.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": &fake{}}}); err != nil {
|
||||
t.Fatalf("a machine off the network was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's container runtime module states the mesh's registry itself (ADR 0222).
|
||||
func TestTheRuntimesModuleTrustsTheMeshsRegistry(t *testing.T) {
|
||||
docker := catalogueManifest(t, "docker")
|
||||
r := Resolution{Node: "workstation", Modules: []Manifest{docker}}
|
||||
out, err := r.Declaration(Rendering{
|
||||
SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
daemon := fileNamed(out, "docker.daemon")
|
||||
if daemon == nil || daemon["into"] != "json" {
|
||||
t.Fatalf("the runtime's file is not written into: %v", daemon)
|
||||
}
|
||||
content, _ := daemon["content"].(string)
|
||||
if !strings.Contains(content, `"insecure-registries": ["anchor.internal:5100"]`) ||
|
||||
!strings.Contains(content, `"live-restore": true`) {
|
||||
t.Fatalf("the runtime's file says %q", content)
|
||||
}
|
||||
|
||||
out, err = r.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if content, _ := fileNamed(out, "docker.daemon")["content"].(string); strings.Contains(content, "insecure-registries") {
|
||||
t.Fatalf("with no store on the network, the runtime is told %q", content)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0223 part 3: a machine's names are one seat's. The `hosts` module holding
|
||||
// `node-hosts-file` became `hostname` holding `node-hostname`, which writes /etc/hostname beside the
|
||||
// machine's own lines in /etc/hosts. The seat was renamed (ADR 0122), so what claims the old name — a
|
||||
// manifest registered before the rename — still holds the one seat.
|
||||
|
||||
func withHostnameAlias(t *testing.T) {
|
||||
t.Helper()
|
||||
was := aliases
|
||||
t.Cleanup(func() { aliases = was })
|
||||
UseAliases(map[string]string{"node-hosts-file": "node-hostname"})
|
||||
}
|
||||
|
||||
func TestTheHostsFilesFormerNameResolvesToTheHostnameSeat(t *testing.T) {
|
||||
if _, known := SeatNamed("node-hostname"); !known {
|
||||
t.Fatal("node-hostname is not in the mesh's set")
|
||||
}
|
||||
withHostnameAlias(t)
|
||||
seat, known := SeatNamed("node-hosts-file")
|
||||
if !known || seat.Name != "node-hostname" || seat.Scope != ScopeNode {
|
||||
t.Fatalf("the former name did not resolve: %+v %v", seat, known)
|
||||
}
|
||||
var verbs []string
|
||||
for _, v := range seat.Serves {
|
||||
verbs = append(verbs, v.Name)
|
||||
}
|
||||
if strings.Join(verbs, " ") != "entries add remove" {
|
||||
t.Errorf("the renamed seat serves %v; its verbs are unchanged", verbs)
|
||||
}
|
||||
}
|
||||
|
||||
// One seat under either name: the module registered before the rename and the one after cannot both
|
||||
// hold it on one machine.
|
||||
func TestTheOldAndTheNewClaimantAreOneSeatOnAMachine(t *testing.T) {
|
||||
withHostnameAlias(t)
|
||||
cat := shelf(
|
||||
mod("hosts", nil, nil, nil, Claim{Name: "node-hosts-file"}),
|
||||
mod("hostname", nil, nil, nil, Claim{Name: "node-hostname"}),
|
||||
)
|
||||
_, err := Resolve(cat, []string{"hosts", "hostname"}, workstation(), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "node-hostname") {
|
||||
t.Errorf("hosts and hostname both held the machine's names on one machine: %v", err)
|
||||
}
|
||||
if _, err := Resolve(cat, []string{"hosts"}, workstation(), World{}); err != nil {
|
||||
t.Errorf("a machine still assigned hosts under the old name does not resolve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's module: /etc/hostname is the operator's `hostname` setting. Without it the module is
|
||||
// left out, naming the key — the mesh never renames a machine on its own — and a mesh-wide setting
|
||||
// naming ${machine:name} gives every machine its mesh name.
|
||||
func TestTheMachinesNameIsItsSetting(t *testing.T) {
|
||||
cat := map[string]Manifest{"hostname": catalogueManifest(t, "hostname")}
|
||||
got, err := Resolve(cat, []string{"hostname"}, Node{Name: "ace", At: "ace.internal"}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
machines := map[string]string{"ace.internal": "10.42.0.2"}
|
||||
nameOf := func(settings SettingsBy) (string, string) {
|
||||
t.Helper()
|
||||
composed, err := got.Compose(Rendering{Names: machines, Machines: machines, Suffix: "internal",
|
||||
Settings: settings})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := composed.LeftOut["hostname"]; why != "" {
|
||||
return "", why
|
||||
}
|
||||
for _, r := range composed.Resources {
|
||||
if r["path"] == "/etc/hostname" {
|
||||
return r["content"].(string), ""
|
||||
}
|
||||
}
|
||||
t.Fatal("no /etc/hostname composed")
|
||||
return "", ""
|
||||
}
|
||||
|
||||
if _, why := nameOf(nil); !strings.Contains(why, "hostname") {
|
||||
t.Errorf("with no setting the machine's name was written, or left out for another reason: %q", why)
|
||||
}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {{From: "ace", Values: map[string]any{"hostname": "Ace"}}}}); name != "Ace\n" {
|
||||
t.Errorf("the operator's name for the machine gave %q (%s)", name, why)
|
||||
}
|
||||
mesh := Layer{From: "the mesh", Values: map[string]any{"hostname": "${machine:name}"}}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {mesh}}); name != "ace\n" {
|
||||
t.Errorf("a mesh-wide ${machine:name} gave %q (%s)", name, why)
|
||||
}
|
||||
node := Layer{From: "ace", Values: map[string]any{"hostname": "Ace"}}
|
||||
if name, why := nameOf(SettingsBy{"hostname": {mesh, node}}); name != "Ace\n" {
|
||||
t.Errorf("a machine's own name over the mesh-wide one gave %q (%s)", name, why)
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package catalogue_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq issue 128, held where the host will see it: the shipped networking module's names,
|
||||
// through the whole composition, and not only through FactsInto.
|
||||
//
|
||||
// Composition prefixes a fact's id with the module that asked for it and passes everything else
|
||||
// through; a step that dropped `into` on the way would send the region as a whole file, and the
|
||||
// host would write the machine's hosts file over again with every unit test above still green.
|
||||
|
||||
// onTheNetwork stands in for the overlay's generator: the node is part of the private network,
|
||||
// and what the generator writes is not what is under test here.
|
||||
type onTheNetwork struct{}
|
||||
|
||||
func (onTheNetwork) Resources(string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{{"id": "overlay-config", "type": "file",
|
||||
"path": "/etc/wireguard/mesh0.conf", "mode": "0600", "content": "[Interface]\n"}}, true, nil
|
||||
}
|
||||
|
||||
func TestTheHostsRegionArrivesAsTheHostWillReadIt(t *testing.T) {
|
||||
shelf := provided(t)
|
||||
// A resolver restarting on the names another module put on the machine, and one resource it
|
||||
// only runs at start — neither of which composition has any business changing.
|
||||
resolver, err := catalogue.ParseManifest([]byte(`{
|
||||
"module": "resolver", "version": "1", "requires": ["mesh-addressing"],
|
||||
"resources": [
|
||||
{"id": "seed", "type": "file", "path": "/etc/resolver/seed", "mode": "0644",
|
||||
"content": "seed\n", "at": "start"},
|
||||
{"id": "daemon", "type": "service", "unit": "resolver.service", "state": "running",
|
||||
"restart-on": ["seed", "mesh-wireguard.fact-node-names"]}
|
||||
]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shelf[resolver.Module] = resolver
|
||||
|
||||
got, err := catalogue.Resolve(shelf, []string{overlay.Domain, "resolver"},
|
||||
catalogue.Node{Name: "homer", At: "homer.internal"}, catalogue.World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names := map[string]string{"homer.internal": "10.42.0.1", "marge.internal": "10.42.0.2"}
|
||||
out, err := got.Declaration(catalogue.Rendering{
|
||||
Names: names, Machines: names, Suffix: "internal",
|
||||
Generators: map[string]catalogue.Generator{overlay.Name: onTheNetwork{}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := map[string]map[string]any{}
|
||||
for _, r := range out {
|
||||
ids[r["id"].(string)] = r
|
||||
}
|
||||
|
||||
hosts := ids[overlay.Name+".fact-node-names"]
|
||||
if hosts == nil {
|
||||
t.Fatalf("no names reached the machine; the declaration has %v", keys(ids))
|
||||
}
|
||||
if hosts["path"] != "/etc/hosts" || hosts["into"] != "block" {
|
||||
t.Fatalf("the hosts file is not written into as a region: %v", hosts)
|
||||
}
|
||||
content := hosts["content"].(string)
|
||||
if !strings.Contains(content, "10.42.0.1\thomer.internal\thomer\t# this machine\n") {
|
||||
t.Errorf("the region does not name the machine:\n%s", content)
|
||||
}
|
||||
for _, floor := range []string{"Generated by the mesh", "localhost", "127.0.1.1"} {
|
||||
if strings.Contains(content, floor) {
|
||||
t.Errorf("the region carries %q, which is the machine's:\n%s", floor, content)
|
||||
}
|
||||
}
|
||||
|
||||
// The resolver's reference to it still names a resource the host will be sent.
|
||||
daemon := ids["resolver.daemon"]
|
||||
if daemon == nil {
|
||||
t.Fatalf("the resolver's service was not composed: %v", keys(ids))
|
||||
}
|
||||
for _, named := range daemon["restart-on"].([]any) {
|
||||
if ids[named.(string)] == nil {
|
||||
t.Errorf("the resolver restarts on %v, which is nothing the host is sent", named)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(daemon["restart-on"], []any{"resolver.seed", overlay.Name + ".fact-node-names"}) {
|
||||
t.Errorf("restart-on is %v", daemon["restart-on"])
|
||||
}
|
||||
|
||||
// And a resource's own `at` passes through as the manifest wrote it.
|
||||
if seed := ids["resolver.seed"]; seed == nil || seed["at"] != "start" {
|
||||
t.Errorf("a resource's at did not survive composition: %v", seed)
|
||||
}
|
||||
}
|
||||
|
||||
func keys(m map[string]map[string]any) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package catalogue
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -56,13 +57,36 @@ func ConsumerIdentity(node, module string) string {
|
||||
return IdentityPrefix + clean(node) + "_" + clean(module)
|
||||
}
|
||||
|
||||
// identityLimit is the shortest identifier limit among the systems these names reach: an S3 access
|
||||
// key's 20 (novox/hq 04-ISSUES/010). PostgreSQL keeps 63 and MinIO 20, so 20 is the one that binds —
|
||||
// the comment used to name PostgreSQL and was wrong. A name over it is refused, with the remedy a
|
||||
// short slug (ADR 0049), not silently cut to fit.
|
||||
const identityLimit = 20
|
||||
// IdentityBound is the longest consumer identity one provision's backend keeps, and what keeps it
|
||||
// (novox/hq ADR 0049, refined by ADR 0225). Max zero means no bound: the provision keeps no name
|
||||
// derived from its consumer, or keeps one in something with no limit the mesh need respect.
|
||||
type IdentityBound struct {
|
||||
// Max is the longest identity that backend keeps, in characters; zero for none.
|
||||
Max int `json:"max,omitempty"`
|
||||
// In is what keeps it, in words a refusal can quote: "an S3 access key", "a PostgreSQL role".
|
||||
In string `json:"in,omitempty"`
|
||||
}
|
||||
|
||||
// CheckIdentity refuses an identity that would not fit the tightest backend a consumer reaches.
|
||||
// Bounded is whether this bound refuses anything.
|
||||
func (b IdentityBound) Bounded() bool { return b.Max > 0 }
|
||||
|
||||
// DefaultIdentityLimit is the bound on a provision whose provider receives its consumers and does
|
||||
// not say how long a name it keeps: an S3 access key's 20 (novox/hq 04-ISSUES/010, 034), the
|
||||
// tightest backend the mesh has met. It was the bound on every provision until ADR 0225; it stays
|
||||
// the bound on any that has not said otherwise, because a provider that is told each consumer's
|
||||
// identity may create a name from it in a backend nobody has measured.
|
||||
const DefaultIdentityLimit = 20
|
||||
|
||||
// DefaultIdentityBound is DefaultIdentityLimit, said as a bound.
|
||||
var DefaultIdentityBound = IdentityBound{Max: DefaultIdentityLimit,
|
||||
In: "a backend that has not said its limit (the tightest known, an S3 access key's)"}
|
||||
|
||||
// identityLimit is the bound CheckIdentity applies, for a caller that does not know which provision
|
||||
// the identity is for.
|
||||
const identityLimit = DefaultIdentityLimit
|
||||
|
||||
// CheckIdentity refuses an identity that would not fit the tightest backend the mesh knows. A caller
|
||||
// that knows the provision uses CheckIdentityWithin and that provision's own bound (ADR 0225).
|
||||
//
|
||||
// **Truncation is not an error in most of these systems** — a name past the limit is cut to fit and
|
||||
// the statement succeeds, so two consumers agreeing for the first N bytes would become one login
|
||||
@@ -70,12 +94,127 @@ const identityLimit = 20
|
||||
// name and is the only thing that can choose another. The remedy is a first-class one: give the
|
||||
// module a short `slug` (ADR 0049), or shorten the machine's name.
|
||||
func CheckIdentity(node, module string) error {
|
||||
return CheckIdentityWithin(node, module, IdentityBound{Max: identityLimit, In: "a backend (an S3 access key)"})
|
||||
}
|
||||
|
||||
// CheckIdentityWithin refuses an identity that would not fit one provision's bound, and accepts any
|
||||
// identity for a provision with none (novox/hq ADR 0225).
|
||||
func CheckIdentityWithin(node, module string, bound IdentityBound) error {
|
||||
if !bound.Bounded() {
|
||||
return nil
|
||||
}
|
||||
got := ConsumerIdentity(node, module)
|
||||
if len(got) <= identityLimit {
|
||||
if len(got) <= bound.Max {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"%s on %s is identified as %q, %d characters where a backend (an S3 access key) keeps %d — "+
|
||||
"%s on %s is identified as %q, %d characters where %s keeps %d — "+
|
||||
"give the module a shorter `slug` or shorten the machine's name",
|
||||
module, node, got, len(got), identityLimit)
|
||||
module, node, got, len(got), bound.In, bound.Max)
|
||||
}
|
||||
|
||||
// Overflow is one consumer whose identity does not fit the provision it requires: left out of its
|
||||
// provider's grants and reported, never a reason to refuse the provider's machine (ADR 0225).
|
||||
type Overflow struct {
|
||||
// Provision is what was required, Provider the machine answering it.
|
||||
Provision string `json:"provision"`
|
||||
Provider string `json:"provider"`
|
||||
// Consumer is the machine, Module the module on it that required it.
|
||||
Consumer string `json:"consumer"`
|
||||
Module string `json:"module"`
|
||||
// Identity is the name the mesh derived, and Bound what it overflows.
|
||||
Identity string `json:"identity"`
|
||||
Bound IdentityBound `json:"bound"`
|
||||
}
|
||||
|
||||
func (o Overflow) String() string {
|
||||
return fmt.Sprintf("%s on %s requires %s from %s and is identified as %q, %d characters where %s "+
|
||||
"keeps %d — left out of %s's grants until the module's `slug` is shorter",
|
||||
o.Module, o.Consumer, o.Provision, o.Provider, o.Identity, len(o.Identity), o.Bound.In,
|
||||
o.Bound.Max, o.Provider)
|
||||
}
|
||||
|
||||
// Overflowing is every requirement of this machine's modules whose identity overflows the bound of
|
||||
// the provision answering it. The same judgement the provider's composition makes before it grants
|
||||
// (grantsFor), made from the consumer's side so `status` can say it about every machine.
|
||||
func (r Resolution) Overflowing() []Overflow {
|
||||
slugs := map[string]string{}
|
||||
for _, m := range r.Modules {
|
||||
slugs[m.Module] = m.Slug
|
||||
}
|
||||
var out []Overflow
|
||||
seen := map[string]bool{}
|
||||
for _, n := range r.Needs {
|
||||
if n.ByRecord || !n.Identity.Bounded() {
|
||||
continue
|
||||
}
|
||||
source := IdentitySource(slugs[n.For], n.For)
|
||||
if CheckIdentityWithin(r.Node, source, n.Identity) == nil {
|
||||
continue
|
||||
}
|
||||
key := n.Name + "\x00" + n.From + "\x00" + n.For
|
||||
if seen[key] {
|
||||
continue // one line per requirement, however many local names it has
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, Overflow{Provision: n.Name, Provider: n.From, Consumer: r.Node, Module: n.For,
|
||||
Identity: ConsumerIdentity(r.Node, source), Bound: n.Identity})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Module != out[j].Module {
|
||||
return out[i].Module < out[j].Module
|
||||
}
|
||||
return out[i].Provision < out[j].Provision
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// DefaultLongestMachine is the machine name the catalogue check judges identities on when it is not
|
||||
// told one: the longest name of the mesh this catalogue is written for, so a catalogue that passes
|
||||
// passes on every machine that mesh has. `module check --longest-machine-name` says another mesh's;
|
||||
// a mesh that names a longer machine raises this in the same change (ADR 0225).
|
||||
const DefaultLongestMachine = 6
|
||||
|
||||
// IdentityProblems is every module whose identity would overflow a provision it wants, on a machine
|
||||
// whose name is `longestMachine` characters — judged before merge, over the catalogue alone, so the
|
||||
// pull request that introduces an overflow is the one refused (novox/hq ADR 0225, issue 263). A
|
||||
// provision no module in the shelf offers is not judged: its bound is not known here.
|
||||
func IdentityProblems(shelf Shelf, longestMachine int) []string {
|
||||
offeredBy := map[string][]string{}
|
||||
for _, name := range shelfOrder(shelf) {
|
||||
for _, o := range shelf[name].Offers() {
|
||||
offeredBy[o] = append(offeredBy[o], name)
|
||||
}
|
||||
}
|
||||
machine := strings.Repeat("n", longestMachine)
|
||||
var problems []string
|
||||
for _, name := range shelfOrder(shelf) {
|
||||
m := shelf[name]
|
||||
source := IdentitySource(m.Slug, m.Module)
|
||||
for _, want := range m.Wants() {
|
||||
// The tightest bound among the modules offering it: whichever one answers on a given
|
||||
// machine, the identity has to fit it.
|
||||
tightest, by := IdentityBound{}, ""
|
||||
for _, provider := range offeredBy[want] {
|
||||
if provider == name {
|
||||
continue // a module answering its own requirement is not its own consumer
|
||||
}
|
||||
b := shelf[provider].IdentityBoundOf(want)
|
||||
if b.Bounded() && (!tightest.Bounded() || b.Max < tightest.Max) {
|
||||
tightest, by = b, provider
|
||||
}
|
||||
}
|
||||
if CheckIdentityWithin(machine, source, tightest) == nil {
|
||||
continue
|
||||
}
|
||||
got := ConsumerIdentity(machine, source)
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s wants %s, and %s keeps its consumers' identities in %s of at most %d characters: "+
|
||||
"on a machine with a %d-character name it is identified as %q, %d — give %s a "+
|
||||
"`slug` of at most %d characters",
|
||||
name, want, by, tightest.In, tightest.Max, longestMachine, got, len(got), name,
|
||||
tightest.Max-len(IdentityPrefix)-longestMachine-1))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Each test names the decision it defends: novox/hq ADR 0225, which refines ADR 0049 after issue 263.
|
||||
|
||||
// The provision a module requires sets the bound on its identity, not the tightest backend anywhere.
|
||||
func TestABoundIsTheProvisionsOwn(t *testing.T) {
|
||||
store := Manifest{Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}
|
||||
database := Manifest{Module: "db", Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 63, In: "a PostgreSQL role"}}},
|
||||
Receives: map[string]string{"postgres-database": "/var/lib/mesh/db/mesh.json"}}
|
||||
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 || b.In != "an S3 access key" {
|
||||
t.Errorf("an object store's stated bound was not taken: %+v", b)
|
||||
}
|
||||
if b := database.IdentityBoundOf("postgres-database"); b.Max != 63 {
|
||||
t.Errorf("a database's stated bound was not taken: %+v", b)
|
||||
}
|
||||
// mesh_workstation_keycloak is 25: refused by the object store, accepted by the database.
|
||||
if CheckIdentityWithin("workstation", "keycloak", store.IdentityBoundOf("s3-bucket")) == nil {
|
||||
t.Error("a 25-character identity fit a 20-character access key")
|
||||
}
|
||||
if err := CheckIdentityWithin("workstation", "keycloak", database.IdentityBoundOf("postgres-database")); err != nil {
|
||||
t.Errorf("a database consumer paid the object store's limit: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// What an offer leaves unsaid follows from whether its provider can keep a name at all.
|
||||
func TestAnUnstatedBoundFollowsWhatTheProviderIsTold(t *testing.T) {
|
||||
// Told nothing about its consumers — no receives, nothing served from their identity: no bound.
|
||||
// The resolver provision is exactly this, and its consumers paid an object store's limit (263).
|
||||
resolver := Manifest{Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}}
|
||||
if b := resolver.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||
t.Errorf("a provision that is told nothing of its consumers bounds them: %+v", b)
|
||||
}
|
||||
// Told each consumer and silent about its backend: the old global bound, not none.
|
||||
told := Manifest{Module: "told", Provides: []Offer{{Name: "thing", Scope: ScopeMesh}},
|
||||
Receives: map[string]string{"thing": "/var/lib/mesh/told/mesh.json"}}
|
||||
if b := told.IdentityBoundOf("thing"); b.Max != DefaultIdentityLimit {
|
||||
t.Errorf("a provider told its consumers and silent about its backend is not held to %d: %+v",
|
||||
DefaultIdentityLimit, b)
|
||||
}
|
||||
// Serving a value built from the identity is being told it, too.
|
||||
serving := Manifest{Module: "serving", Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"bucket": {"name": "b-${consumer:as:dns}"}}}
|
||||
if b := serving.IdentityBoundOf("bucket"); b.Max != DefaultIdentityLimit {
|
||||
t.Errorf("a provider deriving a name from its consumers is not bounded: %+v", b)
|
||||
}
|
||||
// And `false` says it outright, even for a provider that receives.
|
||||
routes := Manifest{Module: "routes", Provides: []Offer{{Name: "route", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{None: true}}},
|
||||
Receives: map[string]string{"route": "/var/lib/mesh/routes/mesh.json"}}
|
||||
if b := routes.IdentityBoundOf("route"); b.Bounded() {
|
||||
t.Errorf("`identity: false` still bounds: %+v", b)
|
||||
}
|
||||
}
|
||||
|
||||
// The field reads as written and writes back the same, and refuses what says nothing.
|
||||
func TestAnOffersIdentityIsParsedStrictly(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
`{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}`,
|
||||
`{"name":"wildcard-resolution","scope":"mesh","identity":false}`,
|
||||
`{"name":"redis-cache","scope":"mesh","identity":{"in":"a Redis ACL user"}}`,
|
||||
} {
|
||||
var o Offer
|
||||
if err := json.Unmarshal([]byte(raw), &o); err != nil {
|
||||
t.Fatalf("%s: %v", raw, err)
|
||||
}
|
||||
back, err := json.Marshal(o)
|
||||
if err != nil || string(back) != raw {
|
||||
t.Errorf("did not round-trip:\n%s\n%s (%v)", raw, back, err)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{
|
||||
`{"name":"x","identity":true}`,
|
||||
`{"name":"x","identity":{"max":20,"in":"y","most":3}}`,
|
||||
} {
|
||||
var o Offer
|
||||
if err := json.Unmarshal([]byte(raw), &o); err == nil {
|
||||
t.Errorf("accepted %s", raw)
|
||||
}
|
||||
}
|
||||
bad := Manifest{Module: "bad", Version: "1", Provides: []Offer{
|
||||
{Name: "unsaid", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 20}},
|
||||
{Name: "tiny", Scope: ScopeMesh, Identity: &OfferIdentity{Max: 4, In: "nothing usable"}},
|
||||
}}
|
||||
raw, err := json.Marshal(bad)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), "without saying what keeps them") ||
|
||||
!strings.Contains(err.Error(), "the shortest the mesh makes") {
|
||||
t.Fatalf("a bound with no `in`, or too short for any identity, was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Refused before merge: the catalogue check judges each module's identity, on the longest machine
|
||||
// name, against the bound of every provision it wants — and names the module and the slug to set.
|
||||
func TestTheCatalogueCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"objects": {Module: "objects", Provides: []Offer{{Name: "s3-bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 20, In: "an S3 access key"}}},
|
||||
Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}},
|
||||
"photoalbum": {Module: "photoalbum", Requires: []string{"s3-bucket"}},
|
||||
"files": {Module: "files", Requires: []string{"s3-bucket"}},
|
||||
}
|
||||
problems := IdentityProblems(shelf, 6)
|
||||
if len(problems) != 1 || !strings.Contains(problems[0], "photoalbum wants s3-bucket") ||
|
||||
!strings.Contains(problems[0], `"mesh_nnnnnn_photoalbum", 22`) ||
|
||||
!strings.Contains(problems[0], "`slug` of at most 8 characters") {
|
||||
t.Fatalf("one overflow, named with its remedy, was expected: %q", problems)
|
||||
}
|
||||
// A longer machine name refuses more: the check is about the mesh's machines, not one.
|
||||
if got := IdentityProblems(shelf, 10); len(got) != 2 {
|
||||
t.Fatalf("on a 10-character name both overflow (mesh_nnnnnnnnnn_files is 21): %q", got)
|
||||
}
|
||||
// And a slug is the remedy it names.
|
||||
album := shelf["photoalbum"]
|
||||
album.Slug = "album"
|
||||
shelf["photoalbum"] = album
|
||||
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||
t.Fatalf("a slug that fits is still refused: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Tonight's case (issue 263): networkmanager, no slug, requiring the mesh's resolver provision on a
|
||||
// machine with a six-character name. Under ADR 0049's one bound it was refused, and its provider's
|
||||
// whole machine with it; the resolver keeps no name, so it is not refused at all.
|
||||
func TestARequirementOnAKeylessProvisionComposesWithALongName(t *testing.T) {
|
||||
shelf := Shelf{
|
||||
"resolver": {Module: "resolver", Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}}},
|
||||
"networkmanager": {Module: "networkmanager", Requires: []string{"wildcard-resolution"}},
|
||||
}
|
||||
if CheckIdentity("laptop", "networkmanager") == nil {
|
||||
t.Fatal("the regression is not reproduced: mesh_laptop_networkmanager fits the old global bound")
|
||||
}
|
||||
if got := IdentityProblems(shelf, 6); len(got) != 0 {
|
||||
t.Fatalf("a requirement on a keyless provision was refused for its length: %q", got)
|
||||
}
|
||||
r := Resolution{Node: "laptop", Modules: []Manifest{shelf["networkmanager"]},
|
||||
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", For: "networkmanager",
|
||||
Identity: shelf["resolver"].IdentityBoundOf("wildcard-resolution")}}}
|
||||
if got := r.Overflowing(); len(got) != 0 {
|
||||
t.Fatalf("a keyless requirement is reported as overflowing: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The consumer's side of the same judgement the provider's composition makes: what `status` says.
|
||||
func TestAnOverflowingRequirementIsNamedFromTheConsumersSide(t *testing.T) {
|
||||
bound := IdentityBound{Max: 20, In: "an S3 access key"}
|
||||
r := Resolution{Node: "laptop",
|
||||
Modules: []Manifest{{Module: "photoalbum"}, {Module: "files"}, {Module: "gallery", Slug: "gal"}},
|
||||
Needs: []Needed{
|
||||
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "photoalbum", Local: "second", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "files", Identity: bound},
|
||||
{Name: "s3-bucket", From: "anchor", For: "gallery", Identity: bound},
|
||||
{Name: "licence", From: "records", For: "photoalbum", ByRecord: true, Identity: bound},
|
||||
}}
|
||||
got := r.Overflowing()
|
||||
if len(got) != 1 || got[0].Module != "photoalbum" || got[0].Identity != "mesh_laptop_photoalbum" ||
|
||||
got[0].Provider != "anchor" {
|
||||
t.Fatalf("one overflow, once, was expected: %+v", got)
|
||||
}
|
||||
if said := got[0].String(); !strings.Contains(said, "an S3 access key keeps 20") ||
|
||||
!strings.Contains(said, "slug") {
|
||||
t.Fatalf("the overflow does not say what keeps it or the remedy: %s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// An identity served as a DNS label is never truncated, so the offer's bound must keep it in one.
|
||||
func TestAnIdentityServedAsADNSLabelIsBoundedToOne(t *testing.T) {
|
||||
serves := map[string]map[string]any{"bucket": {"name": "${consumer:as:dns}"}}
|
||||
wide := Manifest{Module: "wide", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh,
|
||||
Identity: &OfferIdentity{Max: 255, In: "a client id"}}}}
|
||||
if got := CheckServes(wide); len(got) != 1 || !strings.Contains(got[0], "a label keeps 63") {
|
||||
t.Fatalf("a 255-character bound on a DNS label passed: %q", got)
|
||||
}
|
||||
unsaid := Manifest{Module: "unsaid", Serves: serves, Provides: []Offer{{Name: "bucket", Scope: ScopeMesh}}}
|
||||
if got := CheckServes(unsaid); len(got) != 0 {
|
||||
t.Fatalf("the default bound (20) on a DNS label was refused: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ package catalogue
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
@@ -153,6 +154,84 @@ type Offer struct {
|
||||
// a seat's holder gated by the machine's graphical session, and pulling one in for whatever asked
|
||||
// is the misassignment research 026 found. Unmet, the requirement is refused naming who could.
|
||||
Reach string `json:"reach,omitempty"`
|
||||
// Identity is the longest consumer identity this provision's backend keeps (novox/hq ADR 0225):
|
||||
// `{"max": 63, "in": "a PostgreSQL role"}`, or `false` for a provision that keeps no name derived
|
||||
// from its consumer. Unsaid, the mesh assumes the tightest backend it knows when the provider is
|
||||
// told its consumers, and no bound when it is not — see IdentityBoundOf.
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
|
||||
// OfferIdentity is what an offer says about the names its backend keeps for its consumers.
|
||||
type OfferIdentity struct {
|
||||
// None is set by `"identity": false`: the provision keeps no name derived from its consumer.
|
||||
None bool
|
||||
// Max is the longest identity kept, in characters; zero with In set means no limit worth stating.
|
||||
Max int
|
||||
// In is what keeps it, for a refusal to quote.
|
||||
In string
|
||||
}
|
||||
|
||||
// UnmarshalJSON accepts `false` or `{"max": N, "in": "..."}`.
|
||||
func (i *OfferIdentity) UnmarshalJSON(raw []byte) error {
|
||||
var flag bool
|
||||
if err := json.Unmarshal(raw, &flag); err == nil {
|
||||
if flag {
|
||||
return errors.New("an offer's identity is false (it keeps no name) or {max, in}; true says nothing")
|
||||
}
|
||||
*i = OfferIdentity{None: true}
|
||||
return nil
|
||||
}
|
||||
var full struct {
|
||||
Max int `json:"max,omitempty"`
|
||||
In string `json:"in"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("an offer's identity is false or {max, in}: %w", err)
|
||||
}
|
||||
*i = OfferIdentity{Max: full.Max, In: full.In}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON writes it back in the form it was written.
|
||||
func (i OfferIdentity) MarshalJSON() ([]byte, error) {
|
||||
if i.None {
|
||||
return []byte("false"), nil
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
Max int `json:"max,omitempty"`
|
||||
In string `json:"in"`
|
||||
}{i.Max, i.In})
|
||||
}
|
||||
|
||||
// IdentityBoundOf is the bound this module's offer of a provision puts on its consumers' identities
|
||||
// (novox/hq ADR 0225).
|
||||
//
|
||||
// **What an offer says, it gets.** Where it says nothing, the bound follows from whether the
|
||||
// provider can keep a name at all: a provider that receives the provision, or serves its consumers
|
||||
// a value built from their identity, is told who each consumer is and may create a name from it in
|
||||
// a backend nobody measured — so it keeps the old global bound, DefaultIdentityBound. One that does
|
||||
// neither is told nothing about its consumers and keeps nothing of them: no bound. The resolver
|
||||
// provision is that case, and its consumers paid an object store's limit until this (issue 263).
|
||||
func (m Manifest) IdentityBoundOf(provision string) IdentityBound {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name != provision || o.Identity == nil {
|
||||
continue
|
||||
}
|
||||
if o.Identity.None {
|
||||
return IdentityBound{}
|
||||
}
|
||||
return IdentityBound{Max: o.Identity.Max, In: o.Identity.In}
|
||||
}
|
||||
if _, receives := m.Receives[provision]; receives {
|
||||
return DefaultIdentityBound
|
||||
}
|
||||
if served, err := json.Marshal(m.Serves[provision]); err == nil &&
|
||||
bytes.Contains(served, []byte("${consumer:as")) {
|
||||
return DefaultIdentityBound
|
||||
}
|
||||
return IdentityBound{}
|
||||
}
|
||||
|
||||
// MachineReach is whether a provision is usable only on its provider's own machine.
|
||||
@@ -206,20 +285,21 @@ func (o *Offer) UnmarshalJSON(raw []byte) error {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Credential *OfferCredential `json:"credential,omitempty"`
|
||||
Reach string `json:"reach,omitempty"`
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&full); err != nil {
|
||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach}: %w", err)
|
||||
return fmt.Errorf("a provided name is either a string or {name, scope, credential, reach, identity}: %w", err)
|
||||
}
|
||||
o.Name, o.Scope, o.Credential, o.Reach = full.Name, full.Scope, full.Credential, full.Reach
|
||||
o.Name, o.Scope, o.Credential, o.Reach, o.Identity = full.Name, full.Scope, full.Credential, full.Reach, full.Identity
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
|
||||
// went through the mesh comes out looking like the one that went in.
|
||||
func (o Offer) MarshalJSON() ([]byte, error) {
|
||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" {
|
||||
if o.Scope == "" && o.Credential == nil && o.Reach == "" && o.Identity == nil {
|
||||
return json.Marshal(o.Name)
|
||||
}
|
||||
return json.Marshal(struct {
|
||||
@@ -227,7 +307,8 @@ func (o Offer) MarshalJSON() ([]byte, error) {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Credential *OfferCredential `json:"credential,omitempty"`
|
||||
Reach string `json:"reach,omitempty"`
|
||||
}{o.Name, o.Scope, o.Credential, o.Reach})
|
||||
Identity *OfferIdentity `json:"identity,omitempty"`
|
||||
}{o.Name, o.Scope, o.Credential, o.Reach, o.Identity})
|
||||
}
|
||||
|
||||
// Manifest is everything a module says about itself.
|
||||
@@ -237,9 +318,10 @@ type Manifest struct {
|
||||
|
||||
// Slug is a short identifier the mesh uses in place of the module name when it derives a
|
||||
// consumer's login (novox/hq ADR 0049). Optional: a module with a short name needs none. It
|
||||
// exists because `mesh_<node>_<module>` must fit the tightest backend a consumer reaches — an S3
|
||||
// access key is 20 characters — and a long module name would overflow it. A person choosing
|
||||
// `kc` for keycloak keeps the identity legible where a hash would not.
|
||||
// exists because `mesh_<node>_<module>` must fit the bound of every provision the module
|
||||
// requires — an S3 access key's 20 characters is the tightest — and a long module name would
|
||||
// overflow it (ADR 0225: the bound is the provision's own, and a keyless one has none). A person
|
||||
// choosing `kc` for keycloak keeps the identity legible where a hash would not.
|
||||
Slug string `json:"slug,omitempty"`
|
||||
|
||||
// Provides are the names other modules may require. A module always provides its own name;
|
||||
@@ -1265,8 +1347,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
|
||||
}
|
||||
// A slug is a short identifier the mesh derives a login from (novox/hq ADR 0049). The same
|
||||
// charset as a name; its length is checked against a backend's limit at assignment, where the
|
||||
// node it joins is known — a slug that is fine on one machine's short name can overflow another's.
|
||||
// charset as a name; its length is judged against the bound of each provision it wants on the
|
||||
// longest machine name by the catalogue check (IdentityProblems, ADR 0225), and against the
|
||||
// machine it is on when its provider grants it — a slug that is fine on one machine's short name
|
||||
// can overflow another's.
|
||||
if m.Slug != "" && !name.MatchString(m.Slug) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%q is not a usable slug: lower-case letters, digits, dashes and dots", m.Slug))
|
||||
@@ -1303,6 +1387,20 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
||||
if !name.MatchString(p) {
|
||||
problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
|
||||
}
|
||||
// A bound says what keeps the name, so the refusal it causes can say it (ADR 0225); and it
|
||||
// leaves room for the shortest identity the mesh makes, or it would refuse every consumer.
|
||||
if id := offer.Identity; id != nil && !id.None {
|
||||
if strings.TrimSpace(id.In) == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s bounds the identities of %s's consumers without saying what keeps them: `in`",
|
||||
m.Module, p))
|
||||
}
|
||||
if shortest := len(IdentityPrefix) + 3; id.Max < 0 || id.Max > 0 && id.Max < shortest {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s bounds %s's consumers' identities at %d characters, and the shortest the mesh "+
|
||||
"makes is %d", m.Module, p, id.Max, shortest))
|
||||
}
|
||||
}
|
||||
if offer.Credential != nil {
|
||||
own, declared := m.OwnSecrets[offer.Credential.Own]
|
||||
switch {
|
||||
|
||||
@@ -50,12 +50,14 @@ func TestTheShippedNetworkingModulesResolveOnTheirOwn(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// **The names come WITH the network now, not from a third module.** Being on the private
|
||||
// network is what gives a machine a name, so the provider asks for the node-names fact and
|
||||
// there is nothing else to bring in. A module that ran nothing used to be here.
|
||||
// **The network writes no names** (novox/hq ADR 0199): /etc/hosts is the hosts seat holder's
|
||||
// file, and the mesh's resolver answers the machines' names. No fact of the network's module
|
||||
// may name that file.
|
||||
for _, m := range got.Modules {
|
||||
if m.Module == overlay.Name && m.Facts["node-names"].Path == "" {
|
||||
t.Fatalf("the network's provider does not ask for the names: %+v", m.Facts)
|
||||
for name, f := range m.Facts {
|
||||
if m.Module == overlay.Name && f.Path == "/etc/hosts" {
|
||||
t.Fatalf("the network's provider still writes /etc/hosts, as its %q fact", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+116
-13
@@ -194,6 +194,11 @@ type Needed struct {
|
||||
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
|
||||
// licence's manager; empty for every consumer.
|
||||
Manager bool
|
||||
// Identity is the longest consumer identity the answering provision keeps (novox/hq ADR 0225),
|
||||
// from the provider's own offer: what the mesh judges this consumer's identity against, on the
|
||||
// consumer's side for `status` and on the provider's before it grants. No bound for a provision
|
||||
// answered by a record, which keeps no name of anybody's.
|
||||
Identity IdentityBound
|
||||
}
|
||||
|
||||
// Refusal is why a set of assignments cannot become a declaration.
|
||||
@@ -341,7 +346,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
// trust boundary the moment both ends are containers**, and treating it as one gave the
|
||||
// commonest arrangement of all — a service and its database on one node — the weakest
|
||||
// handling, silently.
|
||||
if satisfied[want] && !isModule(catalogue, want) {
|
||||
if satisfied[want] && !isModule(catalogue, want) && !answeredElsewhere(want, node, world, brokered) {
|
||||
here := func(name string) bool { return chosen[name] || assignedHere[name] }
|
||||
local := providersHere(catalogue, here, want)
|
||||
// Which of them it matters to choose between. A plain capability — a shell, a display
|
||||
@@ -403,7 +408,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
needs = append(needs, Needed{
|
||||
Name: want, From: node.Name, At: at,
|
||||
Serves: servedByOne(by, want), For: because[want],
|
||||
SharedOwn: sharedByOne(by, want)})
|
||||
SharedOwn: sharedByOne(by, want), Identity: by.IdentityBoundOf(want)})
|
||||
} else if served := servedByOne(by, want); len(served) > 0 {
|
||||
// Answered here with no credential to mint, but the provider serves facts the
|
||||
// consumer cannot guess — a port, a model name — and so still needs a binding.
|
||||
@@ -447,11 +452,15 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
|
||||
return
|
||||
}
|
||||
shared := ""
|
||||
// A provider whose definition is not in hand is held to the tightest bound the
|
||||
// mesh knows, not to none: what it keeps is not known here (ADR 0225).
|
||||
bound := DefaultIdentityBound
|
||||
if pm, known := catalogue[p.Module]; known {
|
||||
shared, _ = pm.SharedCredentialOf(want)
|
||||
bound = pm.IdentityBoundOf(want)
|
||||
}
|
||||
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At,
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared})
|
||||
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
|
||||
}
|
||||
switch {
|
||||
case world.Unchecked:
|
||||
@@ -767,16 +776,27 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
var problems []string
|
||||
var held []Held
|
||||
|
||||
// onRecord is the recorded holder of a seat, if a handover ever named one.
|
||||
onRecord := func(claim, scope string) (Held, bool) {
|
||||
// onRecord is every recorded holder of a seat, if a handover ever named one: one for most seats,
|
||||
// and as many as were added for a replicated one (novox/hq ADR 0223).
|
||||
onRecord := func(claim, scope string) []Held {
|
||||
var out []Held
|
||||
for _, h := range holdings {
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
cs, cok := SeatNamed(claim)
|
||||
if ok && cok && hs.Name == cs.Name && h.Scope == scope {
|
||||
return h, true
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return Held{}, false
|
||||
return out
|
||||
}
|
||||
// recordedHere says this node's module is one of a seat's holders on record.
|
||||
recordedHere := func(claim, scope, module string) bool {
|
||||
for _, rec := range onRecord(claim, scope) {
|
||||
if rec.Node == node.Name && rec.Module == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
byScope := map[string]map[string]string{} // scope → claim → module
|
||||
@@ -787,21 +807,35 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
// the seat but is not the one on record is eligible, and that is all: it is not a second
|
||||
// holder, so it is not refused, and it does not hold (novox/hq ADR 0131). This is what
|
||||
// lets the next holder stand beside the current one until the seat is handed over.
|
||||
if rec, recorded := onRecord(c.Name, scope); recorded {
|
||||
if rec.Node != node.Name || rec.Module != m.Module {
|
||||
if recs := onRecord(c.Name, scope); len(recs) > 0 {
|
||||
// **A seat held once is on record once** (novox/hq ADR 0223). Only a replicated seat
|
||||
// may have several holders on record; several for any other seat is a store that
|
||||
// disagrees with the mesh's definition of the role, and it is refused, named, rather
|
||||
// than letting two machines answer for what the mesh has one of.
|
||||
if s, known := SeatNamed(c.Name); known && !s.Replicated && len(recs) > 1 {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%q is on record as held by %d assignments, and it is held once per %s — "+
|
||||
"`seat %s --to <node>/<module>` records one", c.Name, len(recs), scope, c.Name))
|
||||
continue
|
||||
}
|
||||
if !recordedHere(c.Name, scope, m.Module) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if byScope[scope] == nil {
|
||||
byScope[scope] = map[string]string{}
|
||||
}
|
||||
if other, taken := byScope[scope][c.Name]; taken {
|
||||
// **One seat under either of its names** (novox/hq ADR 0122): a manifest registered before
|
||||
// a rename claims the former name, and one written after it the current — two claimants of
|
||||
// one seat, compared by the seat they resolve to and not by how each spelled it.
|
||||
seat := canonicalSeat(c.Name)
|
||||
if other, taken := byScope[scope][seat]; taken {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s and %s both claim %q, and only one thing may hold it per %s",
|
||||
other, m.Module, c.Name, scope))
|
||||
other, m.Module, seat, scope))
|
||||
continue
|
||||
}
|
||||
byScope[scope][c.Name] = m.Module
|
||||
byScope[scope][seat] = m.Module
|
||||
held = append(held, Held{Claim: c.Name, Scope: scope, Node: node.Name,
|
||||
Module: m.Module, Site: node.Site})
|
||||
}
|
||||
@@ -810,11 +844,17 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
// And against the rest of the mesh, for the scopes that reach past this machine.
|
||||
for _, h := range held {
|
||||
for _, e := range elsewhere {
|
||||
if e.Node == node.Name || e.Claim != h.Claim || e.Scope != h.Scope {
|
||||
if e.Node == node.Name || canonicalSeat(e.Claim) != canonicalSeat(h.Claim) || e.Scope != h.Scope {
|
||||
continue
|
||||
}
|
||||
switch h.Scope {
|
||||
case ScopeMesh:
|
||||
// **A holder on record is never a second claimant** (novox/hq ADR 0223). Records are
|
||||
// the mesh's settled answer: one for most seats, several only for a replicated seat,
|
||||
// each added by an act. Two holders here are two records, and both hold.
|
||||
if recordedHere(h.Claim, h.Scope, h.Module) {
|
||||
continue
|
||||
}
|
||||
// Both claim and nobody is on record, or this refusal could not have happened.
|
||||
// The remedy is the handover that records the holder (novox/hq ADR 0131,
|
||||
// 04-ISSUES/170), so it is named here rather than left to be found.
|
||||
@@ -835,6 +875,15 @@ func checkClaims(modules []Manifest, node Node, elsewhere []Held, holdings []Hel
|
||||
return held, problems
|
||||
}
|
||||
|
||||
// canonicalSeat is the seat a claimed name refers to, by its current name: itself for a name the mesh
|
||||
// does not know (a module's own seat), its seat's name for a former one.
|
||||
func canonicalSeat(name string) string {
|
||||
if s, known := SeatNamed(name); known {
|
||||
return s.Name
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// checkResources refuses two modules writing the same thing.
|
||||
//
|
||||
// This costs no manifest field: the mesh already holds every resource of every module, so two
|
||||
@@ -912,6 +961,23 @@ func checkResources(modules []Manifest) []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// **A file the mesh renders for a module is that module's path too** (novox/hq ADR 0223). The
|
||||
// machine's resolver file is a fact the uplink's holder asks for, and a second module asking
|
||||
// for it, or declaring it, would have the host write one path twice in every apply, the last
|
||||
// one winning. A fact under the operator's home is placed per account and compared nowhere.
|
||||
for _, name := range sortedFacts(m.Facts) {
|
||||
fact := m.Facts[name]
|
||||
if fact.Home || !strings.HasPrefix(fact.Path, "/") {
|
||||
continue
|
||||
}
|
||||
key := "path " + fact.Path
|
||||
if other, taken := owner[key]; taken && other != m.Module {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s and %s both declare the path %q", other, m.Module, fact.Path))
|
||||
}
|
||||
owner[key] = m.Module
|
||||
ownedPath[fact.Path] = m.Module
|
||||
}
|
||||
}
|
||||
|
||||
// An accessed path is the operator's, so no module may declare it as one of its own
|
||||
@@ -1134,3 +1200,40 @@ func machineReachRemedy(catalogue map[string]Manifest, want, node string) string
|
||||
}
|
||||
return fmt.Sprintf("assign one to %s: %s", node, strings.Join(named, "; "))
|
||||
}
|
||||
|
||||
// answeredElsewhere says that a mesh-wide provision this machine could answer itself is answered by
|
||||
// another machine all the same: one this machine was pinned to, or the holder of the mesh seat that
|
||||
// delivers it (novox/hq ADR 0110, ADR 0194).
|
||||
//
|
||||
// **A provider on the machine was taken before either was asked.** The branch for a provision
|
||||
// answered here bound the consumer to the local provider and consulted a pin only between two local
|
||||
// ones, and the seat's holder never; both were read only for a provider on another machine. So when
|
||||
// every machine ran a provider of `wildcard-resolution` — each machine's own resolver, still assigned
|
||||
// while the mesh moved to one — every machine bound its resolver configuration to itself, with the
|
||||
// mesh's one resolver recorded, held and pinned (novox/hq issue 258). The seat is the mesh's choice of
|
||||
// who answers, made once; a provider that merely runs here does not overrule it, and neither does it
|
||||
// overrule a pin somebody set on this machine.
|
||||
//
|
||||
// Not in the first pass, which asks only what this machine offers and has no providers to read.
|
||||
func answeredElsewhere(want string, node Node, world World, brokered map[string]bool) bool {
|
||||
if world.Unchecked || !brokered[want] {
|
||||
return false
|
||||
}
|
||||
if c, pinned := world.Pinned[want]; pinned {
|
||||
return c.Node != node.Name
|
||||
}
|
||||
// **A machine holding the seat answers itself** (novox/hq ADR 0223). With a replicated seat
|
||||
// another machine holds it too, and the first holder in the providers' order may be that one; a
|
||||
// holder is still where this machine's own requirement is answered, so its resolver file lists
|
||||
// itself first.
|
||||
if seat, delivered := SeatDelivering(want); delivered {
|
||||
for _, h := range append(append([]Held(nil), world.Holdings...), world.Held...) {
|
||||
if hs, ok := SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope &&
|
||||
h.Node == node.Name {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
holder, held := HolderAmong(want, world.Offered[want], world.Held)
|
||||
return held && holder.Node != node.Name
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0223 part 2: /etc/resolv.conf belongs to the module holding node-uplink. The seat that
|
||||
// wrote it, node-resolver-config, and ADR 0220's dependency of it on the uplink retire: one owner for
|
||||
// the file, and it is the program that would otherwise rewrite it.
|
||||
|
||||
func TestTheResolverConfigSeatIsGone(t *testing.T) {
|
||||
if _, known := SeatNamed("node-resolver-config"); known {
|
||||
t.Error("node-resolver-config is still in the mesh's set; the uplink's holder writes the resolver file")
|
||||
}
|
||||
// An uplink's holder needs no seat beside it for the file: it is its own.
|
||||
if got := DependsOn(mod("networkmanager", nil, nil, nil, Claim{Name: "node-uplink"})); len(got) != 0 {
|
||||
t.Errorf("an uplink holder with nothing declared depends on %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it is: nothing claims the retired seat, and every manager the mesh knows holds the
|
||||
// uplink and writes the resolver file.
|
||||
func TestTheCataloguesUplinksWriteTheResolverFileAndNothingElseDoes(t *testing.T) {
|
||||
cat := map[string]Manifest{}
|
||||
for _, m := range theCatalogue(t) {
|
||||
cat[m.Module] = m
|
||||
}
|
||||
if got := PossibleHolders(cat, "node-uplink"); !reflect.DeepEqual(got, uplinks) {
|
||||
t.Errorf("node-uplink can be held by %v, not %v", got, uplinks)
|
||||
}
|
||||
for name, m := range cat {
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == "node-resolver-config" {
|
||||
t.Errorf("%s still claims node-resolver-config", name)
|
||||
}
|
||||
}
|
||||
_, writes := m.Facts["resolvers"]
|
||||
isUplink := false
|
||||
for _, u := range uplinks {
|
||||
isUplink = isUplink || u == name
|
||||
}
|
||||
for _, f := range m.Facts {
|
||||
if f.Path == "/etc/resolv.conf" && !isUplink {
|
||||
t.Errorf("%s writes /etc/resolv.conf and does not hold the uplink", name)
|
||||
}
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["path"] == "/etc/resolv.conf" {
|
||||
t.Errorf("%s declares /etc/resolv.conf as a file of its own", name)
|
||||
}
|
||||
}
|
||||
if isUplink && !writes {
|
||||
t.Errorf("%s holds the uplink and does not write the resolver file", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
// same arrangement, and to the two things a resolver here must never do — read resolv.conf for
|
||||
// its upstreams, or take an address systemd-resolved holds.
|
||||
|
||||
// resolverShelf is the three resolver modules beside something that answers `mesh-addressing`.
|
||||
// resolverShelf is the resolver, an uplink module that writes what the machine asks (novox/hq ADR
|
||||
// 0223), and the container runtime beside something that answers `mesh-addressing`.
|
||||
// The networking module that really does is composed in the controller and cannot be imported
|
||||
// here, so a stand-in offers the same word; what is under test is the manifests, not the network.
|
||||
func resolverShelf(t *testing.T) map[string]Manifest {
|
||||
@@ -23,7 +24,7 @@ func resolverShelf(t *testing.T) map[string]Manifest {
|
||||
shelf := map[string]Manifest{
|
||||
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
|
||||
}
|
||||
for _, name := range []string{"dnsmasq", "resolv-conf", "resolved-split-dns"} {
|
||||
for _, name := range []string{"dnsmasq", "systemd-networkd", "docker"} {
|
||||
shelf[name] = catalogueManifest(t, name)
|
||||
}
|
||||
return shelf
|
||||
@@ -83,31 +84,25 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
||||
t.Errorf("the mesh's resolver still reads or listens on %q", never)
|
||||
}
|
||||
}
|
||||
// And the file that decides what the machine asks names the mesh's resolver first, by address,
|
||||
// and a public one second, asked only when the first is silent (ADR 0196).
|
||||
var resolv string
|
||||
for _, r := range catalogueManifest(t, "resolv-conf").Resources {
|
||||
if r["path"] == "/etc/resolv.conf" {
|
||||
resolv, _ = r["content"].(string)
|
||||
// And the file that decides what the machine asks names every one of the mesh's resolvers, by
|
||||
// address, from the seat's holders, and no public one (ADR 0223): a resolver library that asks
|
||||
// every listed server at once takes the first reply, and a public "no such name" for a mesh name
|
||||
// won it. Written by every uplink module, since the uplink's holder owns the file.
|
||||
for _, uplink := range uplinks {
|
||||
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
|
||||
if !ok || fact.Path != "/etc/resolv.conf" {
|
||||
t.Fatalf("%s's resolver file is not rendered from the roster: %+v", uplink, fact)
|
||||
}
|
||||
}
|
||||
var nameservers []string
|
||||
for _, line := range strings.Split(resolv, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") {
|
||||
nameservers = append(nameservers, strings.TrimPrefix(line, "nameserver "))
|
||||
if !strings.Contains(fact.Template, `{{range index .Holders "mesh-dns-resolver"}}nameserver {{.Address}}`) {
|
||||
t.Errorf("%s's resolv.conf does not list every holder of the mesh's resolver:\n%s", uplink, fact.Template)
|
||||
}
|
||||
}
|
||||
if len(nameservers) != 2 || nameservers[0] != "${bound:wildcard-resolution:address}" || nameservers[1] != "1.1.1.1" {
|
||||
t.Errorf("resolv.conf names %v; the mesh's resolver by address first, a public one second", nameservers)
|
||||
}
|
||||
if !strings.Contains(resolv, "\noptions timeout:1 attempts:1") {
|
||||
t.Errorf("the fallback is not reached after one short attempt:\n%s", resolv)
|
||||
}
|
||||
// The split-DNS alternative points at the same resolver, or a machine that keeps
|
||||
// systemd-resolved in charge would route the mesh's suffix to nothing.
|
||||
for _, r := range catalogueManifest(t, "resolved-split-dns").Resources {
|
||||
if content, _ := r["content"].(string); content != "" && !strings.Contains(content, "DNS=${bound:wildcard-resolution:address}\n") {
|
||||
t.Errorf("resolved-split-dns does not point at the resolver's address:\n%s", content)
|
||||
for _, line := range strings.Split(fact.Template, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") && !strings.Contains(line, "{{") {
|
||||
t.Errorf("%s's resolv.conf names a resolver of its own beside the mesh's: %s", uplink, line)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(fact.Template, "options timeout:1 attempts:2 edns0\n") {
|
||||
t.Errorf("%s: a silent resolver is not passed over after one short wait:\n%s", uplink, fact.Template)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,8 +112,10 @@ func TestTheResolverForwardsToFixedUpstreamsAndNeverReadsResolvConf(t *testing.T
|
||||
// that file, the machine pointed at the resolver by address, and the runtime given no resolver of
|
||||
// its own but kept running across a restart (ADR 0196).
|
||||
func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf"},
|
||||
Node{Name: "anchor", At: "anchor.internal"}, World{})
|
||||
got, err := Resolve(resolverShelf(t), []string{"dnsmasq", "systemd-networkd", "docker"},
|
||||
Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
|
||||
"package-manager": true, "service-manager": true, "privileged": true,
|
||||
"uplink-systemd-networkd": true}}, World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -131,6 +128,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
// happen to be the same map, since nothing routed is part of it.
|
||||
Names: twoMachines, Machines: twoMachines, Suffix: "internal",
|
||||
Zones: []ZoneAt{{Zone: "incus", Address: "10.42.0.2", Port: 5353}},
|
||||
Holders: map[string]map[string]string{"mesh-dns-resolver": {"anchor.internal": "10.42.0.1"}},
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||
Settings: SettingsBy{"dnsmasq": {{From: "the mesh", Values: map[string]any{"listen-addresses": "127.0.0.1"}}}},
|
||||
})
|
||||
@@ -167,13 +165,19 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
t.Errorf("the resolver was not told to forward the zone to its answerer:\n%s", z)
|
||||
}
|
||||
|
||||
// The runtime's own file, written into (novox/hq ADR 0102) with one key, by what decides how the
|
||||
// machine resolves: a restart keeps every container running. No `dns` — a container copies its
|
||||
// machine's resolvers (ADR 0196), and the mesh's resolver is not written into the runtime twice.
|
||||
// The runtime's own file, written into (novox/hq ADR 0102) with one key, by the runtime's own
|
||||
// module — a module does not write another software's configuration (issue 190): a restart keeps
|
||||
// every container running. No `dns` — a container copies its machine's resolvers (ADR 0196), and
|
||||
// neither the resolver nor what decides how the machine resolves writes the runtime's file.
|
||||
if ids["dnsmasq.runtime-dns"] != nil {
|
||||
t.Errorf("the resolver still writes the runtime's dns: %v", ids["dnsmasq.runtime-dns"])
|
||||
}
|
||||
runtime := ids["resolv-conf.runtime-config"]
|
||||
for id := range ids {
|
||||
if strings.HasPrefix(id, "systemd-networkd.runtime") {
|
||||
t.Errorf("what the machine asks still writes the runtime's file: %s", id)
|
||||
}
|
||||
}
|
||||
runtime := ids["docker.daemon"]
|
||||
if runtime == nil || runtime["path"] != "/etc/docker/daemon.json" || runtime["into"] != "json" {
|
||||
t.Fatalf("live-restore is not written into the runtime's file: %v", runtime)
|
||||
}
|
||||
@@ -195,7 +199,7 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
t.Errorf("the runtime is ordered restarted, which stops every container (ADR 0102): %v", r)
|
||||
}
|
||||
for _, on := range asStrings(r["reload-on"]) {
|
||||
if on == "resolv-conf.runtime-config" {
|
||||
if on == "docker.daemon" {
|
||||
reloaded = true
|
||||
}
|
||||
}
|
||||
@@ -204,22 +208,40 @@ func TestTheResolverAndWhatAsksItComposeOnOneMachine(t *testing.T) {
|
||||
t.Errorf("the runtime is not reloaded when its file changes, so live-restore never takes effect")
|
||||
}
|
||||
|
||||
resolv := ids["resolv-conf.resolv"]
|
||||
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\nnameserver 1.1.1.1\n") {
|
||||
t.Fatalf("the machine is not pointed at the resolver by address, with the public fallback: %v", resolv)
|
||||
resolv := ids["systemd-networkd.fact-resolvers"]
|
||||
if resolv == nil || !strings.Contains(resolv["content"].(string), "\nnameserver 10.42.0.1\noptions ") {
|
||||
t.Fatalf("the machine is not pointed at the resolver by address, and only at it: %v", resolv)
|
||||
}
|
||||
}
|
||||
|
||||
// Two modules deciding what a machine asks are refused on one machine, as before — the claim
|
||||
// exists so they never take turns overwriting each other.
|
||||
// Two modules deciding what a machine asks are refused on one machine, as before — now that the file
|
||||
// is the uplink's (novox/hq ADR 0223), by two things: a machine runs one network manager, and no other
|
||||
// module may write the resolver file beside the uplink's, as a file or as a rendered fact.
|
||||
func TestTwoThingsDecidingWhatAMachineAsksAreRefused(t *testing.T) {
|
||||
_, err := Resolve(resolverShelf(t), []string{"dnsmasq", "resolv-conf", "resolved-split-dns"},
|
||||
Node{Name: "anchor", At: "anchor.internal"}, World{})
|
||||
if err == nil {
|
||||
t.Fatal("resolv-conf and resolved-split-dns were both assigned to one machine")
|
||||
shelf := resolverShelf(t)
|
||||
shelf["networkmanager"] = catalogueManifest(t, "networkmanager")
|
||||
node := Node{Name: "anchor", At: "anchor.internal", Capabilities: map[string]bool{
|
||||
"package-manager": true, "service-manager": true,
|
||||
"uplink-systemd-networkd": true, "uplink-networkmanager": true}}
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", "networkmanager"}, node, World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "node-uplink") {
|
||||
t.Fatalf("two network managers were both assigned to one machine: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "node-resolver-config") {
|
||||
t.Fatalf("the refusal does not say what was claimed: %v", err)
|
||||
|
||||
// The second is made up: what is under test is that the resolver file has one owner, so any
|
||||
// module asking the mesh to render it — or declaring it — will do.
|
||||
for name, m := range map[string]Manifest{
|
||||
"a-rendered-one": {Module: "a-rendered-one", Version: "1",
|
||||
Facts: map[string]RosterFile{"mine": {Path: "/etc/resolv.conf", Template: "nameserver 10.42.0.1\n"}}},
|
||||
"a-declared-one": {Module: "a-declared-one", Version: "1", Resources: []map[string]any{
|
||||
{"id": "mine", "type": "file", "path": "/etc/resolv.conf", "content": "nameserver 10.42.0.1\n"}}},
|
||||
} {
|
||||
shelf := resolverShelf(t)
|
||||
shelf[name] = m
|
||||
_, err := Resolve(shelf, []string{"dnsmasq", "systemd-networkd", name}, node, World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "/etc/resolv.conf") {
|
||||
t.Errorf("%s wrote the resolver file beside the uplink's: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -64,6 +64,12 @@ type rosterView struct {
|
||||
// Zones is every zone a module in the mesh answers itself, with where its answerer is (novox/hq
|
||||
// ADR 0199) — what the mesh's resolver forwards. Ordered by zone.
|
||||
Zones []rosterZone
|
||||
// Holders is the machines holding each replicated mesh seat, by seat (novox/hq ADR 0223) — what
|
||||
// a machine's resolver file lists for `mesh-dns-resolver`. **This machine first when it is one
|
||||
// of them**, then the rest by name: the nearest holder is asked first, and two renderings of
|
||||
// one mesh on one machine are one file. A template reads one seat's with
|
||||
// `index .Holders "<seat>"`; a seat nobody holds ranges over nothing.
|
||||
Holders map[string][]rosterEntry
|
||||
}
|
||||
|
||||
// rosterZone is one zone as a template sees it: the zone, and the address and port answering it.
|
||||
@@ -96,21 +102,25 @@ func FactsInto(m Manifest, r Resolution, every, machines, accounts map[string]st
|
||||
// FactsWithZonesInto is FactsInto with the mesh's zones in the view, for a template that ranges them.
|
||||
func FactsWithZonesInto(m Manifest, r Resolution, every, machines, accounts map[string]string, suffix string,
|
||||
zones []ZoneAt) ([]map[string]any, error) {
|
||||
return FactsFrom(m, r, Rendering{Names: every, Machines: machines, Accounts: accounts, Suffix: suffix,
|
||||
Zones: zones})
|
||||
}
|
||||
|
||||
// FactsFrom renders the roster files a module asked for from everything the mesh composed for this
|
||||
// machine: its machines, zones and the holders of each replicated seat.
|
||||
func FactsFrom(m Manifest, r Resolution, with Rendering) ([]map[string]any, error) {
|
||||
if len(m.Facts) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
names := make([]string, 0, len(m.Facts))
|
||||
for name := range m.Facts {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
names := sortedFacts(m.Facts)
|
||||
|
||||
view := rosterView{
|
||||
Node: r.Node,
|
||||
Suffix: strings.TrimPrefix(suffixOr(suffix), "."),
|
||||
Names: entriesFrom(every, accounts, suffix),
|
||||
Machines: entriesFrom(machines, accounts, suffix),
|
||||
Zones: zonesFrom(zones),
|
||||
Suffix: strings.TrimPrefix(suffixOr(with.Suffix), "."),
|
||||
Names: entriesFrom(with.Names, with.Accounts, with.Suffix),
|
||||
Machines: entriesFrom(with.Machines, with.Accounts, with.Suffix),
|
||||
Zones: zonesFrom(with.Zones),
|
||||
Holders: holdersFrom(with.Holders, with.Accounts, with.Suffix, r.Node),
|
||||
}
|
||||
|
||||
out := make([]map[string]any, 0, len(names))
|
||||
@@ -250,3 +260,28 @@ func zonesFrom(zones []ZoneAt) []rosterZone {
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Zone < out[j].Zone })
|
||||
return out
|
||||
}
|
||||
|
||||
// holdersFrom is each replicated seat's holders as a template ranges them: this machine first when
|
||||
// it holds the seat, then the others by name (novox/hq ADR 0223). A machine with no address is left
|
||||
// out, as everywhere in the roster — a resolver named at nothing is a lookup that hangs.
|
||||
func holdersFrom(holders map[string]map[string]string, accounts map[string]string, suffix, node string) map[string][]rosterEntry {
|
||||
out := make(map[string][]rosterEntry, len(holders))
|
||||
for seat, at := range holders {
|
||||
entries := entriesFrom(at, accounts, suffix)
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
return entries[i].Name == node && entries[j].Name != node
|
||||
})
|
||||
out[seat] = entries
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedFacts is a module's fact names in order, so what is said about them is said the same way twice.
|
||||
func sortedFacts(facts map[string]RosterFile) []string {
|
||||
out := make([]string, 0, len(facts))
|
||||
for name := range facts {
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package catalogue
|
||||
|
||||
import "testing"
|
||||
|
||||
// A mesh-wide provision whose seat is held on another machine is answered by that holder, even on a
|
||||
// machine that runs a provider of its own (novox/hq issue 258). Every machine ran its own resolver
|
||||
// while the mesh moved to one; each bound its resolver configuration to itself, with the mesh's
|
||||
// resolver held elsewhere and pinned.
|
||||
func resolverMesh() map[string]Manifest {
|
||||
return map[string]Manifest{
|
||||
"resolver": {Module: "resolver", Version: "1",
|
||||
Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}},
|
||||
Claims: []Claim{{Name: "mesh-dns-resolver", Scope: ScopeMesh}}},
|
||||
"asker": {Module: "asker", Version: "1", Requires: []string{"wildcard-resolution"}},
|
||||
}
|
||||
}
|
||||
|
||||
func resolverWorld(held string, pin *Chosen) World {
|
||||
w := World{
|
||||
Offered: map[string][]Provider{"wildcard-resolution": {
|
||||
{Node: "anchor", At: "anchor.internal", Module: "resolver"},
|
||||
{Node: "laptop", At: "laptop.internal", Module: "resolver"},
|
||||
}},
|
||||
}
|
||||
// Held is who holds it across the mesh; Holdings is the same holder on record, which lets this
|
||||
// machine's own claimant stand beside it (ADR 0131).
|
||||
w.Held = []Held{{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: held, Module: "resolver"}}
|
||||
w.Holdings = w.Held
|
||||
if pin != nil {
|
||||
w.Pinned = map[string]Chosen{"wildcard-resolution": *pin}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func answeredFrom(t *testing.T, world World) string {
|
||||
t.Helper()
|
||||
laptop := Node{Name: "laptop", At: "laptop.internal"}
|
||||
got, err := Resolve(resolverMesh(), []string{"resolver", "asker"}, laptop, world)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range got.Needs {
|
||||
if n.Name == "wildcard-resolution" {
|
||||
return n.From
|
||||
}
|
||||
}
|
||||
t.Fatalf("no binding for wildcard-resolution: %+v", got.Needs)
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestTheSeatsHolderElsewhereAnswersBeforeThisMachinesOwnProvider(t *testing.T) {
|
||||
if from := answeredFrom(t, resolverWorld("anchor", nil)); from != "anchor" {
|
||||
t.Fatalf("bound to %s; the seat is held on anchor", from)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPinElsewhereAnswersBeforeThisMachinesOwnProvider(t *testing.T) {
|
||||
if from := answeredFrom(t, resolverWorld("laptop", &Chosen{Node: "anchor", Module: "resolver"})); from != "anchor" {
|
||||
t.Fatalf("bound to %s; this machine was pinned to anchor", from)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheHolderOnThisMachineStillAnswersHere(t *testing.T) {
|
||||
if from := answeredFrom(t, resolverWorld("laptop", nil)); from != "laptop" {
|
||||
t.Fatalf("bound to %s; the seat is held here", from)
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,10 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A module depends on the node seats that apply its resources (novox/hq ADR 0207), and on the
|
||||
// seats it contributes to (novox/hq ADR 0210).
|
||||
// A module depends on the node seats that apply its resources (novox/hq ADR 0207) and on the
|
||||
// seats it contributes to (novox/hq ADR 0210). A third source — what a seat it holds needs beside it
|
||||
// (novox/hq ADR 0220) — had one user, node-resolver-config needing node-uplink, and went with that
|
||||
// seat when the resolver file became the uplink's own (novox/hq ADR 0223).
|
||||
//
|
||||
// Some of what a module declares is applied through software on the machine that is itself a
|
||||
// module: a service through the service manager, a package through the package manager, a container
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
@@ -42,6 +43,27 @@ import (
|
||||
// ofSeat is where a module asks about a seat: ${seat:<seat>:<the port its holder's software uses>}.
|
||||
var ofSeat = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):([0-9]+)\}`)
|
||||
|
||||
// **Where this machine reaches a mesh seat's holder** (novox/hq ADR 0222, issue 190).
|
||||
//
|
||||
// `${seat:<mesh-seat>:reach}` is host:port — the address this machine dials to reach whatever holds a
|
||||
// seat the mesh holds once. The same reasoning as the port above, one step further: nothing is
|
||||
// required, nothing is granted, no credential is minted, and the answer is an address the mesh
|
||||
// already holds in the clear and composes into every reference it built. What it is for is a module
|
||||
// that must *state* where a mesh service is to software it owns — the container runtime trusting
|
||||
// the mesh's registry is the case — without becoming that service's consumer.
|
||||
//
|
||||
// Answered for the seats in reachedSeats only. Another seat is refused by name rather than answered
|
||||
// with nothing: a module asking where something is that the mesh does not say would otherwise be
|
||||
// given an empty value and never know the question was not understood.
|
||||
//
|
||||
// The answer may be empty: no machine on the private network holds the seat yet (genesis raises
|
||||
// the store before the network). In a file written into as JSON, an empty member is dropped from
|
||||
// its list, and a list left with none is dropped, so the runtime is never told to trust "".
|
||||
var ofSeatReach = regexp.MustCompile(`\$\{seat:([a-z0-9][a-z0-9-]*):reach\}`)
|
||||
|
||||
// reachedSeats is every seat ${seat:…:reach} answers for.
|
||||
var reachedSeats = map[string]bool{"mesh-artifact-store": true}
|
||||
|
||||
// seatInto replaces a resource's ${seat:…} placeholders with where this machine put each seat's
|
||||
// holder — in a file's content, and in a value of a container's or a process's environment. The
|
||||
// same places portInto fills, for the same reason: they are where a program reads a number from.
|
||||
@@ -49,13 +71,24 @@ func seatInto(resource map[string]any, module string, with Rendering) error {
|
||||
switch fmt.Sprint(resource["type"]) {
|
||||
case "file":
|
||||
content, ok := resource["content"].(string)
|
||||
if !ok || !ofSeat.MatchString(content) {
|
||||
if !ok || (!ofSeat.MatchString(content) && !ofSeatReach.MatchString(content)) {
|
||||
return nil
|
||||
}
|
||||
filled, err := seatsFilledInto(content, fmt.Sprintf("%s has a file that", module), with)
|
||||
where := fmt.Sprintf("%s has a file that", module)
|
||||
filled, err := seatsFilledInto(content, where, with)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ofSeatReach.MatchString(filled) {
|
||||
if filled, err = reachFilledInto(filled, where, with); err != nil {
|
||||
return err
|
||||
}
|
||||
if fmt.Sprint(resource["into"]) == "json" {
|
||||
if filled, err = withoutEmptyMembers(filled, where); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
resource["content"] = filled
|
||||
|
||||
case "container", "process":
|
||||
@@ -74,15 +107,18 @@ func seatInto(resource map[string]any, module string, with Rendering) error {
|
||||
var filled map[string]any
|
||||
for _, key := range named {
|
||||
written, ok := env[key].(string)
|
||||
if !ok || !ofSeat.MatchString(written) {
|
||||
if !ok || (!ofSeat.MatchString(written) && !ofSeatReach.MatchString(written)) {
|
||||
continue
|
||||
}
|
||||
value, err := seatsFilledInto(written,
|
||||
fmt.Sprintf("%s's %s %s sets %s to something that",
|
||||
module, resource["type"], resource["name"], key), with)
|
||||
where := fmt.Sprintf("%s's %s %s sets %s to something that",
|
||||
module, resource["type"], resource["name"], key)
|
||||
value, err := seatsFilledInto(written, where, with)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if value, err = reachFilledInto(value, where, with); err != nil {
|
||||
return err
|
||||
}
|
||||
if filled == nil {
|
||||
filled = map[string]any{}
|
||||
for k, v := range env {
|
||||
@@ -118,3 +154,69 @@ func seatsFilledInto(written, where string, with Rendering) (string, error) {
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// reachFilledInto answers every ${seat:…:reach} in one written value with where this machine
|
||||
// reaches the seat's holder, or with nothing when no holder is reachable yet. A seat the mesh does
|
||||
// not answer this for is refused by name.
|
||||
func reachFilledInto(written, where string, with Rendering) (string, error) {
|
||||
for _, m := range ofSeatReach.FindAllStringSubmatch(written, -1) {
|
||||
seat := m[1]
|
||||
if !reachedSeats[seat] {
|
||||
known := make([]string, 0, len(reachedSeats))
|
||||
for s := range reachedSeats {
|
||||
known = append(known, s)
|
||||
}
|
||||
sort.Strings(known)
|
||||
return "", fmt.Errorf("%s says ${seat:%s:reach}, and the mesh says where a seat's holder is "+
|
||||
"reached only for %s (novox/hq ADR 0222)", where, seat, strings.Join(known, ", "))
|
||||
}
|
||||
written = strings.ReplaceAll(written, m[0], with.SeatReach[seat])
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// withoutEmptyMembers drops every empty string from the lists at the top of a JSON object written
|
||||
// into a machine's file, and a list left with no members, so an unanswered ${seat:…:reach} adds
|
||||
// nothing to the machine's list rather than adding "".
|
||||
func withoutEmptyMembers(content, where string) (string, error) {
|
||||
var object map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(content), &object); err != nil {
|
||||
return "", fmt.Errorf("%s is written into as JSON and is not a JSON object: %w", where, err)
|
||||
}
|
||||
changed := false
|
||||
for key, raw := range object {
|
||||
var members []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &members); err != nil {
|
||||
continue // not a list
|
||||
}
|
||||
kept := make([]json.RawMessage, 0, len(members))
|
||||
for _, member := range members {
|
||||
var s string
|
||||
if json.Unmarshal(member, &s) == nil && s == "" {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, member)
|
||||
}
|
||||
if len(kept) == len(members) {
|
||||
continue
|
||||
}
|
||||
changed = true
|
||||
if len(kept) == 0 {
|
||||
delete(object, key)
|
||||
continue
|
||||
}
|
||||
list, err := json.Marshal(kept)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
object[key] = list
|
||||
}
|
||||
if !changed {
|
||||
return content, nil
|
||||
}
|
||||
out, err := json.Marshal(object)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(out) + "\n", nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -211,3 +212,83 @@ func withSeatPorts(m Manifest) Manifest {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Where this machine reaches a mesh seat's holder (novox/hq ADR 0222, issue 190): the container
|
||||
// runtime's module tells the runtime to trust the mesh's registry without binding the store.
|
||||
|
||||
func runtimeTrust() map[string]any {
|
||||
return map[string]any{
|
||||
"type": "file", "id": "daemon", "path": "/etc/docker/daemon.json", "into": "json",
|
||||
"content": `{"live-restore": true, "insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n",
|
||||
}
|
||||
}
|
||||
|
||||
func TestASeatsReachIsWhereThisMachineReachesItsHolder(t *testing.T) {
|
||||
file := runtimeTrust()
|
||||
with := Rendering{SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"}}
|
||||
if err := seatInto(file, "docker", with); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := `{"live-restore": true, "insecure-registries": ["anchor.internal:5100"]}` + "\n"
|
||||
if file["content"] != want {
|
||||
t.Fatalf("content = %q, want %q", file["content"], want)
|
||||
}
|
||||
|
||||
process := map[string]any{"type": "process", "name": "p",
|
||||
"env": map[string]any{"REGISTRY": "${seat:mesh-artifact-store:reach}"}}
|
||||
if err := seatInto(process, "x", with); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := process["env"].(map[string]any)["REGISTRY"]; got != "anchor.internal:5100" {
|
||||
t.Fatalf("an environment value was filled with %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With no holder reachable, the runtime is not told to trust "": the member is dropped, and the
|
||||
// list with it when nothing else is in it.
|
||||
func TestAnUnansweredReachAddsNothingToAList(t *testing.T) {
|
||||
file := runtimeTrust()
|
||||
if err := seatInto(file, "docker", Rendering{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := file["content"]; got != `{"live-restore":true}`+"\n" {
|
||||
t.Fatalf("with no store reachable, the runtime's keys are %q", got)
|
||||
}
|
||||
|
||||
kept := map[string]any{"type": "file", "into": "json",
|
||||
"content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}", "192.0.2.7:5000"]}`}
|
||||
if err := seatInto(kept, "docker", Rendering{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := kept["content"]; got != `{"insecure-registries":["192.0.2.7:5000"]}`+"\n" {
|
||||
t.Fatalf("a list's other members were not kept: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReachTheMeshDoesNotAnswerIsRefused(t *testing.T) {
|
||||
file := map[string]any{"type": "file", "content": "${seat:mesh-store:reach}"}
|
||||
err := seatInto(file, "x", Rendering{SeatReach: map[string]string{"mesh-store": "anchor.internal:5432"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "mesh-artifact-store") {
|
||||
t.Fatalf("a reach the mesh does not answer was not refused by name: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Through the whole composition, as the container runtime's module declares it.
|
||||
func TestTheRuntimesTrustIsComposedFromTheSeatsReach(t *testing.T) {
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{{
|
||||
Module: "docker", Resources: []map[string]any{runtimeTrust()},
|
||||
}}}
|
||||
out, err := r.Declaration(Rendering{
|
||||
SeatReach: map[string]string{"mesh-artifact-store": "anchor.internal:5100"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := fileNamed(out, "docker.daemon")
|
||||
if file == nil {
|
||||
t.Fatalf("no file in %v", out)
|
||||
}
|
||||
if got := file["content"]; !strings.Contains(fmt.Sprint(got), `["anchor.internal:5100"]`) {
|
||||
t.Fatalf("the runtime's file says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
+45
-27
@@ -21,8 +21,16 @@ import (
|
||||
type Seat struct {
|
||||
// Name is what a manifest claims.
|
||||
Name string
|
||||
// Scope is where there may be only one holder.
|
||||
// Scope is where there may be only one holder — unless the seat is Replicated.
|
||||
Scope string
|
||||
// Replicated says a mesh seat may be held on several machines at once, each holder answering the
|
||||
// same thing (novox/hq ADR 0223): the mesh's resolver, held on the anchor and the home server so a
|
||||
// machine's resolver file lists two that give one answer. Each holder is on record, added by an
|
||||
// act (`seat <name> --add <node>/<module>`), never by being assigned: two claimants with nothing on
|
||||
// record are refused exactly as for any mesh seat. One per machine still — two modules on one
|
||||
// node claiming it are refused. Compiled, never stored, like Receives: it is the mesh's definition of
|
||||
// the role, and the store's rows carry no column for it.
|
||||
Replicated bool
|
||||
// Delivers is the provision the seat's holder answers for, or empty. A seat that delivers a
|
||||
// provision may only be held by a module providing it at the seat's scope, and its holder is
|
||||
// what a requirement for that provision resolves to when several modules provide it.
|
||||
@@ -133,18 +141,20 @@ var defaultSeats = append([]Seat{
|
||||
// that machine unresolvable in the meantime. Deleted once no registered manifest claims it.
|
||||
{Name: "mesh-build-machine", Scope: ScopeMesh,
|
||||
Accepts: []string{"build"}, Emits: []string{"started", "built", "log.*"}, Decision: "novox/hq ADR 0190"},
|
||||
// **The mesh's one resolver** (novox/hq ADR 0194, 0196): every node's internal domain, held in one
|
||||
// place, and every node and container asks it first. Delivers what a machine's resolver
|
||||
// configuration requires, so that requirement resolves to the holder wherever it is placed.
|
||||
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Decision: "novox/hq ADR 0194"},
|
||||
// **Retired by ADR 0194, kept while a manifest still claims it** — the same reason as
|
||||
// mesh-build-machine above: a machine still holds it until the mesh's resolver replaces it, and
|
||||
// removing the row first would make that machine unresolvable. Deleted once nothing claims it.
|
||||
{Name: "node-dns-resolver", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||
// **A machine's /etc/hosts is one module's** (novox/hq ADR 0199): its holder writes the machine's
|
||||
// own lines and keeps every other line as the operator's, changed through these three verbs on that
|
||||
// machine alone. The controller holds none of it.
|
||||
{Name: "node-hosts-file", Scope: ScopeNode, Decision: "novox/hq ADR 0199",
|
||||
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
||||
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
||||
// than one machine, each answering the same names from the same roster, and every machine's
|
||||
// resolver file lists every holder — its own first — and no public resolver, so whichever answers
|
||||
// first gives the one answer. Delivers what a machine's resolver configuration requires, so that
|
||||
// requirement resolves to a holder wherever they are placed.
|
||||
{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution", Replicated: true,
|
||||
Decision: "novox/hq ADR 0194, ADR 0223"},
|
||||
// **A machine's names are one module's** (novox/hq ADR 0199, ADR 0223): its holder writes
|
||||
// /etc/hostname and the machine's own lines in /etc/hosts, and keeps every other line of the hosts
|
||||
// file as the operator's, changed through these three verbs on that machine alone. The controller
|
||||
// holds none of it. Named node-hosts-file until ADR 0223; the former name resolves to it as an
|
||||
// alias on a mesh that knew it.
|
||||
{Name: "node-hostname", Scope: ScopeNode, Decision: "novox/hq ADR 0199, ADR 0223",
|
||||
Serves: []Verb{
|
||||
{Name: "entries", Description: "Every line of this machine's /etc/hosts, each marked whose it is: " +
|
||||
"the operator's, or the block of the module or tool that writes it.",
|
||||
@@ -245,11 +255,12 @@ var defaultSeats = append([]Seat{
|
||||
// Deferred (novox/hq ADR 0121): renaming to mesh-private-network is a scope + server/client
|
||||
// model change, not a rename, so it stays until that is built.
|
||||
{Name: "the-private-network", Scope: ScopeNode, Decision: "novox/hq ADR 0110"},
|
||||
{Name: "node-resolver-config", Scope: ScopeNode, Decision: "novox/hq ADR 0121"},
|
||||
// The program that manages the machine's own network. It delivers nothing: its holder only
|
||||
// keeps the manager and the mesh from contradicting each other — the resolver file left to the
|
||||
// mesh, the private network's interface left alone — and never declares a link, an address or
|
||||
// a wireless network, because the link is the only channel a fix could arrive on. A seat
|
||||
// The program that manages the machine's own network. It delivers nothing: its holder keeps the
|
||||
// manager and the mesh from contradicting each other — the private network's interface left
|
||||
// alone — and writes the machine's resolver file itself, because the manager is what would
|
||||
// otherwise rewrite it (novox/hq ADR 0223, which retired node-resolver-config into this seat).
|
||||
// It never declares a link, an address or a wireless network, because the link is the only
|
||||
// channel a fix could arrive on. A seat
|
||||
// rather than a condition in the resolver's module, so a machine running two managers is
|
||||
// refused at assignment instead of found by the resolver being rewritten (novox/hq ADR 0117).
|
||||
{Name: "node-uplink", Scope: ScopeNode, Decision: "novox/hq ADR 0117"},
|
||||
@@ -304,9 +315,11 @@ func UseSeats(s []Seat) {
|
||||
row.Accepts, row.Emits, row.Serves = d.Accepts, d.Emits, d.Serves
|
||||
}
|
||||
}
|
||||
// What a seat receives is never stored (novox/hq ADR 0212), so it is always the compiled one.
|
||||
// What a seat receives and whether it is replicated are never stored (novox/hq ADR 0212, ADR
|
||||
// 0223), so they are always the compiled ones.
|
||||
if d, known := byName[row.Name]; known {
|
||||
row.Receives = d.Receives
|
||||
row.Replicated = d.Replicated
|
||||
}
|
||||
merged = append(merged, row)
|
||||
}
|
||||
@@ -486,19 +499,24 @@ func seatNames() string {
|
||||
// two modules on one node could both provide a provision, and only the one holding the seat
|
||||
// answers for it. Nothing when no seat delivers the provision, when nobody holds
|
||||
// it, or when the holder is not among the providers offered.
|
||||
//
|
||||
// **The first holder in the providers' own order** (novox/hq ADR 0223). A replicated seat has
|
||||
// several, and the answer must not depend on the order the mesh happened to resolve its machines
|
||||
// in: the providers come sorted by machine, so every consumer is bound to the same one. A seat with
|
||||
// one holder gets the same answer as before.
|
||||
func HolderAmong(provision string, providers []Provider, held []Held) (Provider, bool) {
|
||||
seat, delivered := SeatDelivering(provision)
|
||||
if !delivered {
|
||||
return Provider{}, false
|
||||
}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a seat's
|
||||
// former name still matches it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
|
||||
continue
|
||||
}
|
||||
for _, p := range providers {
|
||||
for _, p := range providers {
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
// seat's former name still matches it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != seat.Name || h.Scope != seat.Scope {
|
||||
continue
|
||||
}
|
||||
if p.Node == h.Node && p.Module == h.Module {
|
||||
return p, true
|
||||
}
|
||||
|
||||
@@ -46,14 +46,18 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
|
||||
delivered[s.Delivers] = s.Name
|
||||
}
|
||||
}
|
||||
// Thirty-eight with node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215); thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199); thirty-four
|
||||
// Thirty-six since node-resolver-config retired into node-uplink (novox/hq ADR 0223); thirty-seven
|
||||
// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
|
||||
// node-backup (novox/hq ADR 0214); thirty-seven with node-message-bus (novox/hq ADR 0215);
|
||||
// thirty-six with mesh-dns-resolver (novox/hq ADR 0194) and node-hosts-file (ADR 0199, now
|
||||
// node-hostname); thirty-four
|
||||
// with node-hotkeys (ADR 0212); thirty-three with node-power (ADR 0211); thirty-two since the
|
||||
// graphical session's eleven (ADR 0208); twenty-one with node-package-manager and
|
||||
// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
|
||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). Two fewer once the retired
|
||||
// mesh-build-machine and node-dns-resolver rows go, when no registered manifest claims either.
|
||||
if len(Seats()) != 38 {
|
||||
t.Errorf("the mesh defines %d seats rather than 38; the set is closed, so a change here is "+
|
||||
// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
|
||||
// mesh-build-machine row goes, when no registered manifest claims it.
|
||||
if len(Seats()) != 36 {
|
||||
t.Errorf("the mesh defines %d seats rather than 36; the set is closed, so a change here is "+
|
||||
"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The mesh has two resolvers (novox/hq ADR 0223): `mesh-dns-resolver` is replicated, held on the
|
||||
// anchor and on the home server, each answering the same names; every machine's resolver file lists
|
||||
// every holder — its own first when it is one — and no public resolver. ADR 0196 listed the mesh's
|
||||
// resolver then a public one, and musl asks both at once and takes the first reply: from the home
|
||||
// server the public "no such name" for the anchor's mesh name won, every time, in every Alpine build.
|
||||
// The file is written by the module holding the machine's uplink (ADR 0223 part 2).
|
||||
|
||||
// resolverMachines is the anchor and the home server holding the resolver, and a laptop holding nothing.
|
||||
var resolverMachines = map[string]string{
|
||||
"anchor.internal": "10.42.0.1", "laptop.internal": "10.42.0.2", "home.internal": "10.42.0.3"}
|
||||
|
||||
// bothResolvers is the two holders on record, as `seat mesh-dns-resolver --add` leaves them.
|
||||
var bothResolvers = []Held{
|
||||
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "dnsmasq"},
|
||||
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "home", Module: "dnsmasq"},
|
||||
}
|
||||
|
||||
// uplinks is every module in the catalogue holding node-uplink, and so writing the machine's resolver
|
||||
// file (novox/hq ADR 0223 part 2): the program that would otherwise rewrite it is the one that writes it.
|
||||
var uplinks = []string{"dhcpcd", "networkmanager", "systemd-networkd"}
|
||||
|
||||
// managing is a machine as each uplink module needs it: able to install, run a service and run the
|
||||
// manager that module is for.
|
||||
func managing(node string) Node {
|
||||
caps := map[string]bool{"package-manager": true, "service-manager": true}
|
||||
for _, u := range uplinks {
|
||||
caps["uplink-"+u] = true
|
||||
}
|
||||
return Node{Name: node, At: node + ".internal", Capabilities: caps}
|
||||
}
|
||||
|
||||
// twoResolverShelf is the resolver, the uplink modules that write what a machine asks, and a stand-in
|
||||
// answering `mesh-addressing`.
|
||||
func twoResolverShelf(t *testing.T) map[string]Manifest {
|
||||
t.Helper()
|
||||
out := map[string]Manifest{
|
||||
"net": {Module: "net", Version: "1", Provides: []Offer{{Name: "mesh-addressing"}}},
|
||||
"dnsmasq": catalogueManifest(t, "dnsmasq"),
|
||||
}
|
||||
for _, u := range uplinks {
|
||||
out[u] = catalogueManifest(t, u)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// worldWithout is the rest of the mesh as a plan for one machine sees it: every other holder's claim
|
||||
// and offer, and both holders on record.
|
||||
func worldWithout(node string) World {
|
||||
w := World{Holdings: bothResolvers, Offered: map[string][]Provider{}}
|
||||
for _, h := range bothResolvers {
|
||||
if h.Node == node {
|
||||
continue
|
||||
}
|
||||
w.Held = append(w.Held, h)
|
||||
w.Offered["wildcard-resolution"] = append(w.Offered["wildcard-resolution"],
|
||||
Provider{Node: h.Node, At: h.Node + ".internal", Module: h.Module})
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// resolvConfOn resolves and composes one machine and answers with the nameservers its resolver file
|
||||
// lists, in order, and the file. The file is the uplink's — `uplink` is one of the assigned modules —
|
||||
// and nothing else on the machine declares that path.
|
||||
func resolvConfOn(t *testing.T, node, uplink string, assigned []string) ([]string, string) {
|
||||
t.Helper()
|
||||
got, err := Resolve(twoResolverShelf(t), assigned, managing(node), worldWithout(node))
|
||||
if err != nil {
|
||||
t.Fatalf("%s with %s does not resolve with two resolvers on record: %v", node, uplink, err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{
|
||||
Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
|
||||
Holders: map[string]map[string]string{"mesh-dns-resolver": {
|
||||
"anchor.internal": "10.42.0.1", "home.internal": "10.42.0.3"}},
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("%s with %s does not compose: %v", node, uplink, err)
|
||||
}
|
||||
var file map[string]any
|
||||
for _, r := range out {
|
||||
if r["path"] != "/etc/resolv.conf" {
|
||||
continue
|
||||
}
|
||||
if file != nil {
|
||||
t.Fatalf("%s declares /etc/resolv.conf twice: %v and %v", node, file["id"], r["id"])
|
||||
}
|
||||
file = r
|
||||
}
|
||||
if file == nil || file["id"] != uplink+".fact-resolvers" {
|
||||
t.Fatalf("%s's resolver file is not %s's: %v", node, uplink, file)
|
||||
}
|
||||
content, _ := file["content"].(string)
|
||||
var servers []string
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
if strings.HasPrefix(line, "nameserver ") {
|
||||
servers = append(servers, strings.TrimPrefix(line, "nameserver "))
|
||||
}
|
||||
}
|
||||
return servers, content
|
||||
}
|
||||
|
||||
// Per uplink module: each writes a resolver file listing both holders, the machine's own first on a
|
||||
// holder, and only the holders on a machine that is none.
|
||||
func TestEveryUplinkListsBothResolversItsOwnFirst(t *testing.T) {
|
||||
for _, uplink := range uplinks {
|
||||
for node, want := range map[string][]string{
|
||||
"anchor": {"10.42.0.1", "10.42.0.3"},
|
||||
"home": {"10.42.0.3", "10.42.0.1"},
|
||||
"laptop": {"10.42.0.1", "10.42.0.3"},
|
||||
} {
|
||||
assigned := []string{uplink}
|
||||
if node != "laptop" {
|
||||
assigned = append(assigned, "dnsmasq")
|
||||
}
|
||||
servers, content := resolvConfOn(t, node, uplink, assigned)
|
||||
if strings.Join(servers, " ") != strings.Join(want, " ") {
|
||||
t.Errorf("%s on %s lists %v; want %v\n%s", uplink, node, servers, want, content)
|
||||
}
|
||||
for _, public := range []string{"1.1.1.1", "8.8.8.8", "9.9.9.9"} {
|
||||
if strings.Contains(content, public) {
|
||||
t.Errorf("%s on %s lists a public resolver beside the mesh's (ADR 0223):\n%s", uplink, node, content)
|
||||
}
|
||||
}
|
||||
if !strings.HasSuffix(content, "\noptions timeout:1 attempts:2 edns0\n") {
|
||||
t.Errorf("%s on %s does not end with two short attempts:\n%s", uplink, node, content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One file, whichever manager the machine runs: the three modules carry the same template, so a
|
||||
// machine changing its manager changes nothing in what it asks, and a fix made to one is made to all.
|
||||
func TestEveryUplinkWritesTheSameResolverFile(t *testing.T) {
|
||||
var first, firstOf string
|
||||
for _, uplink := range uplinks {
|
||||
fact, ok := catalogueManifest(t, uplink).Facts["resolvers"]
|
||||
if !ok || fact.Path != "/etc/resolv.conf" || fact.Shared || fact.Home {
|
||||
t.Fatalf("%s does not write the machine's resolver file whole: %+v", uplink, fact)
|
||||
}
|
||||
if first == "" {
|
||||
first, firstOf = fact.Template, uplink
|
||||
continue
|
||||
}
|
||||
if fact.Template != first {
|
||||
t.Errorf("%s's resolver file differs from %s's; the three are kept identical", uplink, firstOf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The requirement stays with what writes the file (ADR 0223 part 1): a machine is refused when nothing
|
||||
// in the mesh resolves, rather than given a file listing nothing.
|
||||
func TestEveryUplinkRequiresTheMeshsResolver(t *testing.T) {
|
||||
for _, uplink := range uplinks {
|
||||
found := false
|
||||
for _, r := range catalogueManifest(t, uplink).Requires {
|
||||
found = found || r == "wildcard-resolution"
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("%s writes the resolver file and does not require wildcard-resolution", uplink)
|
||||
}
|
||||
}
|
||||
_, err := Resolve(twoResolverShelf(t), []string{"networkmanager"}, managing("laptop"), World{})
|
||||
if err == nil || !strings.Contains(err.Error(), "wildcard-resolution") {
|
||||
t.Errorf("an uplink was composed on a mesh with no resolver: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder answers its own requirement, even though the other holder sorts first (issue 258 kept).
|
||||
func TestAHolderAnswersItsOwnRequirement(t *testing.T) {
|
||||
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq", "networkmanager"},
|
||||
managing("home"), worldWithout("home"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range got.Needs {
|
||||
if n.Name == "wildcard-resolution" && n.From != "home" {
|
||||
t.Errorf("the home server's uplink is bound to %s; it holds the seat itself", n.From)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A seat held once is still held once: a second claimant on another machine is refused while
|
||||
// nothing is on record, and a store recording two holders of it is refused, naming the seat.
|
||||
func TestASingleHolderMeshSeatStillRefusesASecondHolder(t *testing.T) {
|
||||
store := shelf(mod("postgres", nil, nil, nil, Claim{Name: "mesh-store", Scope: ScopeMesh}))
|
||||
other := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "postgres"}
|
||||
if _, err := Resolve(store, []string{"postgres"}, workstation(), World{Held: []Held{other}}); err == nil ||
|
||||
!strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Errorf("a second claimant of a seat held once was not refused: %v", err)
|
||||
}
|
||||
here := Held{Claim: "mesh-store", Scope: ScopeMesh, Node: workstation().Name, Module: "postgres"}
|
||||
_, err := Resolve(store, []string{"postgres"}, workstation(),
|
||||
World{Held: []Held{other}, Holdings: []Held{other, here}})
|
||||
if err == nil || !strings.Contains(err.Error(), "on record as held by 2") {
|
||||
t.Errorf("two holders on record for a seat held once were not refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Replicated is not "whoever is assigned": two claimants with nothing on record are refused, as for
|
||||
// any mesh seat, and each holder is added by an act.
|
||||
func TestTwoUnrecordedClaimantsOfTheReplicatedSeatAreRefused(t *testing.T) {
|
||||
w := worldWithout("home")
|
||||
w.Holdings = nil
|
||||
_, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "home", At: "home.internal"}, w)
|
||||
if err == nil || !strings.Contains(err.Error(), "seat mesh-dns-resolver --to") {
|
||||
t.Errorf("a second resolver with nothing on record was not refused, naming the handover: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the mesh's resolver is replicated: a seat being replicated is a decision, recorded.
|
||||
func TestOnlyTheResolverIsReplicated(t *testing.T) {
|
||||
for _, s := range Seats() {
|
||||
if s.Replicated != (s.Name == "mesh-dns-resolver") {
|
||||
t.Errorf("%s replicated = %v; only mesh-dns-resolver is (ADR 0223)", s.Name, s.Replicated)
|
||||
}
|
||||
}
|
||||
UseSeats([]Seat{{Name: "mesh-dns-resolver", Scope: ScopeMesh, Delivers: "wildcard-resolution"}})
|
||||
defer UseSeats(DefaultSeats())
|
||||
if s, _ := SeatNamed("mesh-dns-resolver"); !s.Replicated {
|
||||
t.Error("loading the set from the store, which has no column for it, lost the resolver's replication")
|
||||
}
|
||||
}
|
||||
|
||||
// Every consumer is bound to the same holder whatever order the mesh resolved its machines in.
|
||||
func TestTheFirstHolderIsTheFirstProvider(t *testing.T) {
|
||||
providers := []Provider{{Node: "anchor", Module: "dnsmasq"}, {Node: "home", Module: "dnsmasq"}}
|
||||
for _, held := range [][]Held{bothResolvers, {bothResolvers[1], bothResolvers[0]}} {
|
||||
if p, ok := HolderAmong("wildcard-resolution", providers, held); !ok || p.Node != "anchor" {
|
||||
t.Errorf("held in order %v answered %v", held, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One host record per machine, beside its wildcard (novox/hq issue 262): a name with a host record
|
||||
// says it exists and has no IPv6 address, where the wildcard alone said there is no such name, and
|
||||
// musl reads that as final.
|
||||
func TestTheResolverHasOneHostRecordPerMachine(t *testing.T) {
|
||||
got, err := Resolve(twoResolverShelf(t), []string{"dnsmasq"}, Node{Name: "anchor", At: "anchor.internal"},
|
||||
World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := got.Declaration(Rendering{Names: resolverMachines, Machines: resolverMachines, Suffix: "internal",
|
||||
Needed: map[string]map[string]string{"dnsmasq": {"broker": "sealed"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, _ := byID(out)["dnsmasq.fact-node-zones"]["content"].(string)
|
||||
records := map[string]int{}
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
if strings.HasPrefix(line, "host-record=") {
|
||||
records[strings.TrimPrefix(line, "host-record=")]++
|
||||
}
|
||||
}
|
||||
for name, at := range resolverMachines {
|
||||
if records[name+","+at] != 1 {
|
||||
t.Errorf("%s has %d host records at %s, and has one:\n%s", name, records[name+","+at], at, content)
|
||||
}
|
||||
}
|
||||
if len(records) != len(resolverMachines) {
|
||||
t.Errorf("%d host records for %d machines:\n%s", len(records), len(resolverMachines), content)
|
||||
}
|
||||
}
|
||||
+73
-13
@@ -73,6 +73,13 @@ var ControllerVerbs = []Verb{
|
||||
{Name: "tools", Description: "Every seat's tools, from the mesh's own records: what each role " +
|
||||
"answers, whether or not its holder is up. The mesh's own verbs are the mesh-controller seat's.",
|
||||
Input: schema(nil, nil)},
|
||||
{Name: "calls", Description: "The calls this controller answered lately and what came of each — " +
|
||||
"one still running, one that finished after its caller was told it was running, one whose answer " +
|
||||
"the bus refused — and, given a call's id, its whole answer (novox/hq issue 265). A call that has " +
|
||||
"not finished within ten seconds answers that it is running, with its id; this is where it ends.",
|
||||
Input: schema(map[string]string{
|
||||
"call": "a call's id, as a running answer or `calls` gives it: that call and its whole answer",
|
||||
}, nil)},
|
||||
{Name: "status", Description: "What is wrong, what is quiet, what is out of date, and which " +
|
||||
"machines are behind what the mesh would send them.",
|
||||
Input: schema(nil, nil)},
|
||||
@@ -90,6 +97,7 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{
|
||||
"module": "one module's name; every module when absent",
|
||||
"log": "a build's id (as `builds` lists it): print what the build machine said, line by line",
|
||||
"limit": "how many builds to list (default 20); not with log",
|
||||
}, nil)},
|
||||
{Name: "plans", Description: "What the last merges produced and where each stands (novox/hq ADR 0162): " +
|
||||
"the tiers, the tier a plan is at, what it waits for and since when; one plan whole, given its id.",
|
||||
@@ -101,9 +109,16 @@ var ControllerVerbs = []Verb{
|
||||
"repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules",
|
||||
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
||||
"modules": "with repository: or the modules it would change, comma-separated",
|
||||
"limit": "how many plans to list (default 10); only when listing",
|
||||
"why": "with stop or close: why it is ended by hand — required, and recorded in the hand-act log (novox/hq to-be 45 §7)",
|
||||
"cause": "with stop or close: the cause in a word, or a condition's kind (optional)",
|
||||
}, nil)},
|
||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it.",
|
||||
Input: schema(map[string]string{"node": "the machine's name"}, []string{"node"})},
|
||||
{Name: "plan", Description: "What one machine would run, and why: the declaration the mesh would send it — " +
|
||||
"or, with files, the files it would be given.",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name",
|
||||
"files": "\"true\": the files this machine would be given, instead of the declaration as JSON",
|
||||
}, []string{"node"}, "files")},
|
||||
{Name: "assign", Description: "Put a module on a machine. Refused with the mesh's own words when it cannot resolve there, " +
|
||||
"or when a seat its resources are applied through is held by nothing on the machine (novox/hq ADR 0207).",
|
||||
Input: schema(map[string]string{"node": "the machine's name",
|
||||
@@ -121,9 +136,18 @@ var ControllerVerbs = []Verb{
|
||||
}, []string{"node", "provision", "from", "module"})},
|
||||
{Name: "unpin", Description: "Take that choice back, putting the question to the mesh again.",
|
||||
Input: schema(map[string]string{"node": "the machine's name", "provision": "the provision"}, []string{"node", "provision"})},
|
||||
{Name: "push", Description: "Send a machine everything it should be — or every machine that is behind, when no machine is named.",
|
||||
Input: schema(map[string]string{"node": "the machine's name; every machine behind when absent"}, nil)},
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine, " +
|
||||
{Name: "push", Description: "Send one machine everything it should be. With no machine named it is a push of the " +
|
||||
"WHOLE mesh — every machine that is behind — and the answer says so first; behind says that outright. " +
|
||||
"Answers at once that it is running, with a call id: `calls` with that id says what it sent " +
|
||||
"(a push can reload the bus, which then refuses any answer still to come). A push by hand is a repair, " +
|
||||
"and says why: recorded in the hand-act log (novox/hq to-be 45 §7).",
|
||||
Input: schema(map[string]string{
|
||||
"node": "the machine's name; without it, every machine that is behind",
|
||||
"behind": "\"true\": every machine that is behind, the whole mesh — the same as naming none, said outright; not with node",
|
||||
"why": "why this is pushed by hand: recorded in the hand-act log",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second push for the same reason uses (optional)",
|
||||
}, []string{"why"}, "behind")},
|
||||
{Name: "rotate", Description: "Replace a credential. A pair credential, by provision (and a consuming machine and module, " +
|
||||
"else every holder): both ends are re-sent together. Or a module's own secret, by machine, module and " +
|
||||
"name: made anew and the machine sent, so the module starts again on it — only for a secret its " +
|
||||
"definition says it reads at start; a value given to the mesh, or one the module applies to a backend, is refused with the reason.",
|
||||
@@ -131,7 +155,7 @@ var ControllerVerbs = []Verb{
|
||||
"provision": "a pair credential: the provision whose credential to replace",
|
||||
"consumer": "with provision: only the holder on this machine (optional)",
|
||||
"node": "an own secret: the machine",
|
||||
"module": "an own secret: the module",
|
||||
"module": "an own secret: the module; with provision: only this consuming module's credential (optional)",
|
||||
"secret": "an own secret: its name in the module's definition",
|
||||
}, nil)},
|
||||
{Name: "issue", Description: "Give a module on a machine its account on the bus: minted, and sealed to the " +
|
||||
@@ -148,8 +172,8 @@ var ControllerVerbs = []Verb{
|
||||
"module": "the module's name",
|
||||
"values": "the settings as a JSON object, for set",
|
||||
"node": "one machine; the whole mesh when absent",
|
||||
"clear": "\"true\" to remove the layer instead of setting it",
|
||||
}, []string{"module"})},
|
||||
"clear": "\"true\" to remove the layer instead of setting it; not with values",
|
||||
}, []string{"module"}, "clear")},
|
||||
{Name: "command", Description: "Run one command line of the controller's own, as you would type it at its " +
|
||||
"shell — `node account g14 jochen`, `node show ace`, `module list` — and answer what it printed. The " +
|
||||
"generic verb beside the named ones (novox/hq ADR 0154): everything the binary can do, without a verb " +
|
||||
@@ -167,7 +191,7 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{"id": "the ask's build id, as `queue` lists it"}, []string{"id"})},
|
||||
{Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " +
|
||||
"recorded failed, cancelled by hand. Never touches one in flight.",
|
||||
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil)},
|
||||
Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")},
|
||||
{Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " +
|
||||
"build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.",
|
||||
Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})},
|
||||
@@ -178,7 +202,7 @@ var ControllerVerbs = []Verb{
|
||||
"id": "the build's id",
|
||||
"register": "\"true\" to register what it builds",
|
||||
"older": "\"true\", with register: even though a newer build of the module is registered",
|
||||
}, []string{"id"})},
|
||||
}, []string{"id"}, "register", "older")},
|
||||
{Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " +
|
||||
"announces it failed, killed by hand — settled, never handed to another machine.",
|
||||
Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})},
|
||||
@@ -187,6 +211,26 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
{Name: "resume", Description: "The build seat's holder on one machine — or every holder — takes builds again.",
|
||||
Input: schema(map[string]string{"node": "one machine; every holder when absent"}, nil)},
|
||||
// Acts done by hand, and what the bounds are set from (novox/hq to-be 45 §7, Phase 0).
|
||||
{Name: "hand-act", Description: "Record an act done by hand outside the mesh — a container restarted, a file " +
|
||||
"edited, a service started on a machine — with why and its cause, in the hand-act log beside the pushes and " +
|
||||
"plans ended by hand (novox/hq to-be 45 §7). A cause recorded twice in a fortnight is a healer wanted.",
|
||||
Input: schema(map[string]string{
|
||||
"what": "what was done, in a line",
|
||||
"why": "why it had to be done by hand",
|
||||
"cause": "the cause in a word, or a condition's kind — the word a second act for the same reason uses",
|
||||
"condition": "the key of the condition it addressed, if any (optional)",
|
||||
}, []string{"what", "why", "cause"})},
|
||||
{Name: "hand-acts", Description: "What was done by hand lately — pushes, plans ended, consumers re-made, acts " +
|
||||
"recorded — who, why and the cause of each, and which causes repeat: each repeat is a healer the mesh lacks.",
|
||||
Input: schema(map[string]string{"days": "how many days back (default 14)"}, nil)},
|
||||
{Name: "durations", Description: "How long things take, as the controller measured them: a send to its machine's " +
|
||||
"report (apply), a machine's silence between words (heartbeat-gap), a plan's tier, a build — per machine, " +
|
||||
"repository or module, with median, p90 and max. What the core's bounds are set from (novox/hq to-be 45 Phase 0).",
|
||||
Input: schema(map[string]string{
|
||||
"kind": "one kind: apply, heartbeat-gap, plan-tier or build; every kind when absent",
|
||||
"days": "how many days back (default 14)",
|
||||
}, nil)},
|
||||
{Name: "build", Description: "Have the build machine build a repository. Answers at once with the build's id: " +
|
||||
"`builds` with that id follows it line by line, and the module is registered when the outcome comes.",
|
||||
Input: schema(map[string]string{
|
||||
@@ -197,11 +241,27 @@ var ControllerVerbs = []Verb{
|
||||
}
|
||||
|
||||
// schema is a JSON schema for an object of string properties, which is every argument the verbs
|
||||
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call.
|
||||
func schema(properties map[string]string, required []string) map[string]any {
|
||||
// above take. Kept small on purpose: a schema an agent cannot read is a tool it cannot call. The
|
||||
// switches are the properties that are "true" or "false" and nothing else, said in the schema as an
|
||||
// enum so a caller sees it and the verb can refuse any other word rather than read it as false.
|
||||
//
|
||||
// **The schema is the whole of what a verb takes** (novox/hq issue 244): an argument it does not
|
||||
// name is refused when the verb is called, never passed over.
|
||||
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
|
||||
isSwitch := map[string]bool{}
|
||||
for _, s := range switches {
|
||||
if _, declared := properties[s]; !declared {
|
||||
panic("a switch that is not a property: " + s)
|
||||
}
|
||||
isSwitch[s] = true
|
||||
}
|
||||
props := map[string]any{}
|
||||
for name, description := range properties {
|
||||
props[name] = map[string]any{"type": "string", "description": description}
|
||||
p := map[string]any{"type": "string", "description": description}
|
||||
if isSwitch[name] {
|
||||
p["enum"] = []string{"true", "false"}
|
||||
}
|
||||
props[name] = p
|
||||
}
|
||||
out := map[string]any{"type": "object", "properties": props}
|
||||
if len(required) > 0 {
|
||||
|
||||
@@ -124,7 +124,7 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio
|
||||
|
||||
d := broker.Declared{
|
||||
Module: m.Module,
|
||||
Emits: m.Emits,
|
||||
Emits: m.EmitsAll(),
|
||||
Consumes: fromModules,
|
||||
Watches: watches,
|
||||
// The tools it answers, which is `tools` and not `serves`: the manifest's `serves` is the
|
||||
|
||||
@@ -1158,6 +1158,37 @@ func (i *Inventory) SetUpgradeOf(ctx context.Context, module string, u Upgrade)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CurrentBuild is the build a module is at and whether its policy rolls a new one out (novox/hq
|
||||
// issue 259).
|
||||
type CurrentBuild struct {
|
||||
// Commit is the commit the module's manifest was read at — its `built_from` — and empty for a
|
||||
// module held without a source.
|
||||
Commit string
|
||||
// RollOut is the module's upgrade policy, as Upgrade.RollOut.
|
||||
RollOut bool
|
||||
}
|
||||
|
||||
// CurrentBuilds is every module's current build and upgrade policy, in one read: what a send records
|
||||
// it carried, and what a push compares a machine's last send against.
|
||||
func (i *Inventory) CurrentBuilds(ctx context.Context) (map[string]CurrentBuild, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select name, coalesce(built_from, ''), upgrade = 'roll-out' from module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]CurrentBuild{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var b CurrentBuild
|
||||
if err := rows.Scan(&name, &b.Commit, &b.RollOut); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = b
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Running is every machine assigned a module, in a stable order.
|
||||
//
|
||||
// **Assigned, not reported.** A machine that is assigned the module and has not applied it yet is
|
||||
|
||||
@@ -189,7 +189,7 @@ func TestAMachineIsWaitingWhenWhatItWasSentIsNotWhatItShouldBe(t *testing.T) {
|
||||
}
|
||||
|
||||
// Sent what it should be: not waiting.
|
||||
if err := inv.RecordSent(ctx, anchor.ID, "aaa"); err != nil {
|
||||
if err := inv.RecordSent(ctx, anchor.ID, "aaa", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiting, err = inv.Waiting(ctx, map[string]string{"anchor": "aaa", "laptop": "bbb"})
|
||||
@@ -234,7 +234,7 @@ func TestAMachineWithNothingComputedForItIsNotWaiting(t *testing.T) {
|
||||
// It has been sent something before, which is what makes this the case the guard is for: a
|
||||
// machine with a digest and nothing computed for it would compare against the empty string
|
||||
// and look out of date, when the truth is that nobody worked out what it should be.
|
||||
if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time"); err != nil {
|
||||
if err := inv.RecordSent(ctx, node.ID, "what-it-got-last-time", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waiting, err := inv.Waiting(ctx, map[string]string{})
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
|
||||
|
||||
// The kinds of duration recorded.
|
||||
const (
|
||||
DurationApply = "apply"
|
||||
DurationHeartbeatGap = "heartbeat-gap"
|
||||
DurationPlanTier = "plan-tier"
|
||||
DurationBuild = "build"
|
||||
)
|
||||
|
||||
// DurationKinds are every kind, in the order `durations` shows them.
|
||||
var DurationKinds = []string{DurationApply, DurationHeartbeatGap, DurationPlanTier, DurationBuild}
|
||||
|
||||
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
|
||||
// in Phase 1's first live week.
|
||||
const DurationsKeptFor = 30 * 24 * time.Hour
|
||||
|
||||
// Duration is one measurement.
|
||||
type Duration struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Ref string `json:"ref"`
|
||||
Started time.Time `json:"started"`
|
||||
Took time.Duration `json:"took"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// RecordDuration keeps one measurement, once: the same thing measured again is not a second row.
|
||||
func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error {
|
||||
if d.Took < 0 {
|
||||
return nil // a clock that went back measures nothing
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`,
|
||||
d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send
|
||||
// to the first report of it. A report of anything else, or of a send already measured, measures
|
||||
// nothing — a machine reconciling reports the same declaration every few minutes.
|
||||
func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error {
|
||||
if declared == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'),
|
||||
n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4
|
||||
from node n
|
||||
where n.name = $2 and n.sent = $3 and n.sent_at is not null
|
||||
on conflict do nothing`,
|
||||
DurationApply, node, declared, outcome)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordHeartbeatGap measures the silence before a machine's word: from the last word heard before
|
||||
// it, which the caller read before recording this one.
|
||||
func (i *Inventory) RecordHeartbeatGap(ctx context.Context, node string, before time.Time) error {
|
||||
if before.IsZero() {
|
||||
return nil
|
||||
}
|
||||
now := time.Now()
|
||||
return i.RecordDuration(ctx, Duration{Kind: DurationHeartbeatGap, Subject: node, Node: node,
|
||||
Ref: before.UTC().Format(time.RFC3339Nano), Started: before, Took: now.Sub(before)})
|
||||
}
|
||||
|
||||
// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty.
|
||||
func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select kind, subject, node, ref, started, took_ms, detail from duration
|
||||
where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Duration
|
||||
for rows.Next() {
|
||||
var d Duration
|
||||
var ms int64
|
||||
if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Took = time.Duration(ms) * time.Millisecond
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
|
||||
func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
|
||||
time.Now().Add(-DurationsKeptFor))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// planTierLeft is the measurement a plan's save makes when it leaves a tier: the tier moved on, or
|
||||
// the plan ended. Read in the save's own transaction, so two saves cannot both measure one tier.
|
||||
func planTierLeft(ctx context.Context, tx pgx.Tx, p Plan, now time.Time) (entered time.Time, err error) {
|
||||
var oldTier int
|
||||
var oldState string
|
||||
var since time.Time
|
||||
err = tx.QueryRow(ctx,
|
||||
`select tier, state, coalesce(tier_entered, created) from release_plan where id = $1 for update`,
|
||||
p.ID).Scan(&oldTier, &oldState, &since)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return now, nil // a new plan enters its first tier now
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
wasOpen := oldState == PlanBuilding || oldState == PlanRolling
|
||||
if !wasOpen || (oldTier == p.Tier && p.Open()) {
|
||||
return since, nil // still in the tier, or already ended
|
||||
}
|
||||
var modules []string
|
||||
if oldTier >= 0 && oldTier < len(p.Tiers) {
|
||||
modules = p.Tiers[oldTier]
|
||||
}
|
||||
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, oldTier, len(p.Tiers), modules, p.State)
|
||||
if p.Open() {
|
||||
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, oldTier, len(p.Tiers), modules, p.Tier)
|
||||
}
|
||||
_, err = tx.Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
values ($1, $2, '', $3, $4, $5, $6) on conflict do nothing`,
|
||||
DurationPlanTier, p.Repository, fmt.Sprintf("%s/tier-%d", p.ID, oldTier), since,
|
||||
now.Sub(since).Milliseconds(), detail)
|
||||
return now, err
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a
|
||||
// send measured at its first report and not again at every reconcile that repeats it; a send of
|
||||
// something else measures nothing; a new send is a new measurement.
|
||||
func TestAnApplyIsMeasuredOncePerSend(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
node, err := inv.AddNode(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 3 { // the report, then two reconciles saying the same
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 {
|
||||
t.Fatalf("a second send was not measured: %+v", ds)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's silence is the time since its last word, once per word.
|
||||
func TestASilenceIsMeasuredFromTheLastWord(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
before := time.Now().Add(-90 * time.Second)
|
||||
for range 2 {
|
||||
if err := inv.RecordHeartbeatGap(ctx, "anchor", before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordHeartbeatGap(ctx, "anchor", time.Time{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationHeartbeatGap, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 1 || ds[0].Took < 90*time.Second || ds[0].Took > 2*time.Minute {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then.
|
||||
func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
|
||||
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
|
||||
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.State = PlanRolling // the same tier, saved again
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
|
||||
t.Fatalf("a tier not left was measured: %+v", ds)
|
||||
}
|
||||
p.Tier = 1
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.State = PlanDone
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil { // saved again once ended: nothing more to measure
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" ||
|
||||
ds[1].Ref != "plan-1/tier-1" {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
}
|
||||
@@ -97,3 +97,57 @@ func TestAMachinesMembershipIsOneRowReplacedAndGoesWithTheMachine(t *testing.T)
|
||||
t.Fatalf("a re-told membership did not replace the first: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A replicated seat has several holders on record (novox/hq ADR 0223): each is added beside the
|
||||
// others, adding one twice changes nothing, a handover still leaves exactly one, and unassigning one
|
||||
// takes only its own row.
|
||||
func TestAReplicatedSeatHasSeveralHoldersOnRecord(t *testing.T) {
|
||||
resolver := catalogue.Manifest{Module: "resolver", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}
|
||||
inv, ctx := aMeshWith(t, resolver)
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range []string{"anchor", "home"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, n, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "anchor", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
held, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(held) != 2 || held[0].Node != "anchor" || held[1].Node != "home" {
|
||||
t.Fatalf("the two holders are not both on record, once each: %+v", held)
|
||||
}
|
||||
if err := inv.Unassign(ctx, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "anchor" {
|
||||
t.Fatalf("unassigning one holder took more or less than its own row: %+v", held)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.AddSeatHolder(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, "mesh-dns-resolver", catalogue.ScopeMesh, "home", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if held, _ = inv.Holdings(ctx); len(held) != 1 || held[0].Node != "home" {
|
||||
t.Fatalf("a handover left other holders on record: %+v", held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
-- The per-node resolver's seat is gone (novox/hq ADR 0220, retiring what ADR 0194 decided).
|
||||
--
|
||||
-- ADR 0194 retired `node-dns-resolver` when the mesh moved to one resolver, and kept its row while a
|
||||
-- machine still held it: removing a seat that is claimed makes the claiming machine unresolvable. On
|
||||
-- the mesh this was written for, nothing has held it since every node's resolver moved to the mesh's
|
||||
-- one, and no module in the catalogue claims it, so the row goes.
|
||||
--
|
||||
-- **The compiled defaults no longer carry it, and that alone would not remove it.** Seeding adds a
|
||||
-- seat a release ships and never takes one away (ADR 0122: the table is the live set, and an
|
||||
-- operator's row is left as it is), so a seat the mesh stops defining stays in the table until
|
||||
-- something deletes it — this.
|
||||
--
|
||||
-- A holding on record goes with it by cascade; there is none. No alias is kept: nothing was renamed,
|
||||
-- and a manifest still claiming the old name should be refused at registration, naming it, rather
|
||||
-- than resolve to anything.
|
||||
delete from seat_alias where seat = 'node-dns-resolver' or alias = 'node-dns-resolver';
|
||||
delete from seat where name = 'node-dns-resolver';
|
||||
@@ -0,0 +1,14 @@
|
||||
-- A send records the build of each module it carried (novox/hq issue 259, ADR 0221).
|
||||
--
|
||||
-- A named push ends by sending every other machine whose declaration differs from what it was last
|
||||
-- sent (ADR 0083). Read from the declaration's digest alone, a module whose upgrade policy records
|
||||
-- rather than rolls out, or whose plan sends one machine first (ADR 0218), made every machine running
|
||||
-- it differ, so `push <one machine>` sent all of them the build the policy was holding back. Which
|
||||
-- build of each module a machine was last sent is what tells a held upgrade from a consequence of the
|
||||
-- push, and it is not in a digest.
|
||||
--
|
||||
-- Module name to the commit its build was made from — the module's `built_from` when the declaration
|
||||
-- was composed, empty for a module the mesh holds without a source. NULL for a machine last sent
|
||||
-- before this was kept, or sent a declaration by hand: what it carried is not known, and the push
|
||||
-- treats such a machine as held until it is pushed by name.
|
||||
alter table node add column sent_builds jsonb;
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
-- A replicated seat has several holders on record (novox/hq ADR 0223).
|
||||
--
|
||||
-- 0039 recorded one holder per seat, keyed by the seat: a handover replaced the row, so the seat was
|
||||
-- never without a holder in between. The mesh's resolver is now held on more than one machine, each
|
||||
-- answering the same names, and each of those holders is recorded — added by `seat <name> --add`,
|
||||
-- never by being assigned. So a holding is keyed by the seat and the assignment holding it.
|
||||
--
|
||||
-- Nothing else changes. A handover (`seat <name> --to`) still leaves exactly one row, replacing every
|
||||
-- holder in one transaction; whether a seat may have more than one is the seat's compiled definition,
|
||||
-- judged by the controller before a row is added, and a store holding two for any other seat is
|
||||
-- refused at resolution, naming the seat. Every existing row is one per seat, so it satisfies the new
|
||||
-- key as it stands.
|
||||
alter table seat_holding drop constraint seat_holding_pkey;
|
||||
alter table seat_holding add primary key (seat, node, module);
|
||||
@@ -0,0 +1,17 @@
|
||||
-- What a machine asks for names is the uplink's holder's to write (novox/hq ADR 0223, retiring what
|
||||
-- ADR 0121 and ADR 0220 decided for node-resolver-config).
|
||||
--
|
||||
-- `/etc/resolv.conf` is written by the module holding `node-uplink` — the program that would otherwise
|
||||
-- rewrite it — so the seat whose holder wrote it, and its need of the uplink beside it, go. Its only
|
||||
-- claimant, `resolv-conf`, declared nothing for one release while every machine handed the file to its
|
||||
-- uplink module in one apply, and was then unassigned everywhere and forgotten before this runs.
|
||||
--
|
||||
-- **The compiled defaults no longer carry it, and that alone would not remove it**: seeding adds a
|
||||
-- seat a release ships and never takes one away (ADR 0122), as 0060 found for the per-node resolver.
|
||||
-- A holding on record goes with it by cascade; a node seat has none. No alias is kept: nothing was
|
||||
-- renamed, and a manifest still claiming the old name should be refused at registration, naming it.
|
||||
--
|
||||
-- Numbered after 0063 (node-hosts-file renamed to node-hostname), which is expected to merge first;
|
||||
-- if this one lands first, the two are renumbered so the order they merge in is the order they run.
|
||||
delete from seat_alias where seat = 'node-resolver-config' or alias = 'node-resolver-config';
|
||||
delete from seat where name = 'node-resolver-config';
|
||||
@@ -0,0 +1,23 @@
|
||||
-- A machine's names are one seat's (novox/hq ADR 0223 part 3): `node-hosts-file` is renamed
|
||||
-- `node-hostname`, whose holder writes /etc/hostname beside the machine's own lines in /etc/hosts.
|
||||
--
|
||||
-- A rename is a database update (ADR 0122): the row keeps its verbs, the former name becomes an alias
|
||||
-- that resolves to it, so a manifest registered under the old name — the `hosts` module still assigned
|
||||
-- while machines move to `hostname` — goes on holding the one seat, and two claimants of it on one
|
||||
-- machine are still refused.
|
||||
--
|
||||
-- **Both rows may exist when this runs**, as 0048 found for the artifact store: a controller whose
|
||||
-- compiled defaults carry the new name may seed it before this migration. Then the old row's holding
|
||||
-- moves to it and the old row goes; otherwise the old row is renamed. Either way the old name becomes
|
||||
-- an alias.
|
||||
update seat_holding set seat = 'node-hostname'
|
||||
where seat = 'node-hosts-file'
|
||||
and exists (select 1 from seat where name = 'node-hostname');
|
||||
delete from seat
|
||||
where name = 'node-hosts-file'
|
||||
and exists (select 1 from seat where name = 'node-hostname');
|
||||
update seat set name = 'node-hostname', decided = 'novox/hq ADR 0199, ADR 0223'
|
||||
where name = 'node-hosts-file';
|
||||
insert into seat_alias (alias, seat) values ('node-hosts-file', 'node-hostname')
|
||||
on conflict (alias) do update set seat = excluded.seat;
|
||||
update seat_alias set seat = 'node-hostname' where seat = 'node-hosts-file';
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
-- A provider says which consumer it keeps failing (novox/hq ADR 0224).
|
||||
--
|
||||
-- On 2026-10-05 the identity provider's provisioner failed every consumer 31,000 times in a day and
|
||||
-- only its journal said so (issue 179). A provider now announces a consumer it has failed for minutes
|
||||
-- without one success, and the consumer recovering; the controller keeps the newest failing word per
|
||||
-- provider module, the machine it runs on and the consumer, and removes it on recovery. `status` and
|
||||
-- `node show` read this table: a row is a problem until it is gone.
|
||||
create table provider_standing (
|
||||
module text not null,
|
||||
provider_node text not null,
|
||||
consumer text not null,
|
||||
consumer_node text not null default '',
|
||||
provision text not null default '',
|
||||
class text not null default '',
|
||||
error text not null default '',
|
||||
since timestamptz not null,
|
||||
attempts integer not null default 0,
|
||||
said_at timestamptz not null default now(),
|
||||
primary key (module, provider_node, consumer)
|
||||
);
|
||||
@@ -0,0 +1,31 @@
|
||||
-- The durations the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
--
|
||||
-- Every watchdog of the core's signals table has a bound — how long a machine may take to report
|
||||
-- after a send, how long it may be silent, how long a plan's tier or a build may take — and a bound
|
||||
-- guessed is a condition that cries wolf or one that never fires. So the controller records each
|
||||
-- duration as it is observed, and Phase 1 sets the bounds from what was recorded:
|
||||
--
|
||||
-- apply a declaration sent → the machine's first report of that declaration, per machine
|
||||
-- heartbeat-gap one word from a machine → the next, per machine
|
||||
-- plan-tier a plan entering a tier → leaving it, per repository
|
||||
-- build a build asked → its outcome heard, per module
|
||||
--
|
||||
-- One row per thing measured: `ref` names it (the send, the earlier word, the plan's tier, the
|
||||
-- build), so a report repeated by a reconcile is not a second measurement. Kept a month; read
|
||||
-- through `durations`.
|
||||
create table duration (
|
||||
kind text not null,
|
||||
subject text not null,
|
||||
node text not null default '',
|
||||
ref text not null,
|
||||
started timestamptz not null,
|
||||
took_ms bigint not null,
|
||||
detail text not null default '',
|
||||
recorded timestamptz not null default now(),
|
||||
primary key (kind, subject, ref)
|
||||
);
|
||||
|
||||
create index duration_by_kind on duration (kind, recorded);
|
||||
|
||||
-- When a plan entered the tier it is at, so leaving it measures the tier.
|
||||
alter table release_plan add column tier_entered timestamptz;
|
||||
@@ -851,9 +851,9 @@ func (i *Inventory) DoingOf(ctx context.Context, name string) (Doing, bool, erro
|
||||
var d Doing
|
||||
var failed []byte
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select outcome, refused, failed, applied, at, failing_since, failures
|
||||
`select outcome, refused, failed, applied, at, failing_since, failures, coalesce(declared, '')
|
||||
from node_report where node = $1`, node.ID).
|
||||
Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At, &d.Since, &d.Times)
|
||||
Scan(&d.Outcome, &d.Refused, &failed, &d.Applied, &d.At, &d.Since, &d.Times, &d.Declared)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return Doing{}, false, nil
|
||||
}
|
||||
@@ -909,18 +909,53 @@ func (i *Inventory) LastReports(ctx context.Context) ([]Reported, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RecordSent keeps a digest of the declaration a machine was last sent.
|
||||
// RecordSent keeps a digest of the declaration a machine was last sent, and the build of each module
|
||||
// it carried.
|
||||
//
|
||||
// **A digest rather than the declaration.** The mesh can compute what a machine should be at any
|
||||
// moment; keeping a copy would be a second account of it, able to disagree with the first. What
|
||||
// cannot be recomputed is what was *actually sent*, and that is the whole difference between a
|
||||
// machine that is out of date and one that has never been told.
|
||||
func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error {
|
||||
//
|
||||
// **And which build of each module** (novox/hq issue 259, ADR 0221): module name to the commit its
|
||||
// build was made from. A digest cannot say whether a machine differs because a module moved to a build
|
||||
// its upgrade policy holds back, or because of something a push made — a grant — and only the second
|
||||
// is a push's to send to a machine it did not name. Nil records that it is not known, as for a
|
||||
// declaration sent by hand.
|
||||
func (i *Inventory) RecordSent(ctx context.Context, node, digest string, builds map[string]string) error {
|
||||
var carried *string
|
||||
if builds != nil {
|
||||
raw, err := json.Marshal(builds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := string(raw)
|
||||
carried = &text
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update node set sent = $2, sent_at = now() where id = $1`, node, digest)
|
||||
`update node set sent = $2, sent_at = now(), sent_builds = $3::jsonb where id = $1`, node, digest, carried)
|
||||
return err
|
||||
}
|
||||
|
||||
// SentBuilds is the build of each module a machine was last sent, by its name: module to the commit
|
||||
// its build was made from (novox/hq issue 259). Known is false when that was not kept — a machine
|
||||
// last sent before it was, sent a declaration by hand, or one the mesh does not know.
|
||||
func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) {
|
||||
var raw []byte
|
||||
err = i.store.Pool().QueryRow(ctx, `select sent_builds from node where name = $1`, name).Scan(&raw)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil || raw == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
builds = map[string]string{}
|
||||
if err := json.Unmarshal(raw, &builds); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return builds, true, nil
|
||||
}
|
||||
|
||||
// RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name
|
||||
// (novox/hq issue 249): whether the machine holding the bus must go first is whether this differs
|
||||
// from the list composed now.
|
||||
|
||||
@@ -80,13 +80,29 @@ func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch)
|
||||
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10)
|
||||
// **And how long the tier it left took** (novox/hq to-be 45 Phase 0): measured here, where the
|
||||
// plan moves, in the same transaction as the move, so no save can move a tier unmeasured or
|
||||
// measure one twice.
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
entered, err := planTierLeft(ctx, tx, p, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx,
|
||||
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch, tier_entered)
|
||||
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10, $11)
|
||||
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
||||
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch`,
|
||||
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch)
|
||||
return err
|
||||
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch,
|
||||
tier_entered = excluded.tier_entered`,
|
||||
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch, entered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// OpenPlans is every plan still being worked, oldest first.
|
||||
|
||||
+69
-14
@@ -105,14 +105,27 @@ func (i *Inventory) widenProtocol(ctx context.Context, s catalogue.Seat) error {
|
||||
changed := false
|
||||
row.Accepts, changed = union(row.Accepts, s.Accepts, changed)
|
||||
row.Emits, changed = union(row.Emits, s.Emits, changed)
|
||||
have := map[string]bool{}
|
||||
for _, v := range row.Serves {
|
||||
have[v.Name] = true
|
||||
have := map[string]int{}
|
||||
for n, v := range row.Serves {
|
||||
have[v.Name] = n
|
||||
}
|
||||
for _, v := range s.Serves {
|
||||
if !have[v.Name] {
|
||||
at, kept := have[v.Name]
|
||||
if !kept {
|
||||
row.Serves = append(row.Serves, v)
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
// **The controller's own verbs are described by the binary that runs them** (novox/hq
|
||||
// issue 244, to-be 45 §7). Its seat's verbs are not an operator's to reshape: each is a
|
||||
// command line this binary composes from the arguments its own table declares, and the
|
||||
// console judges a call against the row. A row kept from an older build described `push`
|
||||
// without the `behind` and `why` the binary takes, so the console refused an argument the verb
|
||||
// needs. So a verb this binary defines takes this binary's definition; a verb only the row has
|
||||
// — a newer build's, during a roll-out (ADR 0185) — is left as it is.
|
||||
if s.Name == catalogue.ControllerSeatName && !sameVerb(row.Serves[at], v) {
|
||||
row.Serves[at] = v
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
@@ -215,23 +228,50 @@ func (i *Inventory) RenameSeat(ctx context.Context, from, to string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HoldSeat records that one assignment holds a seat, replacing whoever held it — as one write, so
|
||||
// the seat is never without a holder in between (novox/hq ADR 0131, design 28 task 5.3). The
|
||||
// assignment must exist; the store refuses otherwise, and that refusal is the right one: a seat
|
||||
// cannot be handed to something that is not running anywhere.
|
||||
// HoldSeat records that one assignment holds a seat, replacing whoever held it — every holder, for a
|
||||
// replicated seat (novox/hq ADR 0223) — as one transaction, so the seat is never without a holder in
|
||||
// between (novox/hq ADR 0131, design 28 task 5.3). The assignment must exist; the store refuses
|
||||
// otherwise, and that refusal is the right one: a seat cannot be handed to something that is not
|
||||
// running anywhere.
|
||||
func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module string) error {
|
||||
node, err := i.NodeByName(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
|
||||
on conflict (seat) do update set scope = excluded.scope, node = excluded.node,
|
||||
module = excluded.module, since = now()`,
|
||||
seat, scope, node.ID, module)
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||
if _, err := tx.Exec(ctx,
|
||||
`delete from seat_holding where seat = $1 and not (node = $2 and module = $3)`,
|
||||
seat, node.ID, module); err != nil {
|
||||
return fmt.Errorf("handing %s to %s on %s: %w", seat, module, nodeName, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
|
||||
on conflict (seat, node, module) do update set scope = excluded.scope, since = now()`,
|
||||
seat, scope, node.ID, module); err != nil {
|
||||
return fmt.Errorf("recording %s on %s as the holder of %s: %w", module, nodeName, seat, err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// AddSeatHolder records one more assignment holding a replicated seat, beside those already on
|
||||
// record (novox/hq ADR 0223). Whether the seat may have several holders is the caller's to judge —
|
||||
// the seat's definition is compiled, and the store holds no column for it. Recording a holder
|
||||
// already on record changes nothing.
|
||||
func (i *Inventory) AddSeatHolder(ctx context.Context, seat, scope, nodeName, module string) error {
|
||||
node, err := i.NodeByName(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := i.store.Pool().Exec(ctx,
|
||||
`insert into seat_holding (seat, scope, node, module) values ($1, $2, $3, $4)
|
||||
on conflict (seat, node, module) do nothing`,
|
||||
seat, scope, node.ID, module); err != nil {
|
||||
return fmt.Errorf("adding %s on %s as a holder of %s: %w", module, nodeName, seat, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -240,7 +280,7 @@ func (i *Inventory) HoldSeat(ctx context.Context, seat, scope, nodeName, module
|
||||
func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select h.seat, h.scope, n.name, h.module, coalesce(n.site, '')
|
||||
from seat_holding h join node n on n.id = h.node order by h.seat`)
|
||||
from seat_holding h join node n on n.id = h.node order by h.seat, n.name, h.module`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -255,3 +295,18 @@ func (i *Inventory) Holdings(ctx context.Context) ([]catalogue.Held, error) {
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// sameVerb is whether two definitions of a verb say the same, read as the row stores them.
|
||||
func sameVerb(a, b catalogue.Verb) bool {
|
||||
ja, errA := json.Marshal(a)
|
||||
jb, errB := json.Marshal(b)
|
||||
if errA != nil || errB != nil {
|
||||
return false
|
||||
}
|
||||
var ra, rb any
|
||||
_ = json.Unmarshal(ja, &ra)
|
||||
_ = json.Unmarshal(jb, &rb)
|
||||
ca, _ := json.Marshal(ra)
|
||||
cb, _ := json.Marshal(rb)
|
||||
return string(ca) == string(cb)
|
||||
}
|
||||
|
||||
@@ -113,3 +113,46 @@ func TestRenameSeatKeepsTheFormerNameAsAnAlias(t *testing.T) {
|
||||
t.Fatal("renaming a seat to its own name was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// **The controller's verbs in the row are the binary's** (novox/hq issue 244, to-be 45 §7): a row
|
||||
// seeded by an older build describes `push` without the arguments this one takes, and the console
|
||||
// judges a call against the row — so re-seeding brings the controller's verbs to this binary's
|
||||
// definition, and keeps a verb only the row has, which a newer build added (ADR 0185).
|
||||
func TestTheControllersVerbsInTheRowAreTheBinarys(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := `[{"name":"push","description":"an older push","input":{"type":"object","properties":{"node":{"type":"string"}}}},
|
||||
{"name":"newer","description":"a verb of a newer build"}]`
|
||||
if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`,
|
||||
[]byte(old), catalogue.ControllerSeatName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seats, err := inv.Seats(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verbs := map[string]catalogue.Verb{}
|
||||
for _, s := range seats {
|
||||
if s.Name == catalogue.ControllerSeatName {
|
||||
for _, v := range s.Serves {
|
||||
verbs[v.Name] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
props, _ := verbs["push"].Input["properties"].(map[string]any)
|
||||
if _, takesWhy := props["why"]; !takesWhy || verbs["push"].Description == "an older push" {
|
||||
t.Fatalf("push in the row is still the older build's: %+v", verbs["push"])
|
||||
}
|
||||
if _, kept := verbs["newer"]; !kept {
|
||||
t.Fatal("a verb only the row has was dropped")
|
||||
}
|
||||
if _, added := verbs["durations"]; !added {
|
||||
t.Fatal("a verb this binary adds was not added")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq issue 259: a send keeps which build of each module it carried. A machine never sent
|
||||
// anything, or sent with nothing recorded — before this was kept, or by hand — is not known, which
|
||||
// is not the same as having been sent nothing.
|
||||
func TestASendKeepsTheBuildsItCarried(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
node, err := inv.AddNode(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
|
||||
t.Fatalf("a machine never sent anything has known builds %v (%v): %v", builds, known, err)
|
||||
}
|
||||
|
||||
carried := map[string]string{"resolver": "abc123", "network": ""}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builds, known, err := inv.SentBuilds(ctx, "anchor")
|
||||
if err != nil || !known || !reflect.DeepEqual(builds, carried) {
|
||||
t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err)
|
||||
}
|
||||
|
||||
// Sent with nothing carried: known, and empty.
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", map[string]string{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || !known || len(builds) != 0 {
|
||||
t.Fatalf("an empty send: %v %v %v", builds, known, err)
|
||||
}
|
||||
|
||||
// Sent by hand: not known, and the digest still recorded.
|
||||
if err := inv.RecordSent(ctx, node.ID, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
|
||||
t.Fatalf("a send whose builds are not known read as %v %v: %v", builds, known, err)
|
||||
}
|
||||
if sent, err := inv.Outstanding(ctx, "anchor"); err != nil || sent != "d3" {
|
||||
t.Fatalf("the digest was not recorded with it: %q %v", sent, err)
|
||||
}
|
||||
|
||||
if _, known, err := inv.SentBuilds(ctx, "nobody"); err != nil || known {
|
||||
t.Fatalf("a machine the mesh does not know: %v %v", known, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's current build is the commit its manifest was read at, with its upgrade policy.
|
||||
func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "resolver", Version: "1"},
|
||||
Source{Repository: "novox/mesh-catalog", BuiltFrom: "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "by-hand", Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, "by-hand", Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1"}) {
|
||||
t.Errorf("resolver is at %+v", got)
|
||||
}
|
||||
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true}) {
|
||||
t.Errorf("a module with no source is at %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProviderStanding is a consumer a provider says it keeps failing (novox/hq ADR 0224).
|
||||
type ProviderStanding struct {
|
||||
// Module is the provider's module, and ProviderNode the machine it runs on.
|
||||
Module string `json:"module"`
|
||||
ProviderNode string `json:"provider-node"`
|
||||
// Provision is the interface it provides, e.g. `oidc-client`.
|
||||
Provision string `json:"provision"`
|
||||
// Consumer is the identity the mesh derived for the consumer, ConsumerNode its machine.
|
||||
Consumer string `json:"consumer"`
|
||||
ConsumerNode string `json:"consumer-node"`
|
||||
// Class is what kind of failure: credentials-rejected, unreachable, secret-unreadable, refused.
|
||||
Class string `json:"class"`
|
||||
Error string `json:"error"`
|
||||
// Since is when the unbroken run of failures began; Attempts how many it has been.
|
||||
Since time.Time `json:"since"`
|
||||
Attempts int `json:"attempts"`
|
||||
// SaidAt is when the controller last heard it. A provider says it again every quarter of an hour
|
||||
// while it lasts, so an old one is a provider that stopped saying anything.
|
||||
SaidAt time.Time `json:"said-at"`
|
||||
}
|
||||
|
||||
// SayAgainWithin is how long a failing standing stays current without being said again: twice the
|
||||
// quarter of an hour a provider repeats it at. Older, and status says the provider has gone quiet.
|
||||
const SayAgainWithin = 30 * time.Minute
|
||||
|
||||
// Quiet says the provider has not repeated this standing for longer than it would while it lasts.
|
||||
func (s ProviderStanding) Quiet(now time.Time) bool { return now.Sub(s.SaidAt) > SayAgainWithin }
|
||||
|
||||
// KeepStanding records a provider's newest word: failing keeps it, recovered removes it, and says
|
||||
// whether a recovery removed anything.
|
||||
func (i *Inventory) KeepStanding(ctx context.Context, failing bool, s ProviderStanding) (bool, error) {
|
||||
if !failing {
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`delete from provider_standing where module = $1 and provider_node = $2 and consumer = $3`,
|
||||
s.Module, s.ProviderNode, s.Consumer)
|
||||
return err == nil && tag.RowsAffected() > 0, err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx, `
|
||||
insert into provider_standing
|
||||
(module, provider_node, consumer, consumer_node, provision, class, error, since, attempts, said_at)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
||||
on conflict (module, provider_node, consumer) do update set
|
||||
consumer_node = excluded.consumer_node, provision = excluded.provision,
|
||||
class = excluded.class, error = excluded.error, since = excluded.since,
|
||||
attempts = excluded.attempts, said_at = excluded.said_at`,
|
||||
s.Module, s.ProviderNode, s.Consumer, s.ConsumerNode, s.Provision, s.Class, s.Error, s.Since, s.Attempts)
|
||||
return false, err
|
||||
}
|
||||
|
||||
// FailingProviders is every consumer a provider last said it keeps failing, oldest run first.
|
||||
func (i *Inventory) FailingProviders(ctx context.Context) ([]ProviderStanding, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `
|
||||
select module, provider_node, provision, consumer, consumer_node, class, error, since, attempts, said_at
|
||||
from provider_standing order by since, module, consumer`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []ProviderStanding
|
||||
for rows.Next() {
|
||||
var s ProviderStanding
|
||||
if err := rows.Scan(&s.Module, &s.ProviderNode, &s.Provision, &s.Consumer, &s.ConsumerNode,
|
||||
&s.Class, &s.Error, &s.Since, &s.Attempts, &s.SaidAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A provider's standing (novox/hq ADR 0224), from the grant that lets it say so to the row status
|
||||
// reads.
|
||||
|
||||
// The broker spells the events itself because it cannot import the catalogue; the two agree.
|
||||
func TestTheBrokerAndTheCatalogueNameTheSameStandingEvents(t *testing.T) {
|
||||
if broker.ProvisionerFailing != catalogue.ProvisionerFailing ||
|
||||
broker.ProvisionerRecovered != catalogue.ProvisionerRecovered {
|
||||
t.Fatal("the broker and the catalogue disagree about what a provider's standing is called")
|
||||
}
|
||||
}
|
||||
|
||||
// **Every provider may say it, whatever its manifest lists**: a provider whose manifest forgot the
|
||||
// events would have its announcement refused by the bus, and fail its consumers as silently as on
|
||||
// 2026-10-05 (issue 179). A module that receives no contributions provides nothing and is given
|
||||
// nothing.
|
||||
func TestEveryProviderIsGrantedItsStandingAndNothingElseIs(t *testing.T) {
|
||||
provider := catalogue.Manifest{Module: "keycloak", Version: "1",
|
||||
Emits: []string{"client.created"}, Receives: map[string]string{"oidc-client": "/x/mesh.json"}}
|
||||
consumer := catalogue.Manifest{Module: "grafana", Version: "1", Emits: []string{"dashboard.saved"}}
|
||||
|
||||
d := declaredFor(provider, nil)
|
||||
for _, e := range []string{"client.created", catalogue.ProvisionerFailing, catalogue.ProvisionerRecovered} {
|
||||
if !slices.Contains(d.Emits, e) {
|
||||
t.Fatalf("a provider is not granted %s: %v", e, d.Emits)
|
||||
}
|
||||
}
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindModule, Node: "anchor",
|
||||
Module: "keycloak", Emits: d.Emits})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(perms.Publish, "mesh.mod.keycloak.event.provisioner.failing") {
|
||||
t.Fatalf("the bus would refuse a provider's standing: %v", perms.Publish)
|
||||
}
|
||||
|
||||
if got := declaredFor(consumer, nil).Emits; slices.Contains(got, catalogue.ProvisionerFailing) {
|
||||
t.Fatalf("a module that provides nothing was granted a provider's standing: %v", got)
|
||||
}
|
||||
// Declared by hand as well: said once.
|
||||
provider.Emits = append(provider.Emits, catalogue.ProvisionerFailing)
|
||||
n := 0
|
||||
for _, e := range provider.EmitsAll() {
|
||||
if e == catalogue.ProvisionerFailing {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("%v", provider.EmitsAll())
|
||||
}
|
||||
}
|
||||
|
||||
// And the controller may hear it from every provider, and only those two events.
|
||||
func TestTheControllerHearsEveryProvidersStanding(t *testing.T) {
|
||||
perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered"} {
|
||||
if !slices.Contains(perms.Subscribe, want) {
|
||||
t.Fatalf("the controller may not hear %s: %v", want, perms.Subscribe)
|
||||
}
|
||||
}
|
||||
if slices.Contains(perms.Subscribe, "mesh.mod.*.event.>") {
|
||||
t.Fatal("the controller hears every event in the mesh")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailingStandingIsKeptUntilItRecovers(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
since := time.Date(2026, 10, 5, 0, 49, 0, 0, time.UTC)
|
||||
s := ProviderStanding{Module: "keycloak", ProviderNode: "anchor", Provision: "oidc-client",
|
||||
Consumer: "mesh_home_grafana", ConsumerNode: "home-server", Class: "credentials-rejected",
|
||||
Error: "401 invalid_grant", Since: since, Attempts: 60}
|
||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Said again: one row, the newest word.
|
||||
s.Attempts = 31000
|
||||
if _, err := inv.KeepStanding(ctx, true, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Attempts != 31000 || !got[0].Since.Equal(since) || got[0].ConsumerNode != "home-server" ||
|
||||
got[0].Class != "credentials-rejected" || got[0].SaidAt.IsZero() {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if got[0].Quiet(time.Now()) {
|
||||
t.Fatal("a standing just said reads as quiet")
|
||||
}
|
||||
if !got[0].Quiet(time.Now().Add(SayAgainWithin + time.Minute)) {
|
||||
t.Fatal("a standing not said again for longer than a provider repeats it does not read as quiet")
|
||||
}
|
||||
|
||||
// The same consumer from another machine's provider is its own row.
|
||||
other := s
|
||||
other.ProviderNode = "laptop"
|
||||
if _, err := inv.KeepStanding(ctx, true, other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cleared, err := inv.KeepStanding(ctx, false, s)
|
||||
if err != nil || !cleared {
|
||||
t.Fatalf("recovered cleared nothing: %v %v", cleared, err)
|
||||
}
|
||||
cleared, err = inv.KeepStanding(ctx, false, s)
|
||||
if err != nil || cleared {
|
||||
t.Fatalf("a recovery for nothing kept said it cleared something: %v %v", cleared, err)
|
||||
}
|
||||
got, err = inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].ProviderNode != "laptop" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A role's tool call, kept after it is answered (novox/hq issue 265).
|
||||
//
|
||||
// **A call that outlasts its caller must not end in silence.** A caller waits a bounded time (the
|
||||
// console thirty seconds), and the bus lets a holder answer a request exactly once and only while the
|
||||
// server still remembers it was asked (`allow_responses`). Before this, a push that ran longer than
|
||||
// its caller waited did everything it was asked and its caller read "did not answer in time"; and a
|
||||
// push whose first act sent the bus's own machine a changed user list had its answer refused
|
||||
// outright — a broker reloading its users forgets every reply it was about to permit — so the
|
||||
// answer went nowhere and the only trace was the client library's line on the controller's standard
|
||||
// error. So every call is answered within AnswerWithin, with its whole answer when it has one by
|
||||
// then and with "still running as call <id>" when it does not; and every call is kept here, with
|
||||
// what came of it, including an answer the bus refused, so `calls` can say it.
|
||||
|
||||
// AnswerWithin is how long a call runs before its caller is answered that it is still running. Well
|
||||
// inside the shortest wait of a caller the mesh ships (the console's thirty seconds) and the bus's
|
||||
// own window for an answer (broker.ResponseTTL), so the one answer a call has is never late for
|
||||
// either. A variable so a test need not wait.
|
||||
var AnswerWithin = 10 * time.Second
|
||||
|
||||
// KeptCalls is how many calls this process keeps in memory, newest first — to match a refusal to its
|
||||
// call, and to answer at once; the bus keeps the last thousand (Durably). keptAnswer is the largest
|
||||
// answer kept of a call whose caller was sent it. One its caller never had is kept whole.
|
||||
const (
|
||||
KeptCalls = 100
|
||||
keptAnswer = 64 << 10
|
||||
)
|
||||
|
||||
// The states a kept call is in.
|
||||
const (
|
||||
CallRunning = "running"
|
||||
CallAnswered = "answered"
|
||||
// CallFinishedAfter is a call that finished after its caller was told it was still running: its
|
||||
// answer is here and nowhere else.
|
||||
CallFinishedAfter = "finished after its caller was answered"
|
||||
// CallAbandoned is a call whose controller stopped before it finished (novox/hq to-be 45 §6): a
|
||||
// controller starting finds it running under another and says so, rather than leaving it running
|
||||
// for ever in the record.
|
||||
CallAbandoned = "abandoned: the controller running it stopped before it finished"
|
||||
)
|
||||
|
||||
// Call is one call of a role's tool, as `calls` shows it.
|
||||
type Call struct {
|
||||
ID string `json:"call"`
|
||||
Seat string `json:"seat"`
|
||||
Verb string `json:"verb"`
|
||||
Args json.RawMessage `json:"arguments,omitempty"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished *time.Time `json:"finished,omitempty"`
|
||||
State string `json:"state"`
|
||||
// Failed is the call's own answer being an error — an answer, not a timeout.
|
||||
Failed bool `json:"failed,omitempty"`
|
||||
// Answer is what the call answered, or would have: kept for a call whose caller did not get it.
|
||||
Answer json.RawMessage `json:"answer,omitempty"`
|
||||
// Refused is the bus refusing the answer this holder sent: the caller got nothing, and this is
|
||||
// the only place that says what it would have.
|
||||
Refused string `json:"answer refused by the bus,omitempty"`
|
||||
// Caller is the bus principal that asked, read from the inbox its answer went to — every principal
|
||||
// is granted only its own (novox/hq to-be 45 §7: a hand act says who).
|
||||
Caller string `json:"caller,omitempty"`
|
||||
// Holder is the controller process that served it, so one starting can tell its own running
|
||||
// calls from those a stopped one left.
|
||||
Holder string `json:"holder,omitempty"`
|
||||
|
||||
reply string // the subject the answer went to, which the bus names when it refuses it
|
||||
}
|
||||
|
||||
// CallKeeper keeps calls where the process serving them does not: the controller's bucket on the
|
||||
// bus (novox/hq to-be 45 §6). A call is kept whole on every change — begun, finished, refused — so a
|
||||
// controller replaced at any moment leaves the last word on each.
|
||||
type CallKeeper interface {
|
||||
Keep(ctx context.Context, c Call) error
|
||||
// Kept is one call with its whole answer.
|
||||
Kept(ctx context.Context, id string) (Call, bool, error)
|
||||
// Recent is the kept calls, newest first, without their answers.
|
||||
Recent(ctx context.Context) ([]Call, error)
|
||||
}
|
||||
|
||||
// CallLog keeps the latest calls a holder served.
|
||||
type CallLog struct {
|
||||
mu sync.Mutex
|
||||
calls []*Call // oldest first
|
||||
next uint64
|
||||
now func() time.Time
|
||||
// Follow is the tool that reads this log back, named in a running answer — set by a holder that
|
||||
// serves one (the controller's `calls`); without it, the answer points at the holder's journal.
|
||||
Follow string
|
||||
|
||||
// keeper keeps every call beyond this process, when Durably was given one; writes go through
|
||||
// one goroutine, in order, so a call's last state is the one kept.
|
||||
keeper CallKeeper
|
||||
holder string
|
||||
writes chan Call
|
||||
logger *log.Logger
|
||||
lost int // writes the keeper could not take, said once each
|
||||
keepErr error
|
||||
}
|
||||
|
||||
// Calls is this process's log: one holder process serves its seats on one connection.
|
||||
var Calls = NewCallLog()
|
||||
|
||||
func NewCallLog() *CallLog { return &CallLog{now: time.Now} }
|
||||
|
||||
// keepTries is how many times one call's state is offered to the keeper before it is said lost: the
|
||||
// bus reloading its user list refuses for a moment, and that is exactly when a push runs.
|
||||
const keepTries = 5
|
||||
|
||||
// Durably keeps every call from now on with keeper as well as in memory, under this process's name,
|
||||
// and marks running the calls a controller before this one left running: it stopped, so they cannot
|
||||
// finish (novox/hq to-be 45 §6). Said, naming each.
|
||||
func (l *CallLog) Durably(ctx context.Context, keeper CallKeeper, holder string, logger *log.Logger) error {
|
||||
l.mu.Lock()
|
||||
l.keeper, l.holder, l.logger = keeper, holder, logger
|
||||
if l.writes == nil {
|
||||
l.writes = make(chan Call, 256)
|
||||
go l.keepWrites()
|
||||
}
|
||||
l.mu.Unlock()
|
||||
kept, err := keeper.Recent(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading the calls kept on the bus: %w", err)
|
||||
}
|
||||
for _, c := range kept {
|
||||
if c.State != CallRunning || c.Holder == holder {
|
||||
continue
|
||||
}
|
||||
whole, found, err := keeper.Kept(ctx, c.ID)
|
||||
if err != nil || !found {
|
||||
whole = c
|
||||
}
|
||||
whole.State = CallAbandoned
|
||||
if err := keeper.Keep(ctx, whole); err != nil {
|
||||
return fmt.Errorf("marking %s abandoned: %w", c.ID, err)
|
||||
}
|
||||
if logger != nil {
|
||||
logger.Printf("%s (%s.%s, asked %s by %s) was running under %s, which stopped: marked abandoned — "+
|
||||
"it may have done part of what it was asked, and nothing will finish it", c.ID, c.Seat, c.Verb,
|
||||
c.Started.Format(time.RFC3339), orSomebody(c.Caller), orSomebody(c.Holder))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func orSomebody(s string) string {
|
||||
if s == "" {
|
||||
return "an unnamed caller"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// keep queues one call's state for the keeper. Never blocks a call: a queue that is full is a keeper
|
||||
// that is not taking writes, and that is said rather than waited on.
|
||||
func (l *CallLog) keep(c Call) {
|
||||
if l.writes == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case l.writes <- c:
|
||||
default:
|
||||
l.lost++
|
||||
if l.logger != nil {
|
||||
l.logger.Printf("%s (%s.%s) is kept in memory only: the bus is not taking calls' records (%d not kept)",
|
||||
c.ID, c.Seat, c.Verb, l.lost)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *CallLog) keepWrites() {
|
||||
for c := range l.writes {
|
||||
var err error
|
||||
for try := 0; try < keepTries; try++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
err = l.keeper.Keep(ctx, c)
|
||||
cancel()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Duration(try+1) * time.Second)
|
||||
}
|
||||
if err != nil && l.logger != nil {
|
||||
l.logger.Printf("%s (%s.%s, %s) could not be kept on the bus after %d tries: %v — `calls` "+
|
||||
"answers it from memory until this controller stops", c.ID, c.Seat, c.Verb, c.State, keepTries, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// callerOf is the bus principal an answer goes to: every principal's inbox is `_INBOX.<its user>.`
|
||||
// followed by the client's own random token, and a user may itself hold dots.
|
||||
func callerOf(reply string) string {
|
||||
rest, ok := strings.CutPrefix(reply, "_INBOX.")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
tokens := strings.Split(rest, ".")
|
||||
for i, t := range tokens {
|
||||
if i > 0 && isNUID(t) {
|
||||
return strings.Join(tokens[:i], ".")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isNUID is the client library's random inbox token: twenty-two letters and digits.
|
||||
func isNUID(t string) bool {
|
||||
if len(t) != 22 {
|
||||
return false
|
||||
}
|
||||
for _, r := range t {
|
||||
if !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (l *CallLog) begin(seat, verb string, args json.RawMessage, reply string) *Call {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.next++
|
||||
c := &Call{ID: "call-" + strconv.FormatInt(l.now().UnixNano(), 10) + "-" + strconv.FormatUint(l.next, 10),
|
||||
Seat: seat, Verb: verb, Args: args, Started: l.now(), State: CallRunning, reply: reply,
|
||||
Caller: callerOf(reply), Holder: l.holder}
|
||||
l.calls = append(l.calls, c)
|
||||
if len(l.calls) > KeptCalls {
|
||||
l.calls = l.calls[len(l.calls)-KeptCalls:]
|
||||
}
|
||||
l.keep(*c)
|
||||
return c
|
||||
}
|
||||
|
||||
// finish records a call's answer; answeredAlready is its caller having been told it was running.
|
||||
func (l *CallLog) finish(c *Call, answer []byte, failed, answeredAlready bool) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
at := l.now()
|
||||
c.Finished = &at
|
||||
c.Failed = failed
|
||||
c.Answer = append(json.RawMessage(nil), answer...)
|
||||
if !answeredAlready && len(answer) > keptAnswer {
|
||||
// Its caller has it; kept only in case the bus refuses it, and a whole declaration a
|
||||
// hundred times over is memory nobody asked for.
|
||||
c.Answer, _ = json.Marshal(map[string]any{"not kept": fmt.Sprintf(
|
||||
"an answer of %d bytes, sent to its caller in full", len(answer))})
|
||||
}
|
||||
if answeredAlready {
|
||||
c.State = CallFinishedAfter
|
||||
} else {
|
||||
c.State = CallAnswered
|
||||
}
|
||||
l.keep(*c)
|
||||
}
|
||||
|
||||
// Recent is the kept calls, newest first, as copies: this process's from memory, and, when they are
|
||||
// kept durably, every other the bus holds — a call a controller before this one served included.
|
||||
// Memory wins for a call in both, being the newer word on it. A bus that cannot be read is said in
|
||||
// the error beside what memory holds, never answered as no calls.
|
||||
func (l *CallLog) Recent() ([]Call, error) {
|
||||
l.mu.Lock()
|
||||
out := make([]Call, 0, len(l.calls))
|
||||
seen := map[string]bool{}
|
||||
for i := len(l.calls) - 1; i >= 0; i-- {
|
||||
out = append(out, *l.calls[i])
|
||||
seen[l.calls[i].ID] = true
|
||||
}
|
||||
keeper := l.keeper
|
||||
l.mu.Unlock()
|
||||
if keeper == nil {
|
||||
return out, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
kept, err := keeper.Recent(ctx)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the calls kept on the bus could not be read, so only this controller's own "+
|
||||
"are listed: %w", err)
|
||||
}
|
||||
for _, c := range kept {
|
||||
if !seen[c.ID] {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Started.After(out[j].Started) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Get is one kept call: from memory, or from the bus when this process did not serve it.
|
||||
func (l *CallLog) Get(id string) (Call, bool, error) {
|
||||
l.mu.Lock()
|
||||
for _, c := range l.calls {
|
||||
if c.ID == id {
|
||||
found := *c
|
||||
l.mu.Unlock()
|
||||
return found, true, nil
|
||||
}
|
||||
}
|
||||
keeper := l.keeper
|
||||
l.mu.Unlock()
|
||||
if keeper == nil {
|
||||
return Call{}, false, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
return keeper.Kept(ctx, id)
|
||||
}
|
||||
|
||||
// IsDurable says whether calls outlive this process.
|
||||
func (l *CallLog) IsDurable() bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.keeper != nil
|
||||
}
|
||||
|
||||
// kept is what a call's arguments are kept as: each argument by name, a value only when it is short
|
||||
// and not one that carries settings or a secret — `calls` answers anyone who may call the seat.
|
||||
func kept(args json.RawMessage) json.RawMessage {
|
||||
var given map[string]any
|
||||
if json.Unmarshal(args, &given) != nil {
|
||||
return nil
|
||||
}
|
||||
out := map[string]any{}
|
||||
for k, v := range given {
|
||||
s, isString := v.(string)
|
||||
switch {
|
||||
case k == "values" || k == "secret":
|
||||
out[k] = "(given, not kept)"
|
||||
case isString && len(s) <= 120:
|
||||
out[k] = s
|
||||
case isString:
|
||||
out[k] = s[:120] + "…"
|
||||
default:
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(out)
|
||||
return body
|
||||
}
|
||||
|
||||
// refusedPublish is the bus's words for a publish it refused, with the subject.
|
||||
var refusedPublish = regexp.MustCompile(`Permissions Violation for Publish to "([^"]+)"`)
|
||||
|
||||
// Refusal records the bus refusing an answer, from the error the client library hands the
|
||||
// connection's error handler. It reports whether the error was the refusal of a kept call's answer.
|
||||
func (l *CallLog) Refusal(err error, logger *log.Logger) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
m := refusedPublish.FindStringSubmatch(err.Error())
|
||||
if m == nil {
|
||||
return false
|
||||
}
|
||||
l.mu.Lock()
|
||||
var hit *Call
|
||||
for i := len(l.calls) - 1; i >= 0; i-- {
|
||||
if c := l.calls[i]; c.reply != "" && c.reply == m[1] {
|
||||
hit = c
|
||||
break
|
||||
}
|
||||
}
|
||||
if hit == nil {
|
||||
l.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
at := l.now()
|
||||
hit.Refused = fmt.Sprintf("%s: %s", at.Format(time.RFC3339), err)
|
||||
id, verb, took := hit.ID, hit.Seat+"."+hit.Verb, at.Sub(hit.Started).Round(time.Second)
|
||||
l.keep(*hit)
|
||||
l.mu.Unlock()
|
||||
if logger != nil {
|
||||
// Said in the mesh's words, beside the library's own line: which call, and where its answer is.
|
||||
logger.Printf("the bus refused the answer to %s (%s, asked %s ago): its caller got no answer. "+
|
||||
"What it answered is kept under %s. A broker reloading its user list while a call runs "+
|
||||
"forgets that it may be answered", id, verb, took, id)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// WatchRefusals chains the connection's error handler so a refused answer is recorded against its
|
||||
// call. The handler the dialler set — the library's default, which prints — still runs after it.
|
||||
func (l *CallLog) WatchRefusals(conn *nats.Conn, logger *log.Logger) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
watched.Lock()
|
||||
defer watched.Unlock()
|
||||
if watched.conns == nil {
|
||||
watched.conns = map[*nats.Conn]bool{}
|
||||
}
|
||||
if watched.conns[conn] {
|
||||
return
|
||||
}
|
||||
watched.conns[conn] = true
|
||||
before := conn.ErrorHandler()
|
||||
conn.SetErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) {
|
||||
if errors.Is(err, nats.ErrPermissionViolation) {
|
||||
l.Refusal(err, logger)
|
||||
}
|
||||
if before != nil {
|
||||
before(c, s, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
var watched struct {
|
||||
sync.Mutex
|
||||
conns map[*nats.Conn]bool
|
||||
}
|
||||
|
||||
// answerNow is how a handler asks for its caller to be answered before it goes on (Acknowledge).
|
||||
type answerNowKey struct{}
|
||||
|
||||
// Acknowledge answers the call's caller now that it is running, rather than after AnswerWithin. For
|
||||
// a handler about to do what makes its answer unsendable: a push sends the bus's own machine first,
|
||||
// and a broker reloading its user list forgets every answer it was about to permit. Without effect
|
||||
// outside a served call, and after the first time.
|
||||
func Acknowledge(ctx context.Context) {
|
||||
if f, ok := ctx.Value(answerNowKey{}).(func()); ok {
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
// running is the interim answer: what a caller reads when the call has not finished.
|
||||
func running(c *Call, within time.Duration, acknowledged bool, follow string) []byte {
|
||||
why := fmt.Sprintf("it has not finished in %s", within)
|
||||
if acknowledged {
|
||||
why = "it answers before it starts, because what it does can leave the bus unable to carry a later answer"
|
||||
}
|
||||
next := "its holder's journal says how it ended."
|
||||
if follow != "" {
|
||||
next = fmt.Sprintf("%s with call %q says what came of it.", follow, c.ID)
|
||||
}
|
||||
said := fmt.Sprintf("%s.%s is running as %s — %s. This is not a failure, and it may already have "+
|
||||
"done what was asked: %s", c.Seat, c.Verb, c.ID, why, next)
|
||||
body, _ := json.Marshal(map[string]any{"result": map[string]any{
|
||||
"running": true, "call": c.ID, "started": c.Started, "output": said,
|
||||
}})
|
||||
return body
|
||||
}
|
||||
|
||||
// serveCall runs one call and answers it once: in full when the handler returns within AnswerWithin
|
||||
// and has not acknowledged, and otherwise that it is running — its answer then kept, and logged.
|
||||
func (l *CallLog) serveCall(seat, verb string, args json.RawMessage, reply string, handle ToolHandler,
|
||||
respond func([]byte) error, logger *log.Logger) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), HandlerTimeout)
|
||||
defer cancel()
|
||||
if len(args) == 0 {
|
||||
args = json.RawMessage(`{}`)
|
||||
}
|
||||
c := l.begin(seat, verb, kept(args), reply)
|
||||
// Who asked travels with the call, so an act it does by hand says so (novox/hq to-be 45 §7).
|
||||
ctx = context.WithValue(ctx, callerKey{}, c.Caller)
|
||||
acknowledged := make(chan struct{})
|
||||
var once sync.Once
|
||||
ctx = context.WithValue(ctx, answerNowKey{}, func() { once.Do(func() { close(acknowledged) }) })
|
||||
|
||||
type outcome struct {
|
||||
body []byte
|
||||
failed bool
|
||||
}
|
||||
done := make(chan outcome, 1)
|
||||
go func() {
|
||||
var body []byte
|
||||
result, err := handle(ctx, args)
|
||||
failed := err != nil
|
||||
if err != nil {
|
||||
body, _ = json.Marshal(map[string]any{"error": err.Error()})
|
||||
} else if body, err = json.Marshal(map[string]any{"result": result}); err != nil {
|
||||
failed = true
|
||||
body, _ = json.Marshal(map[string]any{"error": "the answer could not be written as JSON: " + err.Error()})
|
||||
}
|
||||
done <- outcome{body, failed}
|
||||
}()
|
||||
|
||||
say := func(body []byte) {
|
||||
if err := respond(body); err != nil && logger != nil {
|
||||
logger.Printf("%s.%s: could not answer %s: %v", seat, verb, c.ID, err)
|
||||
}
|
||||
}
|
||||
timer := time.NewTimer(AnswerWithin)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case o := <-done:
|
||||
l.finish(c, o.body, o.failed, false)
|
||||
say(o.body)
|
||||
return
|
||||
case <-acknowledged:
|
||||
say(running(c, AnswerWithin, true, l.Follow))
|
||||
case <-timer.C:
|
||||
say(running(c, AnswerWithin, false, l.Follow))
|
||||
}
|
||||
o := <-done
|
||||
l.finish(c, o.body, o.failed, true)
|
||||
if logger != nil {
|
||||
how := "and it succeeded"
|
||||
if o.failed {
|
||||
how = "and it answered an error"
|
||||
}
|
||||
logger.Printf("%s (%s.%s) finished after %s, after its caller was told it was running, %s — its answer is kept under %s", c.ID,
|
||||
seat, verb, time.Since(c.Started).Round(time.Second), how, c.ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Calls kept on the bus (novox/hq to-be 45 §6).
|
||||
//
|
||||
// **Two keys a call**: `<id>` holds the record — verb, arguments as kept, caller, state, times — and
|
||||
// `<id>.answer` the answer, bounded. A listing watches the records alone, so reading the last thousand
|
||||
// calls reads the last thousand small records and not a thousand answers; one call asked by id reads
|
||||
// both. A call's id is one token, so the record's key never holds a dot and `*` matches records only.
|
||||
|
||||
// BusCalls keeps calls in the controller's calls bucket.
|
||||
type BusCalls struct {
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
// CallsOnTheBus opens the calls bucket the controller asserts at its start.
|
||||
func CallsOnTheBus(ctx context.Context, conn *nats.Conn) (*BusCalls, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.CallsBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the calls bucket %s is not on the bus — the controller asserts it at its "+
|
||||
"start, so one older than this has not: %w", broker.CallsBucket, err)
|
||||
}
|
||||
return &BusCalls{kv: kv}, nil
|
||||
}
|
||||
|
||||
const answerKey = ".answer"
|
||||
|
||||
// Keep writes a call's record, and its answer when it has one.
|
||||
func (b *BusCalls) Keep(ctx context.Context, c Call) error {
|
||||
answer := c.Answer
|
||||
c.Answer = nil
|
||||
if len(answer) > 0 {
|
||||
if len(answer) > broker.CallAnswerBytes {
|
||||
answer, _ = json.Marshal(map[string]any{"cut": fmt.Sprintf("an answer of %d bytes; the first %d "+
|
||||
"are kept", len(answer), broker.CallAnswerBytes), "start": string(answer[:broker.CallAnswerBytes])})
|
||||
}
|
||||
// The answer before the record, so a record saying a call finished never points at an answer
|
||||
// not yet written.
|
||||
if _, err := b.kv.Put(ctx, c.ID+answerKey, answer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
record, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = b.kv.Put(ctx, c.ID, record)
|
||||
return err
|
||||
}
|
||||
|
||||
// Kept is one call, with its answer.
|
||||
func (b *BusCalls) Kept(ctx context.Context, id string) (Call, bool, error) {
|
||||
entry, err := b.kv.Get(ctx, id)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrInvalidKey) {
|
||||
return Call{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Call{}, false, err
|
||||
}
|
||||
var c Call
|
||||
if err := json.Unmarshal(entry.Value(), &c); err != nil {
|
||||
return Call{}, false, fmt.Errorf("the record of %s on the bus is not a call: %w", id, err)
|
||||
}
|
||||
answer, err := b.kv.Get(ctx, id+answerKey)
|
||||
switch {
|
||||
case err == nil:
|
||||
c.Answer = json.RawMessage(answer.Value())
|
||||
case !errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return Call{}, false, err
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// Recent is every call the bucket holds, newest first, without answers: read once through a watch
|
||||
// of the records, which hands over the current value of each and then says it has.
|
||||
func (b *BusCalls) Recent(ctx context.Context) ([]Call, error) {
|
||||
w, err := b.kv.Watch(ctx, "*", jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
var out []Call
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading the calls bucket: %w", ctx.Err())
|
||||
case entry := <-w.Updates():
|
||||
if entry == nil {
|
||||
// Every current value handed over.
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Started.After(out[j].Started) })
|
||||
return out, nil
|
||||
}
|
||||
var c Call
|
||||
if json.Unmarshal(entry.Value(), &c) == nil && c.ID != "" {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The calls bucket against a real server (novox/hq to-be 45 §6): whether a call's outcome is
|
||||
// readable by id from a controller that did not serve it is a claim about what the bus keeps.
|
||||
|
||||
func callsBucket(t *testing.T) *BusCalls {
|
||||
t.Helper()
|
||||
js := aBus(t)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.CallsBucket)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keeper, err := CallsOnTheBus(t.Context(), js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return keeper
|
||||
}
|
||||
|
||||
// waitKept waits until the keeper holds a call in a state, the writes being queued.
|
||||
func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
c, found, err := keeper.Kept(context.Background(), id)
|
||||
if err == nil && found && c.State == state {
|
||||
return c
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("%s never kept as %q: %+v %v %v", id, state, c, found, err)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// **A controller restart keeps every call's outcome** (to-be 45 Phase 0): a call one controller
|
||||
// answered is read whole by id from the next, and a call it left running is said abandoned rather
|
||||
// than running for ever.
|
||||
func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) {
|
||||
keeper := callsBucket(t)
|
||||
first := NewCallLog()
|
||||
if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := newAnswers(t)
|
||||
first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`),
|
||||
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1",
|
||||
func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil)
|
||||
recent, _ := first.Recent()
|
||||
finished := waitKept(t, keeper, recent[0].ID, CallAnswered)
|
||||
// A second call, still running when its controller stops.
|
||||
left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "")
|
||||
waitKept(t, keeper, left.ID, CallRunning)
|
||||
|
||||
var said bytes.Buffer
|
||||
second := NewCallLog()
|
||||
if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, found, err := second.Get(finished.ID)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err)
|
||||
}
|
||||
if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" {
|
||||
t.Fatalf("kept %+v", c)
|
||||
}
|
||||
if strings.Contains(string(c.Args), "secret") {
|
||||
t.Fatalf("a setting was kept: %s", c.Args)
|
||||
}
|
||||
abandoned, _, _ := second.Get(left.ID)
|
||||
if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) {
|
||||
t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String())
|
||||
}
|
||||
listed, err := second.Recent()
|
||||
if err != nil || len(listed) != 2 {
|
||||
t.Fatalf("the next controller lists %d calls: %v", len(listed), err)
|
||||
}
|
||||
// Its own running calls are not its predecessor's: a controller starting again under the same
|
||||
// name leaves them alone.
|
||||
mine := second.begin("mesh-controller", "status", nil, "")
|
||||
waitKept(t, keeper, mine.ID, CallRunning)
|
||||
if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning {
|
||||
t.Fatalf("a controller marked its own running call %q", c.State)
|
||||
}
|
||||
}
|
||||
|
||||
// The bucket holds the last thousand calls or fourteen days: a call is two keys, so the stream under
|
||||
// it holds twice as many messages, and asserting it again keeps the count.
|
||||
func TestNatsTheCallsBucketIsBounded(t *testing.T) {
|
||||
callsBucket(t)
|
||||
js := aBus(t)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor {
|
||||
t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge)
|
||||
}
|
||||
}
|
||||
|
||||
// An answer larger than the bound is cut and says so, and the record is still written.
|
||||
func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) {
|
||||
keeper := callsBucket(t)
|
||||
big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)})
|
||||
c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big}
|
||||
if err := keeper.Keep(t.Context(), c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, found, err := keeper.Kept(t.Context(), "call-1-1")
|
||||
if err != nil || !found || !strings.Contains(string(got.Answer), "the first") {
|
||||
t.Fatalf("%v %v %.200s", found, err, got.Answer)
|
||||
}
|
||||
}
|
||||
|
||||
// Who asked is read from the inbox the answer goes to.
|
||||
func TestTheCallerIsTheInboxsPrincipal(t *testing.T) {
|
||||
for reply, want := range map[string]string{
|
||||
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1": "node-tools.g14",
|
||||
"_INBOX.jochen.Pe3sGzAtv8jUBYQ6sKSvKz": "jochen",
|
||||
"_INBOX.x.1": "",
|
||||
"mesh.control.one": "",
|
||||
"": "",
|
||||
} {
|
||||
if got := callerOf(reply); got != want {
|
||||
t.Errorf("%q: %q, want %q", reply, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// answers collects what a call answered, and fails a second answer: the bus permits one.
|
||||
type answers struct {
|
||||
t *testing.T
|
||||
mu sync.Mutex
|
||||
got [][]byte
|
||||
sent chan struct{}
|
||||
}
|
||||
|
||||
func newAnswers(t *testing.T) *answers { return &answers{t: t, sent: make(chan struct{}, 4)} }
|
||||
|
||||
func (a *answers) respond(body []byte) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.got = append(a.got, body)
|
||||
if len(a.got) > 1 {
|
||||
a.t.Errorf("a call was answered %d times; the bus refuses every answer after the first", len(a.got))
|
||||
}
|
||||
a.sent <- struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *answers) only() map[string]any {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if len(a.got) != 1 {
|
||||
a.t.Fatalf("%d answers, want exactly one", len(a.got))
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(a.got[0], &out); err != nil {
|
||||
a.t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func shortWindow(t *testing.T, d time.Duration) {
|
||||
t.Helper()
|
||||
was := AnswerWithin
|
||||
AnswerWithin = d
|
||||
t.Cleanup(func() { AnswerWithin = was })
|
||||
}
|
||||
|
||||
// A call that finishes in time answers in full, once, and is kept as answered.
|
||||
func TestACallThatFinishesInTimeAnswersInFull(t *testing.T) {
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
l.serveCall("mesh-controller", "status", nil, "_INBOX.x.1", func(context.Context, json.RawMessage) (any, error) {
|
||||
return "all well", nil
|
||||
}, a.respond, nil)
|
||||
if got := a.only()["result"]; got != "all well" {
|
||||
t.Fatalf("answered %v", got)
|
||||
}
|
||||
recent, _ := l.Recent()
|
||||
if len(recent) != 1 || recent[0].State != CallAnswered || string(recent[0].Args) != "{}" {
|
||||
t.Fatalf("kept %+v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
// **A call that outlasts AnswerWithin is answered that it is running, with its id** — before its
|
||||
// caller gives up — and what it finally answered is kept under that id, not dropped (issue 265).
|
||||
func TestACallThatOutlastsTheWindowSaysItIsRunningAndKeepsItsAnswer(t *testing.T) {
|
||||
shortWindow(t, 20*time.Millisecond)
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
var logged bytes.Buffer
|
||||
release := make(chan struct{})
|
||||
finished := make(chan struct{})
|
||||
go func() {
|
||||
l.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor"}`), "_INBOX.x.2",
|
||||
func(context.Context, json.RawMessage) (any, error) {
|
||||
<-release
|
||||
return "anchor told", nil
|
||||
}, a.respond, log.New(&logged, "", 0))
|
||||
close(finished)
|
||||
}()
|
||||
<-a.sent
|
||||
got := a.only()["result"].(map[string]any)
|
||||
id, _ := got["call"].(string)
|
||||
if got["running"] != true || id == "" || !strings.Contains(got["output"].(string), id) {
|
||||
t.Fatalf("the running answer does not name its call: %v", got)
|
||||
}
|
||||
if c, _, _ := l.Get(id); c.State != CallRunning {
|
||||
t.Fatalf("while it runs it is kept as %q", c.State)
|
||||
}
|
||||
close(release)
|
||||
<-finished
|
||||
c, ok, _ := l.Get(id)
|
||||
if !ok || c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "anchor told") {
|
||||
t.Fatalf("its answer was not kept: %+v", c)
|
||||
}
|
||||
if !strings.Contains(logged.String(), id) {
|
||||
t.Errorf("finishing late was not said: %q", logged.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A handler that acknowledges is answered then, not when it ends: a push answers before it sends.
|
||||
func TestAnAcknowledgedCallIsAnsweredBeforeItGoesOn(t *testing.T) {
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
l.serveCall("mesh-controller", "push", nil, "_INBOX.x.3", func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||
Acknowledge(ctx)
|
||||
Acknowledge(ctx) // a second time is nothing
|
||||
select {
|
||||
case <-a.sent: // the caller was answered before the work goes on
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("acknowledging did not answer the caller")
|
||||
}
|
||||
return "sent", nil
|
||||
}, a.respond, nil)
|
||||
close(done)
|
||||
}()
|
||||
<-done
|
||||
if got := a.only()["result"].(map[string]any); got["running"] != true {
|
||||
t.Fatalf("an acknowledged call answered %v", got)
|
||||
}
|
||||
if c := recentOf(l)[0]; c.State != CallFinishedAfter || !strings.Contains(string(c.Answer), "sent") {
|
||||
t.Fatalf("kept %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// **A refused answer is recorded against its call, in the mesh's words** — not only as the client
|
||||
// library's line on standard error, which is all there was on 2026-10-05.
|
||||
func TestARefusedAnswerIsKeptAgainstItsCall(t *testing.T) {
|
||||
l, a := NewCallLog(), newAnswers(t)
|
||||
reply := "_INBOX.laptop.node-tools.jJ4DRJFnZYvkPUBKYwUG5v.LNRyztuX"
|
||||
l.serveCall("mesh-controller", "push", nil, reply, func(context.Context, json.RawMessage) (any, error) {
|
||||
return "told", nil
|
||||
}, a.respond, nil)
|
||||
var logged bytes.Buffer
|
||||
refusal := errors.New(`nats: permissions violation: Permissions Violation for Publish to "` + reply + `" on connection [838]`)
|
||||
if !l.Refusal(refusal, log.New(&logged, "", 0)) {
|
||||
t.Fatal("the refusal of a kept call's answer was not recognised")
|
||||
}
|
||||
c := recentOf(l)[0]
|
||||
if c.Refused == "" || !strings.Contains(logged.String(), c.ID) {
|
||||
t.Fatalf("the refusal is not kept or not said: %+v / %q", c, logged.String())
|
||||
}
|
||||
other := errors.New(`nats: permissions violation: Permissions Violation for Publish to "mesh.node.x" on connection [1]`)
|
||||
if l.Refusal(other, nil) || l.Refusal(errors.New("nats: timeout"), nil) {
|
||||
t.Error("an unrelated error was taken for a refused answer")
|
||||
}
|
||||
}
|
||||
|
||||
// What `calls` keeps of the arguments never carries settings or a secret.
|
||||
func TestACallKeepsNoSettingsOrSecrets(t *testing.T) {
|
||||
got := string(kept(json.RawMessage(`{"module":"m","values":"{\"token\":\"s3cret\"}","secret":"s3cret"}`)))
|
||||
if strings.Contains(got, "s3cret") || !strings.Contains(got, `"module":"m"`) {
|
||||
t.Fatalf("kept %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the newest KeptCalls are kept.
|
||||
func TestTheLogKeepsTheNewest(t *testing.T) {
|
||||
l := NewCallLog()
|
||||
for i := 0; i < KeptCalls+5; i++ {
|
||||
l.begin("s", "v", nil, "")
|
||||
}
|
||||
recent, _ := l.Recent()
|
||||
if len(recent) != KeptCalls || !strings.HasSuffix(recent[0].ID, "-105") || !strings.HasSuffix(recent[KeptCalls-1].ID, "-6") {
|
||||
t.Fatalf("kept %d, newest %s, oldest %s", len(recent), recent[0].ID, recent[len(recent)-1].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func recentOf(l *CallLog) []Call {
|
||||
out, _ := l.Recent()
|
||||
return out
|
||||
}
|
||||
@@ -380,6 +380,10 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
if report.Applied == nil && report.Refused == "" && len(report.Failed) == 0 {
|
||||
if report.Superseded != "" {
|
||||
log.Printf("%s set aside declaration %s for the newer %s", report.Node, report.Declared, report.Superseded)
|
||||
} else if err := e.Inventory.RecordHeartbeatGap(ctx, report.Node, node.LastSeen); err != nil {
|
||||
// The silence before this word, which the machine-silent bound will be set from (novox/hq
|
||||
// to-be 45 Phase 0). A measurement lost is said and costs the report nothing.
|
||||
log.Printf("the silence before %s's word could not be recorded: %v", report.Node, err)
|
||||
}
|
||||
return false, e.Inventory.Seen(ctx, node.ID)
|
||||
}
|
||||
@@ -410,12 +414,36 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
|
||||
if err := e.Inventory.RecordCarried(ctx, report.Node, report.Carried); err != nil {
|
||||
return false, err
|
||||
}
|
||||
// **An account of a declaration the mesh has moved past never replaces the account it keeps**
|
||||
// (novox/hq issue 267). The machine-facts half of such a report is kept above, whenever it
|
||||
// arrives; this half is the machine's word on what it did with what it was sent, and the release
|
||||
// plan reads it as such. A reconcile that held a machine to the older declaration a moment before
|
||||
// the newer one arrived reported *after* the newer apply's report, and stored last, it read as the
|
||||
// machine never having applied what it was sent — the plan waited until somebody pushed by hand.
|
||||
// One row per node means the last write wins, so the order of arrival must not decide it.
|
||||
if report.Declared != "" {
|
||||
sent, err := e.Inventory.Outstanding(ctx, report.Node)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if Superseded(report.Declared, sent) {
|
||||
log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
|
||||
"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
|
||||
return false, e.Inventory.Seen(ctx, node.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// **Whether this is news** is the store's answer: it holds the previous report, and a machine
|
||||
// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
|
||||
news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// And how long the machine took, from the send to this first account of it (novox/hq to-be 45
|
||||
// Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure.
|
||||
if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, doing.Outcome); err != nil {
|
||||
log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err)
|
||||
}
|
||||
|
||||
// A refusal, a failure, or a bare word that the node is there — none of them is an account of
|
||||
// what the machine holds, so each moves last_seen and nothing else. Recording a partial list
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The hand-act log (novox/hq to-be 45 §7).
|
||||
//
|
||||
// **A repair a person makes by hand is the record of a healer the mesh does not have yet.** Tonight's
|
||||
// pushes of one machine, plans closed because a report never came, a consumer re-made because it fell
|
||||
// a week behind — each was done, and the only trace was a chat. So every verb that repairs by hand
|
||||
// asks why, and writes one entry: who, which verb and arguments, why, when, the condition it
|
||||
// addresses if one is named, and a cause — a word that, recorded twice in a fortnight, says a healer
|
||||
// is wanted (S15, from Phase 3). `hand-act record` is the same entry for an act done outside the mesh.
|
||||
// The controller is the bucket's only writer; its verbs are the way in.
|
||||
|
||||
// HandAct is one entry.
|
||||
type HandAct struct {
|
||||
ID string `json:"id"`
|
||||
At time.Time `json:"at"`
|
||||
// By is who: the bus principal a seat call came from, or the account and machine at a shell.
|
||||
By string `json:"by"`
|
||||
// Verb and Args are the act as given: `push`, `plans close`, `broker consumer-reset`, or
|
||||
// `hand-act record` with what was done outside the mesh.
|
||||
Verb string `json:"verb"`
|
||||
Args []string `json:"arguments,omitempty"`
|
||||
Why string `json:"why"`
|
||||
// Cause is the condition kind, or a word the person gives; the verb's own name when neither.
|
||||
Cause string `json:"cause"`
|
||||
// Condition is the condition's key the act addresses, when it names one.
|
||||
Condition string `json:"condition,omitempty"`
|
||||
}
|
||||
|
||||
// CallerVar carries a seat call's caller to the command the controller runs for it, so an act done
|
||||
// through the console says who asked rather than "the controller".
|
||||
const CallerVar = "MESH_CALLER"
|
||||
|
||||
// Caller is who is acting in this process: the seat call's caller when the controller ran it for
|
||||
// one, otherwise the account and machine at the shell.
|
||||
func Caller() string {
|
||||
if c := strings.TrimSpace(os.Getenv(CallerVar)); c != "" {
|
||||
return c
|
||||
}
|
||||
user := os.Getenv("USER")
|
||||
if user == "" {
|
||||
user = "an unnamed account"
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("%s at a shell on %s", user, host)
|
||||
}
|
||||
|
||||
type callerKey struct{}
|
||||
|
||||
// CallerIn is the caller of the seat call ctx belongs to, empty outside one.
|
||||
func CallerIn(ctx context.Context) string {
|
||||
c, _ := ctx.Value(callerKey{}).(string)
|
||||
return c
|
||||
}
|
||||
|
||||
var handActSeq atomic.Uint64
|
||||
|
||||
// RecordHandAct writes one entry. Its key is its time and a sequence, so the bucket lists in order.
|
||||
func RecordHandAct(ctx context.Context, conn *nats.Conn, act HandAct) (HandAct, error) {
|
||||
if strings.TrimSpace(act.Why) == "" {
|
||||
return act, errors.New("an act by hand says why: --why <text>")
|
||||
}
|
||||
if act.At.IsZero() {
|
||||
act.At = time.Now().UTC()
|
||||
}
|
||||
if act.ID == "" {
|
||||
act.ID = "act-" + strconv.FormatInt(act.At.UnixNano(), 10) + "-" + strconv.FormatUint(handActSeq.Add(1), 10)
|
||||
}
|
||||
if act.By == "" {
|
||||
act.By = Caller()
|
||||
}
|
||||
if act.Cause == "" {
|
||||
act.Cause = act.Verb
|
||||
}
|
||||
kv, err := handActs(ctx, conn)
|
||||
if err != nil {
|
||||
return act, err
|
||||
}
|
||||
body, err := json.Marshal(act)
|
||||
if err != nil {
|
||||
return act, err
|
||||
}
|
||||
_, err = kv.Put(ctx, act.ID, body)
|
||||
return act, err
|
||||
}
|
||||
|
||||
// HandActs is every entry since a moment, oldest first.
|
||||
func HandActs(ctx context.Context, conn *nats.Conn, since time.Time) ([]HandAct, error) {
|
||||
kv, err := handActs(ctx, conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
var out []HandAct
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("reading the hand-act log: %w", ctx.Err())
|
||||
case entry := <-w.Updates():
|
||||
if entry == nil {
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out, nil
|
||||
}
|
||||
var a HandAct
|
||||
if json.Unmarshal(entry.Value(), &a) == nil && !a.At.Before(since) {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RepeatedCauses are the causes recorded more than once within the fortnight before now, with how
|
||||
// often: each is a repair done by hand again, which is what S15 will raise as a healer wanted.
|
||||
func RepeatedCauses(acts []HandAct, now time.Time) map[string]int {
|
||||
counts := map[string]int{}
|
||||
for _, a := range acts {
|
||||
if now.Sub(a.At) <= 14*24*time.Hour {
|
||||
counts[a.Cause]++
|
||||
}
|
||||
}
|
||||
for c, n := range counts {
|
||||
if n < 2 {
|
||||
delete(counts, c)
|
||||
}
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
func handActs(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.HandActsBucket)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand-act log %s is not on the bus — the controller asserts it at its "+
|
||||
"start, so one older than this has not: %w", broker.HandActsBucket, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The hand-act log against a real server (novox/hq to-be 45 §7): an act is written with who, why and
|
||||
// its cause, read back in order, and a cause recorded twice within a fortnight is found.
|
||||
func TestNatsAnActByHandIsKeptWithWhyAndARepeatIsFound(t *testing.T) {
|
||||
js := aBus(t)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = api.DeleteKeyValue(t.Context(), broker.HandActsBucket)
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(CallerVar, "node-tools.g14, through the mesh-controller seat")
|
||||
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"anchor"}}); err == nil {
|
||||
t.Fatal("an act without why was written")
|
||||
}
|
||||
first, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"anchor"},
|
||||
Why: "it never reported the send", Cause: "sent-not-reported"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.By != "node-tools.g14, through the mesh-controller seat" || first.ID == "" {
|
||||
t.Fatalf("written as %+v", first)
|
||||
}
|
||||
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "plans close", Args: []string{"plan-1"},
|
||||
Why: "waiting on the same report"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RecordHandAct(t.Context(), js.Conn(), HandAct{Verb: "push", Args: []string{"ace"},
|
||||
Why: "again", Cause: "sent-not-reported", At: time.Now().UTC().Add(time.Second)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
acts, err := HandActs(t.Context(), js.Conn(), time.Now().Add(-time.Hour))
|
||||
if err != nil || len(acts) != 3 || acts[0].ID != first.ID || acts[1].Cause != "plans close" {
|
||||
t.Fatalf("%v %+v", err, acts)
|
||||
}
|
||||
repeated := RepeatedCauses(acts, time.Now())
|
||||
if len(repeated) != 1 || repeated["sent-not-reported"] != 2 {
|
||||
t.Fatalf("repeated %v", repeated)
|
||||
}
|
||||
if old, _ := HandActs(t.Context(), js.Conn(), time.Now().Add(time.Hour)); len(old) != 0 {
|
||||
t.Fatalf("acts before the moment asked were listed: %+v", old)
|
||||
}
|
||||
if !strings.HasPrefix(acts[2].ID, "act-") {
|
||||
t.Fatalf("%q", acts[2].ID)
|
||||
}
|
||||
}
|
||||
@@ -100,7 +100,7 @@ func TestABareAliveDoesNotWipeTheDeclarationThatSaysANodeIsCurrent(t *testing.T)
|
||||
}
|
||||
// The mesh sent this node a declaration, and the node applied it and named which by digest.
|
||||
const digest = "d640d1b6a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f6071829304152"
|
||||
if err := inv.RecordSent(ctx, node.ID, digest); err != nil {
|
||||
if err := inv.RecordSent(ctx, node.ID, digest, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{
|
||||
@@ -260,3 +260,67 @@ func TestWhatFiltersAMachineIsKeptFromItsReport(t *testing.T) {
|
||||
t.Fatalf("the next report did not replace what filters the machine: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
// Measured on the home server (novox/hq issue 267): the mesh sent d2; the machine applied it and
|
||||
// reported, and a reconcile's report about d1 — made just before d2 arrived — reached the mesh after
|
||||
// it. Stored last, it read as the machine never having applied d2, and the release plan waited on a
|
||||
// report it had already been given.
|
||||
func TestAnAccountOfAnOlderDeclarationDoesNotReplaceTheNewer(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := context.Background()
|
||||
node, err := inv.AddNode(ctx, "home-server")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard := link.Enrolment{Inventory: inv}
|
||||
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2",
|
||||
Applied: []string{"a", "b"}, Outward: []string{"eth0"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1",
|
||||
Applied: []string{"a"}, Outward: []string{"eth1"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doing, said, err := inv.DoingOf(ctx, "home-server")
|
||||
if err != nil || !said {
|
||||
t.Fatalf("no account kept: %v", err)
|
||||
}
|
||||
if doing.Declared != "d2" || doing.Applied != 2 {
|
||||
t.Fatalf("the older report replaced the account of what was sent: %+v", doing)
|
||||
}
|
||||
// What it says about the machine is kept whenever it arrives, as before.
|
||||
if links, err := inv.OutwardLinksOf(ctx, "home-server"); err != nil || len(links) != 1 || links[0] != "eth1" {
|
||||
t.Fatalf("what the older report said about the machine was not kept: %v %v", links, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The account of the declaration that is outstanding replaces whatever was kept, and so does one
|
||||
// from a machine nothing was ever recorded as sent to.
|
||||
func TestAnAccountOfTheSentDeclarationReplacesTheKeptOne(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := context.Background()
|
||||
node, err := inv.AddNode(ctx, "home-server")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heard := link.Enrolment{Inventory: inv}
|
||||
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d1", Applied: []string{"a"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d2", Applied: []string{"a", "b"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doing, _, err := inv.DoingOf(ctx, "home-server")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if doing.Declared != "d2" || doing.Applied != 2 {
|
||||
t.Fatalf("the account of what was sent was not kept: %+v", doing)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// AnnouncedMerge is one merge the forge announced, as the events stream holds it: what it said, and
|
||||
// when the bus took it.
|
||||
type AnnouncedMerge struct {
|
||||
SourceMoved
|
||||
// At is when the bus took the announcement — the stream's own time, not the forge's.
|
||||
At time.Time
|
||||
// Seq is its place in the events stream.
|
||||
Seq uint64
|
||||
}
|
||||
|
||||
// MergedSubject is where the forge's merges land: the controller's own follow of them.
|
||||
var MergedSubject = broker.ControllerFollows[3]
|
||||
|
||||
// readQuiet is how long a read of the stream waits for one more message before it takes the stream
|
||||
// as read to its end. The stream answers at once when it holds something; this is only the wait at
|
||||
// the end.
|
||||
const readQuiet = 2 * time.Second
|
||||
|
||||
// AnnouncedMerges is every merge the forge announced since a moment, oldest first, read back from
|
||||
// the events stream (novox/hq issue 266).
|
||||
//
|
||||
// **Read on a consumer of its own, filtered on the one subject.** The controller's durable consumer
|
||||
// carries several filters, and the bus server the mesh ran when this was written (2.10) skips a
|
||||
// message now and then on a consumer with more than one filter: it moves its delivered pointer past
|
||||
// the message without ever handing it over, so the controller never hears of it and nothing says so.
|
||||
// A consumer filtered on a single subject reads the stream another way and was not seen to skip. This
|
||||
// one is ordered, ephemeral and acknowledges nothing, so reading it changes nothing on the bus.
|
||||
func (s *Server) AnnouncedMerges(ctx context.Context, since time.Time) ([]AnnouncedMerge, error) {
|
||||
if s.js == nil {
|
||||
return nil, errors.New("this control plane is not on the bus, so it cannot read what the forge announced")
|
||||
}
|
||||
sub, err := s.js.Context().SubscribeSync(MergedSubject, nats.OrderedConsumer(), nats.StartTime(since))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the forge's merges from the events stream: %w", err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
var out []AnnouncedMerge
|
||||
for {
|
||||
wait, cancel := context.WithTimeout(ctx, readQuiet)
|
||||
msg, err := sub.NextMsgWithContext(wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// Nothing more within the quiet wait: the stream has been read to its end.
|
||||
break
|
||||
}
|
||||
meta, err := msg.Metadata()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var moved SourceMoved
|
||||
if json.Unmarshal(msg.Data, &moved) == nil && moved.Commit != "" && moved.Repo != "" {
|
||||
out = append(out, AnnouncedMerge{SourceMoved: moved, At: meta.Timestamp, Seq: meta.Sequence.Stream})
|
||||
}
|
||||
if meta.NumPending == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Seq < out[j].Seq })
|
||||
return out, nil
|
||||
}
|
||||
@@ -34,6 +34,9 @@ const (
|
||||
// built without anybody telling the mesh (novox/hq 04-ISSUES/131).
|
||||
KindSourceMoved = "source-moved"
|
||||
KindCatchUp = "catch-up"
|
||||
// KindProvisioner is a provider saying a consumer has failed for minutes, or recovered
|
||||
// (novox/hq ADR 0224).
|
||||
KindProvisioner = "provisioner"
|
||||
)
|
||||
|
||||
// Control is one thing a node or a module said, as the controller must act on it.
|
||||
@@ -54,6 +57,11 @@ type Control interface {
|
||||
// Body is the message itself — the payload alone, never the envelope.
|
||||
Body() []byte
|
||||
|
||||
// Subject is where it was published. For a module's event it names the emitter, which the bus
|
||||
// enforces (only a module may publish into its own namespace), so who said it is read from here
|
||||
// and never from the body.
|
||||
Subject() string
|
||||
|
||||
// Redelivered says the bus has handed this message over before. An enrolment cares and
|
||||
// nothing else does: one already spent is not finished a second time.
|
||||
Redelivered() bool
|
||||
|
||||
@@ -61,6 +61,7 @@ func (c *fakeInbound) retries(ctx context.Context, s *Server) {
|
||||
|
||||
type fakeControl struct {
|
||||
kind string
|
||||
subject string
|
||||
body []byte
|
||||
tag uint64
|
||||
to *settled
|
||||
@@ -71,6 +72,7 @@ type fakeControl struct {
|
||||
|
||||
func (m *fakeControl) Kind() string { return m.kind }
|
||||
func (m *fakeControl) Body() []byte { return m.body }
|
||||
func (m *fakeControl) Subject() string { return m.subject }
|
||||
func (m *fakeControl) Redelivered() bool { return m.redelivered }
|
||||
func (m *fakeControl) About(string) {}
|
||||
func (m *fakeControl) Answer(context.Context, []byte) error { return nil }
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user