Compare commits

..
Author SHA1 Message Date
mesh-admin 660649eeed Test config end to end into a composed file, and hold the settings tests to no whole-mesh module layer (hq issue 630)
mesh/delivery stopped: the pull request closed unmerged
mesh/merge-gate out of date: judged against trunk 6f1a05366c1d, the trunk is now 8ddf8b700558; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk 6f1a05366c1d, the trunk is now 8ddf8b700558; judged again on a recheck or when it is merged
2026-10-13 13:19:30 +00:00
mesh-admin c9b63f61bd Serve config as a seat verb, and say each value's source as one of the four (hq issue 630)
The listings named a node's own value "the node", which now could be node config; they say "<module> on <node>" for the module's own value instead.
2026-10-13 13:18:33 +00:00
mesh-admin 3f9b935c25 Keep the list from a catalogue build, and carry it and mesh and node config through the facts and the gate (hq issue 630)
The merge gate must compose with the same config the mesh holds, and judge a catalogue change against the list it brings.
2026-10-13 13:17:48 +00:00
mesh-admin 0c31ca0eaf Add the config command: mesh, node and module values in one call, with why and history (hq issue 630)
One place to set each of the three sources over a default, refusing what is not a predefined key and a module value without a node, recorded in the hand-act log like any act by hand.
2026-10-13 13:16:54 +00:00
mesh-admin ea4c318404 Test the four sources, the refusals and the move of whole-mesh module layers (hq issue 630) 2026-10-13 13:15:46 +00:00
mesh-admin df5254f308 Keep mesh and node config with their history, and move whole-mesh module layers onto each assigned node (hq issue 630)
An assignment is one module on one node, so a new whole-mesh layer is refused and the ones held are laid under each assigned node's layer, every replaced layer kept, so no machine composes differently.
2026-10-13 13:14:42 +00:00
mesh-admin fba6a7c454 Carry the catalogue's predefined.json with a build, and declare the config verb (hq issue 630)
The controller only learns manifests through builds, so the list of predefined keys travels the same way: the builder reads it at the catalogue's root, judges the manifest against it and hands it on.
2026-10-13 13:13:24 +00:00
mesh-admin 3d16515771 Declare preferences with a type and a predefined key, and lay mesh and node config between default and node (hq issue 630)
A preference may fall back to one reviewed mesh-wide or node-wide value instead of a module layer for the whole mesh; like a default, config fills ${setting:} only.
2026-10-13 13:08:37 +00:00
159 changed files with 2558 additions and 14246 deletions
-6
View File
@@ -1,6 +0,0 @@
# Release 57: config and provisioning
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
-146
View File
@@ -1,146 +0,0 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
// **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its
// container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it
// replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes
// no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check
// it runs, instead of the build queue handing it out again from the start (issue 594).
//
// Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts
// without them, and a restarted one removes them before it takes work again.
//
// - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed
// when the build ends.
// - drained: written by the ask, before it reads whether a build is in hand.
//
// Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand,
// or the agent sees the drain and hands the request back; never neither.
func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") }
func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") }
// The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it
// really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped —
// takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer
// than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never
// came, which is a fault to see.
const (
drainLapse = 2 * time.Minute
drainTooLong = 15*time.Minute + 5*time.Minute
)
// drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark.
func drainedAt() (began, asked time.Time, ok bool) {
info, err := os.Stat(drainMark())
if err != nil {
return time.Time{}, time.Time{}, false
}
asked = info.ModTime()
began = asked
if raw, err := os.ReadFile(drainMark()); err == nil {
if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) {
began = t
}
}
return began, asked, true
}
// drained says the agent was asked whether it is busy within drainLapse: it takes no new build request.
func drained() bool { return drainedSince(time.Now()) }
func drainedSince(now time.Time) bool {
_, asked, ok := drainedAt()
return ok && now.Sub(asked) < drainLapse
}
// drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for
// longer than its node-engine's bound and the slack.
func drainedTooLong(now time.Time) error {
began, asked, ok := drainedAt()
if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong {
return nil
}
return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+
"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
}
// claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
// back to the queue.
func claim() bool {
// Made, never written over: one build at a time, and a mark already there is a build in hand.
f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
// Unmarked, an ask would say idle while this build runs: the build is not taken.
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
return false
}
_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
if closeErr := f.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = os.Remove(inHandMark())
fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
return false
}
if drained() {
_ = os.Remove(inHandMark())
fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
return false
}
return true
}
// holding says what is in hand, for the ask to print: written once the request is read.
func holding(what string) {
_ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644)
}
// released ends the build in hand.
func released() {
if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
}
}
// undrained clears both marks at the start of a holder: a container started again is asked afresh.
func undrained() error {
for _, mark := range []string{drainMark(), inHandMark()} {
if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("cannot remove %s: %w", mark, err)
}
}
return nil
}
// busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is.
func busy() (string, error) {
// The mark keeps when the drain began while it is asked within drainLapse, and is written again on every
// ask: its time is the last ask.
began := time.Now().UTC()
if was, _, ok := drainedAt(); ok && drained() {
began = was.UTC()
}
if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil {
// Not drained, it could take the next build: said, and answered busy only for what is in hand now.
fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err)
}
held, err := os.ReadFile(inHandMark())
if err != nil {
return "", errors.New("idle: no build in hand, and this agent takes no new one")
}
since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ")
if what == "" {
what = "a build"
}
return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil
}
-162
View File
@@ -1,162 +0,0 @@
package main
import (
"os"
"strings"
"sync"
"testing"
"time"
)
// `mesh-builder busy` answers 0 while a build is in hand and non-zero when idle, and after its first ask the
// agent takes no new build request (novox/hq ADR 0335 §5).
func TestBusyAnswersWhileABuildIsInHandAndDrainsTheAgent(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
if err := undrained(); err != nil {
t.Fatal(err)
}
if drained() {
t.Fatal("an agent never asked is drained")
}
// A build is claimed and runs.
if !claim() {
t.Fatal("an agent never asked refused a build")
}
holding("novox/mesh-controller (check-280)")
said, err := busy()
if err != nil || !strings.Contains(said, "novox/mesh-controller (check-280)") {
t.Fatalf("an agent with a build in hand did not answer busy with what it holds: %q %v", said, err)
}
// From that ask on it takes nothing new: the fetch is not made, and one fetched is handed back.
if !drained() {
t.Fatal("the ask did not drain the agent")
}
if claim() {
t.Fatal("a drained agent took a new build")
}
// The build in hand is still in hand after the refused claim, and busy goes on saying so.
if _, err := busy(); err != nil {
t.Fatalf("a refused claim ended the build in hand: %v", err)
}
// It ends: idle, and still drained.
released()
if _, err := busy(); err == nil {
t.Fatal("an agent with nothing in hand answered busy")
}
if claim() {
t.Fatal("a drained agent took a build after its last one ended")
}
// Started again, it is asked afresh.
if err := undrained(); err != nil {
t.Fatal(err)
}
if drained() || !claim() {
t.Fatal("a holder started again is still drained")
}
released()
}
// An ask racing a fetch sees the build in hand, or the agent sees the drain and hands the request back: never
// neither, which would replace the container under a build just taken.
func TestAnAskRacingAFetchNeverMissesTheBuild(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
for i := 0; i < 200; i++ {
if err := undrained(); err != nil {
t.Fatal(err)
}
var claimed bool
var askErr error
var wg sync.WaitGroup
wg.Add(2)
go func() { defer wg.Done(); claimed = claim() }()
go func() { defer wg.Done(); _, askErr = busy() }()
wg.Wait()
if claimed && askErr != nil {
t.Fatalf("round %d: the agent took a build and the ask said idle", i)
}
released()
}
}
// A drain no longer asked lapses: an agent whose replacement was withdrawn after the first ask (a put-back, a
// revert, a stopped walk) takes work again after drainLapse, rather than staying drained with its health green.
func TestADrainNoLongerAskedLapses(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
if err := undrained(); err != nil {
t.Fatal(err)
}
if _, err := busy(); err == nil {
t.Fatal("an idle agent answered busy")
}
if !drained() || claim() {
t.Fatal("an agent just asked takes work")
}
// The last ask was three minutes ago.
old := time.Now().Add(-3 * time.Minute)
if err := os.Chtimes(drainMark(), old, old); err != nil {
t.Fatal(err)
}
if drained() {
t.Fatal("a drain no longer asked for three minutes still holds")
}
if !claim() {
t.Fatal("an agent whose drain lapsed takes no work")
}
released()
}
// An agent drained, and asked all along, for longer than its node-engine's bound and the slack is not ready: the
// replacement it was drained for never came. One asked again after a lapse starts a new drain.
func TestAnAgentDrainedTooLongIsNotReady(t *testing.T) {
t.Setenv("TMPDIR", t.TempDir())
if err := undrained(); err != nil {
t.Fatal(err)
}
taking("mesh-build-machine")
if _, err := busy(); err == nil {
t.Fatal("an idle agent answered busy")
}
now := time.Now()
if err := drainedTooLong(now); err != nil {
t.Fatalf("an agent drained a moment ago is not ready: %v", err)
}
// The drain began 25 minutes ago, and the node-engine asked a moment ago.
began := now.Add(-25 * time.Minute).UTC().Format(time.RFC3339)
if err := os.WriteFile(drainMark(), []byte(began+"\n"), 0o644); err != nil {
t.Fatal(err)
}
err := drainedTooLong(time.Now())
if err == nil || !strings.Contains(err.Error(), "has not come") {
t.Fatalf("an agent drained for 25 minutes reads ready: %v", err)
}
// Each ask keeps when the drain began.
if _, err := busy(); err == nil {
t.Fatal("an idle agent answered busy")
}
if err := drainedTooLong(time.Now()); err == nil {
t.Fatal("an ask started the drain again")
}
// The asks stopped: the drain lapsed, and so did the fault.
old := time.Now().Add(-3 * time.Minute)
if err := os.Chtimes(drainMark(), old, old); err != nil {
t.Fatal(err)
}
if err := drainedTooLong(time.Now()); err != nil {
t.Fatalf("a lapsed drain still reads not ready: %v", err)
}
}
// An ask this build does not know exits non-zero: an older image asked `busy` reads idle, not busy for the
// node-engine's whole bound.
func TestAnUnknownCommandFails(t *testing.T) {
was := os.Args
t.Cleanup(func() { os.Args = was })
os.Args = []string{"mesh-builder", "some-later-word"}
if err := run(); err == nil {
t.Fatal("an unknown command exited 0")
}
os.Args = []string{"mesh-builder", "help"}
if err := run(); err != nil {
t.Fatalf("help failed: %v", err)
}
}
-31
View File
@@ -41,9 +41,6 @@ type holder struct {
say func(link.HolderState) error
// remove runs what a kill needs outside the build: the containers left behind.
remove builder.Runner
// scratch is where check builds are kept (novox/hq ADR 0350 §5): what the week's sweep lets go of. Nil where
// this machine publishes to no artifact store.
scratch builder.Scratch
mu sync.Mutex
paused bool
@@ -134,34 +131,6 @@ func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
}
}
// expireChecks lets go of every check build older than a week (novox/hq ADR 0350 §5), at start and every while
// after. Every holder sweeps the one store; what one let go of, another finds gone.
func (h *holder) expireChecks(ctx context.Context, every time.Duration, now func() time.Time) {
sweep := func() {
if h.scratch == nil {
return
}
n, err := h.scratch.Expire(ctx, builder.CheckBuildsKept, now())
switch {
case err != nil:
fmt.Fprintf(os.Stderr, "check builds older than %s: %d let go of, and not all: %v\n", builder.CheckBuildsKept, n, err)
case n > 0:
fmt.Fprintf(os.Stderr, "check builds older than %s: %d check(s)' builds let go of\n", builder.CheckBuildsKept, n)
}
}
sweep()
tick := time.NewTicker(every)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
sweep()
}
}
}
// begin records the build this machine took, with the cancel that ends it.
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
+12 -141
View File
@@ -75,12 +75,6 @@ digest of their own configuration. The result is printed as JSON.
exits 0 once the agent running in this container is bound to its build work and pulls it, and
1 before: its container's health check.
mesh-builder busy
exits 0 while the agent running in this container has a build in hand, and 1 when it has none;
from the first ask the agent takes no new build: its container's busy command, asked before the
container is replaced (novox/hq ADR 0335).
`
func run() error {
@@ -92,41 +86,17 @@ func run() error {
case "build":
return buildOnce(context.Background(), os.Args[2:])
case "ready":
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work, and
// once it has been drained longer than its node-engine would wait to replace it (ADR 0335).
if err := ready(); err != nil {
return err
}
return drainedTooLong(time.Now())
case "busy":
// The container's busy command (novox/hq ADR 0335): drains the agent, and exits non-zero once
// nothing is in hand.
said, err := busy()
if err != nil {
return err
}
fmt.Println(said)
return nil
case "help", "-h", "--help":
// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
return ready()
default:
fmt.Print(usage)
return nil
default:
// **An ask this build does not know fails** (novox/hq ADR 0335): an image older than `busy`, asked
// it, must read as idle — exiting 0 would hold its container for the node-engine's whole bound.
fmt.Fprint(os.Stderr, usage)
return fmt.Errorf("%q is not a command this mesh-builder knows", os.Args[1])
}
}
if err := notReady(); err != nil {
return err
}
// **The builder's own version is part of every reuse key** (novox/hq ADR 0350 §1): a builder that builds
// differently never takes a build another one made.
builder.Version = builder.SelfVersion(version)
if err := undrained(); err != nil {
return err
}
credential, err := brokerFrom()
if err != nil {
return err
@@ -179,13 +149,8 @@ func run() error {
// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
// recreated mid-check left the check's store and bus running, and the ask went to another machine.
h.removeLeftBehind()
// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
// in hand and then says idle, and its container is replaced between builds.
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
Paused: func() bool { return h.Paused() || drained() },
Taking: func() { taking(seat) },
Claim: claim,
})
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
Taking: func() { taking(seat) }})
defer machine.Close()
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
@@ -209,13 +174,8 @@ func run() error {
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
publisher := builder.Registry{Address: registry, Run: builder.Command}
// Check builds are let go of after a week (novox/hq ADR 0350 §5).
h.scratch = publisher
go h.expireChecks(ctx, time.Hour, time.Now)
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
defer released()
holding(work.Request().Repository + " (" + work.Request().ID + ")")
answer(ctx, publisher, on, workspace, work, h)
})
}
@@ -302,7 +262,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
if err == nil {
say("packages", saysRegistry(npmrc))
}
registryHere, isRegistry := publisher.(builder.Registry)
var built builder.Result
if request.Check != nil {
// **A pull request's merge check, not a build** (novox/hq to-be 45 §9): nothing is built,
@@ -312,32 +271,17 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
if r, ok := publisher.(builder.Registry); ok {
registry = r.Address
}
spec := checkSpecOf(request)
// What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the
// scratch namespace. **Never with the package registry's credential**: this is code nobody has approved
// yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages
// at build time has no reuse key anyway, so nothing is lost by refusing it here.
if isRegistry && len(spec.Builds) > 0 {
spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held,
Seats: request.Seats}
}
var v builder.CheckVerdict
v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say)
v, err = builder.Check(building, builder.Command, checkSpecOf(request), workspace, registry, forgeFrom(), say)
if err == nil {
result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report,
Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo),
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built)}
Head: v.Head, Trunk: v.Trunk, Beside: v.Beside}
// What each of a group's heads was judged as (novox/hq ADR 0323 §6).
for _, m := range v.Members {
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
Commit: m.Head, Trunk: m.Trunk})
}
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
if request.Check.Try {
if line, err := json.Marshal(triedOf(v)); err == nil {
say(link.StepTried, string(line))
}
}
}
} else if err == nil {
// The package-registry credential is a build input, so it is resolved before the clone: a
@@ -345,30 +289,9 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
// after a clone that then fails at npm ci.
// Every container it starts is labelled with its id, so a kill finds what outlived the
// docker client (ADR 0219).
if request.Reuse != nil && isRegistry && !request.DryRun {
// **A walk's build takes the check's build of the same key** (novox/hq ADR 0350 §2).
reuse := builder.Reuse{Scratch: registryHere}
for _, p := range request.Reuse.Pulls {
reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number})
}
// Only what a verdict names is taken (ADR 0350 §2).
for _, b := range request.Reuse.Builds {
named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo,
Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key,
Digests: map[string]string{}}
for _, m := range b.Built.Made {
named.Digests[m.Name] = builder.DigestOf(m.Reference)
}
reuse.Named = append(reuse.Named, named)
}
built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
} else {
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
}
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
forgeFrom(), say, request.Seats)
}
// What it copied into the store, whatever became of the build (novox/hq ADR 0257).
result.Mirrored = built.Mirrored
@@ -404,19 +327,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
}
result.Against = built.Against
result.SourceFingerprint = built.Source
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
result.ReuseKey = built.ReuseKey
if built.Reused != nil {
f := built.Reused.From
result.Reused = built.Reused.Words()
result.ReusedFrom = &link.ReusedFrom{Pull: link.PullRef{Owner: f.Pull.Owner, Repo: f.Pull.Repo,
Number: f.Pull.Number}, Head: f.Head, Trunk: f.Trunk, Check: f.Check, Key: f.Key}
} else if request.Reuse != nil {
result.BuiltWhy = built.Why
if result.BuiltWhy == "" {
result.BuiltWhy = "it has no artifact to build"
}
}
result.Predefined = built.Predefined
result.Trunk, result.OnTrunk, result.Branches = built.Trunk, built.OnTrunk, built.Branches
for _, r := range built.Read {
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
@@ -424,11 +335,6 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
for _, s := range built.Sources {
result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
}
// The list of predefined keys the manifest was judged against, for the controller to publish
// (novox/hq ADR 0348 §7).
if p := built.Predefined; p != nil {
result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation}
}
say("built", built.Manifest.Module+" from "+short(built.Commit))
}
}
@@ -468,15 +374,10 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
Tests: c.Tests, StoreOnly: c.StoreOnly}
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
for dir, b := range c.Beside {
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
}
for _, b := range c.Builds {
spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path,
Ref: b.Ref, Merged: b.Merged, Tier: b.Tier, Skip: b.Skip})
}
for _, m := range c.Members {
spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository,
Ref: m.Ref, Paths: m.Paths, Base: m.Base})
@@ -484,36 +385,6 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
return spec
}
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
func triedOf(v builder.CheckVerdict) link.Tried {
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
Head: v.Head, Trunk: v.Trunk}
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
}
if v.Repo != nil && v.Repo.Failed != "" {
t.Failed = strings.Split(v.Repo.Failed, "\n")
}
if t.Verdict == "" {
t.Verdict = "error"
}
return t
}
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
var out []link.CheckBuilt
for _, b := range built {
c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String()}
for _, m := range b.Made {
c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference})
}
out = append(out, c)
}
return out
}
// layerOf is one layer of a check as the outcome carries it.
func layerOf(l *builder.Layer) *link.CheckLayer {
if l == nil {
-1
View File
@@ -16,7 +16,6 @@ func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
"version)": true, // --version
`"stopping")`: true, // the loop.s shutdown line
"usage)": true, // --help text, for a human
"said)": true, // busy: the answer its node-engine reads (novox/hq ADR 0335)
}
for _, file := range []string{"once.go", "main.go"} {
src, err := os.ReadFile(file)
-31
View File
@@ -1,31 +0,0 @@
package main
import (
"testing"
"time"
"github.com/novox/mesh-controller/internal/builder"
)
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
// check's verdict, how long it took, and each line naming what failed.
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
tried.Summary != "its merge-check.sh failed: TestX" {
t.Fatalf("a failed try: %+v", tried)
}
// A head that does not merge fails both layers: the check's own word.
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
t.Fatalf("a try that does not merge: %+v", conflict)
}
// No verdict is an error, never a pass.
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
t.Fatalf("a try with no verdict: %+v", none)
}
}
+2 -3
View File
@@ -424,9 +424,8 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string
}
// And its settings on this machine, composed against its definition (rule 1, rule 6).
for _, layer := range c.layers {
said := layer.Whole()
keys := make([]string, 0, len(said))
for k := range said {
keys := make([]string, 0, len(layer.Values))
for k := range layer.Values {
keys = append(keys, k)
}
sort.Strings(keys)
+30 -53
View File
@@ -15,7 +15,6 @@ import (
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
@@ -183,10 +182,8 @@ func buildFrom(result link.BuildResult) inventory.Build {
kept.Manifest = recordedManifest(result.Manifest, announced)
// The module name comes from the manifest, which only exists when the build got that far.
if len(kept.Manifest) > 0 {
if list, err := judgedWith(result); err == nil {
if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
kept.Module = m.Module
}
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
kept.Module = m.Module
}
}
return kept
@@ -486,7 +483,6 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
Seats: seatBases(ctx),
DryRun: dryRun,
}
request.Reuse = reuseAsked(ctx, dryRun)
fmt.Printf("asked for %s", source)
if source.Seat != "" {
fmt.Printf(" (%s)", repository)
@@ -605,11 +601,6 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module
// written with the same values.
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
catalogue.Manifest, inventory.Build, error) {
// The list of predefined keys as the store keeps it now: another controller process may have kept a newer one.
if err := loadPredefined(ctx, inv); err != nil {
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+
"read: %w", err)
}
kept := buildFrom(result)
if err := inv.RecordBuild(ctx, kept); err != nil {
return catalogue.Manifest{}, kept, err
@@ -620,15 +611,7 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
result.On, result.Repository, result.Failed)
}
// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
// mesh publishes, before it is registered.
list, err := judgedWith(result)
if err != nil {
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
result.Repository, short(result.Commit), err)
}
manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
manifest, err := catalogue.ParseManifest(kept.Manifest)
if err != nil {
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
result.On, result.Repository, err)
@@ -686,12 +669,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
manifest.Module, short(result.Commit), err)
}
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
// registered only as the walk's copy in the module's own namespace.
if scratch := inScratch(result.Made); scratch != "" {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
}
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
// and the next push would send it.
@@ -702,23 +679,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
short(result.Commit))
}
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
// catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module
// is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list
// does not hold, or of another type, is refused here and never enters the catalogue.
if carried, err := carriedList(result); err != nil {
return manifest, kept, err
} else if carried != nil {
if err := publishPredefined(ctx, inv, result, carried); err != nil {
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
}
}
if manifest.NamesPredefined() {
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+
"list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err)
}
}
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
// what the build the mesh stands on was made from registers that build's artifacts at the new
// commit, so no machine is sent a new digest for a source nobody changed — an image is not
@@ -736,6 +696,20 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
}
}
}
// **The list of predefined keys arrives with a build of the catalogue** (novox/hq issue 630): the
// builder reads predefined.json at the catalogue repository's root at the commit built, and it is kept
// before the module is registered, so a preference falling back to a key added in the same commit is
// judged against the list that adds it. Only from the catalogue: another repository names no list.
if len(result.Predefined) > 0 && isCatalogueRepository(result.Repository) {
list, err := catalogue.ParsePredefined(result.Predefined)
if err != nil {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
manifest.Module, short(result.Commit), err)
}
if err := inv.SetPredefined(ctx, list, result.Repository, result.Commit); err != nil {
return manifest, kept, err
}
}
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
if errors.Is(err, inventory.ErrSuperseded) {
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
@@ -754,16 +728,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
return manifest, kept, nil
}
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
func inScratch(made []link.MadeArtifact) string {
for _, m := range made {
if path, kept := catalogue.InArtifactStore(catalogue.Recorded(m.Reference)); kept && strings.HasPrefix(path, builder.ScratchPrefix) {
return m.Reference
}
}
return ""
}
// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
"never published (ADR 0238) — merge it, and the merge builds it")
@@ -1075,3 +1039,16 @@ func buildLog(ctx context.Context, id string) error {
}
return nil
}
// catalogueRepository is the repository the catalogue is: the one whose predefined.json is the list of
// predefined keys (novox/hq issue 630).
const catalogueRepository = "mesh-catalog"
// isCatalogueRepository is whether a repository, as a path on the forge or a URL, is the catalogue.
func isCatalogueRepository(repository string) bool {
trimmed := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSpace(repository), "/"), ".git")
if i := strings.LastIndex(trimmed, "/"); i >= 0 {
trimmed = trimmed[i+1:]
}
return trimmed == catalogueRepository
}
-103
View File
@@ -1,103 +0,0 @@
package main
import (
"fmt"
"time"
"github.com/novox/mesh-controller/internal/inventory"
)
// A first-node gate waits for a busy container (novox/hq ADR 0335 §4).
//
// When a machine reports, on what the gate sent it, that a container of a judged module was left running
// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it
// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its
// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded
// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped
// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and
// its gate starts as before.
// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the
// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here.
var busyGateSlack = 5 * time.Minute
// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this
// controller's clock from when it first read the wait.
const busyEngineBound = 15 * time.Minute
// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of
// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the
// machine nor a wrong word from it holds a walk longer.
func busyGateEnds(b *inventory.GateBusy) time.Time {
until := b.Seen.Add(busyEngineBound)
if !b.Until.IsZero() && b.Until.Before(until) {
until = b.Until
}
return until.Add(busyGateSlack)
}
// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges:
// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the
// gate's Since to its end. Pure but for the gate it is handed.
func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool {
if g.Busy != nil && g.Busy.Ended != nil {
return false
}
for _, j := range pairs {
b, waits := busyOn(j, f)
if !waits {
continue
}
if g.Busy == nil {
g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since,
Seen: now}
}
if g.Busy.Seen.IsZero() {
g.Busy.Seen = now
}
g.Busy.Until, g.Busy.Said = b.Until, b.Said
if now.After(busyGateEnds(g.Busy)) {
// The engine said it would have replaced it by now: the judging starts, and finds the module not
// yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound.
end := now
g.Busy.Ended, g.Busy.PastBound = &end, true
g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil
return false
}
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil
g.Read(now, false, g.Busy.Words())
return true
}
if g.Busy == nil {
return false
}
// Replaced: the judging starts at the report that said so.
end := now
if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) {
end = r.At.UTC()
}
g.Busy.Ended = &end
g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil
return false
}
// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left
// running because it said busy.
func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) {
r, said := f.reports[j.node]
if !said || r.At == nil || !f.reportedOn(j.node, r) {
return inventory.BusyWait{}, false
}
return inventory.WaitingFor(r.Busy, j.module)
}
// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting
// for it: not yet what was sent, so not healthy, whatever its health says of the container still running.
func busyStill(j judged, f gateFacts) (string, bool) {
b, waits := busyOn(j, f)
if !waits {
return "", false
}
return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s",
j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true
}
-148
View File
@@ -1,148 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A walk whose module's container waits as busy on its first machine says so, and starts that machine's
// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4).
func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) {
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent,
Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}}
pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}}
reported := sent.Add(20 * time.Second)
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute),
Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}}
facts := func(at time.Time, busy []inventory.BusyWait) gateFacts {
// As judgeMoves hands it: the report held against the gate's own send.
return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox",
Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}}
}
now := reported.Add(time.Minute)
if !busyWait(g, pairs, facts(reported, waiting), now) {
t.Fatal("a gate whose module's container waits as busy judged it")
}
if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil {
t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy)
}
for _, said := range []string{gateLine(g), g.Last} {
if !strings.Contains(said, "waits for a busy container on novox") {
t.Errorf("the walk does not say it waits for a busy container there: %s", said)
}
}
if !strings.Contains(g.Last, "check of #280") {
t.Errorf("the wait does not say what the container holds: %s", g.Last)
}
// Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run.
now = reported.Add(10 * time.Minute)
if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 {
t.Fatalf("the gate stopped waiting within the bound: %+v", g)
}
// The machine reports it replaced: the judging starts at that report.
replacedAt := reported.Add(12 * time.Minute)
now = replacedAt.Add(5 * time.Second)
replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}}
if busyWait(g, pairs, facts(replacedAt, replaced), now) {
t.Fatal("the gate still waits after the container was replaced")
}
if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) {
t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy)
}
if strings.Contains(gateLine(g), "waits for a busy container") {
t.Errorf("a gate judging still says it waits: %s", gateLine(g))
}
// And a later report listing it waiting again does not hold a gate that started.
if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) {
t.Error("a gate that started waits again")
}
}
// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack,
// the judging starts, and finds the module not yet healthy while its container is still listed waiting.
func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) {
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
pairs := []judged{{module: "build-agent", node: "novox"}}
waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}}
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
At: &sent, Current: true, Busy: waiting}}}
now := sent.Add(15*time.Minute + busyGateSlack + time.Second)
if busyWait(g, pairs, f, now) {
t.Fatal("a gate waited past the engine's bound and the slack")
}
if !g.Busy.PastBound || !g.Since.Equal(now) {
t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since)
}
if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") {
t.Fatalf("a module still running its busy container reads as what was sent: %q", why)
}
}
// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a
// report not about the gate's send says nothing of it.
func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) {
sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent,
Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}}
pairs := []judged{{module: "build-agent", node: "ace"}}
at := sent.Add(time.Minute)
older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
At: &at, Declared: "d2", ReportedSequence: 9}}}
if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) {
t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since)
}
elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied,
At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server",
State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}}
if busyWait(g, pairs, elsewhere, at) {
t.Fatal("a wait reported about an earlier declaration held the gate")
}
}
// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until
// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait.
func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) {
seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen}
pairs := []judged{{module: "build-agent", node: "novox"}}
ahead := seen.Add(time.Hour)
f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied,
At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent",
State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}}
if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) {
t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy)
}
if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) {
t.Fatal("the gate stopped waiting within the bound on its own clock")
}
if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound {
t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy)
}
}
// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent.
func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) {
modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{
{"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}},
{"id": "agent-env", "type": "file"}}}}
lines := busyNotSent(modules, false)
if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") ||
!strings.Contains(lines[0], "replaced at once") {
t.Fatalf("the plan does not say the busy not sent: %v", lines)
}
if lines := busyNotSent(modules, true); len(lines) != 0 {
t.Fatalf("a busy sent is said as not sent: %v", lines)
}
}
+45 -29
View File
@@ -47,23 +47,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
shelf := catalogue.Shelf{}
faulted := map[string]bool{}
failed := 0
// The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read
// once per repository: a manifest naming a key is judged against the list its own tree holds.
lists := map[string]catalogue.PredefinedList{}
listOf := func(path string) (catalogue.PredefinedList, error) {
root := catalogue.RepositoryRoot(filepath.Dir(path))
if root == "" {
return nil, nil
}
if list, read := lists[root]; read {
return list, nil
}
list, err := catalogue.ReadPredefined(root)
if err != nil {
return nil, err
}
lists[root] = list
return list, nil
// The list of predefined keys a preference may fall back to (novox/hq issue 630): the predefined.json
// at the root of the repository the manifests are in, found above the first of them.
list, listAt, listErr := predefinedAbove(paths[0])
if listErr != nil {
fmt.Fprintf(out, "%s: %v\n", listAt, listErr)
failed++
}
for _, path := range paths {
raw, err := os.ReadFile(path)
@@ -72,18 +61,7 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
failed++
continue
}
list, err := listOf(path)
if err != nil {
fmt.Fprintf(out, "%s: %v\n", path, err)
failed++
continue
}
m, err := catalogue.ParseManifestWith(raw, list)
if err == nil {
// Each template's text read from the tree beside the manifest, and judged as inline content is
// (novox/hq ADR 0349 §2): a missing template is refused by name.
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
fmt.Fprintf(out, "%s: %v\n", path, err)
failed++
@@ -121,6 +99,20 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error {
failed += len(named)
faulted[m.Module] = true
}
// A preference falls back only to a key on the list, with its type (novox/hq issue 630).
if named := catalogue.PredefinedNamed(m); len(named) > 0 {
if listAt == "" {
fmt.Fprintf(out, "note: %s: its preferences fall back to %s, and no %s was found above it; "+
"registration judges them against the list the mesh holds\n", path, strings.Join(named, ", "),
catalogue.PredefinedFile)
} else if wrong := catalogue.PredefinedProblems(m, list); len(wrong) > 0 {
for _, p := range wrong {
fmt.Fprintf(out, "%s: %s\n", path, p)
}
failed += len(wrong)
faulted[m.Module] = true
}
}
shelf[m.Module] = m
}
@@ -314,3 +306,27 @@ func manifestsUnder(dir string) ([]string, error) {
sort.Strings(found)
return found, err
}
// predefinedAbove reads the predefined.json found in the directory of a manifest or one above it, up to
// the repository's root (the directory holding .git). The path it was found at is "" when there is none.
func predefinedAbove(manifest string) (catalogue.Predefined, string, error) {
dir, err := filepath.Abs(filepath.Dir(manifest))
if err != nil {
return nil, "", nil
}
for {
at := filepath.Join(dir, catalogue.PredefinedFile)
if raw, err := os.ReadFile(at); err == nil {
list, err := catalogue.ParsePredefined(raw)
return list, at, err
}
if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil {
return nil, "", nil
}
up := filepath.Dir(dir)
if up == dir {
return nil, "", nil
}
dir = up
}
}
-87
View File
@@ -1,87 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"os"
"strconv"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/inventory"
)
// A pull request's check builds go when it closes (novox/hq ADR 0350 §5): the build seat keeps them in the artifact
// store's scratch namespace for the walk of its merge to take. **Let go of where the moment is heard**: the
// controller ends the walk answering a merge — after that walk took what it reuses — and the delivery's owner, which
// hears the forge say a pull request closed unmerged, asks the `forget-checks` verb; both let go through the store,
// as the controller's own collection does. The build seat's holders let go of anything a week old on their own,
// so a word lost here leaves a week's builds, never more.
// checkBuildsIn is the scratch namespace as the controller reaches the artifact store.
func checkBuildsIn(ctx context.Context, inv *inventory.Inventory) (builder.Scratch, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
address, err := artifactStoreAddress(ctx, inv, shelf, "")
if err != nil {
return nil, err
}
if address == "" {
return nil, errors.New("the artifact store is not on the mesh's network")
}
return builder.Registry{Address: address}, nil
}
// forgetChecks lets go of a pull request's check builds and answers what it did.
var forgetChecks = func(ctx context.Context, inv *inventory.Inventory, pull builder.PullRef) (string, error) {
scratch, err := checkBuildsIn(ctx, inv)
if err != nil {
return "", err
}
n, err := scratch.Forget(ctx, pull)
if err != nil {
return "", fmt.Errorf("%s: %d check(s)' builds let go of, and not all: %w", pull, n, err)
}
return fmt.Sprintf("%s: %d check(s)' builds let go of", pull, n), nil
}
// forgetChecksOfWalk lets go, on the save that ends a walk, of the check builds of every pull request whose merge
// it answers: the walk took what it reuses. Said, never fatal.
func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) {
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
return
}
for _, pull := range reuseAskOf(&p).Pulls {
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
if err != nil {
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
continue
}
fmt.Printf("%s: %s\n", p.ID, said)
}
}
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
func forgetChecksCommand(ctx context.Context, args []string) error {
if len(args) != 3 {
return errors.New("forget-checks <owner> <repo> <number>")
}
number, err := strconv.Atoi(args[2])
if err != nil || number <= 0 {
return fmt.Errorf("%q is not a pull request's number", args[2])
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
if err != nil {
return err
}
fmt.Println(said)
return nil
}
-216
View File
@@ -1,216 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// One build per change (novox/hq ADR 0350), as the controller asks it and takes it in.
// The check of a change is asked to build what a merge of it would move, in the walk's order: a module of the
// repository checked from the merge, one standing on it from its own repository as the walk builds it, and one
// that reads the repository through a context not at all. A change that moves no module builds nothing.
func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) {
entries := []inventory.Entry{
{Manifest: catalogue.Manifest{Module: "app"}, Source: inventory.Source{Repository: "novox/shop", Seat: "git", Path: "modules/app"}},
{Manifest: catalogue.Manifest{Module: "web"}, Source: inventory.Source{Repository: "novox/web", Seat: "git", Ref: "main"}},
{Manifest: catalogue.Manifest{Module: "docs"}, Source: inventory.Source{Repository: "novox/docs", Seat: "git"}},
}
p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Base: "main", Commit: "abc"}
scope := checkScope{Reach: mergeReach{Packaging: entries[2:], Plan: inventory.Plan{Tiers: [][]string{{"app", "docs"}, {"web"}}}}}
clone := func(s inventory.Source) (string, error) { return "http://forge.invalid/" + s.Repository + ".git", nil }
got := checkBuildsOf(p, scope, entries, clone)
want := []link.CheckBuild{
{Module: "app", Path: "modules/app", Merged: true, Tier: 0},
{Module: "docs", Tier: 0, Skip: "its build reads novox/shop through a context, which a check cannot hand the merge"},
{Module: "web", Repository: "http://forge.invalid/novox/web.git", Ref: "main", Tier: 1},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("the check is asked to build\n%+v\nwant\n%+v", got, want)
}
if got := checkBuildsOf(p, checkScope{}, entries, clone); got != nil {
t.Fatalf("a change that moves no module asks for builds: %+v", got)
}
}
// A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry
// run never does, and neither does an ask that is not a walk's.
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
asked := reuseAskOf(p)
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
t.Fatalf("the walk names %+v", asked.Pulls)
}
walk := withReuse(t.Context(), asked)
if reuseAsked(walk, false) != asked {
t.Fatal("a walk's ask does not carry its leave")
}
if reuseAsked(walk, true) != nil {
t.Fatal("a dry run carries a leave to take a check's build")
}
if reuseAsked(t.Context(), false) != nil {
t.Fatal("an ask that is not a walk's carries a leave")
}
}
// A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses.
func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) {
var forgot []builder.PullRef
was := forgetChecks
forgetChecks = func(_ context.Context, _ *inventory.Inventory, pull builder.PullRef) (string, error) {
forgot = append(forgot, pull)
return "", nil
}
t.Cleanup(func() { forgetChecks = was })
p := inventory.Plan{ID: "plan-1", Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
{Repository: "novox/shop", Number: 7}}}}
p.WalkSaid = inventory.WalkStarted
forgetChecksOfWalk(t.Context(), nil, p)
p.WalkSaid = ""
forgetChecksOfWalk(t.Context(), nil, p)
if len(forgot) != 0 {
t.Fatalf("a walk not ended let go of %v, which it may still take", forgot)
}
p.WalkSaid = inventory.WalkEnded
forgetChecksOfWalk(t.Context(), nil, p)
if want := []builder.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}; !reflect.DeepEqual(forgot, want) {
t.Fatalf("the walk ended let go of %v", forgot)
}
}
// A build in the scratch namespace is never registered: only the walk's copy in the module's own namespace is.
func TestABuildInTheScratchNamespaceIsNeverRegistered(t *testing.T) {
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: "store.internal:5000/app/server@sha256:aa"}}); got != "" {
t.Fatalf("a module's own build reads as a scratch build: %s", got)
}
scratch := "store.internal:5000/scratch/app/server@sha256:aa"
if got := inScratch([]link.MadeArtifact{{Name: "server", Reference: scratch}}); got != scratch {
t.Fatalf("a scratch build was not found: %q", got)
}
if got := inScratch([]link.MadeArtifact{{Name: "look", Reference: "http://store.internal:5000/v2/scratch/app/look/blobs/sha256:bb"}}); got == "" {
t.Fatal("a scratch archive was not found")
}
}
// The walk says per module what it did: "reused from …" or "built: <why>" — and a build seat from before says
// neither, which is said.
func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
if got := (link.BuildResult{Reused: "reused from the check of novox/shop#7"}).MadeWords(); got != "reused from the check of novox/shop#7" {
t.Fatalf("said %q", got)
}
if got := (link.BuildResult{BuiltWhy: "no check built it"}).MadeWords(); got != "built: no check built it" {
t.Fatalf("said %q", got)
}
if got := (link.BuildResult{}).MadeWords(); !strings.Contains(got, "predates ADR 0350") {
t.Fatalf("said %q", got)
}
}
// A walk's module the check's build stood in for is registered by the digest the check built — the copy in the
// module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
// is named in testdata/store-tests.
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
gateEvery = 0
digest := "sha256:" + strings.Repeat("c", 64)
reused := func(id, reference string) link.BuildResult {
manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{
{"id": "svc", "type": "container", "name": "app", "image": reference}}})
return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor",
Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"},
Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}},
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"},
Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest,
ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9},
Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}}
}
if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil ||
!strings.Contains(err.Error(), "scratch") {
t.Fatalf("a build in the scratch namespace was registered: %v", err)
}
id := link.NewBuildID(time.Now().Add(time.Second))
result := reused(id, "store.internal:5000/app/server@"+digest)
if _, _, err := takeIn(ctx, inv, result); err != nil {
t.Fatal(err)
}
current, err := inv.CurrentBuilds(ctx)
if err != nil || current["app"].Commit != "c3" {
t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err)
}
shelf, err := inv.Catalogue(ctx)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(shelf["app"].Resources)
if !strings.Contains(string(raw), "app/server@"+digest) {
t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw)
}
// The walk asked it: the walk keeps what its build did, and its gate judges the registered build.
asked := time.Now().UTC()
p := g.plan(t)
p.Commit = "c3"
p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords())
p = g.plan(t)
if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") {
t.Fatalf("the walk keeps %+v", m)
}
for i := 0; i < 6; i++ {
advancePlans(ctx, g.open)
}
p = g.plan(t)
gate := p.Modules["app"].Gate
if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed {
t.Fatalf("the gate judged %+v, not the reused build", gate)
}
if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" {
t.Fatalf("the first machine was sent %v, not the reused build", sent)
}
}
// A walk's leave names the builds its own pull requests' verdicts made, and none when they cannot be read: the
// builder takes nothing a verdict does not name (novox/hq ADR 0350 §2).
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
was := checkedBuildsOf
t.Cleanup(func() { checkedBuildsOf = was })
var asked []link.PullRef
named := []link.CheckedBuild{{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 7}, Check: "build-1",
Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}}
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
asked = pulls
return named, nil
}
pulls := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}}
if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); !reflect.DeepEqual(got.Builds, named) ||
!reflect.DeepEqual(asked, pulls) {
t.Fatalf("the leave names %+v, read for %+v", got.Builds, asked)
}
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
return nil, errors.New("no bus")
}
if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); got.Builds != nil {
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
}
}
+1 -51
View File
@@ -282,59 +282,10 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
Source: sourceOnSeat(source),
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
Manifests: scope.Manifests, Judge: scope.Judge, Builds: checkBuildsOf(p, scope, entries, clone)},
Manifests: scope.Manifests, Judge: scope.Judge},
}, nil
}
// checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them
// for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges
// into, from the merge the check makes; any other — a module standing on one — from its own repository at the
// branch it follows, as the walk builds it. A module whose build reads the repository checked through a context,
// or that the merge deletes, cannot be built as the walk would: said, and nothing after it is built. A change that
// moves no module builds nothing.
func checkBuildsOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry,
clone func(inventory.Source) (string, error)) []link.CheckBuild {
if len(scope.Reach.Plan.Tiers) == 0 {
return nil
}
m := link.SourceMoved{Owner: p.Owner, Repo: p.Repo, Base: p.Base, CloneURL: p.CloneURL}
byName := map[string]inventory.Entry{}
for _, e := range entries {
byName[e.Manifest.Module] = e
}
skip := map[string]string{}
for _, e := range scope.Reach.Packaging {
skip[e.Manifest.Module] = "its build reads " + p.Owner + "/" + p.Repo + " through a context, which a check " +
"cannot hand the merge"
}
for _, e := range scope.Reach.Deleted {
skip[e.Manifest.Module] = "the merge deletes it"
}
var out []link.CheckBuild
for tier, names := range scope.Reach.Plan.Tiers {
for _, name := range names {
e, known := byName[name]
b := link.CheckBuild{Module: name, Tier: tier, Skip: skip[name]}
switch {
case !known:
b.Skip = "the catalogue does not hold it"
case b.Skip != "":
case sourceIs(e.Source, m):
b.Merged, b.Path = true, e.Source.Path
default:
url, err := clone(e.Source)
if err != nil {
b.Skip = "its repository cannot be cloned: " + err.Error()
break
}
b.Repository, b.Path, b.Ref = url, e.Source.Path, followedBranch(e.Source.Ref)
}
out = append(out, b)
}
}
return out
}
// besideRefs is the ref each repository is cloned at beside a check, by the directory it is found under:
// a core repository at the commit the mesh runs of the module built from it (`running`) — but the
// catalogue, whose checkout beside is what tests read its files from, at its main, what the next merge
@@ -431,7 +382,6 @@ func checkedOf(result link.BuildResult) link.Checked {
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and the commits of the
// repositories read beside it.
c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took
c.Built = result.Check.Built
if c.Gate == nil {
// A build seat from before the layers: its verdict is the gate's.
c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary}
+383
View File
@@ -0,0 +1,383 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// configCommand reads and sets config values (novox/hq issue 630). A value resolves from four sources, the
// last set winning: the module's default, mesh config, node config, and the module's own value on a node
// (its module-assignment config). Mesh and node config hold values only for the predefined keys, the
// catalogue's list; a module's own value is set on one node, never for the whole mesh.
//
// config [list] [--node <node>] [--module <module>]
// config set <key> <value> [--node <node> [--module <module>]] --why <text>
// config clear <key> [--node <node> [--module <module>]] --why <text>
// config history <key> [--node <node>]
//
// A set or clear is recorded in the hand-act log with the caller and why, kept in the history, and takes
// effect at the next push of the machines it reaches, which it names — as `settings set` does.
func configCommand(ctx context.Context, args []string) error {
sub := "list"
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
sub, args = args[0], args[1:]
}
set := flag.NewFlagSet("config", flag.ContinueOnError)
node := set.String("node", "", "one node: its node config, or with --module the module's own value there")
module := set.String("module", "", "a module: its own value on --node (module-assignment config)")
acts := addHandActFlags(set)
positionals, err := parseAround(set, args)
if err != nil {
return err
}
if *module != "" && *node == "" && sub != "list" {
return fmt.Errorf("config: %s names no node, and there is no module layer for the whole mesh — an "+
"assignment is one module on one node (novox/hq issue 630). Give node, or set mesh config for a "+
"predefined key without module. Nothing was changed", *module)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
inv := open.inventory
switch sub {
case "list":
if len(positionals) > 0 {
return errors.New("config list [--node <node>] [--module <module>]")
}
return configList(ctx, inv, *node, *module)
case "history":
if len(positionals) != 1 || *module != "" {
return errors.New("config history <key> [--node <node>] — a module's own values are in `settings show <module> --node <node> --history`")
}
past, err := inv.ConfigHistory(ctx, positionals[0], *node)
if err != nil {
return err
}
where := configWhere(*node, "")
if len(past) == 0 {
fmt.Printf("%s, %s: no value has been replaced\n", positionals[0], where)
return nil
}
for _, p := range past {
v, _ := json.Marshal(p.Value)
fmt.Printf("%s, %s: %s until %s (%s", positionals[0], where, v,
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy)
if p.ReplacedWhy != "" {
fmt.Printf(": %s", p.ReplacedWhy)
}
fmt.Printf("); %s, because %s\n", orUnknown(p.SetBy), orUnknown(p.Why))
}
return nil
case "set", "clear":
default:
return fmt.Errorf("config has no %q; it has list, set, clear and history", sub)
}
// set and clear.
if err := acts.require("config " + sub); err != nil {
return err
}
why := strings.TrimSpace(*acts.why)
var key string
var value any
switch {
case sub == "set" && len(positionals) == 2:
key = positionals[0]
if err := json.Unmarshal([]byte(positionals[1]), &value); err != nil {
// A bare word is a string: `config set theme dark` means "dark".
value = positionals[1]
}
if value == nil {
return errors.New("config set takes a value; to remove one, config clear")
}
case sub == "clear" && len(positionals) == 1:
key = positionals[0]
default:
return errors.New("config set <key> <value> [--node <node> [--module <module>]] --why <text>, or " +
"config clear <key> [--node <node> [--module <module>]] --why <text>")
}
if *node != "" {
if _, err := inv.NodeByName(ctx, *node); err != nil {
return err
}
}
where := configWhere(*node, *module)
// The module's own value on one node: a preference it declares, of its type, judged as `settings set`
// judges a layer, its trusted keys refused through a verb exactly as there (novox/hq issue 339, 340).
if *module != "" {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return err
}
m, known := shelf[*module]
if !known {
return fmt.Errorf("%w: %s", inventory.ErrNoSuchModule, *module)
}
p, declared := catalogue.PreferencesOf(m)[key]
if !declared {
return fmt.Errorf("config: %s declares no preference %q; config sets a module's preferences (%s) — "+
"any other setting is the settings verb's. Nothing was changed", *module, key,
preferencesOrNone(sortedPreferenceKeys(m)))
}
if value != nil {
if err := catalogue.ConfigValueProblem(fmt.Sprintf("%s's preference %q", *module, key), p.Type, value); err != nil {
return err
}
}
before, _, err := inv.Layer(ctx, *node, *module)
if err != nil {
return err
}
after := map[string]any{}
for k, v := range before {
after[k] = v
}
if value == nil {
delete(after, key)
} else {
after[key] = value
}
if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, after, *module, *node); err != nil {
return err
}
acts.record(ctx, "config", configActArgs(sub, positionals, *node, *module))
was, had, err := inv.SetModuleValue(ctx, *node, *module, key, value, setByWords()+": "+why)
if err != nil {
return err
}
sayConfigChange(key, where, value, was, had)
fmt.Printf(" takes effect at the next push: run `plan %s --diff` to see it, `push %s` to send it\n", *node, *node)
return nil
}
// Mesh config or node config: a predefined key only, judged before the act is recorded.
if value != nil {
if err := inv.JudgeConfig(ctx, key, *node, value); err != nil {
return err
}
}
acts.record(ctx, "config", configActArgs(sub, positionals, *node, ""))
before, had := configValueOf(ctx, inv, key, *node)
if value == nil {
removed, err := inv.ClearConfig(ctx, key, *node, setByWords(), why)
if err != nil {
return err
}
if !removed {
fmt.Printf("%s, %s: no value was set; nothing changed\n", key, where)
return nil
}
} else if err := inv.SetConfig(ctx, key, *node, value, setByWords(), why); err != nil {
return err
}
sayConfigChange(key, where, value, before, had)
reached, err := reachedBy(ctx, inv, key, *node)
if err != nil {
return err
}
if len(reached) == 0 {
fmt.Println(" no module assigned there falls back to it yet; nothing to push")
return nil
}
fmt.Printf(" takes effect at the next push of %s: run `push %s` to send it\n", strings.Join(reached, ", "),
strings.Join(reached, " "))
return nil
}
// configList lists the predefined keys with their mesh and node config, then each module's preferences
// with the value on each node it is assigned to and where it comes from.
func configList(ctx context.Context, inv *inventory.Inventory, node, module string) error {
if node != "" {
if _, err := inv.NodeByName(ctx, node); err != nil {
return err
}
}
list, err := inv.Predefined(ctx)
if err != nil {
return err
}
values, err := inv.Config(ctx)
if err != nil {
return err
}
var b strings.Builder
if len(list) == 0 {
fmt.Fprintf(&b, "no predefined keys: the list arrives with a build of the catalogue carrying %s\n",
catalogue.PredefinedFile)
} else {
fmt.Fprintf(&b, "predefined keys (the catalogue's %s), with mesh config and node config:\n", catalogue.PredefinedFile)
}
for _, key := range list.Keys() {
k := list[key]
fmt.Fprintf(&b, " %s (%s): %s\n", key, k.Type, k.Why)
held := false
for _, v := range values {
if v.Key != key || (node != "" && v.Node != "" && v.Node != node) {
continue
}
held = true
shown, _ := json.Marshal(v.Value)
fmt.Fprintf(&b, " %s: %s — %s, because %s\n", configWhere(v.Node, ""), shown, orUnknown(v.SetBy), v.Why)
}
if !held {
b.WriteString(" no mesh or node config: each module's default\n")
}
}
for _, key := range inventory.ConfigKeysSorted(values) {
if _, on := list[key]; on {
continue
}
for _, v := range values {
if v.Key == key {
shown, _ := json.Marshal(v.Value)
fmt.Fprintf(&b, " %s is no longer a predefined key, and %s still holds %s for it: it reaches nothing; "+
"config clear %s%s\n", key, configWhere(v.Node, ""), shown, key, nodeFlag(v.Node))
}
}
}
entries, err := inv.Catalogued(ctx)
if err != nil {
return err
}
var listed []preferencesOf
for _, e := range entries {
m := e.Manifest
if len(catalogue.PreferencesOf(m)) == 0 || (module != "" && m.Module != module) {
continue
}
if node != "" && !containsString(e.On, node) {
continue
}
p := preferencesOf{Manifest: m, On: map[string][]catalogue.SettingSource{}}
for _, n := range e.On {
if node != "" && n != node {
continue
}
p.Nodes = append(p.Nodes, n)
layers, err := inv.SettingsFor(ctx, n, m.Module)
if err != nil {
return err
}
p.On[n] = catalogue.Effective(m, layers)
}
listed = append(listed, p)
}
b.WriteString("\npreferences, and on each node the value and where it comes from (default, mesh config, " +
"node config, or the module on the node):\n")
b.WriteString(describePreferences(listed))
fmt.Print(b.String())
return nil
}
// configWhere is how a config value's place is said.
func configWhere(node, module string) string {
switch {
case module != "":
return module + " on " + node
case node != "":
return catalogue.NodeConfigLayer(node)
}
return catalogue.MeshConfigLayer
}
// configValueOf is the value mesh config (node "") or a node's config holds for a key.
func configValueOf(ctx context.Context, inv *inventory.Inventory, key, node string) (any, bool) {
values, err := inv.Config(ctx)
if err != nil {
return nil, false
}
for _, v := range values {
if v.Key == key && v.Node == node {
return v.Value, true
}
}
return nil, false
}
// reachedBy is the nodes a value of mesh config (node "") or a node's config for a key reaches: those
// running a module whose preference falls back to it.
func reachedBy(ctx context.Context, inv *inventory.Inventory, key, node string) ([]string, error) {
entries, err := inv.Catalogued(ctx)
if err != nil {
return nil, err
}
nodes := map[string]bool{}
for _, e := range entries {
if !containsString(catalogue.PredefinedNamed(e.Manifest), key) {
continue
}
for _, n := range e.On {
if node == "" || n == node {
nodes[n] = true
}
}
}
out := make([]string, 0, len(nodes))
for n := range nodes {
out = append(out, n)
}
sort.Strings(out)
return out, nil
}
func sayConfigChange(key, where string, value, before any, had bool) {
was := "nothing"
if had {
w, _ := json.Marshal(before)
was = string(w)
}
if value == nil {
fmt.Printf("%s, %s: cleared (was %s); the next source down applies\n", key, where, was)
return
}
now, _ := json.Marshal(value)
fmt.Printf("%s, %s: %s (was %s); the value it replaced is kept\n", key, where, now, was)
}
func configActArgs(sub string, positionals []string, node, module string) []string {
args := append([]string{sub}, positionals...)
if node != "" {
args = append(args, "--node", node)
}
if module != "" {
args = append(args, "--module", module)
}
return args
}
func sortedPreferenceKeys(m catalogue.Manifest) []string {
declared := catalogue.PreferencesOf(m)
keys := make([]string, 0, len(declared))
for k := range declared {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
func orUnknown(s string) string {
if strings.TrimSpace(s) == "" {
return "not kept"
}
return s
}
func preferencesOrNone(keys []string) string {
if len(keys) == 0 {
return "it declares none"
}
return strings.Join(keys, ", ")
}
+187
View File
@@ -0,0 +1,187 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The config verb's line: a listing without key, a set or clear with why, and a module value refused
// without a node (novox/hq issue 630).
func TestTheConfigVerbsLine(t *testing.T) {
for _, c := range []struct {
args map[string]any
want string
}{
{map[string]any{}, "config list"},
{map[string]any{"node": "g14", "module": "dunst"}, "config list --node g14 --module dunst"},
{map[string]any{"key": "font-size", "value": "12", "why": "bigger"}, "config set font-size 12 --why bigger"},
{map[string]any{"key": "font-size", "value": "13", "node": "g14", "why": "dense"},
"config set font-size 13 --node g14 --why dense"},
{map[string]any{"key": "font-size", "value": "14", "node": "g14", "module": "dunst", "why": "x"},
"config set font-size 14 --node g14 --module dunst --why x"},
{map[string]any{"key": "font-size", "clear": "true", "node": "g14", "why": "back"},
"config clear font-size --node g14 --why back"},
{map[string]any{"key": "font-size", "history": "true"}, "config history font-size"},
} {
argv, err := argvFor("config", c.args)
if err != nil || strings.Join(argv, " ") != c.want {
t.Errorf("%v: %v %v, want %s", c.args, argv, err, c.want)
}
}
for want, args := range map[string]map[string]any{
"no module layer for the whole mesh": {"key": "font-size", "value": "12", "module": "dunst", "why": "x"},
"why": {"key": "font-size", "value": "12"},
"names its key": {"value": "12"},
"not both": {"key": "font-size", "value": "12", "clear": "true", "why": "x"},
} {
if _, err := argvFor("config", args); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%v: %v, want %q", args, err, want)
}
}
}
// End to end through the verb, as the operator's acceptance reads it: cleared → the default; mesh font-size,
// then the node's, then dunst's own on the node — each in the node's dunstrc as composed (novox/hq issue 630).
func TestConfigReachesTheComposedFileFromEachSource(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
if err := open.inventory.SetPredefined(ctx, catalogue.Predefined{
"font-size": {Type: catalogue.TypeNumber, Why: "the size text is drawn at"}}, "", ""); err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "dunst", Version: "1",
Preferences: map[string]catalogue.Preference{
"font-size": {Type: catalogue.TypeNumber, Default: float64(10), Predefined: "font-size", Why: "100 DPI"},
},
Resources: []map[string]any{{"id": "dunstrc", "type": "file", "path": "/etc/dunst/dunstrc", "mode": "0644", "trusted": false,
"content": "font = Inter ${setting:font-size}\n"}}})
for _, n := range []string{"laptop", "anchor"} {
if _, err := assign(ctx, open, n, "dunst"); err != nil {
t.Fatal(err)
}
}
dunstrc := func(node string) string {
t.Helper()
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
declared, problems, err := composedAndValidated(ctx, open, node, gens, Allocating)
if err != nil {
t.Fatalf("%s does not compose: %v %v", node, err, problems)
}
for _, r := range declared.Resources {
if r["path"] == "/etc/dunst/dunstrc" {
return fmt.Sprint(r["content"])
}
}
t.Fatalf("%s has no dunstrc: %v", node, problems)
return ""
}
want := func(node, size string) {
t.Helper()
if got := dunstrc(node); got != "font = Inter "+size+"\n" {
t.Fatalf("%s's dunstrc: %q, want size %s", node, got, size)
}
}
verb := func(args map[string]any) error {
t.Helper()
return throughVerb(t, "config", args)
}
want("laptop", "10")
if err := verb(map[string]any{"key": "font-size", "value": "11", "why": "a larger default"}); err != nil {
t.Fatal(err)
}
want("laptop", "11")
want("anchor", "11")
if err := verb(map[string]any{"key": "font-size", "value": "13", "node": "laptop", "why": "a dense screen"}); err != nil {
t.Fatal(err)
}
want("laptop", "13")
want("anchor", "11")
if err := verb(map[string]any{"key": "font-size", "value": "15", "node": "laptop", "module": "dunst",
"why": "dunst alone larger"}); err != nil {
t.Fatal(err)
}
want("laptop", "15")
listing := stdoutOf(t, func() error { return verb(map[string]any{}) })
for _, said := range []string{"font-size (number): the size text is drawn at", "mesh config: 11",
"node config (laptop): 13", "laptop: 15 (dunst on laptop)", "anchor: 11 (mesh config)",
"falls back to the predefined key font-size"} {
if !strings.Contains(listing, said) {
t.Errorf("the listing does not say %q:\n%s", said, listing)
}
}
// Cleared, each falls back to the next source down.
for _, step := range []struct {
args map[string]any
size string
}{
{map[string]any{"key": "font-size", "clear": "true", "node": "laptop", "module": "dunst", "why": "x"}, "13"},
{map[string]any{"key": "font-size", "clear": "true", "node": "laptop", "why": "x"}, "11"},
{map[string]any{"key": "font-size", "clear": "true", "why": "x"}, "10"},
} {
if err := verb(step.args); err != nil {
t.Fatal(err)
}
want("laptop", step.size)
}
// Refused: a key not on the list for mesh config, a module value without a node, a type not the key's.
if err := verb(map[string]any{"key": "colour", "value": "blue", "why": "x"}); err == nil ||
!strings.Contains(err.Error(), "predefined") {
t.Fatalf("mesh config for a key not on the list: %v", err)
}
if err := runLine(t, []string{"config", "set", "font-size", "12", "--module", "dunst", "--why", "x"}); err == nil ||
!strings.Contains(err.Error(), "no module layer for the whole mesh") {
t.Fatalf("a module value without a node: %v", err)
}
if err := verb(map[string]any{"key": "font-size", "value": `"large"`, "node": "laptop", "module": "dunst",
"why": "x"}); err == nil || !strings.Contains(err.Error(), "is a number") {
t.Fatalf("a string for a number: %v", err)
}
if err := verb(map[string]any{"key": "nothing", "value": "1", "node": "laptop", "module": "dunst",
"why": "x"}); err == nil || !strings.Contains(err.Error(), "declares no preference") {
t.Fatalf("a module value for a key it does not declare: %v", err)
}
}
// The module check reads the catalogue's predefined.json at the repository's root and refuses a preference
// falling back to a key not on it (novox/hq issue 630).
func TestTheModuleCheckJudgesPreferencesAgainstTheCataloguesList(t *testing.T) {
root := t.TempDir()
for _, dir := range []string{".git", "modules/dunst"} {
if err := os.MkdirAll(filepath.Join(root, dir), 0o755); err != nil {
t.Fatal(err)
}
}
manifest := filepath.Join(root, "modules/dunst/module.json")
write := func(path, body string) {
t.Helper()
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
write(manifest, `{"module": "dunst", "version": "1",
"preferences": {"font-size": {"type": "number", "default": 10, "predefined": "font-size", "why": "100 DPI"}},
"resources": [{"id": "rc", "type": "file", "path": "/etc/rc", "mode": "0644", "trusted": false,
"content": "font ${setting:font-size}"}]}`)
write(filepath.Join(root, "predefined.json"), `{"theme": {"type": "string", "why": "light or dark"}}`)
var out strings.Builder
if err := moduleCheck([]string{manifest}, &out); err == nil || !strings.Contains(out.String(), "not on the catalogue's list") {
t.Fatalf("a key not on the list passed: %v\n%s", err, out.String())
}
write(filepath.Join(root, "predefined.json"), `{"font-size": {"type": "number", "why": "the size text is drawn at"}}`)
out.Reset()
if err := moduleCheck([]string{manifest}, &out); err != nil {
t.Fatalf("a key on the list was refused: %v\n%s", err, out.String())
}
}
+40 -2
View File
@@ -263,6 +263,37 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
f.Controller.Commit = c.Commit
}
// The list of predefined keys, and mesh and node config (novox/hq issue 630).
predefined, err := inv.Predefined(ctx)
if err != nil {
return snapshot.Facts{}, err
}
for key, k := range predefined {
if f.Predefined == nil {
f.Predefined = map[string]snapshot.PredefinedKey{}
}
f.Predefined[key] = snapshot.PredefinedKey{Type: k.Type, Why: k.Why}
}
config, err := inv.Config(ctx)
if err != nil {
return snapshot.Facts{}, err
}
nodeConfig := map[string]map[string]any{}
meshConfig := map[string]any{}
for _, c := range config {
if c.Node == "" {
meshConfig[c.Key] = c.Value
continue
}
if nodeConfig[c.Node] == nil {
nodeConfig[c.Node] = map[string]any{}
}
nodeConfig[c.Node][c.Key] = c.Value
}
if len(meshConfig) > 0 {
f.Config = scrub.Values(meshConfig)
}
gens, gensErr := generators(ctx, open)
hostShelf := shelf[hostModule]
meshWide := map[string]bool{}
@@ -280,6 +311,9 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
if n.HostVersion != "" {
engines[n.HostVersion] = true
}
if values := nodeConfig[n.Name]; len(values) > 0 {
m.Config = scrub.Values(values)
}
m.System = systemOf(hostShelf, n.HostVersion)
m.Libc = libcOf(m.System)
reported, err := inv.DescribedOf(ctx, n.Name)
@@ -345,14 +379,18 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
return snapshot.Facts{}, err
}
for _, layer := range layers {
if layer.Config {
// Mesh and node config are said once, beside the machine and the mesh (novox/hq issue 630).
continue
}
if layer.From == catalogue.MeshWideLayer {
if !meshWide[module] {
meshWide[module] = true
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
}
continue
}
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())})
m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)})
}
}
m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub)
+1 -1
View File
@@ -31,7 +31,7 @@ func aMeshWithSecrets(t *testing.T) (*stores, []string) {
}
}
secrets := []string{"Hunter2-Is-Not-A-Password-9f8e7d", "0123456789abcdefABCDEF0123456789zz"}
if err := open.inventory.SetSettings(ctx, "", "files", map[string]any{
if err := open.inventory.KeepSettings(ctx, "", "files", map[string]any{
"admin_password": secrets[0], "upstream": "10.77.0.9", "hub": "anchor"}); err != nil {
t.Fatal(err)
}
+3 -7
View File
@@ -81,13 +81,9 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
}
return f, err
}
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
gateSettle, gateEvery, gateBound = 0, 0, 300*time.Millisecond
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
advancePlans(ctx, b.open)
+36 -211
View File
@@ -28,12 +28,9 @@ import (
// that machine reported it applied. A build that applies and then does nothing, crashes, serves no
// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by
// the component's health — the core's health definitions, or a module's own — within ten minutes of the
// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third
// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the
// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states
// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304
// decision 3).
// Only then are the rest sent.
// apply: a core module passing three times over at least two minutes, a leaf module once, as soon as the
// node-engine states it healthy (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it). Only then
// are the rest sent.
//
// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed
// at its gate — registration refuses it and nothing sends it again on its own — the module's registered
@@ -44,30 +41,22 @@ import (
// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes.
var (
// gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the
// gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR
// 0352). Readings gateEvery apart, the first at the apply, put three inside it.
gateWithin = 2 * time.Minute
// gateSettle is how long the first machine must stay healthy, at the least.
gateSettle = 2 * time.Minute
// gateBound is how long after the apply the build may take to become healthy.
gateBound = 10 * time.Minute
// gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at
// the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the
// node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282
// decision 4 as ADR 0304 decision 3 changed it, issue 524).
// gatePasses is how many consecutive judgings must find a core module healthy, gateEvery apart at the
// least, after gateSettle. gateLeafPasses is a leaf module's: one, taken once the node-engine states every
// resource of it healthy by its own check, with no settle (novox/hq ADR 0282 decision 4 as ADR 0304
// decision 3 changed it, issue 524).
gatePasses = 3
gateLeafPasses = 1
gateLeafSettles = false
gateEvery = 40 * time.Second
// gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its
// start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either
// counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR
// 0351 decisions 1 and 4).
gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin)
)
// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
// delivery budget; how many readings its gate takes is gateRuleOf's.
// own path, or one holding the mesh's resolver. Every other module is a leaf.
func coreModule(module string, m catalogue.Manifest) bool {
if _, own := onTheControllersPath[module]; own {
return true
@@ -80,72 +69,12 @@ func coreModule(module string, m catalogue.Manifest) bool {
return coreComponent(module) != ""
}
// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the
// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it;
// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading
// (novox/hq ADR 0352). said names the rule in the gate's text.
type gateRule struct {
passes int
within time.Duration
witnessed bool
said string
}
// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its
// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send,
// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never
// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner,
// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4);
// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304
// decision 3).
func gateRuleOf(module string, m catalogue.Manifest) gateRule {
switch {
case coreComponent(module) == lease.ComponentController:
return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " +
"after the node-engine's witness window (novox/hq ADR 0351)"}
case coreComponent(module) == lease.ComponentNodeTools:
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
"after the node-engine's witness window (novox/hq ADR 0351)"}
case module == catalogue.DeliverySeat:
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
case coreModule(module, m):
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
"or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)}
// gateWants is how many passes a module needs, and whether it must stay healthy gateSettle first.
func gateWants(module string, m catalogue.Manifest) (passes int, settles bool) {
if coreModule(module, m) {
return gatePasses, true
}
r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"}
if gateLeafSettles {
r.within = gateWithin
r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send",
gateLeafPasses, gateWithin)
}
return r
}
// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the
// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A
// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it
// applied (applied, as the gate first read it) and, for the controller, before the new controller took the
// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end.
// A controller that starts again takes the lease again, and so moves its floor on. Pure.
func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time,
holder *lease.Holder) time.Time {
floor := since
if !rule.witnessed {
return floor
}
for _, n := range machines {
start := since
if at, read := applied[n]; read && at.After(start) {
start = at
}
if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) {
start = holder.Taken
}
if end := start.Add(gateWitnessWindow); end.After(floor) {
floor = end
}
}
return floor
return gateLeafPasses, gateLeafSettles
}
// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions
@@ -572,13 +501,6 @@ func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module stri
return judgeMoves(ctx, open, g, pairs, now)
}
// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one
// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352).
func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool {
most := g.Since.Add(gateWithin)
return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most)
}
// judged is one module on one machine, as a gate judges it.
type judged struct{ module, node string }
@@ -589,9 +511,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
if g.Verdict != "" {
return g.Verdict, nil
}
// A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing:
// a module healthy throughout is never held past it (novox/hq ADR 0352).
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) {
if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery {
return "", nil
}
for _, c := range g.Carried {
@@ -624,11 +544,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
facts.sentBuilds[j.node] = builds
}
}
// **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the
// walk says it waits there, and its gate starts at the report that says the container replaced.
if busyWait(g, pairs, facts, now) {
return "", nil
}
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
// gate happens to be kept on.
@@ -653,9 +568,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
for _, j := range pairs {
w := words[j.node]
h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts)
if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) {
h, said = healthNotYet, why
}
if h == healthGood || h == healthPerson {
if on, named := w.on[j.module]; named {
h, said = healthNotYet, on
@@ -696,51 +608,6 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
why = said
}
}
// **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read
// each machine report its send applied, and who holds the controller lease, the earliest its reading counts.
if g.Applied == nil {
g.Applied = map[string]time.Time{}
}
for node := range byMachine {
r, said := facts.reports[node]
if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied &&
facts.reportedOn(node, r) {
g.Applied[node] = *r.At
}
}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
var on []string
for _, j := range pairs {
if j.module == m {
on = append(on, j.node)
}
}
rules[m] = gateRuleOf(m, shelf[m])
floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder)
}
tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why,
rules: rules, floors: floors}, now)
return g.Verdict, nil
}
// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading
// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's
// rule and the earliest its reading counts.
type gateJudging struct {
modules []string
reading map[string]health
waits map[string]string
worst health
why string
rules map[string]gateRule
floors map[string]time.Time
}
// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but
// for the gate it writes.
func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why
if g.Healthy == nil {
g.Healthy = map[string]int{}
}
@@ -756,53 +623,38 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
delete(g.HealthyAt, m)
continue
}
if w := waits[m]; w != "" {
if g.Waits == nil {
g.Waits = map[string]string{}
}
g.Waits[m] = w
}
// **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the
// controller's and the tool runner's after the witness window, any other module's from the send.
if now.Before(j.floors[m]) {
continue
}
// **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more
// often while another module beside it is not yet healthy counts no faster.
if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery {
g.Healthy[m]++
g.HealthyAt[m] = now
}
if w := waits[m]; w != "" {
if g.Waits == nil {
g.Waits = map[string]string{}
}
g.Waits[m] = w
}
}
// What the send needs is what its most demanding module needs — the most readings, then the latest
// floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout
// its two minutes (novox/hq ADR 0352).
wants, floor, within, said := 0, time.Time{}, time.Duration(0), ""
settled := now.Sub(*g.Since) >= gateSettle
// What the send needs is what its most demanding module needs: a send carrying a core module is judged
// as the core.
wants, settles := 0, false
for _, m := range modules {
r, f := j.rules[m], j.floors[m]
if r.passes > wants || (r.passes == wants && f.After(floor)) {
wants, within, said = r.passes, r.within, r.said
}
if f.After(floor) {
floor = f
}
p, s := gateWants(m, shelf[m])
wants, settles = max(wants, p), settles || s
}
if len(modules) == 0 {
// A send judged on no machine of its modules (its first machine not running them) has no class to
// be judged by: it is judged as the core, never passed at once without a reading.
wants, floor, within = gatePasses, *g.Since, gateWithin
said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin)
}
// heldLongEnough is a rule's most: within after the send, healthy at every reading since it.
heldLongEnough := func(within time.Duration) bool {
return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within))
wants, settles = gatePasses, true
}
g.Wants = wants
// A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after
// A module's own pass is its own class's: a leaf beside a core module that fails keeps its build after
// its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524).
passedAlone := func(m string) bool {
r := j.rules[m]
return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within)))
p, s := gateWants(m, shelf[m])
return (settled || !s) && reading[m] <= healthPerson && g.Healthy[m] >= p
}
// fail decides the send failed: what passed on its own keeps its pass, everything else is put back.
fail := func(why string) {
@@ -819,7 +671,6 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
pastBound := now.Sub(*g.Since) > gateBound
switch {
case worst == healthBroken:
g.Faulted = true
g.Read(now, false, g.BrokenWhy)
var judging []string
for _, m := range modules {
@@ -838,36 +689,25 @@ func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) {
case worst == healthWaiting:
// Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either —
// the provider's own condition says what is wrong (ADR 0240 rule 5).
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
g.Read(now, false, why)
case worst == healthNotYet:
g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting
g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing
g.Read(now, false, why)
if pastBound {
fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why))
}
case now.Before(floor):
// Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351).
g.HealthySeen = true
// The bound still holds: a module that cannot be read healthy after its floor fails at it.
g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s",
floor.UTC().Format(time.RFC3339), said), nil
if pastBound {
fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last))
}
default:
// Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict.
g.HealthySeen = true
g.Passes++
g.LastPass, g.Last, g.Failing = &now, "", nil
g.Read(now, true, "")
if g.Passes >= wants || heldLongEnough(within) {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes,
now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now)
if g.Passes >= wants && (settled || !settles) {
decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s", g.Passes,
now.Sub(*g.Since).Round(time.Second))+waitsSaid(g.Waits), now)
}
}
return g.Verdict, nil
}
// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each
@@ -1443,12 +1283,6 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
}
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
out = append(out, backlogObservation()...)
// And a rest that went silent (novox/hq ADR 0353 §4).
silent, err := silentObservation(ctx, d.open.inventory)
if err != nil {
return nil, err
}
out = append(out, silent...)
return sortedFound(dedupeObservations(out)), nil
}
@@ -1538,15 +1372,6 @@ func gateLine(g *inventory.PlanGate) string {
case inventory.GateFailed:
what = "FAILED"
}
switch {
case g.Verdict == "" && g.Pending:
what = "kept, not yet sent" // novox/hq ADR 0353 §8
case g.Verdict == "" && g.Silent:
what = "silent on its send, left" // ADR 0353 §4
}
if g.Verdict == "" && g.Waiting() {
what = "waits for a busy container on " + g.Busy.Machine
}
line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what)
if g.Component != "" {
line += " (core: " + g.Component + ")"
+7 -11
View File
@@ -21,15 +21,11 @@ import (
// wait, and a provider waiting for a login says who waits on it.
// withGateBounds sets the gate's bounds for one test.
func withGateBounds(t *testing.T, within, every, bound time.Duration) {
func withGateBounds(t *testing.T, settle, every, bound time.Duration) {
t.Helper()
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateWithin, gateEvery, gateBound = within, every, bound
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery, gateBound = settle, every, bound
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
}
// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says
@@ -60,7 +56,7 @@ func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
judgeLeavesAsTheCore(t)
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second)
withGateBounds(t, 150*time.Millisecond, 20*time.Millisecond, 10*time.Second)
since := time.Now().Add(-time.Second)
base := factsOn(t, "laptop", since, healthyContainer("app"))
wasGather := gatherGateFacts
@@ -101,7 +97,7 @@ const broker_runtime = "node-tools"
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
withGateBounds(t, time.Hour, time.Hour, time.Hour)
withGateBounds(t, 0, time.Hour, time.Hour)
since := time.Now().Add(-time.Minute)
base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late",
Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"})
@@ -380,7 +376,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
return f, nil
}
withGateBounds(t, time.Hour, 0, 10*time.Second)
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
release := func() inventory.Plan {
t.Helper()
-338
View File
@@ -1,338 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass
// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's
// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so.
// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left.
func withTheGatesTimes(t *testing.T) {
t.Helper()
wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow
wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles
gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second
gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false
t.Cleanup(func() {
gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow
gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles
})
}
func TestEachModuleHasItsGateRule(t *testing.T) {
withTheGatesTimes(t)
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
within time.Duration
witnessed bool
said string
}{
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
{"app", nil, 1, 0, false, "a leaf module's rule"},
} {
r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) {
t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within,
c.witnessed, c.said)
}
}
}
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
func TestTheCoreStaysTheCore(t *testing.T) {
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
if !coreModule(m, catalogue.Manifest{Module: m}) {
t.Errorf("%s is no longer judged as the core for its class", m)
}
}
if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) {
t.Error("the resolver's holder is no longer judged as the core for its class")
}
}
// gateOn judges one module on one machine, healthy at every reading.
type gateOn struct {
module, node string
g *inventory.PlanGate
holder *lease.Holder
}
func aGateOn(module, node string, since time.Time) *gateOn {
return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since,
Applied: map[string]time.Time{}}}
}
func (o *gateOn) read(at time.Time, h health, why string) {
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
}
func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(15 * time.Second)
o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)}
// Ready, but inside the witness window, which opened before it took the lease: not counted.
for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} {
o.read(sent.Add(after), healthGood, "")
if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") {
t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g)
}
}
o.read(sent.Add(80*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 {
t.Fatalf("after the witness window: %+v; want passed on one reading", o.g)
}
if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") ||
!strings.Contains(o.g.Why, "ADR 0351") {
t.Errorf("the verdict says %q; want the rule it applied", o.g.Why)
}
}
// A controller that starts again takes the lease again: its witness window starts again, and so does the
// gate's wait.
func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.g.Applied["novox"] = sent.Add(10 * time.Second)
o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)}
o.read(sent.Add(90*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g)
}
o.read(sent.Add(130*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed {
t.Fatalf("not passed after the new window: %+v", o.g)
}
}
// Without the lease's word (a controller carried by another gate), the window counts from the applied report,
// and from the send at the least.
func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
r := gateRuleOf("mesh-controller", catalogue.Manifest{})
if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) {
t.Errorf("floor %s after the send; want 1m", f.Sub(sent))
}
if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"},
map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) {
t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent))
}
}
// A controller that a witness put back, or that never becomes ready, fails its gate as before.
func TestAControllerThatFailsIsStillFailed(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back")
if o.g.Verdict != inventory.GateFailed {
t.Fatalf("a controller put back by its witness: %+v; want failed", o.g)
}
o = aGateOn("mesh-controller", "novox", sent)
o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != "" {
t.Fatalf("failed before the bound: %+v", o.g)
}
o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready")
if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") {
t.Fatalf("never ready: %+v; want failed at the bound", o.g)
}
}
// threeReadings judges a module under the three-reading rule on one machine, the send at sent.
func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) {
t.Helper()
g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent}
rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}})
floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil)
return g, func(after time.Duration, h health) {
why := ""
if h != healthGood {
why = "the node-engine reports another build"
}
tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why,
rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after))
}
}
// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two
// minutes (novox/hq ADR 0352).
func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) {
withTheGatesTimes(t)
for _, module := range []string{"mesh-host", "dnsmasq"} {
g, judge := threeReadings(t, module, time.Now())
judge(15*time.Second, healthGood)
judge(55*time.Second, healthGood)
if g.Verdict != "" || g.Passes != 2 {
t.Fatalf("%s: after two readings %+v; want still judging", module, g)
}
judge(95*time.Second, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 ||
!strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") {
t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g)
}
}
}
// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at
// two minutes comes sooner than the spacing after the last pass.
func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
last := sent.Add(100 * time.Second)
held := &inventory.PlanGate{Since: &sent, LastPass: &last}
if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) {
t.Fatal("the judging at two minutes is not due, or is due before")
}
held.Faulted = true
if dueAtTheMost(held, sent.Add(2*time.Minute)) {
t.Fatal("a send that was not healthy at a reading is judged at two minutes")
}
g, judge := threeReadings(t, "mesh-host", time.Now())
judge(100*time.Second, healthGood)
judge(2*time.Minute, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 2 {
t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g)
}
g, judge = threeReadings(t, "mesh-host", time.Now())
judge(2*time.Minute, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 1 {
t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g)
}
}
// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold.
func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) {
withTheGatesTimes(t)
g, judge := threeReadings(t, "mesh-host", time.Now())
judge(10*time.Second, healthNotYet)
judge(100*time.Second, healthGood)
judge(2*time.Minute, healthGood)
if g.Verdict != inventory.GatePassed || g.Faulted {
t.Fatalf("%+v; want passed at two minutes", g)
}
}
// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one
// never healthy fails at the bound and is put back; one found broken fails.
func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) {
withTheGatesTimes(t)
g, judge := threeReadings(t, "mesh-host", time.Now())
judge(20*time.Second, healthNotYet)
judge(60*time.Second, healthGood)
judge(100*time.Second, healthGood)
judge(2*time.Minute+10*time.Second, healthGood)
if g.Verdict != inventory.GatePassed || g.Passes != 3 {
t.Fatalf("%+v; want passed only on its third healthy reading", g)
}
g, judge = threeReadings(t, "mesh-host", time.Now())
judge(60*time.Second, healthGood)
judge(100*time.Second, healthNotYet)
judge(2*time.Minute, healthGood)
if g.Verdict != "" || !g.Faulted {
t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g)
}
judge(10*time.Minute+time.Second, healthNotYet)
if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") {
t.Fatalf("never healthy again: %+v; want failed at the bound", g)
}
g, judge = threeReadings(t, "mesh-host", time.Now())
judge(60*time.Second, healthGood)
judge(100*time.Second, healthBroken)
if g.Verdict != inventory.GateFailed {
t.Fatalf("a broken reading: %+v; want failed, to be put back", g)
}
}
// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once.
func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
o := aGateOn("node-tools", "ace", sent)
o.g.Applied["ace"] = sent.Add(10 * time.Second)
o.read(sent.Add(30*time.Second), healthGood, "")
if o.g.Verdict != "" {
t.Fatalf("the tool runner passed inside its witness window: %+v", o.g)
}
o.read(sent.Add(70*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") {
t.Fatalf("the tool runner: %+v; want passed on one reading", o.g)
}
o = aGateOn(catalogue.DeliverySeat, "novox", sent)
o.read(sent.Add(5*time.Second), healthGood, "")
if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") {
t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g)
}
}
// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at
// two minutes (novox/hq ADR 0352 decision 3).
func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
modules := []string{"mesh-controller", "mesh-host"}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
}
floors["mesh-host"] = sent
floors["mesh-controller"] = sent.Add(90 * time.Second)
judge := func(after time.Duration, h health) {
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h},
worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after))
}
judge(30*time.Second, healthGood)
if g.Passes != 0 {
t.Fatalf("a reading before the controller's floor was counted: %+v", g)
}
judge(70*time.Second, healthNotYet)
judge(2*time.Minute, healthGood)
if !g.Faulted || g.Verdict != "" {
t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g)
}
}
// A send carrying the controller and the node-engine is judged by the node-engine's three readings.
func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) {
withTheGatesTimes(t)
sent := time.Now()
g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent}
modules := []string{"mesh-controller", "mesh-host"}
rules, floors := map[string]gateRule{}, map[string]time.Time{}
for _, m := range modules {
rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m})
floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil)
}
for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} {
tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood,
"mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after))
}
if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 {
t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g)
}
}
+29 -8
View File
@@ -124,14 +124,10 @@ func aGateMesh(t *testing.T) *gateMesh {
}
t.Cleanup(func() { gatherGateFacts = wasGather })
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateWithin, gateEvery = time.Hour, time.Hour
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
built := time.Now().UTC()
plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
@@ -276,7 +272,7 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
g := aGateMesh(t)
ctx := t.Context()
gateWithin = time.Hour // a core module could not pass on fewer readings for an hour
gateSettle = time.Hour // a core module could not pass for an hour
advancePlans(ctx, g.open) // the first machine is sent the new build
advancePlans(ctx, g.open) // one healthy judging
p := g.plan(t)
@@ -289,6 +285,31 @@ func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
}
}
// Which modules are judged as the core: those on the controller's own path and the resolver's holder, three
// readings after the settle time; every other module one (novox/hq ADR 0282 decision 1, ADR 0304 decision 3).
func TestAModuleIsJudgedByItsClass(t *testing.T) {
for _, c := range []struct {
module string
claims []catalogue.Claim
passes int
settles bool
}{
{"app", nil, 1, false},
{"nzbget", nil, 1, false},
{"mesh-controller", nil, 3, true},
{"mesh-host", nil, 3, true},
{"node-tools", nil, 3, true},
{"nats", nil, 3, true},
{catalogue.DeliverySeat, nil, 3, true},
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, true},
} {
p, s := gateWants(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims})
if p != c.passes || s != c.settles {
t.Errorf("%s: %d passes, settles %v; want %d, %v", c.module, p, s, c.passes, c.settles)
}
}
}
// A first machine that never becomes healthy fails its gate at the bound, and is put back.
func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) {
judgeLeavesAsTheCore(t)
+2
View File
@@ -89,6 +89,8 @@ var handActVerbs = []handActVerb{
{Verb: handActIssued, Decision: "a credential the provider's software issued, kept for its consumers (ADR 0305)"},
{Verb: "hand-act drill", Decision: "a drill is a person's deliberate test of the mesh, never a repair"},
// A person's decisions by design.
// A config value is what a person chooses the mesh's modules use (novox/hq issue 630), never a repair.
{Verb: "config", Decision: "a config value is a person's choice of what the modules use, not a repair (issue 630)"},
{Verb: "retire approve", Decision: "nothing is retired past its bound without a person (ADR 0230)"},
{Verb: "retire reject", Decision: "keeping a consumer active is a person's word (ADR 0230)"},
{Verb: "cleanup delete", Decision: "nothing retired is deleted without a person (ADR 0230)"},
+1 -1
View File
@@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
for _, mv := range f.moves(node, modules, same, known, false) {
if planStillToSend(plans, mv.Module, node) == "" {
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
}
}
if len(why) > 0 {
-87
View File
@@ -1,87 +0,0 @@
package main
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
// send that last carried that build before the walk got there, and nothing will make them report again:
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
inv := g.open.inventory
// app is the build machine, so a module of the next tier is built by it: the tier gates the next on app
// running on its machines.
for _, m := range []catalogue.Manifest{
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
{Module: "web", Version: "w1"},
} {
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
t.Fatal(err)
}
}
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
for _, b := range []inventory.Build{
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
} {
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
}
// Each machine answered that send, applied, before the walk's tier was built.
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
At: time.Now().Add(-10 * time.Second)}); err != nil {
t.Fatal(err)
}
}
built := time.Now().UTC()
p := g.plan(t)
p.Tiers = [][]string{{"app"}, {"web"}}
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if len(g.sent) != 0 {
t.Fatalf("a no-move build was sent: %v", g.sent)
}
p = g.plan(t)
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
t.Fatalf("the build was not marked no move: %+v", s)
}
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
}
// A machine that has not answered its last send is still waited on: no move is not "never sent".
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
p = g.plan(t)
if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") {
t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note)
}
}
+3 -18
View File
@@ -104,10 +104,6 @@ func run() error {
return replayCommand(ctx, args[1:])
case "kill":
return killCommand(ctx, args[1:])
case "forget-checks":
return forgetChecksCommand(ctx, args[1:])
case "try":
return tryCommand(ctx, args[1:])
case "pause", "resume":
return pauseCommand(ctx, args[0], args[1:])
case "collection":
@@ -175,6 +171,8 @@ func run() error {
return adoptCommand(ctx, args[1:])
case "settings":
return settingsCommand(ctx, args[1:])
case "config":
return configCommand(ctx, args[1:])
case "secret":
return secretCommand(ctx, args[1:])
case "operator":
@@ -383,13 +381,6 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
// recorded or registered.
if result.Check != nil || result.Checked != nil {
if isTry(result) {
// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
// build's log. Never said as `checked`, so the forge's holder sets no status.
fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
result.Repository, result.Ref)
return nil
}
checked(ctx, result)
return nil
}
@@ -440,14 +431,8 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
}
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
// What the walk's build did, kept on the walk whose ask it answers (novox/hq ADR 0350 §4); a build seat from
// before it says nothing, and that is what is kept.
made := result.MadeWords()
if result.Reused != "" || result.BuiltWhy != "" {
fmt.Printf("%s: %s %s\n", result.ID, manifest.Module, made)
}
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
planBuiltSaying(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID, made)
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
// A module registered may be one a machine is now behind: `status` is composed again.
statusFrom.nudge()
return nil
+63 -22
View File
@@ -296,6 +296,24 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
}
v.Failures = append(v.Failures, failures...)
}
// The list of predefined keys a head's catalogue tree carries, which the change composes with
// (novox/hq issue 630); the mesh's own list otherwise.
var changedList catalogue.Predefined
for _, h := range heads {
if h.Tree == "" {
continue
}
raw, err := os.ReadFile(filepath.Join(h.Tree, catalogue.PredefinedFile))
if err != nil {
continue
}
list, err := catalogue.ParsePredefined(raw)
if err != nil {
v.Failures = append(v.Failures, err.Error())
continue
}
changedList = list
}
if len(in.group) > 0 {
say("a delivery group: %d head(s) composed together", len(heads))
}
@@ -319,7 +337,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
// Both compositions, each in a store of its own.
say("composing every machine as the mesh is")
before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil)
before, _, notes, err := composeEveryMachine(ctx, in, base, sources, nil, nil)
if err != nil {
return v, fmt.Errorf("the mesh as the snapshot says it is could not be raised: %w", err)
}
@@ -332,7 +350,7 @@ func judgeChange(ctx context.Context, in mergeCheckInput) (mergeVerdict, error)
}
}
sort.Strings(added)
after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added)
after, tried, notes, err := composeEveryMachine(ctx, in, change, sources, added, changedList)
if err != nil {
return v, fmt.Errorf("the mesh with the change could not be raised: %w", err)
}
@@ -550,17 +568,8 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
out[k] = m
}
var failures []string
// The list of predefined keys at the tree's root (novox/hq ADR 0348 §7): what the build of this tree judges each
// manifest against and registration publishes, so what the gate judges each against too. A list that is
// wrong is refused by name, and every manifest is then judged against none.
predefined, err := catalogue.ReadPredefined(tree)
if err != nil {
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
catalogue.PredefinedFile, version, oneLine(err.Error())))
predefined = nil
}
found := map[string]string{} // module → its directory in the tree
err = filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
@@ -584,12 +593,7 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
if err != nil {
return err
}
m, err := catalogue.ParseManifestWith(raw, predefined)
if err == nil {
// Each template's text from the tree beside the manifest, as the build reads it back from the archive it
// packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module.
m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path))
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
if inScope(dir) {
failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s",
@@ -619,7 +623,7 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve
// As registration reads a built manifest: strictly, again (takeIn).
var raw []byte
if raw, err = json.Marshal(resolved); err == nil {
resolved, err = catalogue.ParseManifestWith(raw, predefined)
resolved, err = catalogue.ParseManifest(raw)
}
}
if err == nil {
@@ -762,13 +766,13 @@ func standInBuild(m catalogue.Manifest) (catalogue.Manifest, error) {
// grant to compose — and answers each machine's composition. Notes are what of the snapshot could not
// be raised: a refusal to register a module, keep a setting or a pin.
func composeEveryMachine(ctx context.Context, in mergeCheckInput, shelf map[string]catalogue.Manifest,
sources map[string]inventory.Source, trials []string) (map[string]mergeComposed, map[string]mergeTrial, []string, error) {
sources map[string]inventory.Source, trials []string, list catalogue.Predefined) (map[string]mergeComposed, map[string]mergeTrial, []string, error) {
open, drop, err := throwawayStores(ctx, in.admin)
if err != nil {
return nil, nil, nil, err
}
defer drop()
notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources)
notes, err := raiseFromFacts(ctx, open, in.facts, shelf, sources, list)
if err != nil {
return nil, nil, notes, err
}
@@ -964,9 +968,22 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error)
// for each standing in for the one it holds, its assignments, the seats it holds, its pins and its
// settings, and a stand-in for every secret a person gave.
func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf map[string]catalogue.Manifest,
sources map[string]inventory.Source) ([]string, error) {
sources map[string]inventory.Source, list catalogue.Predefined) ([]string, error) {
inv := open.inventory
var notes []string
// The list of predefined keys before any module, which registration judges its preferences against
// (novox/hq issue 630): the change's own when its catalogue tree carries one, the mesh's otherwise.
if list == nil && len(f.Predefined) > 0 {
list = catalogue.Predefined{}
for key, k := range f.Predefined {
list[key] = catalogue.PredefinedKey{Type: k.Type, Why: k.Why}
}
}
if len(list) > 0 {
if err := inv.SetPredefined(ctx, list, "", ""); err != nil {
notes = append(notes, fmt.Sprintf("the list of predefined keys is not kept: %s", oneLine(err.Error())))
}
}
names := make([]string, 0, len(shelf))
for name := range shelf {
names = append(names, name)
@@ -1059,6 +1076,20 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m
}
}
}
// Mesh and node config, as the mesh holds them (novox/hq issue 630).
for _, key := range sortedAnyKeys(f.Config) {
if err := inv.SetConfig(ctx, key, "", f.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil {
notes = append(notes, fmt.Sprintf("the mesh config of %s is not kept: %s", key, oneLine(err.Error())))
}
}
for _, m := range f.Machines {
for _, key := range sortedAnyKeys(m.Config) {
if err := inv.SetConfig(ctx, key, m.Name, m.Config[key], "the facts snapshot", "raised from the facts snapshot"); err != nil {
notes = append(notes, fmt.Sprintf("%s's node config of %s is not kept: %s", m.Described(), key,
oneLine(err.Error())))
}
}
}
for _, s := range f.Settings {
if err := inv.KeepSettings(ctx, "", s.Module, standInPaths(s.Values)); err != nil {
notes = append(notes, fmt.Sprintf("the mesh's settings of %s are not kept: %s", s.Module, oneLine(err.Error())))
@@ -1326,3 +1357,13 @@ func firstOr(items, otherwise []string) string {
}
return "it says nothing more"
}
// sortedAnyKeys is a map's keys, sorted.
func sortedAnyKeys(in map[string]any) []string {
keys := make([]string, 0, len(in))
for k := range in {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
+1 -15
View File
@@ -33,21 +33,7 @@ import (
func aMesh(t *testing.T) *stores {
t.Helper()
inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
return aMeshOnTheStore(t)
}
// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the
// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651).
func aMeshBefore(t *testing.T, number int) (*stores, func()) {
t.Helper()
_, migrate := inventory.ForTestBefore(t, number)
return aMeshOnTheStore(t), migrate
}
// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at.
func aMeshOnTheStore(t *testing.T) *stores {
t.Helper()
licences.ForTest(t) // planning reaches this one too, by name and never by connection
licences.ForTest(t) // planning reaches this one too, by name and never by connection
open, err := openStores(t.Context())
if err != nil {
+31 -29
View File
@@ -8,7 +8,6 @@ import (
"fmt"
"net"
"os"
"path/filepath"
"sort"
"strings"
"time"
@@ -128,10 +127,6 @@ func moduleCommand(ctx context.Context, args []string) error {
if err != nil {
return err
}
// A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2).
if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil {
return err
}
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
if err != nil {
return err
@@ -542,10 +537,6 @@ func settingsCommand(ctx context.Context, args []string) error {
if p.SetBy != "" {
by = "; " + p.SetBy
}
if p.Why != "" {
// A layer whose placement moved out of it (migration 0097): still the layer, as it stands.
by += "; " + p.Why
}
fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where,
p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown)
}
@@ -580,18 +571,14 @@ func settingsCommand(ctx context.Context, args []string) error {
return err
}
m, ok := known[positionals[0]]
if !ok || len(m.Settings) == 0 {
if !ok || len(catalogue.PreferencesOf(m)) == 0 {
return nil
}
var layers []catalogue.Layer
if *node != "" {
if mesh, has, err := inv.Layer(ctx, "", positionals[0]); err != nil {
// Every source on the node: mesh config, node config, then the module's own (novox/hq issue 630).
if layers, err = inv.SettingsFor(ctx, *node, positionals[0]); err != nil {
return err
} else if has {
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: mesh})
}
if has {
layers = append(layers, catalogue.Layer{From: *node, Values: values})
}
} else if has {
layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: values})
@@ -621,7 +608,7 @@ func settingsCommand(ctx context.Context, args []string) error {
var listed []preferencesOf
for _, e := range entries {
m := e.Manifest
if len(m.Settings) == 0 || (only != "" && m.Module != only) {
if len(catalogue.PreferencesOf(m)) == 0 || (only != "" && m.Module != only) {
continue
}
if *node != "" && !containsString(e.On, *node) {
@@ -691,7 +678,11 @@ func describeEffective(module, where string, values []catalogue.SettingSource) s
fmt.Fprintf(&b, "%s on %s, every value and where it comes from:\n", module, where)
for _, v := range values {
value, _ := json.Marshal(v.Value)
fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, v.From)
from := v.From
if v.From == where && where != "the whole mesh" {
from = valueSource(module, where, v)
}
fmt.Fprintf(&b, " %s = %s (%s", v.Key, value, from)
if v.HasDefault && !v.FromDefault {
d, _ := json.Marshal(v.Default)
fmt.Fprintf(&b, "; the default is %s", d)
@@ -732,28 +723,27 @@ func describePreferences(modules []preferencesOf) string {
on = "on " + strings.Join(p.Nodes, ", ")
}
fmt.Fprintf(&b, "%s (%s)\n", p.Manifest.Module, on)
keys := make([]string, 0, len(p.Manifest.Settings))
for k := range p.Manifest.Settings {
declared := catalogue.PreferencesOf(p.Manifest)
keys := make([]string, 0, len(declared))
for k := range declared {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
d := p.Manifest.Settings[k]
d := declared[k]
def, _ := json.Marshal(d.Default)
fmt.Fprintf(&b, " %s, default %s: %s\n", k, def, d.Why)
falls := ""
if d.Predefined != "" {
falls = ", falls back to the predefined key " + d.Predefined
}
fmt.Fprintf(&b, " %s, default %s%s: %s\n", k, def, falls, d.Why)
for _, n := range p.Nodes {
for _, s := range p.On[n] {
if s.Key != k {
continue
}
v, _ := json.Marshal(s.Value)
from := s.From
if s.FromDefault {
from = catalogue.DefaultLayer
} else if s.From != catalogue.MeshWideLayer {
from = "the node"
}
fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, from)
fmt.Fprintf(&b, " %s: %s (%s)\n", n, v, valueSource(p.Manifest.Module, n, s))
}
}
}
@@ -761,6 +751,18 @@ func describePreferences(modules []preferencesOf) string {
return b.String()
}
// valueSource is where a value on a node comes from, as the four sources are said (novox/hq issue 630):
// "default", "mesh config", "node config (<node>)", or "<module> on <node>" for the module's own value.
func valueSource(module, node string, s catalogue.SettingSource) string {
switch {
case s.FromDefault:
return catalogue.DefaultLayer
case s.From == catalogue.MeshConfigLayer, s.From == catalogue.NodeConfigLayer(node), s.From == catalogue.MeshWideLayer:
return s.From
}
return module + " on " + node
}
// nodeFlag is ` --node <node>` for a machine's layer, nothing for the whole mesh's.
func nodeFlag(node string) string {
if node == "" {
+1 -1
View File
@@ -684,7 +684,7 @@ func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
if !offers {
continue
}
serves, _, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
if err != nil {
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
}
+1 -1
View File
@@ -88,7 +88,7 @@ func checkWait(module, machine string, w inventory.Wait, f operatorWaitFacts) er
case w.Provision != "":
return checkProvisionWait(module, machine, w, m, f)
case w.Setting != "":
if _, declared := m.Settings[w.Setting]; !declared {
if !catalogue.DeclaresPreference(m, w.Setting) {
return fmt.Errorf("says it waits for the setting %s, which %s does not declare", w.Setting, module)
}
return nil
-113
View File
@@ -1,113 +0,0 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause
// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a
// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then
// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is
// byte for byte what the controller before this change composed from the same seed: the golden files were
// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1),
// where there was no migration 0097 to run.
func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) {
open, migrate := aMeshBefore(t, 97)
ctx := t.Context()
inv := open.inventory
image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64)
register(t, open, catalogue.Manifest{Module: "library", Version: "1",
Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}},
Resources: []map[string]any{
{"id": "config", "type": "directory", "mode": "0750"},
{"id": "server", "type": "container", "image": image,
"volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}},
// Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3).
{"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640",
"content": `{"log":"info"}`, "merge": "json"},
}})
register(t, open, catalogue.Manifest{Module: "found", Version: "1",
Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}})
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
t.Fatal(err)
}
for _, l := range []struct{ node, module, values string }{
{"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`},
{"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}},
"accesses": {"series": "/storage/media/series"}, "workers": 4}`},
{"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`},
{"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`},
} {
if l.node == "" {
inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values)
continue
}
inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`,
l.node, l.module, l.values)
}
migrate()
// Assigned once the store is migrated: an assignment reads the layers, as this controller holds them.
for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} {
if _, err := assign(ctx, open, a[0], a[1:]...); err != nil {
t.Fatalf("assign %v: %v", a, err)
}
}
gens, err := generators(ctx, open)
if err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
plan, settings, err := planFor(ctx, open, node)
if err != nil {
t.Fatal(err)
}
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating)
if err != nil {
t.Fatal(err)
}
composed, err := plan.Compose(with)
if err != nil {
t.Fatal(err)
}
if len(composed.LeftOut) > 0 {
t.Fatalf("%s leaves out %v", node, composed.LeftOut)
}
var told []map[string]any
for _, r := range composed.Resources {
if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" {
told = append(told, r)
}
}
sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) })
got, err := json.MarshalIndent(told, "", " ")
if err != nil {
t.Fatal(err)
}
golden := filepath.Join("testdata", "placement-moved", node+".json")
if os.Getenv("MESH_WRITE_GOLDEN") != "" {
if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil {
t.Fatal(err)
}
continue
}
want, err := os.ReadFile(golden)
if err != nil {
t.Fatal(err)
}
if string(want) != string(got)+"\n" {
t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want)
}
}
}
-16
View File
@@ -456,9 +456,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
"release-held": worded(func(o conditions.Observation) words {
return releaseHeldWords(nil, o.Also)
}),
"release-silent": worded(func(o conditions.Observation) words {
return releaseSilentWords(o.Also)
}),
"facts-stale": worded(func(o conditions.Observation) words {
return words{Headline: "Merge checks use outdated facts",
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
@@ -844,19 +841,6 @@ func releaseHeldWords(modules, machines []string) words {
Resolved: "Resolved: the held updates are released"}
}
// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
func releaseSilentWords(machines []string) words {
where := "Some machines"
if len(machines) > 0 {
where = conditions.Capital(namesWords(machines, 4))
}
return words{Headline: "Machines did not answer an update",
Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
"they were left as they were. What they run is not known to be the update.",
Resolved: "Resolved: the next walk reached them"}
}
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
func reloginNeeds(node string) string {
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
+7 -72
View File
@@ -289,14 +289,9 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
continue
}
caps, _ := inv.ProfileOf(ctx, n.Name)
// Its public domain too (novox/hq ADR 0346): a provider elsewhere serves its own composed names,
// and composes them as its own plan does, under its own domain.
// Read as its profile is: a machine whose domain cannot be read (removed since the list was read)
// composes no public name, which leaves the key out and says so, and fails no other machine's plan.
domain, _ := inv.PublicDomainOf(ctx, n.Name)
others = append(others, candidate{
catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps,
At: onNetwork[n.Name], PublicDomain: domain}, theirs})
At: onNetwork[n.Name]}, theirs})
}
offered := map[string][]catalogue.Provider{}
@@ -315,18 +310,12 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
// What that module says a consumer needs to know, with that node's settings on
// it: a port somebody moved on the provider is a port its consumers must be told
// about, and the two coming from different places is how they come to disagree.
serves, layers, err := servedOnNode(ctx, inv, o.node.Name, m, name)
if err != nil {
return catalogue.World{}, err
}
// And its own composed names, as that node composes them (novox/hq ADR 0346): a name it
// does not compose leaves the key out, said under each consumer's need.
serves, leftOut, err := got.ServedOwnNames(m, name, serves, catalogue.WithDefaults(m, layers))
serves, err := servedOnNode(ctx, inv, o.node.Name, m, name)
if err != nil {
return catalogue.World{}, err
}
offered[name] = append(offered[name], catalogue.Provider{
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module, LeftOut: leftOut})
Node: o.node.Name, At: o.node.At, Serves: serves, Module: m.Module})
}
}
}
@@ -1008,14 +997,9 @@ func renderingFor(ctx context.Context, open *stores, node string,
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
readsBusy, err := engineReadsBusy(ctx, inv, node)
if err != nil {
return catalogue.Rendering{}, inventory.Node{}, err
}
return catalogue.Rendering{
ReadsHealth: readsHealth,
JudgesRoot: judgesRoot,
ReadsBusy: readsBusy,
BusMembership: memberships[node],
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports,
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
@@ -1049,35 +1033,6 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string
return had && stated.Contract >= link.RootContract, nil
}
// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its
// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return had && stated.Contract >= link.BusyContract, nil
}
// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
if reads {
return nil
}
var out []string
for _, m := range modules {
for _, r := range m.Resources {
if _, has := r[catalogue.BusyField]; has {
out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+
"statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+
"(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract))
}
}
}
return out
}
// zonesInTheMesh is every zone a module in the mesh declares, where the mesh placed it (novox/hq ADR
// 0199): the zone settled from that node's settings, the node's private address, the port the
// answering listen is published on there.
@@ -1471,29 +1426,10 @@ func planCommand(ctx context.Context, args []string) error {
// in this output would have told anybody that.
for _, n := range plan.Needs {
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
// And a served key the provider left out, because its machine composes no name it asks for
// (novox/hq ADR 0346 §3).
for _, line := range n.LeftOut {
fmt.Printf(" %s\n", line)
}
}
// What the providers on this machine leave out of what they serve, and why: said where the remedy
// is (novox/hq ADR 0346 §3).
for _, line := range plan.ServedLeftOut(settings) {
fmt.Printf("%s: %s\n", args[0], line)
}
for _, line := range unboundLines(plan) {
fmt.Println(line)
}
// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
// container is replaced at once, as before. A statement that cannot be read is said as such.
if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
} else {
for _, line := range busyNotSent(plan.Modules, reads) {
fmt.Printf("%s: %s\n", args[0], line)
}
}
declared, err := declarationFor(ctx, open, args[0], plan, settings)
if err != nil {
return err
@@ -1630,20 +1566,19 @@ func managerPublicKeyFor(
// artifact store's trust and the references composed through it (04-ISSUES/102). Unreadable
// given ports are that node's refusal to report, not this reader's.
func servedOnNode(ctx context.Context, inv *inventory.Inventory, node string,
m catalogue.Manifest, provision string) (map[string]any, []catalogue.Layer, error) {
m catalogue.Manifest, provision string) (map[string]any, error) {
ports, layers, err := portsGivenOn(ctx, inv, node, m)
if err != nil {
return nil, nil, err
return nil, err
}
serves := catalogue.ServedOn(m, provision, ports)
if len(serves) > 0 {
serves, err = catalogue.Settle(serves, catalogue.WithDefaults(m, layers))
if err != nil {
return nil, nil, err
return nil, err
}
}
// The layers too, read once, for what is filled after (novox/hq ADR 0346).
return serves, layers, nil
return serves, nil
}
// portsGivenOn is where a node puts a module's ports — assigned, then given over them — and the
-93
View File
@@ -1,93 +0,0 @@
package main
import (
"context"
"fmt"
"path"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2, issue 661).
//
// **The newest trunk build of the catalogue wins.** A build of the catalogue reads predefined.json at the commit
// it builds and carries it with the commit's generation (link.PredefinedList). Registration keeps it in the
// controller's store when that generation is larger than the one kept (inventory.KeepPredefined), so an older
// build's list heard later never replaces a newer one, and every manifest the controller reads again is judged
// against the list kept (catalogue.UsePredefined). mesh-config publishes it on the bus from there (design 57
// package K3); the controller keeps no config value, only the keys.
// fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by
// its name, in the URL form or the seat form.
func fromTheCatalogue(result link.BuildResult) bool {
for _, repository := range []string{result.Repository, sourceRepository(result)} {
name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git")
if name == catalogue.CatalogueRepository {
return true
}
}
return false
}
func sourceRepository(result link.BuildResult) string {
if result.Source == nil {
return ""
}
return result.Source.Repository
}
// carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another
// repository, or one that carried none.
func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) {
if result.Predefined == nil || !fromTheCatalogue(result) {
return nil, nil
}
list, err := catalogue.ParsePredefined(result.Predefined.Keys)
if err != nil {
return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err)
}
return list, nil
}
// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the
// commit built, which its build judged it against, where it carried one; the list kept otherwise.
func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) {
list, err := carriedList(result)
if err != nil || list != nil {
return list, err
}
return catalogue.PublishedPredefined(), nil
}
// loadPredefined makes the list kept in the store the one this controller judges manifests against. A store
// keeping none leaves the list it has.
func loadPredefined(ctx context.Context, inv *inventory.Inventory) error {
kept, has, err := inv.PredefinedKept(ctx)
if err != nil || !has {
return err
}
catalogue.UsePredefined(kept.Keys)
return nil
}
// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the
// store keeps then — this build's, or a newer one another controller process kept.
func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult,
list catalogue.PredefinedList) error {
kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository,
Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID})
if err != nil {
return err
}
if kept {
fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID,
short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", "))
} else {
fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+
"stands\n", result.ID, short(result.Commit), result.Predefined.Generation)
}
return loadPredefined(ctx, inv)
}
-197
View File
@@ -1,197 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): `module check` and the
// merge gate judge each manifest against the list at the root of its tree; registration publishes the newest
// trunk build's list of the catalogue and judges every module against the list the mesh publishes.
const theFirstKeys = `{
"font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
"agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
}`
// notifierFallingBackTo is the notifier in the config form, its font size falling back to the key named.
func notifierFallingBackTo(key, kind string) string {
def := "10"
if kind == "string" {
def = `"10"`
}
return `{"module":"notifier","version":"1",
"config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points",
"default":` + def + `,"predefined":"` + key + `"}},
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644",
"content":"font = ${config:font-size}\n"}]}`
}
// withoutAPublishedList judges the test against no list the mesh published, and puts the process's back after.
func withoutAPublishedList(t *testing.T) {
t.Helper()
was := catalogue.ReplacePublished(nil)
t.Cleanup(func() { catalogue.ReplacePublished(was) })
}
// aRepositoryTree is a repository's checkout holding one module at modules/notifier, and the list when given.
func aRepositoryTree(t *testing.T, manifest string, list *string) string {
t.Helper()
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil {
t.Fatal(err)
}
if list != nil {
if err := os.WriteFile(filepath.Join(root, catalogue.PredefinedFile), []byte(*list), 0o644); err != nil {
t.Fatal(err)
}
}
return aTemplatedModule(t, filepath.Join(root, "modules", "notifier"), manifest, nil)
}
func TestModuleCheckJudgesAManifestAgainstTheListAtItsRepositoryRoot(t *testing.T) {
withoutAPublishedList(t)
list := theFirstKeys
check := func(path string) (string, error) {
var out bytes.Buffer
err := moduleCheck([]string{path}, &out)
return out.String(), err
}
if out, err := check(aRepositoryTree(t, notifierFallingBackTo("font-size", "number"), &list)); err != nil ||
!strings.Contains(out, "notifier: ok") {
t.Fatalf("a module naming font-size was refused: %v\n%s", err, out)
}
for _, c := range []struct {
manifest string
list *string
says string
}{
{notifierFallingBackTo("font-scale", "number"), &list, `"font-scale", which is not on the list`},
{notifierFallingBackTo("font-size", "string"), &list, "which is a number, and this value is a string"},
{notifierFallingBackTo("font-size", "number"), nil, "is not present"},
{notifierFallingBackTo("font-size", "number"), ptr(`{"font-size": {"type": "number"}}`), "no description"},
} {
if out, err := check(aRepositoryTree(t, c.manifest, c.list)); err == nil || !strings.Contains(out, c.says) {
t.Errorf("wanted the check refused saying %q, got %v:\n%s", c.says, err, out)
}
}
}
func ptr(s string) *string { return &s }
// writeTheList puts the list at the root of a tree the gate judges.
func writeTheList(t *testing.T, tree, list string) string {
t.Helper()
if err := os.WriteFile(filepath.Join(tree, catalogue.PredefinedFile), []byte(list), 0o644); err != nil {
t.Fatal(err)
}
return tree
}
func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) {
withoutAPublishedList(t)
f, manifests := templatedMesh(t)
v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier",
notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json")
if v.Verdict == "fail" || v.Modules["notifier"] != "changed" {
t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report())
}
for _, c := range []struct{ manifest, list, says string }{
{notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`},
{notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"},
{inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`},
} {
v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list),
"modules/notifier/module.json", catalogue.PredefinedFile)
if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) {
t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report())
}
}
}
// catalogueBuild is a trunk build of the catalogue carrying the list at a generation.
func catalogueBuild(id, manifest, list string, generation int) link.BuildResult {
r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
Path: "modules/notifier", Ref: "main", On: "anchor", Commit: strings.Repeat(id[len(id)-1:], 12),
Manifest: json.RawMessage(manifest), Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
if list != "" {
r.Predefined = &link.PredefinedList{Keys: json.RawMessage(list), Generation: generation}
}
return r
}
func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) {
withoutAPublishedList(t)
open := aMesh(t)
ctx := t.Context()
// A manifest naming font-size registers, and its build's list is published.
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory,
catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil {
t.Fatalf("a module naming font-size was not registered: %v", err)
}
kept, has, err := open.inventory.PredefinedKept(ctx)
if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" {
t.Fatalf("the list was not published: %+v %v %v", kept, has, err)
}
if got := catalogue.PublishedPredefined(); len(got) != 2 {
t.Fatalf("the controller judges against %v, not the list published", got)
}
// One naming a key not on the list, or of another type, is refused and never enters the catalogue.
for i, c := range []struct{ manifest, says string }{
{notifierFallingBackTo("font-scale", "number"), "not on the list"},
{notifierFallingBackTo("font-size", "string"), "which is a number"},
} {
id := []string{"b-2", "b-3"}[i]
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil ||
!strings.Contains(err.Error(), c.says) {
t.Errorf("wanted the registration refused saying %q, got %v", c.says, err)
}
}
src, err := open.inventory.SourceOf(ctx, "notifier")
if err != nil || src.BuiltFrom != strings.Repeat("1", 12) {
t.Fatalf("a refused build replaced the registered one: %+v %v", src, err)
}
// A newer build's list, which drops agent-managed-settings, replaces the published one.
smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}`
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4",
notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil {
t.Fatal(err)
}
// The older build's outcome heard later registers nothing new about the list: the newer stands.
if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5",
notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil {
t.Fatal(err)
}
kept, _, _ = open.inventory.PredefinedKept(ctx)
if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 {
t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept)
}
// A build of another repository publishes nothing, and its manifest is judged against the published list.
other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99)
other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier"
other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`,
`"chimes"`, 1))
if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil {
t.Fatalf("a module of another repository naming a published key: %v", err)
}
if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" {
t.Fatalf("a build of another repository published its list: %+v", kept)
}
other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings",
"number"), `"notifier"`, `"chimes"`, 1))
if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") {
t.Fatalf("a key only another repository's list holds was registered: %v", err)
}
}
+1 -1
View File
@@ -211,7 +211,7 @@ func (r *rootReader) executeServed(ctx context.Context, machine string) (bool, s
}
holders++
var setting *any
if _, declared := e.Manifest.Settings[loginShellVerb]; declared {
if catalogue.DeclaresPreference(e.Manifest, loginShellVerb) {
layers, err := r.settings(ctx, machine, e.Manifest.Module)
if err != nil {
served = true
+70 -247
View File
@@ -18,9 +18,8 @@ package main
// it beside the layer (`settings` says it back), and records the warrant in the hand-act log on the bus, where a
// person's decisions are read; the router edits the ask on every channel to its outcome. No seat event of its
// own: nothing consumes one, and the installer's first user list in the node-engine's repository names every
// controller event, so one would cost a node-engine change for a fact without a reader. No push is left for
// afterwards: the warrant that sets the layer opens the walk that sends it (novox/hq ADR 0344, settings_walk.go),
// so the operator's one Approve is the act that makes the change take effect.
// controller event, so one would cost a node-engine change for a fact without a reader. The push afterwards is a
// separate act, as it is for a layer set at the terminal.
//
// **What the operator reads** is the module, the machine, each key with its exact new value and, for a changed
// key, the value it replaces. A value that may not leave the mesh (an address, a path, a secret's shape: the
@@ -28,13 +27,6 @@ package main
// ask says so, and the whole is read with `settings proposals <id>` — whose fingerprint must be the one on the
// phone. Fail closed: a proposal nothing can carry to the operator is refused at once, in words, and never left
// waiting for an answer that cannot come.
//
// **On a channel that proves who answers, the values are shown WHOLE** (novox/hq issue 383, the operator's
// decision of 2026-10-10): a mount point, a share name or a private address is the thing being approved and must
// be readable, so the ask carries them whole beside the explanation (asks.Ask.Whole), the router shows that only
// on a kind that verifies its sender, and only a value shaped like a secret is withheld there. The message is the
// headline, one line per key, who proposed it and when; the fingerprint and how to read the proposal whole are
// the Details answer's (asks.Ask.Details). The masking stays for conditions and for channels that prove nothing.
import (
"context"
@@ -55,7 +47,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/asks"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/outward"
)
@@ -163,48 +154,47 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
if len([]rune(headline)) > asks.HeadlineLength {
headline = verb + " settings?"
}
// The message reads at a glance (issues 383 and 639): a summary of what changes where, then at most three
// points, a key each; every key with its whole value, who proposed it, the fingerprint and how to read it at the
// terminal are its Details, which the router notes after the answers.
shown, shownAll, shownWhole := p.change(machines, false)
whole, wholeAll, _ := p.change(machines, true)
summary := p.summary()
compose := func(points []string) string {
return strings.Join(append([]string{summary}, points...), "\n")
shown, whole := p.change(machines)
var b strings.Builder
if p.Clear {
fmt.Fprintf(&b, "Clear the settings of %s on %s, back to what the module says?\n\n", p.Module, p.where())
} else {
fmt.Fprintf(&b, "Set the settings of %s on %s to these values?\n\n", p.Module, p.where())
}
// The Details: every setting it sets, one line each, uncut — whole where the sender is proven (WholeDetails),
// masked elsewhere — then who proposed it, the fingerprint and how to read it at the terminal.
details := func(lines []string) string {
var d strings.Builder
d.WriteString(strings.Join(lines, "\n") + "\n")
d.WriteString(p.onApprove() + "\n")
fmt.Fprintf(&d, "Proposed by %s at %s. Fingerprint %s.\n", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"),
fingerprint(p.Digest))
if !shownWhole {
d.WriteString("Where who answers is not proven, its values are shown as ‹address›, ‹path› or ‹withheld›.\n")
}
fmt.Fprintf(&d, "Read it whole, with this fingerprint: settings proposals %s at the controller's terminal, or the "+
"controller's settings verb with proposal %s through the mesh MCP server.", id, id)
return d.String()
fmt.Fprintf(&b, "Proposed by %s, at %s. ", sayable(p.From, machines), p.At.Local().Format("15:04 on 2 Jan"))
switch {
case p.Clear && p.HadLayer:
b.WriteString("The layer it removes:\n\n")
case p.Clear:
b.WriteString("There is no layer to remove; approving changes nothing.")
case !p.HadLayer:
b.WriteString("There is no layer yet; this is the whole of it:\n\n")
default:
b.WriteString("The layer is replaced whole; what changes against it:\n\n")
}
q := asks.Ask{ID: id, Headline: headline, Explanation: compose(shown), Who: asks.Operator,
b.WriteString(shown)
fmt.Fprintf(&b, "\n\nFingerprint %s.", fingerprint(p.Digest))
if !whole {
b.WriteString(" Parts shown as ‹address›, ‹path› or ‹withheld› may not leave the mesh: read it whole, with " +
"this fingerprint, through the mesh MCP server (mesh-controller.settings, proposal " + id + ") or with " +
"mesh-cli settings proposals " + id + ".")
}
if p.Clear {
b.WriteString(" Approved, the layer is removed at once and the machine takes it at its next push.")
} else {
b.WriteString(" Approved, the layer is set at once and the machine takes it at its next push.")
}
q := asks.Ask{ID: id, Headline: headline, Explanation: b.String(), Who: asks.Operator,
Expires: p.At.Add(askApproveFor), OnExpiry: "the proposal is discarded; nothing changes",
About: p.about(), Details: details(shownAll)}
// Only where a value was masked: an ask whose values all pass the content rule shows the same everywhere.
if w := compose(whole); w != q.Explanation {
q.Whole = w
}
if w := details(wholeAll); w != q.Details {
q.WholeDetails = w
}
About: p.about()}
options := map[string]int{}
for i, act := range p.actions(id) {
binds, _ := asks.ActDigest(boundAct(act))
oid := optionID(act.Label)
options[oid] = i
does := "the settings are set and sent to " + p.reaches() + " at once, each machine judged"
does := "the settings are set; nothing is pushed yet"
if p.Clear {
does = "the layer is removed and that is sent to " + p.reaches() + " at once, each machine judged"
does = "the layer is removed; nothing is pushed yet"
}
if act.Arguments["answer"] == answerDecline {
does = "nothing changes; the proposal is discarded"
@@ -215,141 +205,55 @@ func (p settingsProposal) ask(id string, machines []string) (asks.Ask, map[strin
return q, options
}
// summary is the proposal's first line, what changes where: how many settings, how many stay, and that Approve sends
// the layer at once by a walk, one machine at a time (novox/hq ADR 0344); the rest of what Approve does
// (onApprove) is the Details'.
func (p settingsProposal) summary() string {
if p.Clear {
if !p.HadLayer {
return "There is no layer to remove; approving changes nothing."
}
return fmt.Sprintf("Removes the %d setting(s) of %s set on %s; approved, the layer is removed and that is sent at once "+
"by a walk to %s, one machine at a time, each judged.", len(leaves(p.Before, "")), p.Module, p.where(), p.reaches())
}
added, changed, removed := settingsChange(p.Before, p.Values)
n := len(added) + len(changed) + len(removed)
unchanged := len(leaves(p.Values, "")) - len(added) - len(changed)
if n == 0 {
return fmt.Sprintf("Nothing changes: the %d setting(s) of %s on %s are as they stand.", unchanged, p.Module, p.where())
}
stay := ""
if unchanged > 0 {
stay = fmt.Sprintf(", and %d stay as they are", unchanged)
}
return fmt.Sprintf("Changes %d setting(s) of %s on %s%s; approved, the layer is set and sent at once by a walk to %s, "+
"one machine at a time, each judged.", n, p.Module, p.where(), stay, p.reaches())
}
// shownMost is the most of a change the phone is shown, in bytes; the rest is read whole with `settings proposals`.
const shownMost = 1400
// change is what changes: as at most asks.MostPoints points for the glance, as every line whole for the note, and
// whether every value was shown exactly: "+ key: value"
// added, "~ key: value (was: old)" changed, "- key (was: old)" removed. Each value is shown under the content rule
// (sayableValue), or — exact, for a channel that proves who answers — as it is, withheld only when shaped like a
// secret (wholeValue). More changes than points: the first two, and the keys of the rest; a point longer than a
// phone shows at a glance is cut and says so. The note carries every line uncut (the review of #304: Approve sets
// exactly what the operator can read).
func (p settingsProposal) change(machines []string, exact bool) ([]string, []string, bool) {
// change is what changes, line by line, each value shown under the content rule, and whether every value was
// shown whole: "+ key: value" added, "~ key: value (was: old)" changed, "- key (was: old)" removed, and the
// count of keys unchanged.
func (p settingsProposal) change(machines []string) (string, bool) {
whole := true
say := func(v any) string {
s, w := sayableValue(v, machines)
if exact {
s, w = wholeValue(v, machines)
}
whole = whole && w
return s
}
type line struct{ key, text string }
var lines []line
var lines []string
if p.Clear {
was := leaves(p.Before, "")
for _, k := range layerKeys(was) {
lines = append(lines, line{k, fmt.Sprintf("- %s: %s", k, say(was[k]))})
lines = append(lines, fmt.Sprintf("- %s: %s", k, say(was[k])))
}
} else {
added, changed, removed := settingsChange(p.Before, p.Values)
was, now := leaves(p.Before, ""), leaves(p.Values, "")
for _, k := range added {
lines = append(lines, line{k, fmt.Sprintf("+ %s: %s", k, say(now[k]))})
lines = append(lines, fmt.Sprintf("+ %s: %s", k, say(now[k])))
}
for _, k := range changed {
lines = append(lines, line{k, fmt.Sprintf("~ %s: %s (was: %s)", k, say(now[k]), say(was[k]))})
lines = append(lines, fmt.Sprintf("~ %s: %s (was: %s)", k, say(now[k]), say(was[k])))
}
for _, k := range removed {
lines = append(lines, line{k, fmt.Sprintf("- %s (was: %s)", k, say(was[k]))})
lines = append(lines, fmt.Sprintf("- %s (was: %s)", k, say(was[k])))
}
unchanged := len(now) - len(added) - len(changed)
switch {
case len(lines) == 0 && unchanged > 0:
lines = append(lines, fmt.Sprintf("= nothing changes: the %d key(s) are as they stand", unchanged))
case unchanged > 0:
lines = append(lines, fmt.Sprintf("= %d key(s) unchanged", unchanged))
}
}
// **Each line is judged as the router will judge it** (the review of #304, ADR 0347 §1 and §3): a line the
// content rule (masked) or the secret rule (whole) would refuse — a value given to a secret's name, say
// "+ password: hunter2" — is said with its key still named and its value withheld, never left for the
// router to withhold whole and unnamed. A line refused even so is the propose's refusal (judgeLines).
for i, l := range lines {
if lineLeaves(l.text, exact, machines) {
continue
out := strings.Join(lines, "\n")
if len(out) > shownMost {
cut := shownMost
for cut > 0 && out[cut] != '\n' {
cut--
}
lines[i].text = l.text[:2] + l.key + " (value withheld)"
out = out[:cut] + fmt.Sprintf("\n… NOT SHOWN IN FULL here: %d more bytes", len(strings.Join(lines, "\n"))-cut)
whole = false
}
all := make([]string, len(lines))
for i, l := range lines {
all[i] = l.text
}
// What the note holds of a value: whole where the glance is the whole one (shown only where the sender is
// proven, with the whole details beside it), under the content rule elsewhere — so the marker says no more than
// the note it points to (the review of ADR 0347).
inNote := "in the note"
if exact {
inNote = "whole in the note"
}
var out []string
for i, l := range lines {
if len(lines) > asks.MostPoints && i == asks.MostPoints-1 {
var rest []string
for _, r := range lines[i:] {
rest = append(rest, r.key)
}
out = append(out, fmt.Sprintf("and %d more, each %s: %s", len(rest), inNote, strings.Join(rest, ", ")))
break
}
out = append(out, l.text)
}
// A point too long for a glance is cut, and says so: its value is the note's.
cut := "… (cut: " + inNote + ")"
for i, l := range out {
if r := []rune(l); len(r) > asks.PointLength {
out[i] = string(r[:asks.PointLength-len([]rune(cut))]) + cut
}
}
return out, all, whole
}
// lineLeaves says whether a line may leave the mesh as the router judges it: under the content rule for the masked
// form, the secret rule alone for the whole one.
func lineLeaves(line string, exact bool, machines []string) bool {
if exact {
_, ok := outward.Secret(line, machines...)
return ok
}
_, ok := outward.Check(line, machines...)
return ok
}
// judgeLines is the first line of an ask's details the router would refuse, or withhold, and why: masked under
// the content rule line by line, whole under the secret rule. Empty when every line may leave.
func judgeLines(details string, exact bool, machines []string) (string, string) {
for _, line := range strings.Split(details, "\n") {
if strings.TrimSpace(line) == "" {
continue
}
if exact {
if refusal, ok := outward.Secret(line, machines...); !ok {
return line, refusal.String()
}
continue
}
if refusal, ok := outward.Check(line, machines...); !ok {
return line, refusal.String()
}
}
return "", ""
return out, whole
}
func layerKeys(m map[string]any) []string {
@@ -396,27 +300,6 @@ func sayableValue(v any, machines []string) (string, bool) {
return out, out == text
}
// wholeValue is a value as a channel that proves who answers is shown it (asks.Ask.Whole), and whether it was
// shown whole: as it is, unless it is shaped like a secret (outward.Secret), which is withheld whole — never in
// part, so no half of a key reaches the phone.
func wholeValue(v any, machines []string) (string, bool) {
text, ok := v.(string)
if !ok {
raw, err := json.Marshal(v)
if err != nil {
return markWithheld, false
}
text = string(raw)
}
if text == "" {
return `""`, true
}
if _, ok := outward.Secret(text, machines...); !ok {
return markWithheld, false
}
return text, true
}
// sayable is a text with what may not leave the mesh replaced in place by a marker; what the markers cannot make
// pass is withheld whole.
func sayable(text string, machines []string) string {
@@ -549,23 +432,6 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error)
return refuse("the ask's words would carry %s, which may not leave the mesh, and the change could not be "+
"shown without it; %s", refusal, atTheTerminalInstead(in))
}
// The whole words are shown only where the sender is proven, and never a secret's shape: wholeValue withholds
// one, and the router would refuse the ask if one were left, so it is refused here first, in words.
if refusal, ok := outward.Secret(q.Whole, machines...); !ok {
return refuse("the change shown whole would carry %s, which may not leave the mesh on any channel; %s",
refusal, atTheTerminalInstead(in))
}
// So are the details (ADR 0347 §1, §3): every line the router would withhold from the note, or refuse the ask
// for, is refused here, in the controller's own words, naming the line's key — Approve never sets a value the
// operator was shown nothing of.
if line, why := judgeLines(q.Details, false, machines); line != "" {
return refuse("the change's line %q would carry %s, which the operator's note may not show; %s",
keyOfLine(line), why, atTheTerminalInstead(in))
}
if line, why := judgeLines(q.WholeDetails, true, machines); line != "" {
return refuse("the change's line %q shown whole would carry %s, which may not leave the mesh on any channel; %s",
keyOfLine(line), why, atTheTerminalInstead(in))
}
// Fail closed, before anything is kept: no router, no grant, no channel means no ask.
if pr.routerHere != nil {
here, err := pr.routerHere(ctx)
@@ -653,29 +519,15 @@ func (pr proposer) propose(ctx context.Context, in proposeInput) (string, error)
fingerprint(p.Digest))
fmt.Fprintf(&b, " until %s nothing changes: the layer is set only on Approve, and discarded on Decline or at expiry\n",
q.Expires.Local().Format("2006-01-02 15:04"))
fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, a walk sends it to %s at once, one machine at "+
"a time, each judged, and its end is told as every walk's is (novox/hq ADR 0344)", id, p.reaches())
fmt.Fprintf(&b, " `settings proposals %s` shows it whole; once approved, `push %s` sends it", id, pushWord(p.Node))
return b.String(), nil
}
// reaches is the machines an approved layer is sent to, in words: the one it names, or every machine running the
// module.
func (p settingsProposal) reaches() string {
if p.Node == "" {
return "every machine running " + p.Module
func pushWord(node string) string {
if node == "" {
return "--behind"
}
return p.Node
}
// onApprove is what the ask says Approve does (novox/hq ADR 0344 §4): the one Approve sets the layer and sends it.
func (p settingsProposal) onApprove() string {
what := "the layer is set"
if p.Clear {
what = "the layer is removed"
}
return fmt.Sprintf("Approved, %s and sent at once to %s, one machine at a time, each judged, every build there "+
"kept; its end is told as every walk's is. A module that reads its settings only when it starts takes them at "+
"its next start.", what, p.reaches())
return node
}
func newProposalID() string {
@@ -690,6 +542,11 @@ const (
// proposeCommand is `settings propose`, on this controller's stores and bus.
func proposeCommand(ctx context.Context, module, node, valuesArg string, clear, replace bool) error {
if node == "" && !clear {
// There is no module layer for the whole mesh (novox/hq issue 630); clearing one left over still works.
return fmt.Errorf("%s: there is no module layer for the whole mesh — an assignment is one module on one "+
"node (novox/hq issue 630). Propose it for one node with --node. Nothing was proposed", module)
}
var values map[string]any
if !clear {
if valuesArg == "" {
@@ -936,40 +793,6 @@ func setLayerIn(open *stores) setOnWarrant {
changed = "nothing changed"
}
}
// **And sends it** (novox/hq ADR 0344): the layer the operator approved is walked to the machines it reaches
// at once. A walk that cannot be opened is said as the act's failure — the layer is set and nothing sends it.
walked, err := openSettingsWalk(ctx, open, inventory.PlanSettings{Module: p.Module, Node: p.Node, Ask: askID,
Change: changed})
if err != nil {
return "", fmt.Errorf("%s (ask %s) is set, and the walk that sends it could not be opened: %v — `push %s` sends it",
changed, askID, err, pushTo(p.Node))
}
return changed + " (ask " + askID + "); " + walked, nil
return changed + " (ask " + askID + ")", nil
}
}
// pushTo is the push that sends a layer by hand when its walk could not: the machine, or every one behind.
func pushTo(node string) string {
if node == "" {
return "--behind"
}
return node
}
// keyOfLine is what a refusal names of a line: the key of a change's line ("+ key: …"), or the line's first words.
func keyOfLine(line string) string {
if len(line) > 2 && strings.ContainsRune("+~-=", rune(line[0])) && line[1] == ' ' {
rest := line[2:]
if i := strings.IndexAny(rest, ": "); i > 0 {
rest = rest[:i]
}
if _, ok := outward.Secret(rest); !ok {
return "whose key is shaped like a secret"
}
return rest
}
if r := []rune(line); len(r) > 40 {
return string(r[:40]) + "…"
}
return line
}
+14 -247
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"slices"
@@ -117,43 +116,24 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
if q.Options[0].Binds == q.Options[1].Binds {
t.Error("Approve and Decline bind the same act")
}
// The message reads at a glance (issues 383 and 639): a summary, then one point per key — and nothing else:
// who proposed it, the fingerprint and the how-to are the Details, which the router notes after the answers.
summary := "Changes 3 setting(s) of mounts on shanks; approved, the layer is set and sent at once by a walk to shanks, one machine at a time, each judged."
if q.Explanation != summary+"\n+ sources: recalbox=‹address›@‹path›:ro\n~ shares: library=‹path› (was: none)\n- old (was: x)" {
t.Errorf("the explanation:\n%s", q.Explanation)
}
if err := asks.Glance(q.Explanation); err != nil {
t.Error(err)
for _, line := range []string{
"Set the settings of mounts on shanks to these values?",
"Proposed by g14/claude-code, through the mesh-controller seat, at " + r.now.Local().Format("15:04 on 2 Jan") + ".",
"+ sources: recalbox=‹address›@‹path›:ro",
"~ shares: library=‹path› (was: none)",
"- old (was: x)",
"Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
"read it whole, with this fingerprint",
} {
if !strings.Contains(q.Explanation, line) {
t.Errorf("the explanation lacks %q:\n%s", line, q.Explanation)
}
}
for _, leak := range []string{"nas.lan", "/mnt/recalbox", "/mnt/library", "smb://"} {
if strings.Contains(q.Explanation, leak) {
t.Errorf("the explanation carries %q, which may not leave the mesh", leak)
}
}
// Where the sender is proven, the values whole: the mount point and the share are what is approved.
if q.Whole != summary+"\n+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox:ro\n~ shares: library=/mnt/library (was: none)\n- old (was: x)" ||
asks.Glance(q.Whole) != nil {
t.Errorf("the whole words:\n%s", q.Whole)
}
for _, line := range []string{
"Proposed by g14/claude-code, through the mesh-controller seat at " + r.now.Local().Format("15:04 on 2 Jan") +
". Fingerprint " + fingerprint(layerDigest(mountsSources)) + ".",
"Where who answers is not proven, its values are shown as ‹address›, ‹path› or ‹withheld›.",
"Read it whole, with this fingerprint: settings proposals " + kept(r).ID + " at the controller's terminal",
} {
if !strings.Contains(q.Details, line) {
t.Errorf("Details lack %q:\n%s", line, q.Details)
}
}
if refusal, ok := outward.Check(q.Details, "anchor", "laptop", "shanks"); !ok {
t.Errorf("the router would withhold a line of the details: %s", refusal)
}
for _, howTo := range []string{"ingerprint", "settings proposals", "settings verb", "not proven", "Proposed by"} {
if strings.Contains(q.Explanation, howTo) || strings.Contains(q.Whole, howTo) {
t.Errorf("the message carries the how-to %q", howTo)
}
}
all := []string{q.Headline, q.Explanation, q.OnExpiry}
for _, o := range q.Options {
all = append(all, o.Label, o.Does)
@@ -161,9 +141,6 @@ func TestAProposalAsksAtTheLevelApproveWithTheExactChange(t *testing.T) {
if refusal, ok := outward.Check(strings.Join(all, "\n"), "anchor", "laptop", "shanks"); !ok {
t.Errorf("the router would refuse the ask: %s", refusal)
}
if refusal, ok := outward.Secret(q.Whole, "anchor", "laptop", "shanks"); !ok {
t.Errorf("the router would refuse the whole words: %s", refusal)
}
// Kept as the controller's own ask, about no condition, with the proposal whole and its digests.
kept := r.theProposal(t)
p := kept.Proposal
@@ -196,98 +173,11 @@ func TestAMeshWideLayerIsProposed(t *testing.T) {
}
q := r.askSent(t)
if q.Headline != "Set notes on the whole mesh?" || q.About != "settings.notes.mesh" ||
q.Explanation != "Changes 1 setting(s) of notes on the whole mesh; approved, the layer is set and sent at once by a walk to every machine running notes, one machine at a time, each judged.\n+ x: 1" ||
q.Whole != "" {
!strings.Contains(q.Explanation, "Set the settings of notes on the whole mesh to these values?") {
t.Errorf("%+v", q)
}
}
// kept is the one proposal the rig keeps.
func kept(r *proposerRig) asked {
for _, a := range r.store {
if a.Proposal != nil {
return a
}
}
return asked{}
}
// The switch between the masked and the whole change (issue 383): the same change, under the content rule and
// exact, differs in the masked values alone; a value shaped like a secret is withheld in both, whole, with its
// key still named; and a change no value of which is masked carries no whole words of its own.
func TestAProposalShowsItsValuesWholeOnlyWhereTheSenderIsProvenAndNeverASecret(t *testing.T) {
machines := []string{"anchor", "laptop", "shanks"}
r := newProposerRig(t)
values := map[string]any{"shares": "media=/storage/media", "sources": "recalbox=smb://nas.lan/recalbox@/mnt/recalbox", "font": "Inter 13"}
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: values}); err != nil {
t.Fatal(err)
}
q := r.askSent(t)
masked, _, _ := kept(r).Proposal.change(machines, false)
whole, _, _ := kept(r).Proposal.change(machines, true)
if !strings.HasSuffix(q.Explanation, "\n"+strings.Join(masked, "\n")) || !strings.HasSuffix(q.Whole, "\n"+strings.Join(whole, "\n")) {
t.Fatalf("the ask does not carry the change masked and whole:\n%s\n--\n%s", q.Explanation, q.Whole)
}
if want := []string{"+ font: Inter 13", "+ shares: media=‹path›", "+ sources: recalbox=‹address›@‹path›"}; strings.Join(masked, "\n") != strings.Join(want, "\n") {
t.Errorf("masked: %q", masked)
}
if want := []string{"+ font: Inter 13", "+ shares: media=/storage/media", "+ sources: recalbox=smb://nas.lan/recalbox@/mnt/recalbox"}; strings.Join(whole, "\n") != strings.Join(want, "\n") {
t.Errorf("whole: %q", whole)
}
if _, ok := outward.Secret(q.Whole, "shanks"); !ok {
t.Error("the router would refuse the whole words")
}
// Mutation: the masked and the whole change differ in exactly the lines whose value was masked.
differ := 0
for i := range masked {
if masked[i] != whole[i] {
differ++
if !strings.Contains(masked[i], "‹") || strings.Contains(whole[i], "‹") {
t.Errorf("the lines differ otherwise than by the mask:\n%s\n%s", masked[i], whole[i])
}
}
}
if differ != 2 {
t.Errorf("%d lines differ, and the mask covered 2", differ)
}
// A value shaped like a secret is withheld in both, whole, with its key still named.
r1 := newProposerRig(t)
secrets := map[string]any{"github": "ghp_abcdefghijklmnopqrstuvwxyz0123456789", "cert": "-----BEGIN CERTIFICATE-----"}
if _, err := r1.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: secrets}); err != nil {
t.Fatal(err)
}
q1 := r1.askSent(t)
for _, line := range []string{"+ cert: ‹withheld›", "+ github: ‹withheld›"} {
if !strings.Contains(q1.Explanation, line) {
t.Errorf("lacks %q:\n%s", line, q1.Explanation)
}
}
for _, secret := range []string{"ghp_", "BEGIN CERTIFICATE"} {
if strings.Contains(q1.Explanation, secret) || strings.Contains(q1.Whole, secret) || strings.Contains(q1.Details, secret) {
t.Errorf("the secret %q reaches the phone", secret)
}
}
// More keys than points (issue 639): the first two, then the keys of the rest, read whole at the terminal.
r3 := newProposerRig(t)
five := map[string]any{"a": "1", "b": "2", "c": "3", "d": "4", "e": "5"}
if _, err := r3.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: five}); err != nil {
t.Fatal(err)
}
if q3 := r3.askSent(t); !strings.HasSuffix(q3.Explanation, "\n+ a: 1\n+ b: 2\nand 3 more, each in the note: c, d, e") || asks.Glance(q3.Explanation) != nil ||
!strings.Contains(q3.Details, "settings proposals") {
t.Errorf("five keys:\n%s", q3.Explanation)
}
// No value masked: the message is the same everywhere, and the ask says no whole words.
r2 := newProposerRig(t)
if _, err := r2.pr.propose(context.Background(), proposeInput{module: "dunst", node: "laptop", values: map[string]any{"font-size": 13, "width": 500}}); err != nil {
t.Fatal(err)
}
if q2 := r2.askSent(t); q2.Whole != "" || !strings.HasSuffix(q2.Explanation, "\n+ font-size: 13\n+ width: 500") ||
strings.Contains(q2.Details, "not proven") {
t.Errorf("%+v", q2)
}
}
// A proposal expired unanswered is kept so by the reconciling, and a warrant for it afterwards sets nothing.
func TestAnExpiredProposalIsKeptExpired(t *testing.T) {
r := newAskerRig(t)
@@ -315,8 +205,7 @@ func TestAClearIsProposedAndShowsWhatItRemoves(t *testing.T) {
}
q := r.askSent(t)
if q.Headline != "Clear notes on laptop?" || !strings.Contains(q.Explanation, "- places.data.owner: 1001:1001") ||
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") || !strings.Contains(q.Whole, "- places.data.path: /srv/notes") ||
!strings.HasPrefix(q.Explanation, "Removes the 2 setting(s) of notes set on laptop; approved, the layer is removed and that is sent at once by a walk to laptop, one machine at a time") {
!strings.Contains(q.Explanation, "- places.data.path: ‹path›") {
t.Errorf("%+v", q)
}
if p := r.theProposal(t).Proposal; !p.Clear || p.Digest != layerDigest(r.before) || len(r.judged) != 0 {
@@ -855,125 +744,3 @@ func captureStdout(t *testing.T, f func()) string {
_ = w.Close()
return <-done
}
// The review of #304: Approve sets exactly what the operator can read. A proposal of five keys, one value 300
// characters long, shows two keys at a glance, says what it cut, and carries every key with its whole value in its
// Details for a channel that proves who answers (WholeDetails), and every key masked for one that does not.
func TestEveryKeyAndItsWholeValueReachTheOperator(t *testing.T) {
var long strings.Builder
long.WriteString("library=/storage")
for i := 0; long.Len() < 300; i++ {
long.WriteString(fmt.Sprintf("/album-%02d", i))
}
value := long.String()[:300]
values := map[string]any{"a": "1", "b": "2", "c": "3", "d": "4", "shares": value}
r := newProposerRig(t)
if _, err := r.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: values}); err != nil {
t.Fatal(err)
}
q := r.askSent(t)
if err := q.Check(r.now); err != nil {
t.Fatal(err)
}
// What the operator reads on a proving channel: the whole words, then the note's whole details.
visible := q.Whole + "\n" + q.WholeDetails
if q.Whole == "" {
visible = q.Explanation + "\n" + q.WholeDetails
}
for k, v := range values {
if !strings.Contains(visible, "+ "+k+": "+v.(string)) {
t.Errorf("the operator never reads %s whole:\n%s", k, visible)
}
}
// The masked glance points to a note that holds the values masked, and says no more; the whole glance, shown only
// beside the whole details, says they are whole there.
if asks.Glance(q.Explanation) != nil || !strings.HasSuffix(q.Explanation, "\nand 3 more, each in the note: c, d, shares") ||
strings.Contains(q.Explanation, "whole in the note") || !strings.HasSuffix(q.Whole, "\nand 3 more, each whole in the note: c, d, shares") {
t.Errorf("the glance:\n%s", q.Explanation)
}
for k := range values {
if !strings.Contains(q.Details, "+ "+k+": ") {
t.Errorf("the masked details lack %s:\n%s", k, q.Details)
}
}
if strings.Contains(q.Details, "/storage") {
t.Errorf("the masked details show a path:\n%s", q.Details)
}
if refusal, ok := outward.Check(q.Details, "shanks"); !ok {
t.Errorf("the router would withhold the masked details: %s", refusal)
}
if _, ok := outward.Secret(q.WholeDetails, "shanks"); !ok {
t.Error("the router would refuse the whole details")
}
// A value too long for a glance is cut there, and says so.
r2 := newProposerRig(t)
if _, err := r2.pr.propose(context.Background(), proposeInput{module: "mounts", node: "shanks", values: map[string]any{"shares": value}}); err != nil {
t.Fatal(err)
}
q2 := r2.askSent(t)
// The cut point is 160 characters, its marker inside the bound.
if last := q2.Whole[strings.LastIndex(q2.Whole, "\n")+1:]; len([]rune(last)) != asks.PointLength {
t.Errorf("the cut point is %d characters, not %d: %q", len([]rune(last)), asks.PointLength, last)
}
if !strings.HasSuffix(q2.Whole, "… (cut: whole in the note)") || !strings.HasSuffix(q2.Explanation, "‹path›") && !strings.HasSuffix(q2.Explanation, "… (cut: in the note)") || !strings.Contains(q2.WholeDetails, "+ shares: "+value) || asks.Glance(q2.Whole) != nil {
t.Errorf("the cut is not said, or the note lacks the whole:\n%s\n--\n%s", q2.Whole, q2.WholeDetails)
}
}
// The review of #304: a value given to a secret's name, past the second key, is said with its key named and its
// value withheld, in the note on every channel; the router has nothing left to withhold unnamed or refuse. A line
// the router would refuse even so is refused at propose, in the controller's words.
func TestASecretNamedKeyIsNamedAndItsValueWithheld(t *testing.T) {
r := newProposerRig(t)
values := map[string]any{"a": "1", "b": "2", "c": "3", "password": "hunter2"}
if _, err := r.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop", values: values}); err != nil {
t.Fatal(err)
}
q := r.askSent(t)
for _, form := range []string{q.Details, q.WholeDetails, q.Explanation, q.Whole} {
if strings.Contains(form, "hunter2") {
t.Errorf("the value reaches the phone:\n%s", form)
}
}
if !strings.Contains(q.Details, "+ password (value withheld)") {
t.Errorf("the key is not named with its value withheld:\n%s", q.Details)
}
if q.WholeDetails != "" && !strings.Contains(q.WholeDetails, "+ password (value withheld)") {
t.Errorf("the whole details do not name the key:\n%s", q.WholeDetails)
}
if line, why := judgeLines(q.Details, false, nil); line != "" {
t.Errorf("the router would withhold %q: %s", line, why)
}
if line, why := judgeLines(q.WholeDetails, true, nil); line != "" {
t.Errorf("the router would refuse %q: %s", line, why)
}
if !strings.HasSuffix(q.Explanation, "\nand 2 more, each in the note: c, password") {
t.Errorf("the glance:\n%s", q.Explanation)
}
// A key shaped like a secret cannot be named on any channel: refused at propose, in the controller's words, and
// the key itself not echoed.
r2 := newProposerRig(t)
secretKey := "ghp_abcdefghijklmnopqrstuvwxyz0123456789"
_, err := r2.pr.propose(context.Background(), proposeInput{module: "notes", node: "laptop",
values: map[string]any{"a": "1", "b": "2", secretKey: "3"}})
if err == nil || !strings.Contains(err.Error(), "Nothing was proposed") || strings.Contains(err.Error(), secretKey) {
t.Errorf("a secret-shaped key: %v", err)
}
if len(r2.sent) != 0 {
t.Errorf("the refused proposal was asked")
}
}
// keyOfLine names a change's key, and never one shaped like a secret.
func TestARefusalNamesTheKeyAndNeverASecret(t *testing.T) {
for line, want := range map[string]string{
"+ password (value withheld)": "password",
"~ shares: ‹path› (was: none)": "shares",
"+ ghp_abcdefghijklmnopqrstuvwxyz0123456789: 1": "whose key is shaped like a secret",
"Proposed by g14 at 15:04.": "Proposed by g14 at 15:04.",
} {
if got := keyOfLine(line); got != want {
t.Errorf("%q: %q, want %q", line, got, want)
}
}
}
+1 -7
View File
@@ -249,13 +249,7 @@ func serve(ctx context.Context) (err error) {
// (novox/hq to-be 45 §9).
checkEvents = bus
// And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239).
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) {
sayPlanMoved(ctx, bus, open.inventory, p)
// A walk ended has taken what it reuses: its merges' check builds go (novox/hq ADR 0350 §5).
if p.WalkSaid == inventory.WalkEnded {
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
}
}
inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { sayPlanMoved(ctx, bus, open.inventory, p) }
if err := server.Checks(following{open}); err != nil {
return err
}
+1 -30
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"errors"
"fmt"
"sort"
@@ -84,27 +83,6 @@ func everyBuildKept(ctx context.Context, node string) bool {
return on != "" && on == node
}
type keepForSettingsKey struct{}
// keepingBuildsForSettings is a context whose sends compose every module of one machine — recorded or rolling out —
// at the build that machine was last sent, as keepingEveryBuild does, for a settings walk's send (novox/hq ADR 0344):
// the layer the operator approved, and nothing of any module's new code. A build held for a person, or waiting
// for a gate, stays where it is; the walk that is its own carries it.
func keepingBuildsForSettings(ctx context.Context, node string) context.Context {
return context.WithValue(ctx, keepForSettingsKey{}, node)
}
// errSettingsCannotKeep is a settings walk's send refused because a build of its machine cannot be kept: what the
// machine was last sent is not known, or a build it runs is gone from the records. It does not clear by itself, so
// the walk fails on it rather than trying again (ADR 0344).
var errSettingsCannotKeep = errors.New("the settings cannot be sent with every build kept")
// settingsKeptOn is whether this context is a settings walk's send to this machine.
func settingsKeptOn(ctx context.Context, node string) bool {
on, _ := ctx.Value(keepForSettingsKey{}).(string)
return on != "" && on == node
}
// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a
// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context
// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere —
@@ -114,7 +92,7 @@ func settingsKeptOn(ctx context.Context, node string) bool {
// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and
// composing the mesh's builds would be the very move the grants step must not make.
func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) {
every := everyBuildKept(ctx, node) || settingsKeptOn(ctx, node)
every := everyBuildKept(ctx, node)
skip, on := keptExcept(ctx)
if !on && !every {
return nil, nil
@@ -128,9 +106,6 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st
return nil, err
}
if !known {
if settingsKeptOn(ctx, node) {
return nil, fmt.Errorf("%w: what %s was last sent is not known (novox/hq ADR 0344)", errSettingsCannotKeep, node)
}
if every {
return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+
"there alone with every build kept (novox/hq issue 490)", node)
@@ -191,10 +166,6 @@ func keepRecorded(ctx context.Context, open *stores, node string, shelf map[stri
if err != nil {
return nil, err
}
if !found && settingsKeptOn(ctx, node) {
return nil, fmt.Errorf("%w: %s runs %s's build %s, which the build records no longer hold (novox/hq ADR 0344)",
errSettingsCannotKeep, node, m, short(kept[m]))
}
if !found && everyBuildKept(ctx, node) {
return nil, fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+
"list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m]))
+42 -546
View File
@@ -39,8 +39,8 @@ import (
// from the same manifest is no move.
//
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest
// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own:
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
// sendScope is what a send may carry that no gate has seen.
@@ -81,15 +81,6 @@ type moveFacts struct {
srcs map[string]map[string]string
passed map[string]map[string]bool
plans []inventory.Plan
// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
self string
}
type walkingKey struct{}
// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
func withWalking(ctx context.Context, id string) context.Context {
return context.WithValue(ctx, walkingKey{}, id)
}
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
@@ -108,7 +99,6 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
return f, err
}
f.plans, err = inv.OpenPlans(ctx)
f.self, _ = ctx.Value(walkingKey{}).(string)
return f, err
}
@@ -203,24 +193,17 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
// feet. A release keeps no module records: its open gate's carried moves are its walk.
func (f moveFacts) walkedBy(module, node, to string) string {
for _, p := range f.plans {
if !p.Open() || (p.Release != nil && p.ID == f.self) {
if !p.Open() {
continue
}
if p.Release != nil {
// The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353).
gates := []*inventory.PlanGate{p.Release.Gate}
for _, g := range p.Release.Gates {
gates = append(gates, g)
g := p.Release.Gate
if g == nil || g.Verdict != "" {
continue
}
for _, g := range gates {
// A send kept and not yet made walks nothing yet (ADR 0353 §8).
if g == nil || g.Verdict != "" || g.Silent || g.Pending {
continue
}
for _, c := range g.Carried {
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
return p.ID
}
for _, c := range g.Carried {
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
return p.ID
}
}
continue
@@ -320,8 +303,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
}
}
if len(refused) > 0 {
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+
"0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
}
return kept, nil
}
@@ -337,35 +320,19 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
// answered for the gate to keep, nil when there was none.
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
moves, err := gatedMoves(ctx, open, node, owns)
if err != nil || len(moves) == 0 {
return nil, nil, nil, err
}
sayRecreations(ctx, open, moves)
sent, grants, err := sendJudged(ctx, open, node)
if err != nil {
return nil, nil, nil, err
}
return moves, sent, grants, nil
}
// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing:
// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way
// while a plan that has started walks one of those builds elsewhere.
func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
inv := open.inventory
f, err := readMoveFacts(ctx, inv)
if err != nil {
return nil, err
return nil, nil, nil, err
}
moves, err := machineMoves(ctx, open, f, node, true)
if err != nil {
return nil, err
return nil, nil, nil, err
}
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
return nil, err
return nil, nil, nil, err
}
own := func(module string) bool {
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
@@ -375,7 +342,7 @@ func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory
continue
}
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
mv.Module, short(mv.To), node, id)
}
}
@@ -383,7 +350,7 @@ func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
for _, o := range owns {
if id := f.walkedBy(o.Module, node, o.To); id != "" {
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
o.Module, short(o.To), node, id)
}
}
@@ -396,26 +363,30 @@ func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory
moves[i].Build = o.Build
}
}
if len(moves) == 0 {
return nil, nil
if len(moves) == 0 && len(owns) == 0 {
return nil, nil, nil, nil
}
for i := range moves {
if moves[i].Build == "" {
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
return nil, err
return nil, nil, nil, err
}
}
}
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
return moves, nil
sayRecreations(ctx, open, moves)
sent, grants, err := sendJudged(ctx, open, node)
if err != nil {
return nil, nil, nil, err
}
return moves, sent, grants, nil
}
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
seen := map[string]bool{}
for _, c := range g.Carried {
// A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3).
if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) {
if c.Module == except || c.Build == "" || seen[c.Build] {
continue
}
seen[c.Build] = true
@@ -645,10 +616,6 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
}
for _, p := range plans {
if p.Release != nil {
if by != "" && p.Release.Settings != nil {
return nil, fmt.Errorf("%s is sending %s; what waits is released once it ends — `plans %s` says where it is",
p.ID, p.Repository, p.ID)
}
if by != "" {
return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID)
}
@@ -674,9 +641,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
return nil, err
}
for _, p := range recent {
// A settings walk that failed holds nothing back (novox/hq ADR 0344 §6): its failure is its module's
// settings on one machine, said on its own, not a build no gate passed.
if p.Release == nil || p.Release.Settings != nil {
if p.Release == nil {
continue
}
if p.State == inventory.PlanFailed {
@@ -724,7 +689,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
Release: &inventory.PlanRelease{Order: order, By: by},
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names),
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
strings.Join(order, ", "))}
if err := inv.SavePlan(ctx, &p); err != nil {
return nil, err
@@ -733,23 +698,11 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
return &p, nil
}
// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a
// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR
// 0353 §1), and judged; the plan done when every machine is.
//
// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with
// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it
// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged.
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
// gate, or the machine being judged judged once more; the plan done when every machine is.
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
r := p.Release
if r.Settings != nil {
return advanceSettingsWalk(ctx, open, p)
}
ctx = withWalking(ctx, p.ID)
now := time.Now().UTC()
if r.Rest {
return advanceRest(ctx, open, p, now)
}
if r.Gate == nil {
heard, err := releaseHeard(ctx, open)
if err != nil {
@@ -762,7 +715,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
r.Next++
continue
}
moves, err := gatedMoves(ctx, open, node, nil)
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
note := fmt.Sprintf("waiting before %s: %v", node, err)
changed := p.Note != note
@@ -770,15 +723,23 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
return changed, nil
}
if err != nil {
return false, fmt.Errorf("reading what waits on %s: %w", node, err)
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
}
if len(moves) == 0 {
r.Done = append(r.Done, node)
r.Next++
continue
}
r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
Grants: grants}
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
if said := grantsSaid(grants); said != "" {
p.Note += "; " + said
}
if said := recreationsSaid(moves); said != "" {
p.Note += "; " + said
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
@@ -789,19 +750,6 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
return true, nil
}
g := r.Gate
if g.Pending {
node := firstOf(g.Machines)
out, err := sendKept(ctx, open, p, node, g, now)
if err != nil || out == keptWaiting {
return out == keptWaiting, err
}
if out == keptNothing {
r.Gate = nil
r.Done = append(r.Done, node)
r.Next++
}
return true, nil
}
verdict, err := judgeMoves(ctx, open, g, nil, now)
if err != nil {
return false, err
@@ -820,11 +768,6 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
r.Done = append(r.Done, firstOf(g.Machines))
r.Next++
r.Gate = nil
// **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own
// gate, in place of one machine at a time (ADR 0236 §22).
r.Rest = true
p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
case inventory.GateSuperseded:
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
@@ -845,453 +788,6 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
return true, nil
}
// What sendKept did with a kept send.
const (
keptSent = iota // sent now
keptAlready // the machine was already sent it: judged from that send, nothing sent
keptNothing // nothing waits there any more: no gate
keptWaiting // another walk judges a build it would carry: it waits
)
// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send
// already carries what the gate is for, made before a controller was replaced and before it was kept: then
// nothing is sent, and the gate judges from that send (§6).
func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate,
now time.Time) (int, error) {
inv := open.inventory
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
p.Note = fmt.Sprintf("waiting before %s: %v", node, err)
return keptWaiting, nil
}
if err != nil {
return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err)
}
if len(moves) > 0 {
g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false
g.Sent = sentNow(ctx, inv, sent)
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
if said := grantsSaid(grants); said != "" {
p.Note += "; " + said
}
if said := recreationsSaid(moves); said != "" {
p.Note += "; " + said
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
return keptSent, nil
}
if !alreadySent(ctx, inv, node, g.Carried) {
return keptNothing, nil
}
since := now
if s, found, err := inv.SentTo(ctx, node); err == nil && found {
if !s.At.IsZero() {
since = s.At
}
g.Sent = map[string]inventory.SentDeclaration{node: s}
}
g.Since, g.Pending = &since, false
p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again",
node, len(g.Carried), since.Local().Format("15:04:05"))
fmt.Printf("%s: %s\n", p.ID, p.Note)
return keptAlready, nil
}
// alreadySent says a machine's last send carries every build of the moves: a send made and not kept.
func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool {
if len(moves) == 0 {
return false
}
sent, known, err := inv.SentBuilds(ctx, node)
if err != nil || !known {
return false
}
for _, mv := range moves {
if !sameCommit(sent[mv.Module], mv.To) {
return false
}
}
return true
}
// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every
// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their
// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent.
func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) {
r := p.Release
// The sends kept for this step, in the plan's order: the control node, last in it, last (§1).
var pending []string
for _, n := range r.Order {
if g := r.Gates[n]; g != nil && g.Pending {
pending = append(pending, n)
}
}
if len(pending) > 0 {
// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
return false, err
} else if grew != "" {
for _, n := range pending {
delete(r.Gates, n)
}
opened, waiting, err := openRestStep(ctx, open, p)
if err != nil {
return true, err
}
p.Note = "chosen again: " + grew
if !opened && waiting != "" {
p.Note += "; " + waiting
}
return true, nil
}
var sent, already, waiting []string
for _, node := range pending {
out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
if err != nil {
return true, err
}
switch out {
case keptSent:
sent = append(sent, node)
case keptAlready:
already = append(already, node)
case keptNothing:
delete(r.Gates, node)
r.Done = append(r.Done, node)
case keptWaiting:
waiting = append(waiting, node)
}
}
var said []string
if len(sent) > 0 {
said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there")
}
if len(already) > 0 {
said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send")
}
if len(waiting) > 0 {
said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries")
}
if len(said) > 0 {
p.Note = strings.Join(said, "; ")
}
return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil
}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return false, err
}
byNode := map[string]inventory.Reported{}
for _, rep := range reports {
byNode[rep.Node] = rep
}
changed, noteWas := false, p.Note
var notes, judging []string
batched, back := batchingRollbacks(ctx)
for _, node := range r.Order {
g := r.Gates[node]
if g == nil || g.Verdict != "" || g.Silent {
continue
}
// What another machine's failure already put back is not this gate's to judge (ADR 0353 §3).
if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) {
decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now)
r.Failed = append(r.Failed, node)
changed = true
continue
}
// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
// until it reports, and past ApplySilentAfter it is listed and left.
if !reportedOnSend(byNode[node], node, g) {
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
g.Silent = true
r.Silent = append(r.Silent, node)
changed = true
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
inventory.ApplySilentAfter))
continue
}
judging = append(judging, node+": not yet reported on its send")
continue
}
verdict, err := judgeMoves(ctx, open, g, nil, now)
if err != nil {
return changed, err
}
switch verdict {
case "":
if putBackBroken(ctx, open, p, g, "", nil) {
changed = true
notes = append(notes, p.Note)
}
judging = append(judging, node+": "+gateLine(g))
case inventory.GatePassed:
passCarried(ctx, open, p, g, "")
r.Done = append(r.Done, node)
changed = true
case inventory.GateSuperseded:
// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
// it; what still waits is released again.
r.Done = append(r.Done, node)
changed = true
notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
default:
// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
// machine of the step goes on to its own verdict.
stateWas := p.State
p.Note = ""
failCarried(batched, open, p, g, "")
p.State = stateWas
r.Failed = append(r.Failed, node)
changed = true
notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
returnedElsewhere(ctx, open, r, node, g)
}
}
p.Note = ""
sendRollbacks(ctx, open, p, back)
if p.Note != "" {
notes = append(notes, strings.TrimPrefix(p.Note, "; "))
}
if len(judging) > 0 {
note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
p.Note = note
return changed || noteWas != note, nil
}
// Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3).
if len(r.Failed) == 0 {
opened, waiting, err := openRestStep(ctx, open, p)
if err != nil {
return changed, err
}
if opened {
return true, nil
}
if waiting != "" {
note := strings.Join(append(notes, waiting), "; ")
p.Note = note
return changed || noteWas != note, nil
}
}
p.Tier = len(p.Tiers)
end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
if len(r.Silent) > 0 {
end += "; silent on their send, left: " + strings.Join(r.Silent, ", ")
}
if len(r.Skipped) > 0 {
end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
}
p.State = inventory.PlanDone
if len(r.Failed) > 0 {
p.State = inventory.PlanFailed
end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
p.State = inventory.PlanFailed
end = "no machine of the rest reported on its send; " + end
}
p.Note = strings.Join(append(notes, end), "; ")
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
// empty when none would.
func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return "", err
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
}
for _, node := range pending {
moves, err := gatedMoves(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
continue // sendKept says it waits
}
if err != nil {
return "", err
}
kept := r.Gates[node].Carried
for _, mv := range moves {
// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
return c.Module == mv.Module && sameCommit(c.To, mv.To)
}) {
continue
}
return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
}
}
return "", nil
}
// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
if r.At == nil {
return false
}
if s, kept := g.Sent[node]; kept {
return s.ReportsOn(r)
}
return r.Current
}
// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
for _, c := range g.Carried {
if c.Build == "" {
continue
}
if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
continue
}
for node, other := range r.Gates {
if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
continue
}
for _, oc := range other.Carried {
if oc.Module == c.Module && sameCommit(oc.To, c.To) {
other.Returned = append(other.Returned, c.Module)
}
}
}
}
}
// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) {
r := p.Release
heard, err := releaseHeard(ctx, open)
if err != nil {
return false, "", err
}
f, err := readMoveFacts(ctx, open.inventory)
if err != nil {
return false, "", err
}
waiting := map[string][]inventory.CarriedMove{}
var nodes, walked []string
for _, node := range r.Order[min(r.Next, len(r.Order)):] {
if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) ||
slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) {
continue
}
if !heard[node] {
r.Skipped = append(r.Skipped, node)
continue
}
moves, err := gatedMoves(ctx, open, node, nil)
if errors.Is(err, errWalkedElsewhere) {
walked = append(walked, fmt.Sprintf("%s (%v)", node, err))
continue
}
if err != nil {
return false, "", fmt.Errorf("reading what waits on %s: %w", node, err)
}
if len(moves) == 0 {
r.Done = append(r.Done, node)
continue
}
waiting[node] = moves
nodes = append(nodes, node)
}
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return false, "", err
}
step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
if len(step) == 0 {
if len(walked) > 0 {
return false, "waiting before " + strings.Join(walked, "; "), nil
}
return false, "", nil
}
if r.Gates == nil {
r.Gates = map[string]*inventory.PlanGate{}
}
for _, node := range step {
r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
}
p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
if left := len(nodes) - len(step); left > 0 {
p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
}
return true, "", nil
}
// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
func oneAtATime(module string, m catalogue.Manifest) bool {
if module == hostModule {
return true
}
return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
}
// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
// first machine. Pure.
//
// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
// shows only as its silence and no rollback reaches it.
func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
alone func(module string) bool) []string {
first := map[string]string{}
var out []string
for _, n := range nodes {
var unpassed []string
for _, mv := range waiting[n] {
if !passed(mv.Module, mv.To) || alone(mv.Module) {
unpassed = append(unpassed, mv.Module+"@"+mv.To)
}
}
if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
continue
}
for _, k := range unpassed {
first[k] = n
}
out = append(out, n)
}
return out
}
// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
recent, err := inv.RecentPlans(ctx, 20)
if err != nil {
return nil, err
}
for _, p := range recent {
if p.Release == nil || p.Release.Settings != nil || p.Open() {
continue
}
if len(p.Release.Silent) == 0 {
return nil, nil
}
machines := append([]string(nil), p.Release.Silent...)
sort.Strings(machines)
o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
w := releaseSilentWords(machines)
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
return []conditions.Observation{o}, nil
}
return nil, nil
}
// backlogObservation is what the gate's probe says of a release held after a failure.
func backlogObservation() []conditions.Observation {
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
+12 -275
View File
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/builder"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -251,20 +250,6 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
return out
}
// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine
// running it runs (novox/hq issue 280): nothing is sent for it.
// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader.
const (
noMovePrefix = "no move"
noMoveWhy = noMovePrefix + ": made from the source every machine running it runs"
)
// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says,
// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648).
func noMove(s *inventory.PlanModule) bool {
return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix)
}
// applied says whether every machine running the module has reported since the module was built.
// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
// after that machine was sent the build (novox/hq issue 256).
@@ -328,86 +313,6 @@ func init() {
}
}
// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2).
type reuseKey struct{}
// withReuse is ctx carrying a walk's leave to take a check's build.
func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context {
return context.WithValue(ctx, reuseKey{}, r)
}
// reuseIn is the leave ctx carries; nil for every ask but a walk's.
func reuseIn(ctx context.Context) *link.ReuseAsk {
r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk)
return r
}
// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it
// takes nothing either (novox/hq ADR 0350 §2, §6).
func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk {
if dryRun {
return nil
}
return reuseIn(ctx)
}
// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers.
func reuseAskOf(p *inventory.Plan) *link.ReuseAsk {
r := &link.ReuseAsk{}
if p.Delivery == nil {
return r
}
seen := map[link.PullRef]bool{}
for _, m := range p.Delivery.Merges {
pull, ok := pullOf(m)
if !ok {
continue
}
if !seen[pull] {
seen[pull] = true
r.Pulls = append(r.Pulls, pull)
}
}
return r
}
// checkedBuildsOf are the builds the build seat's checks of those pull requests said they made for their merges,
// within the week check builds are kept (novox/hq ADR 0350 §2): read from the seat's own outcomes on the bus. A
// variable so a test needs no bus.
var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
if len(pulls) == 0 {
return nil, nil
}
js, err := aBus()
if err != nil {
return nil, err
}
defer js.Close()
return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept))
}
// reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder
// may take (ADR 0350 §2). When they cannot be read the leave names none, and the walk builds, saying why.
func reuseWithVerdicts(ctx context.Context, r *link.ReuseAsk) *link.ReuseAsk {
builds, err := checkedBuildsOf(ctx, r.Pulls)
if err != nil {
fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err)
return r
}
r.Builds = builds
return r
}
// pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled
// (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none.
func pullOf(m inventory.PlanMerge) (link.PullRef, bool) {
parts := strings.Split(strings.Trim(strings.TrimSuffix(m.Repository, ".git"), "/"), "/")
if len(parts) < 2 || m.Number <= 0 || parts[len(parts)-2] == "" || parts[len(parts)-1] == "" {
return link.PullRef{}, false
}
return link.PullRef{Owner: parts[len(parts)-2], Repo: parts[len(parts)-1], Number: m.Number}, true
}
// askModule asks the build machine for one module of a plan and marks it asked, with the id it was
// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked.
func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) {
@@ -435,9 +340,7 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
ref = carried
}
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
// requests it answers, whose checks name the part of a differing key.
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, reuseAskOf(p))), source, e.Source.Path, ref)
id, err := askABuild(ctx, source, e.Source.Path, ref)
if err != nil {
state.State = "failed"
state.Why = err.Error()
@@ -456,7 +359,7 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
state.State = "asked"
state.AskedAt = &now
state.Build = id
state.Why, state.Commit, state.BuiltAt, state.Made = "", "", nil, ""
state.Why, state.Commit, state.BuiltAt = "", "", nil
}
// planBuilt marks a module built (or failed) in every open plan whose current tier holds it, and
@@ -468,12 +371,6 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
// would send machines, and the next tier, what the later merge replaced. asked is zero when the
// build's request time is not known, and such an outcome is taken as before.
func planBuilt(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id string) {
planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
}
// planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
// ADR 0350 §4) — on the module of the walk it answers.
func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
inv := open.inventory
// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
// than write over what the holder is about to save. Not taken, it is still in the build records,
@@ -539,7 +436,6 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
state.State = "built"
state.BuiltAt = &now
state.Commit = commit
state.Made = made
}
if err := inv.SavePlan(ctx, p); err != nil {
fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err)
@@ -609,7 +505,7 @@ func advancePlans(ctx context.Context, open *stores) {
return
}
defer release()
// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
// What waits for a gate, released one machine at a time (ADR 0236).
if _, err := releaseBacklog(ctx, open, ""); err != nil {
fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
}
@@ -819,7 +715,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
} else if same {
now := time.Now().UTC()
state.SentAt = &now
state.Why = noMoveWhy
state.Why = "no move: made from the source every machine running it runs"
fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
continue
}
@@ -1009,25 +905,6 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
if state.BuiltAt != nil {
since = *state.BuiltAt
}
// **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648).
// Nothing was sent for it, so no report follows its marking or its build: a machine that does not
// happen to report again held the walk for ever. Each machine runs it once it answered its last
// send — the one whose builds made it no move — whenever that was.
if noMove(state) {
byNode := map[string]inventory.Reported{}
for _, r := range reports {
byNode[r.Node] = r
}
if ok, on := appliedEach(m, func(node string) time.Time {
if s := byNode[node].Sent; s != nil {
return *s
}
return time.Time{}
}, running, reports); !ok {
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
}
continue
}
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
// a module is sent twice — the first machine, then the rest — and SentAt is the second.
// Asked of every machine, the first machine's report, made between the two sends, read as
@@ -1089,36 +966,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
runningOf map[string][]string) error {
inv := open.inventory
before, beforeKnown, _ := inv.SentBuilds(ctx, node)
// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
// the readings of a gate already judging it there.
// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
appliedLastSend(ctx, inv, node) {
if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
adoptSend(ctx, open, p, node, modules, before, runningOf)
return nil
}
}
// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
// between the send and keeping it reads it here, and a failure still puts the machine back.
kept := false
for _, m := range modules {
s := p.Modules[m]
if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
s.Previous, kept = before[m], true
}
}
if kept {
if err := inv.SavePlan(ctx, p); err != nil {
return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
}
}
owns := make([]inventory.CarriedMove, 0, len(modules))
for _, m := range modules {
s := p.Modules[m]
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
}
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
// all of it there.
@@ -1133,7 +984,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
sentWhat := sentNow(ctx, inv, sent)
for _, m := range modules {
s := p.Modules[m]
// What the first machine ran before (ADR 0236), kept above before the send.
// What the first machine ran before: what a failed gate puts back (ADR 0236).
if beforeKnown {
s.Previous = before[m]
}
s.First, s.FirstAt = sent, &now
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
@@ -1162,113 +1016,6 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
return nil
}
// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
// there — whose carried move also says what the machine ran before, for a rollback.
func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
before map[string]string, runningOf map[string][]string) {
inv := open.inventory
since := time.Now().UTC()
last, found, err := inv.SentTo(ctx, node)
if err == nil && found && !last.At.IsZero() {
since = last.At
}
lead := modules[0]
other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
for _, m := range modules {
s := p.Modules[m]
// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
switch {
case s.Previous != "":
case m == lead && from != "":
s.Previous = from
default:
s.Previous = before[m]
}
at := since
s.First, s.FirstAt = []string{node}, &at
s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
From: s.Previous, To: s.Commit, Since: &at}
if found {
s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
}
s.GatedBy = ""
if m == lead {
s.Gate.TakeReadings(other, modules, id)
} else {
s.GatedBy = lead
}
}
p.State = inventory.PlanRolling
p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
p.Tier, node, strings.Join(modules, ", "))
if id != "" {
p.Note += ", going on from the readings of " + id
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
}
// appliedLastSend says a machine reported it applied the declaration it was sent last.
func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
reports, err := inv.LastReports(ctx)
if err != nil {
return false
}
for _, r := range reports {
if r.Node == node {
return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
}
}
return false
}
// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
// walk's id, and what the machine ran before; nil when none does.
func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
plans, err := inv.OpenPlans(ctx)
if err != nil {
return nil, "", ""
}
// The same build, by its id where both name one, else by its commit.
same := func(b, c string) bool {
if b != "" && build != "" {
return b == build
}
return sameCommit(c, commit)
}
usable := func(g *inventory.PlanGate) bool {
return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
slices.Contains(g.Machines, node)
}
for _, q := range plans {
if q.ID == self {
continue
}
if r := q.Release; r != nil {
gates := []*inventory.PlanGate{r.Gate}
for _, g := range r.Gates {
gates = append(gates, g)
}
for _, g := range gates {
if !usable(g) {
continue
}
for _, c := range g.Carried {
if c.Module == module && c.Node == node && same(c.Build, c.To) {
return g, q.ID, c.From
}
}
}
continue
}
if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
return s.Gate, q.ID, s.Previous
}
}
return nil, "", ""
}
// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
// on the first of them passed, and its pass was kept for every build it carried (passCarried).
func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
@@ -1757,19 +1504,9 @@ func plansCommand(ctx context.Context, args []string) error {
// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
if len(r.Failed)+len(r.Silent) > 0 {
fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
}
gates := []*inventory.PlanGate{r.Gate}
for _, n := range r.Order {
gates = append(gates, r.Gates[n])
}
for _, g := range gates {
if g == nil {
continue
}
fmt.Printf(" %s\n", gateLine(g))
for _, c := range g.Carried {
if r.Gate != nil {
fmt.Printf(" %s\n", gateLine(r.Gate))
for _, c := range r.Gate.Carried {
fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
if c.Recreates != "" {
fmt.Printf(" %-22s %s\n", "", c.Recreates)
+3 -7
View File
@@ -133,15 +133,11 @@ func aBacklog(t *testing.T) *backlogMesh {
}
return f, nil
}
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateWithin, gateEvery = time.Hour, time.Hour
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
})
return b
}
+3 -7
View File
@@ -112,14 +112,10 @@ func TestReplay318(t *testing.T) {
}
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
})
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
@@ -188,7 +184,7 @@ func TestReplay318(t *testing.T) {
return f, nil
}
// The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once.
gateWithin, gateEvery, gateBound = time.Hour, 0, 0
gateSettle, gateEvery, gateBound = 0, 0, 0
for i := 0; i < 12; i++ {
advancePlans(ctx, open)
+3 -7
View File
@@ -248,13 +248,9 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
return names, nil
}
t.Cleanup(func() { sendRollout = wasSend })
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateWithin, gateEvery = time.Hour, 0
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
gateSettle, gateEvery = 0, 0
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
built := time.Now().UTC()
plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
-372
View File
@@ -1,372 +0,0 @@
package main
import (
"context"
"slices"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by
// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
//
// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
// eleven minutes; the new controller its last send started sent the control node again the controller build it
// had just applied, and judged it from its first reading.
// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
func aBacklogOnThree(t *testing.T) *backlogMesh {
t.Helper()
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
if _, err := inv.AddNode(ctx, "desk"); err != nil {
t.Fatal(err)
}
if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
wasHeard := releaseHeard
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
}
t.Cleanup(func() { releaseHeard = wasHeard })
return b
}
// events records every send and every judging, in order.
func events(t *testing.T) *[]string {
t.Helper()
var said []string
wasSend, wasGather := sendRollout, gatherGateFacts
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
said = append(said, "send "+strings.Join(names, ","))
return wasSend(ctx, open, names)
}
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
said = append(said, "judge")
return wasGather(ctx, open, component)
}
t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
return &said
}
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
// was judged to its verdict before the third was sent.
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklogOnThree(t)
ctx := t.Context()
said := events(t)
gateEvery = 0
for i := 0; i < 12; i++ {
advancePlans(ctx, b.open)
}
p := b.release(t)
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
t.Fatalf("sends and judgings: %v", *said)
}
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
}
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
}
}
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklogOnThree(t)
ctx := t.Context()
inv := b.open.inventory
gateEvery = 0
b.broken["desk"] = true // desk fails what it is sent in the rest's step
advancePlans(ctx, b.open)
for i := 0; i < 12 && b.release(t).Open(); i++ {
advancePlans(ctx, b.open)
}
p := b.release(t)
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
t.Fatalf("laptop was never sent: %v", b.sent)
}
if !slices.Contains(p.Release.Done, "laptop") {
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
}
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
}
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
t.Fatal("app's build is not marked failed")
}
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
t.Fatalf("desk was not put back: %v", sent)
}
}
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
release := b.release(t)
judging := release.Release.Gate
if judging == nil || judging.Pending || judging.Passes == 0 {
t.Fatalf("the release is not judging anchor: %+v", judging)
}
built := time.Now().UTC()
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
Build: "build-app-c2"}}}
if err := inv.SavePlan(ctx, &plan); err != nil {
t.Fatal(err)
}
advancePlans(ctx, b.open)
if len(b.sent) != 1 {
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
}
p, err := inv.PlanByID(ctx, "plan-app")
if err != nil {
t.Fatal(err)
}
g := p.Modules["app"].Gate
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
}
if p.Modules["app"].Previous != "c1" {
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
}
}
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
judgeLeavesAsTheCore(t)
g := aGateMesh(t)
ctx := t.Context()
// The old controller sends anchor app c2 and is replaced before it keeps the send.
made := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
if _, err := made(ctx, open, names); err != nil {
return nil, err
}
return nil, context.Canceled
}
advancePlans(ctx, g.open)
sendRollout = made
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
}
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
advancePlans(ctx, g.open)
if len(g.sent) != 1 {
t.Fatalf("the send was made again: %v", g.sent)
}
s := g.plan(t).Modules["app"]
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
}
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
if s.Previous != "c1" || s.Gate.From != "c1" {
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
}
}
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
// made again (ADR 0353 §8).
func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
p, err := releaseBacklog(ctx, b.open, "")
if err != nil || p == nil {
t.Fatalf("no release opened: %v", err)
}
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
}
if err := inv.SavePlan(ctx, p); err != nil {
t.Fatal(err)
}
// The send is made, and the controller is replaced before it keeps it.
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
t.Fatal(err)
}
advancePlans(ctx, b.open)
if len(b.sent) != 1 {
t.Fatalf("anchor was sent again: %v", b.sent)
}
g := b.release(t).Release.Gate
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
t.Fatalf("anchor is not judged on the send made: %+v", g)
}
}
// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
// for the next step, and one whose every build passed goes now (ADR 0353 §2).
func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
waiting := map[string][]inventory.CarriedMove{
"desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
"shanks": {{Module: "app", To: "c2"}},
"novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
}
passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
never := func(string) bool { return false }
step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
}
if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
t.Fatalf("a machine whose every build passed waits: %v", step)
}
}
// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
// carries either to one machine only, while what else passed goes everywhere at once.
func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
waiting := map[string][]inventory.CarriedMove{
"desk": {{Module: hostModule, To: "h2"}},
"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
"shanks": {{Module: "app", To: "c2"}},
"novox": {{Module: "dns", To: "d2"}},
"ace": {{Module: "dns", To: "d2"}},
}
resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
alone := func(m string) bool {
if m == "dns" {
return oneAtATime(m, resolver)
}
return oneAtATime(m, catalogue.Manifest{Module: m})
}
step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
func(string, string) bool { return true }, alone)
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
}
if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
t.Fatal("a leaf module is held to one machine at a time")
}
}
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
b := aBacklogOnThree(t)
ctx := t.Context()
gateEvery = 0
b.broken["anchor"] = true
advancePlans(ctx, b.open)
for i := 0; i < 6 && b.release(t).Open(); i++ {
advancePlans(ctx, b.open)
}
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
for _, s := range b.sent {
if !slices.Equal(s, []string{"anchor"}) {
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
}
}
}
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
// ordered after every other.
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklogOnThree(t)
ctx := t.Context()
register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
t.Fatal(err)
}
said := events(t)
gateEvery = 0
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
advancePlans(ctx, b.open)
}
if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
t.Fatalf("the order is %v", p.Release.Order)
}
laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
}
}
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
judgeLeavesAsTheCore(t)
b := aBacklog(t)
ctx := t.Context()
inv := b.open.inventory
reporting := sendRollout
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
b.sent = append(b.sent, names)
current, _ := open.inventory.CurrentBuilds(ctx)
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
}
return reporting(ctx, open, names)
}
t.Cleanup(func() { sendRollout = reporting })
gateEvery = 0
for i := 0; i < 8; i++ {
advancePlans(ctx, b.open)
}
p := b.release(t)
g := p.Release.Gates["laptop"]
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
}
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
g.Since = &since
if err := inv.SavePlan(ctx, &p); err != nil {
t.Fatal(err)
}
advancePlans(ctx, b.open)
p = b.release(t)
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
}
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
t.Fatal("a silent machine's build was marked failed")
}
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
t.Fatalf("laptop was put back: %v", sent)
}
obs, err := silentObservation(ctx, inv)
if err != nil {
t.Fatal(err)
}
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
t.Fatalf("the silence is not said: %+v", obs)
}
}
+43 -33
View File
@@ -452,33 +452,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
case "queue":
// As data, the table its output (novox/hq issue 455).
return []string{"queue", "--json"}, nil
case "forget-checks":
if err := need("owner", "repo", "number"); err != nil {
return nil, err
}
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
case "try":
// A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered.
if id := str("id"); id != "" {
return []string{"try", "--id", id}, nil
}
if a.given["repository"] == "" && a.given["ref"] == "" {
// Neither shape named: the command says its usage, which names both, as `build` does.
return []string{"try"}, nil
}
if err := need("repository", "ref"); err != nil {
return nil, err
}
argv := []string{"try", str("repository"), str("ref")}
for _, flag := range []string{"base", "tests"} {
if v := str(flag); v != "" {
argv = append(argv, "--"+flag, v)
}
}
if on("store-only") {
argv = append(argv, "--store-only")
}
return argv, nil
case "cancel", "kill":
if err := need("id"); err != nil {
return nil, err
@@ -908,6 +881,48 @@ func (a *verbArguments) commandLine() ([]string, error) {
argv = append(argv, "--adopted")
}
return argv, nil
case "config":
// Config values from four sources (novox/hq issue 630): mesh config and node config for a predefined
// key, and a module's own value on one node. A listing without key; a set or clear says why.
key := str("key")
if str("module") != "" && str("node") == "" && (key != "" || on("clear")) {
return nil, errors.New("config: there is no module layer for the whole mesh — an assignment is one module " +
"on one node (novox/hq issue 630); give node, or leave module out to set mesh config for a predefined key")
}
place := func(argv []string) []string {
if n := str("node"); n != "" {
argv = append(argv, "--node", n)
}
if m := str("module"); m != "" {
argv = append(argv, "--module", m)
}
return argv
}
switch {
case key == "":
if str("value") != "" || on("clear") || on("history") || str("why") != "" {
return nil, errors.New("config: a set, a clear or a history names its key")
}
return place([]string{"config", "list"}), nil
case on("history"):
if str("value") != "" || on("clear") || str("module") != "" || str("why") != "" {
return nil, errors.New("config: history reads a key's mesh or node config, and takes only node")
}
return place([]string{"config", "history", key}), nil
case on("clear") && str("value") != "":
return nil, errors.New("config: a value or clear, not both")
case on("clear"):
if err := need("why"); err != nil {
return nil, err
}
return append(place([]string{"config", "clear", key}), "--why", str("why")), nil
case str("value") != "":
if err := need("why"); err != nil {
return nil, err
}
return append(place([]string{"config", "set", key, str("value")}), "--why", str("why")), nil
}
return nil, errors.New("config: with key, give value, clear or history")
case "settings":
// Every module's preferences, their defaults and each machine's value (novox/hq ADR 0262):
// the one interface for them, so no module builds a settings tool of its own. Asked for by
@@ -1040,8 +1055,6 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
"delivery": true,
// The build queue, each ask with its position among the waiting (novox/hq issue 455).
"queue": true,
// A branch tried before its pull request (novox/hq issue 686).
"try": true,
// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
"plans": true}
@@ -1332,10 +1345,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
link.AnswersWithin(ctx, bound+planWaitStart)
}
}
if verb == "try" && len(argv) > 1 && argv[1] == "--id" {
// Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229).
link.AnswersWithin(ctx, tryWait+planWaitStart)
}
if answer, read := readHere(ctx, argv); read {
return answer, nil
}
@@ -1647,6 +1656,7 @@ var commandReadForms = map[string]func(rest []string) bool{
"node": func(r []string) bool { return subIn(r, "list", "show") },
"module": func(r []string) bool { return subIn(r, "list") },
"settings": func(r []string) bool { return subIn(r, "show", "preferences", "proposals") },
"config": func(r []string) bool { return flagsOnly(r) || subIn(r, "list", "history") },
"retire": func(r []string) bool { return subIn(r, "list") },
"cleanup": func(r []string) bool { return subIn(r, "list") },
"delivery": func(r []string) bool { return subIn(r, "plan", "walks") },
@@ -1,60 +0,0 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// novox/hq ADR 0346, issue 624: a provider on another machine serves its consumers its own composed
// names, as that machine composes them; one it does not compose is left out, said under the need.
func TestAProviderElsewhereServesItsOwnComposedName(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
forge, err := catalogue.ParseManifest([]byte(`{"module":"forge","version":"1","requires":["route"],
"provides":[{"name":"npm-package-registry","scope":"mesh"}],
"contributes":{"route":{"web":{"label":"git","port":3000}}},
"serves":{"npm-package-registry":{"scheme":"http","port":3000,"npm-path":"/api/packages/novox/npm/",
"public-root":"https://${bound:route:name-web}/api/packages/novox/npm/"}}}`))
if err != nil {
t.Fatal(err)
}
register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"),
Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}})
register(t, open, forge)
register(t, open, catalogue.Manifest{Module: "lab", Version: "1", Requires: []string{"npm-package-registry"},
Binds: map[string]string{"npm-package-registry": "/var/lib/lab/package-registry.json"}})
for _, a := range [][2]string{{"anchor", "proxy"}, {"anchor", "forge"}, {"laptop", "lab"}} {
if _, err := assign(ctx, open, a[0], a[1]); err != nil {
t.Fatalf("assign %s %s: %v", a[0], a[1], err)
}
}
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
t.Fatal(err)
}
plan, _, err := planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
n := need(t, plan, "lab", "npm-package-registry")
if n.Serves["public-root"] != "https://git.example.test/api/packages/novox/npm/" || len(n.LeftOut) != 0 {
t.Fatalf("the consumer elsewhere is served the forge's public root: %v, left out %v", n.Serves, n.LeftOut)
}
if err := open.inventory.SetPublicDomain(ctx, "anchor", ""); err != nil {
t.Fatal(err)
}
plan, _, err = planFor(ctx, open, "laptop")
if err != nil {
t.Fatal(err)
}
n = need(t, plan, "lab", "npm-package-registry")
if _, there := n.Serves["public-root"]; there {
t.Fatalf("anchor composes no public name, so the key is left out: %v", n.Serves)
}
if len(n.LeftOut) != 1 || !strings.Contains(n.LeftOut[0], `without "public-root"`) {
t.Fatalf("and the need says so: %v", n.LeftOut)
}
}
@@ -14,7 +14,7 @@ func TestSettingsSayWhereEachValueComesFrom(t *testing.T) {
{Key: "width", Value: float64(250), From: catalogue.DefaultLayer, FromDefault: true, Default: float64(250), HasDefault: true},
})
want := "dunst on laptop, every value and where it comes from:\n" +
" font-size = 13 (laptop; the default is 10)\n" +
" font-size = 13 (dunst on laptop; the default is 10)\n" +
" width = 250 (default)\n"
if got != want {
t.Fatalf("said\n%s\nwant\n%s", got, want)
@@ -66,7 +66,7 @@ func TestPreferencesSayEachMachinesValueAndItsSource(t *testing.T) {
want := "dunst (on desk, laptop)\n" +
" font-size, default 10: readable at 100 DPI\n" +
" desk: 10 (default)\n" +
" laptop: 16 (the node)\n" +
" laptop: 16 (dunst on laptop)\n" +
" width, default 250: forty characters\n" +
" desk: 300 (the mesh)\n" +
" laptop: 250 (default)\n"
@@ -96,7 +96,7 @@ func TestPreferencesListedFromTheStores(t *testing.T) {
t.Fatal(err)
}
all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) })
if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") ||
if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (notes on laptop)") ||
!strings.Contains(all, "font-size, default 10: readable at 100 DPI") {
t.Fatalf("the listing:\n%s", all)
}
-281
View File
@@ -1,281 +0,0 @@
package main
import (
"context"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// An approved settings change is sent by a walk at once, and the operator's one Approve is its push (novox/hq ADR
// 0344, issue 641).
//
// On 2026-10-13 the operator approved a betterbird proposal on Telegram and nothing happened: the warrant set the
// layer, and the push stayed a separate act (ADR 0277 §18) that nobody took. Now the warrant that sets the layer
// opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the
// builds. Its order is every machine running the module that the layer reaches, the control node last; each is
// sent and its module judged there, from the machine's report on that send, before the next is sent. The send is
// the layer alone: every module there stays at the build its machine was last sent (keepingBuildsForSettings), so a
// build held for a person (ADR 0242) or waiting for a gate stays where it is — a policy of `record` holds a build,
// never a setting — and a send that would move a running module's data stops the walk (ADR 0217 §3). The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's
// channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing
// else.
// settingsWalkPrefix names a settings walk's id.
const settingsWalkPrefix = "settings-"
// settingsWalkFor is what a settings walk says it is for, where a merge's walk says its repository.
func settingsWalkFor(s inventory.PlanSettings) string {
where := s.Node
if where == "" {
where = "the whole mesh"
}
return fmt.Sprintf("the settings of %s on %s", s.Module, where)
}
// settingsWalkOrder is the machines a layer reaches, in the order a settings walk sends them: every machine
// running the module, or the one the layer names when it runs it there, sorted, the control node last. Pure.
func settingsWalkOrder(node string, running, controllers []string) []string {
var order, last []string
for _, n := range running {
if node != "" && n != node {
continue
}
if slices.Contains(controllers, n) {
last = append(last, n)
} else {
order = append(order, n)
}
}
sort.Strings(order)
sort.Strings(last)
return append(order, last...)
}
// openSettingsWalk opens the walk that sends an approved layer, and answers the words of what it does: the
// machines it sends, or that no machine runs the module and nothing is sent. Called on the operator's warrant,
// after the layer is set.
var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanSettings) (string, error) {
inv := open.inventory
running, err := inv.Running(ctx, s.Module)
if err != nil {
return "", err
}
controllers, err := inv.Running(ctx, catalogue.ControllerSeatName)
if err != nil {
return "", err
}
order := settingsWalkOrder(s.Node, running, controllers)
if len(order) == 0 {
return fmt.Sprintf("no machine runs %s there, so there is nothing to send", s.Module), nil
}
release, err := inv.HoldPlans(ctx, true)
if err != nil {
return "", err
}
defer release()
now := time.Now().UTC()
p := inventory.Plan{ID: fmt.Sprintf("%s%d", settingsWalkPrefix, now.UnixNano()), Repository: settingsWalkFor(s),
Created: now, State: inventory.PlanRolling, Tiers: [][]string{{s.Module}}, Modules: map[string]*inventory.PlanModule{},
Release: &inventory.PlanRelease{Order: order, Settings: &s},
Note: fmt.Sprintf("approved (ask %s): sent to %s, one machine at a time, each judged", s.Ask,
strings.Join(order, ", "))}
if err := inv.SavePlan(ctx, &p); err != nil {
return "", err
}
fmt.Printf("%s: %s — %s\n", p.ID, p.Repository, p.Note)
return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil
}
// settingsDataMoves is what a settings walk's send to a machine would move of a running module's data: composed as
// that send composes it, every build kept, and compared with what the machine was last sent (ADR 0217 §3).
var settingsDataMoves = func(ctx context.Context, open *stores, node string) ([]dataMove, error) {
keeping := keepingBuildsForSettings(keepingRecorded(ctx), node)
gens, err := generators(keeping, open)
if err != nil {
return nil, err
}
plan, settings, err := planFor(keeping, open, node)
if err != nil {
return nil, err
}
declared, err := declarationWith(keeping, open, node, plan, settings, gens, Reading)
if err != nil {
return nil, err
}
body, err := declared.Body()
if err != nil {
return nil, err
}
return heldMoves(keeping, open.inventory, node, body, nil)
}
// judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A
// settings walk sends a machine only when no other walk judges there: its send would set aside the declaration
// that walk's gate reads the machine's report against.
var judgingOn = func(ctx context.Context, open *stores, self, node string) (string, error) {
plans, err := open.inventory.OpenPlans(ctx)
if err != nil {
return "", err
}
return judgingIn(plans, self, node), nil
}
// judgingIn is judgingOn over plans read. Pure.
func judgingIn(plans []inventory.Plan, self, node string) string {
judging := func(g *inventory.PlanGate) bool {
return g != nil && g.Verdict == "" && slices.Contains(g.Machines, node)
}
for _, p := range plans {
if p.ID == self || !p.Open() {
continue
}
if p.Release != nil {
if judging(p.Release.Gate) {
return p.ID
}
continue
}
for _, s := range p.Modules {
if s != nil && judging(s.Gate) {
return p.ID
}
}
}
return ""
}
// advanceSettingsWalk takes one step of a settings walk: the next machine sent and its module judged there, or the
// machine being judged judged once more; done when every machine is.
func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
r, s := p.Release, p.Release.Settings
now := time.Now().UTC()
wait := func(note string) (bool, error) {
changed := p.Note != note
p.Note = note
return changed, nil
}
if r.Gate == nil {
heard, err := releaseHeard(ctx, open)
if err != nil {
return false, err
}
for r.Next < len(r.Order) {
node := r.Order[r.Next]
if !heard[node] {
r.Skipped = append(r.Skipped, node)
r.Next++
continue
}
if id, err := judgingOn(ctx, open, p.ID, node); err != nil {
return false, err
} else if id != "" {
return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id))
}
// **A running module's data that would move is a person's word** (ADR 0217 §3): the walk stops before
// that machine, failed and said, and `push --move` sends it.
held, err := settingsDataMoves(ctx, open, node)
if errors.Is(err, errSettingsCannotKeep) {
return failBefore(p, node, err.Error())
}
if err != nil {
return false, fmt.Errorf("whether %s's data would move could not be read: %w", node, err)
}
if len(held) > 0 {
var said strings.Builder
sayDataHeld(&said, node, held)
return failBefore(p, node, "a running module's data would move there — "+
strings.Join(strings.Fields(said.String()), " "))
}
// A build that cannot be kept is refused by the composition above, which is the send's own, and fails the
// walk there: the send flattens its refusals into words, so it is not looked for again here.
sent, grants, err := sendJudged(keepingBuildsForSettings(ctx, node), open, node)
if err != nil {
return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err)
}
// **The module's build is the one the gate judges** (issue 641 review): kept by this send, it is what a
// crossing send — a merge's walk sending the machine a new build of the module — supersedes, so this gate
// never reads that send's report as its own.
kept := ""
if builds, known, err := open.inventory.SentBuilds(ctx, node); err == nil && known {
kept = builds[s.Module]
}
r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now,
From: kept, To: kept, Sent: sentNow(ctx, open.inventory, sent), Grants: grants}
p.Note = fmt.Sprintf("sent %s %s (ask %s), every build there kept; judging %s there", node,
settingsWalkFor(*s), s.Ask, s.Module)
if said := grantsSaid(grants); said != "" {
p.Note += "; " + said
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, strings.Join(r.Done, ", "))
if len(r.Done) == 0 {
// Sent nowhere is not delivered: said as the walk's failure, so the operator is not told it landed.
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s (ask %s) was sent to no machine", settingsWalkFor(*s), s.Ask)
}
if len(r.Skipped) > 0 {
p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ")
}
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
g := r.Gate
node := r.Order[r.Next]
verdict, err := judgeMoves(ctx, open, g, []judged{{module: s.Module, node: node}}, now)
if err != nil {
return false, err
}
switch verdict {
case "":
return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g)))
case inventory.GatePassed:
r.Done = append(r.Done, node)
r.Next++
r.Gate = nil
return true, nil
case inventory.GateSuperseded:
p.State = inventory.PlanSuperseded
p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; the layer is set, and the "+
"machines not yet sent take it at their next send", node, g.Why)
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+
"replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + unreached(r)
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
// failBefore ends a settings walk failed before it sent a machine, saying why, and that the layer stays.
func failBefore(p *inventory.Plan, node, why string) (bool, error) {
r, s := p.Release, p.Release.Settings
p.State = inventory.PlanFailed
p.Note = fmt.Sprintf("%s not sent to %s: %s; the layer stays as the operator approved it", settingsWalkFor(*s),
node, why) + unreached(r)
fmt.Printf("%s: %s\n", p.ID, p.Note)
return true, nil
}
// unreached is what a stopped settings walk left without the layer, in words: the machines after the one it stopped
// at, and those passed over as away (ADR 0344 §6). Empty when there are none.
func unreached(r *inventory.PlanRelease) string {
var out string
if r.Next+1 < len(r.Order) {
out += "; not sent: " + strings.Join(r.Order[r.Next+1:], ", ")
}
if len(r.Skipped) > 0 {
out += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
}
return out
}
-572
View File
@@ -1,572 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/lease"
)
// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take
// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a
// time, judging the module on each; the ask says so before the Approve, and nothing names a push.
// The order: every machine running the module, or the one the layer names, the control node last.
func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) {
running := []string{"shanks", "novox", "g14", "ace"}
for _, c := range []struct {
node string
want []string
}{
{"", []string{"ace", "g14", "shanks", "novox"}},
{"g14", []string{"g14"}},
{"novox", []string{"novox"}},
{"elsewhere", nil},
} {
if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) {
t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want)
}
}
}
// A settings walk sends a machine only when no other walk judges there.
func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) {
judging := &inventory.PlanGate{Machines: []string{"g14"}}
passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed}
plans := []inventory.Plan{
{ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}},
{ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}},
{ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
{ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
}
if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" {
t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id)
}
if id := judgingIn(plans[1:2], "x", "g14"); id != "" {
t.Errorf("a passed gate judges: %q", id)
}
if id := judgingIn(plans, "x", "shanks"); id != "" {
t.Errorf("on shanks %q judges", id)
}
}
// The ask says, before the Approve, that the layer is sent at once and judged, and that its end is told as every
// walk's is; neither the ask nor the proposer's answer names a push any more.
func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) {
for _, c := range []struct {
in proposeInput
where string
had bool
}{
{proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false},
{proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true},
{proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true},
} {
r := newProposerRig(t)
if c.had {
r.before, r.had = map[string]any{"x": "0"}, true
}
words, err := r.pr.propose(context.Background(), c.in)
if err != nil {
t.Fatal(err)
}
q := r.askSent(t)
if err := q.Check(r.now); err != nil {
t.Fatalf("the ask is refused: %v", err)
}
// At a glance (ADR 0345): the summary says it is sent at once by a walk, one machine at a time; the rest of
// what Approve does is the Details', which the router notes after the answers.
if !strings.Contains(q.Explanation, "sent at once by a walk to "+c.where+", one machine at a time, each judged") ||
!strings.Contains(q.Details, "and sent at once to "+c.where+", one machine at a time, each judged, "+
"every build there kept; its end is told as every walk's is") {
t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation)
}
approve := q.Options[0]
if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") {
t.Errorf("Approve does %q", approve.Does)
}
if !strings.Contains(words, "a walk sends it to "+c.where+" at once") {
t.Errorf("the proposer's answer:\n%s", words)
}
for _, said := range []string{q.Explanation, q.Whole, approve.Does, words} {
if strings.Contains(said, "push") {
t.Errorf("a push is still named, as if a second act were needed:\n%s", said)
}
}
}
}
// ---- on the real stores -----------------------------------------------------------------------------
// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a
// machine that does not run the module opens none and says so.
func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644",
"content": "x = ${setting:x}\n"}}})
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
t.Fatal(err)
}
// A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7).
if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil {
t.Fatal(err)
}
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 {
t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans))
}
set := setLayerIn(open)
now := time.Now()
values := map[string]any{"x": "1"}
p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now,
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
words, err := set(ctx, p, "s9", "approved by the operator (ask s9)")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") {
t.Errorf("the warrant's words: %q", words)
}
plans, err := open.inventory.OpenPlans(ctx)
if err != nil || len(plans) != 1 {
t.Fatalf("open walks: %+v %v", plans, err)
}
w := plans[0]
if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" ||
!reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" ||
!reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) {
t.Fatalf("the settings walk: %+v %+v", w, w.Release)
}
// A machine's layer where the module does not run: set, and nothing to send.
q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now,
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
words, err = set(ctx, q, "s10", "approved by the operator (ask s10)")
if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") {
t.Fatalf("a layer nobody runs: %q %v", words, err)
}
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 {
t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans))
}
// A walk that cannot be opened is the act's failure, said with the push that sends it by hand.
was := openSettingsWalk
openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) {
return "", fmt.Errorf("the plans are not to be had")
}
t.Cleanup(func() { openSettingsWalk = was })
again := map[string]any{"x": "2"}
byHand := map[string]any{"x": "0"}
r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true,
From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)}
if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+
"sends it could not be opened: the plans are not to be had — `push laptop` sends it") {
t.Errorf("an unopened walk: %v", err)
}
}
// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at
// once, recorded, and judged from the reports, with a machine named broken failing what it was sent.
type walkMesh struct {
open *stores
sent [][]string
broken map[string]bool
// kept says, per send, whether it kept every build of its machine; moves is the data a machine's send would move.
kept []bool
moves map[string][]dataMove
heard map[string]bool
}
func aSettledMesh(t *testing.T) *walkMesh {
t.Helper()
open := aMesh(t)
ctx := t.Context()
inv := open.inventory
w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{},
heard: map[string]bool{"anchor": true, "laptop": true}}
withConditionsInMemory(t)
was := doctorFrom
doctorFrom = nil
t.Cleanup(func() { doctorFrom = was })
old := time.Now().Add(-2 * time.Hour)
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1",
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil {
t.Fatal(err)
}
for _, n := range []string{"anchor", "laptop"} {
if _, err := inv.Assign(ctx, n, "app"); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil {
t.Fatal(err)
}
}
wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil }
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
modules, err := open.inventory.Assigned(ctx, node)
if err != nil {
return nil, err
}
sent, known, err := open.inventory.SentBuilds(ctx, node)
if err != nil {
return nil, err
}
return f.moves(node, modules, sent, known, all), nil
}
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil }
n := 0
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
w.sent = append(w.sent, append([]string(nil), names...))
w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0]))
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
return nil, err
}
for _, node := range names {
modules, _ := open.inventory.Assigned(ctx, node)
carried := map[string]string{}
for _, m := range modules {
carried[m] = current[m].Commit
}
n++
digest := fmt.Sprintf("d-%s-%d", node, n)
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
return nil, err
}
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
return nil, err
}
}
return names, nil
}
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
reports, err := open.inventory.LastReports(ctx)
if err != nil {
return f, err
}
for _, r := range reports {
if w.broken[r.Node] {
r.Outcome = inventory.OutcomeFailed
}
f.reports[r.Node] = r
}
return f, nil
}
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
gateWithin, gateEvery = time.Hour, time.Hour
t.Cleanup(func() {
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
settingsDataMoves = wasData
gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound
})
return w
}
func (w *walkMesh) walk(t *testing.T) inventory.Plan {
t.Helper()
plans, err := w.open.inventory.RecentPlans(t.Context(), 10)
if err != nil {
t.Fatal(err)
}
for _, p := range plans {
if p.Release != nil && p.Release.Settings != nil {
return p
}
}
t.Fatal("no settings walk")
return inventory.Plan{}
}
// The walk sends the first machine alone and judges the module there before it sends the next, and ends
// delivered on both; nothing waited for a gate, so each send carried the layer alone.
func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) {
judgeLeavesAsTheCore(t)
w := aSettledMesh(t)
ctx := t.Context()
words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"})
if err != nil || !strings.Contains(words, "sends it to anchor, laptop") {
t.Fatalf("%q %v", words, err)
}
advancePlans(ctx, w.open)
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent)
}
p := w.walk(t)
if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") {
t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release)
}
gateEvery = 0
for i := 0; i < 6; i++ {
advancePlans(ctx, w.open)
}
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) {
t.Fatalf("sent %v", w.sent)
}
p = w.walk(t)
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) ||
p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") {
t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release)
}
// It said its start and its end, so the operator is told once (ADR 0308).
if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil {
t.Fatalf("the walk's start and end are not marked: %+v", p.Times)
}
// Nothing waits, and no walk of waiting builds opened beside it.
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil {
t.Fatalf("%+v %v", r, err)
}
}
// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the
// failure holds back no walk of waiting builds.
func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) {
w := aSettledMesh(t)
ctx := t.Context()
inv := w.open.inventory
w.broken["anchor"] = true
if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil {
t.Fatal(err)
}
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
advancePlans(ctx, w.open)
}
p := w.walk(t)
if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
!strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") ||
!strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") {
t.Fatalf("the walk is %s: %s", p.State, p.Note)
}
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent)
}
if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" {
t.Fatalf("the layer after the failure: %v %v %v", layer, has, err)
}
// A build that waits for a gate is still walked: the failed settings walk holds nothing back.
w.broken["anchor"] = false
now := time.Now().Add(-time.Minute)
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
t.Fatal(err)
}
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err)
}
}
// Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or
// waiting for a gate stays where it is (ADR 0344 §3).
func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) {
w := aSettledMesh(t)
ctx := t.Context()
inv := w.open.inventory
gateEvery = 0
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
advancePlans(ctx, w.open)
}
if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone {
t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note)
}
// What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent.
now := time.Now().Add(-time.Minute)
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
t.Fatal(err)
}
kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor")
if err != nil || kept["app"] != "c1" {
t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err)
}
if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" {
t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept)
}
// And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see.
f, err := readMoveFacts(ctx, inv)
if err != nil {
t.Fatal(err)
}
waiting, _ := machineMoves(ctx, w.open, f, "anchor", false)
if len(waiting) != 1 || waiting[0].Module != "app" {
t.Fatalf("anchor waits for %+v, want app's c2", waiting)
}
if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 {
t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err)
}
}
// A machine whose send would move a running module's data stops the walk there, failed and said with the push that
// sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached
// no machine is not said delivered.
func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) {
w := aSettledMesh(t)
ctx := t.Context()
gateEvery = 0
w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}}
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
advancePlans(ctx, w.open)
}
p := w.walk(t)
if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) ||
!strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") ||
!strings.Contains(p.Note, "push laptop --move app") {
t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note)
}
v := aSettledMesh(t)
v.heard = map[string]bool{"laptop": true}
gateEvery = 0
if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil {
t.Fatal(err)
}
for i := 0; i < 4; i++ {
advancePlans(ctx, v.open)
}
p = v.walk(t)
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) ||
!strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") {
t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note)
}
u := aSettledMesh(t)
u.heard = map[string]bool{}
if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil {
t.Fatal(err)
}
advancePlans(ctx, u.open)
if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
!strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 {
t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note)
}
}
// The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new
// build of the module while the settings gate judges there — supersedes it, and its report is never read as the
// settings send's (issue 641 review, finding 1).
func TestACrossingSendSupersedesASettingsGate(t *testing.T) {
judgeLeavesAsTheCore(t)
w := aSettledMesh(t)
ctx := t.Context()
inv := w.open.inventory
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil {
t.Fatal(err)
}
advancePlans(ctx, w.open)
p := w.walk(t)
if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" {
t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate)
}
// Another walk sends anchor app's c2 while the settings gate judges there.
now := time.Now().Add(-time.Minute)
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil {
t.Fatal(err)
}
if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor",
Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil {
t.Fatal(err)
}
gateEvery = 0
advancePlans(ctx, w.open)
p = w.walk(t)
if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded ||
!strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 {
t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release)
}
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent)
}
}
// A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is
// gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a
// failure names the machines passed over as away (issue 641 review, findings 2 and 3).
func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) {
w := aSettledMesh(t)
ctx := t.Context()
inv := w.open.inventory
// What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on.
if _, err := inv.AddNode(ctx, "fresh"); err != nil {
t.Fatal(err)
}
if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) {
t.Fatalf("a settings send to a machine never sent: %v", err)
}
w.heard = map[string]bool{"laptop": true}
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) {
return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node)
}
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil {
t.Fatal(err)
}
advancePlans(ctx, w.open)
p := w.walk(t)
if p.State != inventory.PlanFailed || len(w.sent) != 0 ||
!strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") ||
!strings.Contains(p.Note, "not heard from, left as they were: anchor") {
t.Fatalf("the walk is %s: %s", p.State, p.Note)
}
// It is not open, so a build waiting for a gate is walked.
now := time.Now().Add(-time.Minute)
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
t.Fatal(err)
}
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err)
}
}
-6
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"fmt"
"os"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
@@ -125,11 +124,6 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
if aliases, err := inv.Aliases(ctx); err == nil {
catalogue.UseAliases(aliases)
}
// And the list of predefined keys the mesh publishes (novox/hq ADR 0348 §7), which every manifest read again
// is judged against. A store a migration has not reached yet keeps none, and nothing names a key then.
if err := loadPredefined(ctx, inv); err != nil {
fmt.Fprintf(os.Stderr, "the list of predefined keys kept cannot be read: %v\n", err)
}
return inv, nil
}
-204
View File
@@ -1,204 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
snapshot "github.com/novox/mesh-controller/internal/facts"
"github.com/novox/mesh-controller/internal/inventory"
)
// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1,
// issue 654): `module check` and the merge gate read it from the tree beside the manifest, so a change to a
// template alone is judged as a change to its module.
const templatedNotifier = `{"module":"notifier","version":"1",
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"files/notifier.conf"}]}`
// inlineNotifier is the same module as the mesh holds it: built, its text inline.
const inlineNotifier = `{"module":"notifier","version":"1",
"resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}`
// aTemplatedModule writes a module's directory: its manifest and its files/.
func aTemplatedModule(t *testing.T, dir, manifest string, files map[string]string) string {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
path := filepath.Join(dir, "module.json")
if err := os.WriteFile(path, []byte(manifest), 0o644); err != nil {
t.Fatal(err)
}
for name, text := range files {
full := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(text), 0o644); err != nil {
t.Fatal(err)
}
}
return path
}
func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) {
good := aTemplatedModule(t, t.TempDir(), templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
var out bytes.Buffer
if err := moduleCheck([]string{good}, &out); err != nil || !strings.Contains(out.String(), "notifier: ok") {
t.Fatalf("a module with its text in files/ was refused: %v\n%s", err, out.String())
}
// A change to the template alone, to text the controller refuses inline, is refused the same.
bad := aTemplatedModule(t, t.TempDir(), templatedNotifier,
map[string]string{"files/notifier.conf": "volume = 3\nstore = ${dir:nowhere}\n"})
out.Reset()
if err := moduleCheck([]string{bad}, &out); err == nil || !strings.Contains(out.String(), "nowhere") {
t.Fatalf("a template whose text is refused inline passed:\n%s", out.String())
}
// A symlink under files/ that no template names: the build's packing refuses it, so the check does too.
linked := t.TempDir()
aTemplatedModule(t, linked, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
if err := os.Symlink("/etc/hostname", filepath.Join(linked, "files", "elsewhere")); err != nil {
t.Fatal(err)
}
out.Reset()
if err := moduleCheck([]string{filepath.Join(linked, "module.json")}, &out); err == nil ||
!strings.Contains(out.String(), "files/elsewhere is a symlink") {
t.Fatalf("a symlink under files/ passed the check:\n%s", out.String())
}
// And a template the tree lacks, by name.
missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil)
out.Reset()
if err := moduleCheck([]string{missing}, &out); err == nil || !strings.Contains(out.String(), "files/notifier.conf") {
t.Fatalf("a missing template was not refused by name:\n%s", out.String())
}
}
// templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline.
func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) {
t.Helper()
open := aMesh(t)
ctx := t.Context()
manifests := map[string]string{
"lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"],
"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`,
"notifier": inlineNotifier,
}
for name, raw := range manifests {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
t.Fatalf("%s: %v", name, err)
}
if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog",
Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", name); err != nil {
t.Fatalf("assign %s: %v", name, err)
}
}
f, err := gatherFacts(ctx, open, "2.11.17")
if err != nil {
t.Fatal(err)
}
return f, manifests
}
// aTemplatedTree is the catalogue's checkout with the notifier's text in files/.
func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has bool) string {
t.Helper()
tree := aTree(t, withEdit(manifests, "notifier", ""))
files := map[string]string{}
if has {
files["files/notifier.conf"] = text
}
aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, files)
return tree
}
const notifierTemplate = "modules/notifier/files/notifier.conf"
func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) {
f, manifests := templatedMesh(t)
// The module moved to its template, the same text: nothing changes on any machine.
v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json",
notifierTemplate)
if v.Verdict == "fail" || v.Modules["notifier"] != "" {
t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report())
}
// The template alone changed: its module changed, and composes.
v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate)
if v.Verdict == "fail" || v.Modules["notifier"] != "changed" {
t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report())
}
if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" {
t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width)
}
// The template alone changed to text the controller refuses: the pull request fails, naming the module.
v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate)
if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") {
t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report())
}
// A symlinked files/ fails: nothing is read from outside the tree.
tree := aTree(t, withEdit(manifests, "notifier", ""))
elsewhere := t.TempDir()
if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil {
t.Fatal(err)
}
aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil)
if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil {
t.Fatal(err)
}
v = gateJudged(t, f, tree, notifierTemplate)
if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") {
t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report())
}
// And the template removed from the tree fails, naming it.
v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate)
if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") {
t.Fatalf("a missing template passed the gate:\n%s", v.Report())
}
}
// What the gate composes for the module in its template form is what the mesh composes for it inline.
func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) {
dir := t.TempDir()
path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"})
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
m, err = catalogue.WithTemplatesFrom(m, dir)
if err != nil {
t.Fatal(err)
}
resolved, err := standInBuild(m)
if err != nil {
t.Fatal(err)
}
inline, err := catalogue.ParseManifest([]byte(inlineNotifier))
if err != nil {
t.Fatal(err)
}
if builtAlike(resolved) != builtAlike(inline) {
got, _ := json.Marshal(resolved)
t.Fatalf("the templated module resolves to %s, not its inline twin", got)
}
}
+15 -5
View File
@@ -263,8 +263,18 @@ func TestTheAgentsManagedSettingsAreRefusedThroughAVerb(t *testing.T) {
if err := atTheTerminal(t, "settings", "set", "claude-code", allow, "--node", "laptop"); err != nil {
t.Fatalf("the managed settings at the terminal: %v", err)
}
if err := atTheTerminal(t, "settings", "set", "claude-code", server); err != nil {
t.Fatalf("a tool server for the mesh at the terminal: %v", err)
// There is no module layer for the whole mesh (novox/hq issue 630), at the terminal either; one the mesh
// still holds from before is kept as it is, and is the terminal's as before.
if err := atTheTerminal(t, "settings", "set", "claude-code", server); err == nil ||
!strings.Contains(err.Error(), "no module layer for the whole mesh") {
t.Fatalf("a tool server for the whole mesh at the terminal: %v", err)
}
var held map[string]any
if err := json.Unmarshal([]byte(server), &held); err != nil {
t.Fatal(err)
}
if err := open.inventory.KeepSettings(ctx, "", "claude-code", held); err != nil {
t.Fatalf("a whole-mesh layer held from before: %v", err)
}
kept := layer("laptop")
// Through a verb they are neither changed, dropped nor cleared.
@@ -359,15 +369,15 @@ func TestOnlyATrustedFilesNamedKeyIsRefusedThroughAVerb(t *testing.T) {
if _, err := assign(ctx, open, "anchor", "mailer"); err != nil {
t.Fatal(err)
}
if err := throughVerb(t, "settings", map[string]any{"module": "mailer", "values": `{"tags":{"todo":{"name":"To do"}}}`}); err != nil {
if err := throughVerb(t, "settings", map[string]any{"module": "mailer", "node": "anchor", "values": `{"tags":{"todo":{"name":"To do"}}}`}); err != nil {
t.Fatalf("a key of a file nothing trusts is the verb's: %v", err)
}
err := throughVerb(t, "settings", map[string]any{"module": "mailer", "replace": "true",
err := throughVerb(t, "settings", map[string]any{"module": "mailer", "node": "anchor", "replace": "true",
"values": `{"tags":{"todo":{"name":"To do"}},"accounts":{"mail":{"address":"me@mail.example"}}}`})
if err == nil || !strings.Contains(err.Error(), "controller's terminal") || !strings.Contains(err.Error(), "accounts") {
t.Fatalf("a key of a trusted file is the terminal's: %v", err)
}
values, _, err := open.inventory.Layer(ctx, "", "mailer")
values, _, err := open.inventory.Layer(ctx, "anchor", "mailer")
if err != nil {
t.Fatal(err)
}
@@ -1,46 +0,0 @@
[
{
"id": "found.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"networks": [
"predecessor_default"
],
"type": "container"
},
{
"id": "library.access-storage/downloads",
"mode": "read",
"path": "/storage/downloads",
"type": "access"
},
{
"id": "library.access-storage/media/series",
"mode": "read",
"path": "/storage/media/series",
"type": "access"
},
{
"id": "library.config",
"mode": "0750",
"owner": "1001:2000",
"path": "/var/lib/library/config",
"type": "directory"
},
{
"content": "{\n \"accesses\": {\n \"downloads\": \"/storage/downloads\",\n \"series\": \"/storage/media/series\"\n },\n \"log\": \"warn\",\n \"places\": {\n \"config\": {\n \"owner\": \"1001:2000\",\n \"path\": \"/var/lib/library/config\"\n }\n },\n \"workers\": 4\n}\n",
"id": "library.runtime-config",
"mode": "0640",
"path": "/var/lib/library/config/runtime.json",
"type": "file"
},
{
"id": "library.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"type": "container",
"volumes": [
"/var/lib/library/config:/config",
"/storage/media/series:/series",
"/storage/downloads:/downloads:ro"
]
}
]
@@ -1,37 +0,0 @@
[
{
"id": "library.access-home/media/series",
"mode": "read",
"path": "/home/media/series",
"type": "access"
},
{
"id": "library.access-storage/downloads",
"mode": "read",
"path": "/storage/downloads",
"type": "access"
},
{
"id": "library.config",
"mode": "0750",
"path": "/var/lib/library/config",
"type": "directory"
},
{
"content": "{\n \"accesses\": {\n \"downloads\": \"/storage/downloads\",\n \"series\": \"/home/media/series\"\n },\n \"log\": \"warn\"\n}\n",
"id": "library.runtime-config",
"mode": "0640",
"path": "/var/lib/library/config/runtime.json",
"type": "file"
},
{
"id": "library.server",
"image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"type": "container",
"volumes": [
"/var/lib/library/config:/config",
"/home/media/series:/series",
"/storage/downloads:/downloads:ro"
]
}
]
+14 -25
View File
@@ -130,14 +130,10 @@ func aTierMesh(t *testing.T, modules ...string) *tierMesh {
}
t.Cleanup(func() { gatherGateFacts = wasGather })
wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound
// The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351).
wasWindow := gateWitnessWindow
gateWitnessWindow = 0
t.Cleanup(func() { gateWitnessWindow = wasWindow })
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
// One judging per advance until a test says otherwise: a pass waits gateEvery for the next.
gateWithin, gateEvery = time.Hour, time.Hour
t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound })
gateSettle, gateEvery = 0, time.Hour
t.Cleanup(func() { gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound })
built := time.Now().UTC()
states := map[string]*inventory.PlanModule{}
@@ -226,9 +222,6 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule)
ctx := t.Context()
inv := tm.open.inventory
// The node tools are held inside their witness window, which the node-engine keeps while they settle: their
// one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4).
gateWitnessWindow = time.Hour
advancePlans(ctx, tm.open)
if len(tm.sent) != 1 {
@@ -412,10 +405,7 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
}
return f, err
}
// app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the
// judgings that find the node tools broken, and is judged to its own pass after them (the hold the
// settle time gave before novox/hq ADR 0352 counted readings from the send).
gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second
gateEvery, gateSettle, gateBound = 0, 100*time.Millisecond, 10*time.Second
for i := 0; i < 20 && len(seen) < 2; i++ {
advancePlans(ctx, tm.open)
}
@@ -450,15 +440,15 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
}
}
// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351
// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool
// runner's rule: one reading, not before the node-engine's witness window for it has ended.
func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) {
// **A send carrying a core module waits for the core's readings** (issue 524 review): the leaf beside the node
// tools has its one reading at once, but the send is judged as the core: not before the settle time, and three
// readings.
func TestASendCarryingTheCoreIsJudgedAsTheCore(t *testing.T) {
tm := aTierMesh(t, broker.RuntimeModule, "app1")
ctx := t.Context()
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
// between readings would count a pass at every look and never let an advance rest.
gateEvery, gateWitnessWindow = time.Hour, time.Hour
gateEvery, gateSettle = time.Hour, time.Hour
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
@@ -467,10 +457,10 @@ func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) {
if lead == nil {
lead = p.Modules["app1"].Gate
}
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 {
t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != gatePasses {
t.Fatalf("passed before the core's settle time: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead)
}
gateEvery, gateWitnessWindow = 0, 0
gateEvery, gateSettle = 0, 0
for i := 0; i < 6; i++ {
advancePlans(ctx, tm.open)
}
@@ -479,9 +469,8 @@ func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) {
t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent)
}
for _, m := range []string{broker.RuntimeModule, "app1"} {
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 ||
!strings.Contains(g.Why, "the tool runner's rule")) {
t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g)
if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < gatePasses) {
t.Fatalf("%s's gate: %+v; want passed on %d readings", m, g, gatePasses)
}
}
}
-353
View File
@@ -1,353 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"net/url"
"os"
"regexp"
"strings"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
)
// A branch is tried before its pull request (novox/hq issue 686).
//
// A repository's store tests skip without a store, and no agent may start a database on its machine. So a
// builder pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time:
// each failure there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312).
//
// `try <owner>/<repository> <ref>` asks the build seat for the repository's own check of a pushed branch or
// commit, as the merge check runs it (internal/builder/check.go): the ref merged into the trunk's head, the
// trunk's merge-check.sh, each part in its toolchain's container, against a throwaway store and bus. It is a
// check and nothing more:
//
// - **no pull request, so no status**: its outcome is never said as `checked`, which is what the forge's
// holder sets statuses from (isTry, in Built);
// - **nothing registered, nothing built**: it names no module of the graph, so the merge gate does not run and
// no build is made for a merge (novox/hq ADR 0350's scratch builds are a pull request's check's alone);
// - **a check in the build queue**: after deliveries (link.KindOf, novox/hq issue 558), and bounded by the
// check's own bound (checkTimeout);
// - **the code a pull request's check would run anyway**: only a repository of the mesh, whose every pull
// request the build seat already checks, in containers that hold no runtime socket.
//
// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id <id>` reads what
// came of it from the build's log, where the build seat says it whole (link.StepTried): holding its answer up
// to tryWait for a verdict, like `plans wait` (novox/hq issue 229). Nothing is kept beside the log.
// tryWait is how long reading a try holds its answer for the verdict. A variable so a test need not wait.
var tryWait = 20 * time.Second
// A ref as the forge is asked for it — a branch or a commit, never an option — and a `go test -run` pattern.
var (
tryRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
tryTests = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`)
)
// tryAnswer is what `try` answers, asking or reading.
type tryAnswer struct {
ID string `json:"id"`
// State is asked (just now), waiting (nothing said of it yet: in the build queue), running, or the verdict:
// pass, warning (the repository declares no merge-check.sh), fail, or error (it could not run).
State string `json:"state"`
Said string `json:"said"`
Repository string `json:"repository,omitempty"`
Ref string `json:"ref,omitempty"`
Commit string `json:"commit,omitempty"`
Base string `json:"base,omitempty"`
Trunk string `json:"trunk,omitempty"`
On string `json:"on,omitempty"`
Took string `json:"took,omitempty"`
Failed []string `json:"failed,omitempty"`
// Last is the last thing the build seat said of one still running.
Last string `json:"last,omitempty"`
}
// isTry says a check's outcome is a try's: marked so, or — echoed by a build seat from before the mark — a
// check of no pull request and no group, which is nobody's status either.
func isTry(result link.BuildResult) bool {
c := result.Checked
return c != nil && (c.Try || (c.Number == 0 && c.Group == ""))
}
// asATry makes a check's ask a try's: the repository's own script alone, for no pull request.
func asATry(request link.BuildRequest, tests string, storeOnly bool) link.BuildRequest {
c := *request.Check
c.Number, c.Try, c.Tests, c.StoreOnly = 0, true, tests, storeOnly
c.Paths, c.Modules, c.New, c.Dependents, c.Manifests, c.Builds = nil, nil, nil, nil, nil, nil
c.Judge, c.Plan, c.Group, c.Members = "", nil, "", nil
request.Check = &c
return request
}
// maxTries is how many tries may wait or run at once, mesh-wide: each holds a build machine for minutes.
const maxTries = 4
// triesIn reads the build queue for a new try of one commit: the id of a try of that commit already waiting or
// running — the answer to a second ask — and whether there is room for another.
func triesIn(q link.Queue, owner, repo, commit string) (same string, room bool) {
n := 0
for _, a := range q.Asks {
if !link.IsTry(a.Request) || (a.State != link.AskWaiting && a.State != link.AskInFlight) {
continue
}
n++
c := a.Request.Check
if same == "" && strings.EqualFold(c.Owner, owner) && strings.EqualFold(c.Repo, repo) && a.Request.Ref == commit {
same = a.ID
}
}
return same, n < maxTries
}
// forgeCommit asks the forge, through its module's tools on the bus, which commit a branch or commit of a
// repository is, and the repository's default branch. A variable so a test needs no forge.
var forgeCommit = func(ctx context.Context, owner, repo, ref string) (commit, trunk string, err error) {
js, err := aBus()
if err != nil {
return "", "", err
}
defer js.Close()
bus := link.OverNATS{Conn: js.Conn()}
ask := func(path string, into any) error {
raw, _ := json.Marshal(map[string]any{"path": path})
answer, err := link.Ask(ctx, bus, "gitea", "gitea_api", raw, 30*time.Second)
if err != nil {
return err
}
if answer.Error != "" {
return errors.New(answer.Error)
}
return json.Unmarshal(answer.Result, into)
}
var commits struct {
Result []struct {
SHA string `json:"sha"`
} `json:"result"`
}
if err := ask("/repos/"+owner+"/"+repo+"/commits?limit=1&stat=false&verification=false&files=false&sha="+
url.QueryEscape(ref), &commits); err != nil {
return "", "", err
}
if len(commits.Result) == 0 || commits.Result[0].SHA == "" {
return "", "", errors.New("the forge named no commit for it")
}
var found struct {
Result struct {
DefaultBranch string `json:"default_branch"`
} `json:"result"`
}
if err := ask("/repos/"+owner+"/"+repo, &found); err != nil {
return "", "", err
}
return commits.Result[0].SHA, found.Result.DefaultBranch, nil
}
// tryCommand is `try <owner>/<repository> <ref> [--base <branch>] [--tests <pattern>] [--store-only]`, and
// `try --id <id>` to read what came of one.
func tryCommand(ctx context.Context, args []string) error {
set := flag.NewFlagSet("try", flag.ContinueOnError)
base := set.String("base", "", "the branch it is merged into (the repository's default branch when none)")
tests := set.String("tests", "", "only the tests this `go test -run` pattern names")
storeOnly := set.Bool("store-only", false, "only the repository's store tests")
id := set.String("id", "", "a try's id: what came of it")
positionals, err := parseAround(set, args)
if err != nil {
return err
}
usage := errors.New("try <owner>/<repository> <branch or commit> [--base <branch>] [--tests <pattern>] " +
"[--store-only], or try --id <id>")
if *id != "" {
if len(positionals) != 0 {
return usage
}
// Before it is a token of a bus subject: an id, never a pattern.
if !link.SafeBuildID(*id) {
return fmt.Errorf("%q is not a try's id", *id)
}
return tryRead(ctx, *id)
}
if len(positionals) != 2 {
return usage
}
owner, repo, isPath := strings.Cut(positionals[0], "/")
ref := positionals[1]
switch {
case !isPath || !tryRef.MatchString(owner) || !tryRef.MatchString(repo) || strings.Contains(repo, "/"):
return fmt.Errorf("%q is not a repository on the forge, owner/name; nothing was asked", positionals[0])
case !tryRef.MatchString(ref):
return fmt.Errorf("%q is not a branch or a commit; nothing was asked", ref)
case *base != "" && !tryRef.MatchString(*base):
return fmt.Errorf("%q is not a branch; nothing was asked", *base)
case *tests != "" && !tryTests.MatchString(*tests):
return fmt.Errorf("%q is not a `go test -run` pattern of test names; nothing was asked", *tests)
}
// **An unknown ref is refused here, loudly** — not sent to a machine to fail at its clone minutes later.
commit, trunk, err := forgeCommit(ctx, owner, repo, ref)
if err != nil {
return fmt.Errorf("the forge holds no branch or commit %s in %s/%s — is it pushed? — so nothing was asked: %w",
ref, owner, repo, err)
}
if *base != "" {
trunk = *base
}
if trunk == "" {
return fmt.Errorf("the forge named no default branch for %s/%s, and no --base says what %s is merged into; "+
"nothing was asked", owner, repo, ref)
}
open, err := openStores(ctx)
if err != nil {
return err
}
defer open.Close()
entries, read, edges, err := theGraph(ctx, open.inventory)
if err != nil {
return err
}
p := link.PullUpdated{Owner: owner, Repo: repo, Base: trunk, Commit: commit}
// Whose repository it is, as a pull request's check reckons it: only the mesh's is tried, the code the
// build seat runs for every pull request already.
mesh := pullScope(p, entries, read, edges)
if !mesh.Mesh {
return fmt.Errorf("%s/%s is no repository of the mesh — no module is built from it and its owner is not "+
"the core's — and only the mesh's are tried on the build seat; nothing was asked", owner, repo)
}
request, err := checkRequestFor(ctx, open, p, checkScope{Mesh: true, From: mesh.From}, entries)
if err != nil {
return err
}
request = asATry(request, *tests, *storeOnly)
seat := buildSeatHeld(ctx)
// **Bounded in number** (novox/hq issue 686): one try of a commit at a time, and maxTries in all.
js, err := dialTheBus()
if err != nil {
return err
}
queue, err := link.ReadQueue(ctx, js, seat)
js.Close()
if err != nil {
return err
}
same, room := triesIn(queue, owner, repo, commit)
if same != "" {
return printTry(tryAnswer{ID: same, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
On: seat, Said: fmt.Sprintf("a try of %s/%s at %.12s already waits or runs, as %s: nothing new was asked. "+
"Read it with try id=%s", owner, repo, commit, same, same)})
}
if !room {
return fmt.Errorf("%d tries already wait or run on the build seat, the most at once (`queue` lists them); "+
"nothing was asked — read one of yours, or ask again when one has ended", maxTries)
}
ask, err := askOverOn(seat)
if err != nil {
return err
}
defer ask.Close()
if err := ask.Ask(ctx, request); err != nil {
return err
}
return printTry(tryAnswer{ID: request.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
Base: trunk, On: seat, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: its own "+
"merge-check.sh, as a check in the build queue, bounded at %s; no status is set and nothing is registered. "+
"Read it with try id=%s — a full run of this kind takes minutes", seat, owner, repo, commit, ref, trunk,
checkTimeout, request.ID)})
}
// tryRead answers what came of a try, holding the answer up to tryWait for its verdict.
func tryRead(ctx context.Context, id string) error {
until := time.Now().Add(tryWait)
for {
lines, err := buildLines(ctx, id)
if err != nil {
return err
}
a := tryAnswerOf(id, lines)
if (a.State != "waiting" && a.State != "running") || !time.Now().Add(2*time.Second).Before(until) {
return printTry(a)
}
if err := sleepOrDone(ctx, 2*time.Second); err != nil {
return err
}
}
}
// tryAnswerOf reads a try's answer from what the build seat said of it: its outcome under StepTried; `failed`
// for one that could not run — an error, never a pass; else how far it is.
func tryAnswerOf(id string, lines []link.BuildLine) tryAnswer {
a := tryAnswer{ID: id, State: "waiting", Said: "nothing is said of " + id + " yet: it waits in the build queue " +
"(`queue` says where), or no try has that id; ask again"}
for _, l := range lines {
switch l.Step {
case link.StepTried:
var t link.Tried
if err := json.Unmarshal([]byte(l.Message), &t); err != nil || t.Verdict == "" {
return tryAnswer{ID: id, State: "error", Said: "the build seat's word on it cannot be read: " + l.Message}
}
return tryAnswer{ID: id, State: t.Verdict, Said: t.Summary, Took: t.Took, Commit: t.Head, Trunk: t.Trunk,
Failed: t.Failed}
case "failed":
return tryAnswer{ID: id, State: "error", Said: "it could not run: " + l.Message}
case "checked":
// A check that is no try, or a build seat from before the verb: its verdict is not a try's.
return tryAnswer{ID: id, State: "error", Said: "the build seat checked it and said no try's outcome — " +
"it predates the verb (novox/hq issue 686), or " + id + " is no try: " + l.Message + "; `builds` with " +
"the id says every line"}
}
a.State, a.Last = "running", l.Message
a.Said = "still running; ask again. `builds` with the id says every line so far"
}
return a
}
func printTry(a tryAnswer) error {
fmt.Fprintln(os.Stderr, a.Said)
for _, l := range a.Failed {
fmt.Fprintln(os.Stderr, " "+l)
}
out, err := json.MarshalIndent(a, "", " ")
if err != nil {
return err
}
fmt.Println(string(out))
return nil
}
// buildLines is everything the build seat said of one build, as the bus keeps it, in order.
func buildLines(ctx context.Context, id string) ([]link.BuildLine, error) {
js, err := aBus()
if err != nil {
return nil, fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
}
defer js.Close()
subject := link.BuildLogOf("*", id)
sub, err := js.Context().PullSubscribe(subject, "",
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
if err != nil {
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
}
defer func() { _ = sub.Unsubscribe() }()
var lines []link.BuildLine
for {
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
return nil, fmt.Errorf("reading a build's log: %w", err)
}
for _, msg := range batch {
var line link.BuildLine
if json.Unmarshal(msg.Data, &line) == nil {
lines = append(lines, line)
}
}
if len(batch) < 200 {
return lines, nil
}
}
}
-194
View File
@@ -1,194 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
"github.com/novox/mesh-controller/internal/link"
)
// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed
// branch or commit, with no pull request.
// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own
// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image.
func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) {
request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a",
Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"},
Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf,
Plan: &link.ChangePlan{}}}
tried := asATry(request, "^TestStore", true)
c := tried.Check
if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly {
t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c)
}
if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil {
t.Fatalf("a try runs the gate or builds for a merge: %+v", c)
}
if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" {
t.Fatalf("a try lost what it merges into what: %+v", tried)
}
if link.KindOf(tried) != link.KindCheck {
t.Fatal("a try does not count as a check in the build queue")
}
}
// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and
// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one).
func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) {
said := &saidOnTheBus{}
checkEvents = said
defer func() { checkEvents = nil }()
for name, result := range map[string]link.BuildResult{
"a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}},
"a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true},
Failed: "cannot clone"},
// A build seat from before the mark echoes the ask without it: a check of no pull request and no
// group is still nobody's status.
"a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"},
Check: &link.CheckOutcome{Verdict: "pass"}},
} {
if err := (builds{}).Built(context.Background(), result); err != nil {
t.Fatalf("%s: %v", name, err)
}
}
if len(said.events) != 0 {
t.Fatalf("a try's outcome was said for the forge: %v", said.events)
}
// A pull request's check is still said.
_ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{
Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}})
if len(said.events) != 1 {
t.Fatalf("a pull request's check was said %d time(s)", len(said.events))
}
}
// saidOnTheBus keeps the seat events said, and nothing else.
type saidOnTheBus struct {
link.Bus
events []string
}
func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error {
s.events = append(s.events, seat+" "+event)
return nil
}
// A ref the forge does not hold is refused before anything is asked, naming it.
func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) {
was := forgeCommit
defer func() { forgeCommit = was }()
forgeCommit = func(_ context.Context, owner, repo, ref string) (string, string, error) {
return "", "", errors.New("404 object does not exist [id: " + ref + "]")
}
err := tryCommand(context.Background(), []string{"novox/mesh-controller", "fix/no-such-branch"})
if err == nil {
t.Fatal("a try of a ref the forge does not hold was asked")
}
for _, want := range []string{"fix/no-such-branch", "novox/mesh-controller", "nothing was asked", "404"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("the refusal does not say %q: %v", want, err)
}
}
for _, args := range [][]string{{"mesh-controller", "main"}, {"novox/mesh-controller"}, {"novox/a", "--upload-pack=x"},
{"novox/a", "main", "--tests", "x; rm -rf /"}, {"novox/a", "main", "--id", "build-1"}} {
if err := tryCommand(context.Background(), args); err == nil {
t.Errorf("try %v was taken", args)
}
}
}
// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took.
func TestATrysAnswerCarriesTheFailingLines(t *testing.T) {
tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s",
Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2",
"FAIL\tgithub.com/novox/a/cmd\t3.1s"}})
lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"},
{Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"},
{Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}}
a := tryAnswerOf("build-1", lines)
if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" ||
a.Trunk != "ca3f43c" || a.ID != "build-1" {
t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a)
}
// Not answered yet: said as running, with how far it is — never as a pass.
if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" {
t.Fatalf("a try still running: %+v", a)
}
if a := tryAnswerOf("build-1", nil); a.State != "waiting" {
t.Fatalf("a try nothing was said of: %+v", a)
}
// One that could not run is an error with the build seat's word, never a pass.
if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed",
Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") {
t.Fatalf("a try that could not run: %+v", a)
}
}
// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id,
// a fifth is refused; a try that ended, and a pull request's check, count for nothing.
func TestTriesAreBoundedInNumber(t *testing.T) {
ask := func(id, repo, ref, state string, try bool) link.QueuedAsk {
return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref,
Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}}
}
q := link.Queue{Asks: []link.QueuedAsk{
ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true),
ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}}
if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room {
t.Fatalf("a second try of a commit: %q %v", same, room)
}
if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room {
t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room)
}
if same, _ := triesIn(q, "novox", "a", "c9"); same != "" {
t.Fatalf("a pull request's check was taken for a try: %q", same)
}
q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true))
if _, room := triesIn(q, "novox", "e", "c5"); room {
t.Fatal("a fifth try was let in")
}
// An id is checked before it is a token of a bus subject.
for _, id := range []string{"x.>", "*", "a b"} {
if err := tryCommand(context.Background(), []string{"--id", id}); err == nil ||
!strings.Contains(err.Error(), "not a try's id") {
t.Errorf("try --id %q: %v", id, err)
}
}
}
// The verb runs its command, and is declared with the shell habit it replaces (novox/hq ADR 0245).
func TestTheTryVerbRunsItsCommand(t *testing.T) {
for _, c := range []struct {
args map[string]any
want []string
}{
{map[string]any{"repository": "novox/a", "ref": "fix/x"}, []string{"try", "novox/a", "fix/x"}},
{map[string]any{"repository": "novox/a", "ref": "fix/x", "base": "main", "tests": "^TestX$", "store-only": "true"},
[]string{"try", "novox/a", "fix/x", "--base", "main", "--tests", "^TestX$", "--store-only"}},
{map[string]any{"id": "build-1"}, []string{"try", "--id", "build-1"}},
} {
got, err := argvFor("try", c.args)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("try %v: %v %v, want %v", c.args, got, err, c.want)
}
}
if _, err := argvFor("try", map[string]any{"repository": "novox/a"}); err == nil {
t.Error("a try of no ref was taken")
}
for _, v := range catalogue.ControllerVerbs {
if v.Name == "try" {
if len(v.Replaces) == 0 {
t.Error("try names no shell habit it replaces")
}
return
}
}
t.Error("try is not a verb of the controller seat")
}
+1 -1
View File
@@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
// now waits for a gate like any other build, and the release plan walks it through the machines, one
// at a time, each judged.
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+
fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+
"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
return nil
}
-6
View File
@@ -177,12 +177,6 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s
reached = appendNew(reached, r.Gate.Machines...)
carried(r.Gate)
}
for _, g := range r.Gates {
if g != nil && !g.Pending {
reached = appendNew(reached, g.Machines...)
carried(g)
}
}
for _, tier := range p.Tiers {
for _, m := range tier {
for _, n := range running(m) {
+2 -2
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
github.com/jackc/pgx/v5 v5.10.0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
@@ -36,4 +36,4 @@ require (
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
// `go mod vendor` fails loudly, at once, everywhere.
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f
+2 -8
View File
@@ -1,13 +1,7 @@
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc h1:h5v2AbdjM/clMY1YxS9p7VuyYtIjkcw85w9Oz1cOYQc=
git.novox.be/novox/mesh-host v0.0.0-20261013220009-b9a293f8c8bc/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f h1:8N5OW2mdTNIck2pe4EciTYX5NsrPsHrTLENGNIWYNTU=
git.novox.be/novox/mesh-host v0.0.0-20261011092230-20d5af9b2d5f/go.mod h1:tcTK4LMs1d6JpZUwy3hSHMFG/MIuDr/XFs7/nbeaW/c=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629 h1:7TYFV1Q6Mcz+lsLP/sds87tysET4rKCn3rRIQV8gNmA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013141520-9588871d6629/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d h1:HEJyqorKktrkJPeeJdwEo0kdrJjH3fm/YVS9MGftCqY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013145622-f4745447ba6d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-91
View File
@@ -1,91 +0,0 @@
package artifacts
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
)
// Listing and letting go of tagged documents: the check builds the scratch namespace keeps (novox/hq ADR 0350
// §5). A check build is found by its tag, and the build seat's holder lets each go when its pull request closes
// or it is seven days old, so the store keeps a week of check builds and no more.
// Tags is every tag repository holds, in the store's order. None for a repository the store does not hold.
func (s Store) Tags(ctx context.Context, repository string) ([]string, error) {
if s.Address == "" {
return nil, fmt.Errorf("this mesh has no artifact store on its network to list %s in", repository)
}
var tags []string
next := "http://" + s.Address + "/v2/" + repository + "/tags/list?n=1000"
for next != "" {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, next, nil)
if err != nil {
return nil, err
}
response, err := s.client().Do(request)
if err != nil {
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
}
switch response.StatusCode {
case http.StatusOK:
case http.StatusNotFound:
response.Body.Close()
return tags, nil
default:
response.Body.Close()
return nil, fmt.Errorf("the artifact store answered %s for the tags of %s", response.Status, repository)
}
var page struct {
Tags []string `json:"tags"`
}
err = json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&page)
link := response.Header.Get("Link")
response.Body.Close()
if err != nil {
return nil, fmt.Errorf("the tags of %s could not be read: %w", repository, err)
}
tags = append(tags, page.Tags...)
next = nextPage(link, s.Address)
}
return tags, nil
}
// nextPage is the registry's `Link: </v2/…>; rel="next"`, made whole; empty when there is no next page.
func nextPage(link, address string) string {
if !strings.Contains(link, `rel="next"`) {
return ""
}
start, end := strings.Index(link, "<"), strings.Index(link, ">")
if start < 0 || end <= start+1 {
return ""
}
target := link[start+1 : end]
if strings.HasPrefix(target, "/") {
return "http://" + address + target
}
return target
}
// DeleteTagged lets go of the document tag names in repository: its manifest, by digest, so the next
// collection takes its layer. Gone when the tag names nothing.
func (s Store) DeleteTagged(ctx context.Context, repository, tag string) error {
if s.Address == "" {
return fmt.Errorf("this mesh has no artifact store on its network to let go of %s:%s in", repository, tag)
}
digest, err := s.manifestDigest(ctx, repository, tag)
if err != nil {
return err
}
if digest == "" {
return Gone
}
err = s.remove(ctx, s.url(repository, "manifests", digest), repository+":"+tag)
if errors.Is(err, Gone) {
return Gone
}
return err
}
+28 -277
View File
@@ -91,16 +91,6 @@ type Result struct {
// (novox/hq issue 280).
Source string
// ReuseKey is the build's reuse key (novox/hq ADR 0350 §1): its source fingerprint's parts with the builder's
// version, and ReuseParts those parts; empty, with NoKey saying why, where the source does not pin the build.
ReuseKey string
ReuseParts map[string]string
NoKey string
// Reused is the check build a walk took instead of building (ADR 0350 §2), and Why, when it built, why it did
// not take one (§3). Both empty for a build not asked to reuse.
Reused *Reused
Why string
// Sources are what this build was made from, as files (novox/hq ADR 0267 rule 1): per repository, the
// entries a changed file is tested against (SourceHolds). The module's own repository has an empty
// Repository. Said only for a build of a commit on the trunk, and only for a repository whose every
@@ -109,16 +99,10 @@ type Result struct {
// sees stays its source, as before.
Sources []BuildSource
// Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR
// 0348 §7), which the manifest was judged against; nil for a repository that holds none.
Predefined *Predefined
}
// Predefined is the list of predefined keys a build read: its text, canonical (catalogue.PredefinedList), and
// the generation of the commit built — how many commits it stands on, which orders two builds of one trunk.
type Predefined struct {
Keys []byte
Generation int
// Predefined is the repository's list of predefined keys (catalogue.PredefinedFile at its root) as
// read at the commit built, when it has one (novox/hq issue 630): what the controller keeps as the
// list mesh and node config hold values for. Empty for a repository without one.
Predefined json.RawMessage
}
// BuildSource is the build source a build read in one repository (novox/hq ADR 0267).
@@ -157,44 +141,14 @@ func Build(ctx context.Context, run Runner, publish Publisher,
// one record that it is there.
var mirrored []string
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
&mirrored, nil, seats...)
result.Mirrored = mirrored
return result, err
}
// Reuse is what a walk's build may take instead of building (novox/hq ADR 0350 §2): the scratch namespace's check
// builds, and the pull requests the walk answers, whose checks name a differing key.
type Reuse struct {
Scratch Scratch
Pulls []PullRef
// Named are the builds the verdicts of the walk's own pull requests name (novox/hq ADR 0350 §2): a scratch build
// is taken only when one of them names its key and every digest. The record in the store alone is never enough.
Named []NamedBuild
// keyFirst is a check's build (§1): its key is taken before anything is built, and a module with none is not
// built at all — nothing could ever take it.
keyFirst bool
}
// errNoKey is a check's build of a module whose source does not pin it: not built (ADR 0350 §1).
var errNoKey = errors.New("it has no reuse key")
// BuildReusing is Build for the walk of a trunk commit (novox/hq ADR 0350 §2): before building, it takes the reuse
// key of the commit's tree, and where a check build holds the same key it copies that build into the module's
// namespace by digest and answers with it, building nothing. Otherwise it builds, and Result.Why says why (§3).
// Only a commit on its trunk takes a check build: anything else builds, as Build does.
func BuildReusing(ctx context.Context, run Runner, publish Publisher, reuse Reuse,
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
forge GitCredential, log Log, seats ...map[string]string) (Result, error) {
var mirrored []string
result, err := build(ctx, run, publish, repository, path, ref, workspace, held, npmrc, forge, log,
&mirrored, &reuse, seats...)
&mirrored, seats...)
result.Mirrored = mirrored
return result, err
}
func build(ctx context.Context, run Runner, publish Publisher,
repository, path, ref, workspace string, held map[string]string, npmrc Npmrc,
forge GitCredential, log Log, mirrored *[]string, reuse *Reuse, seats ...map[string]string) (Result, error) {
forge GitCredential, log Log, mirrored *[]string, seats ...map[string]string) (Result, error) {
// The clone base of each seat a context may name (novox/hq ADR 0155); variadic so the callers
// that hand none — tests of everything but contexts — read as they did.
var seatBases map[string]string
@@ -272,42 +226,37 @@ func build(ctx context.Context, run Runner, publish Publisher,
"%s has no %s at %s, so there is nothing saying what it is: %w",
repository, ManifestName, describe(path), err)
}
// **The list of predefined keys at the commit built** (novox/hq ADR 0348 §7): predefined.json at the
// repository's root, read strictly; a list that is wrong fails the build, and the manifest is judged against
// the list of its own tree, never against one the mesh published from another commit.
predefined, err := catalogue.ReadPredefined(tree)
if err != nil {
say("predefined", "INVALID: %v", err)
return Result{}, err
}
manifest, err := catalogue.ParseManifestWith(raw, predefined)
manifest, err := catalogue.ParseManifest(raw)
if err != nil {
say("manifest", "INVALID: %v", err)
return Result{}, err
}
var listed *Predefined
if predefined != nil {
text, err := predefined.Canonical()
if err != nil {
return Result{}, err
}
listed = &Predefined{Keys: text, Generation: generationOf(ctx, run, tree, commit)}
say("predefined", "%d key(s) at %s, generation %d", len(predefined), short(commit), listed.Generation)
}
// What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is
// where a changed definition is first read, so it is refused here before anything is made of it.
// Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the
// module's directory lacks is refused here, before anything is built.
judged, err := catalogue.WithTemplatesFrom(manifest, within)
if err != nil {
say("manifest", "INVALID: %v", err)
return Result{}, err
}
if problems := catalogue.WrittenSettingsProblems(judged); len(problems) > 0 {
if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 {
err := fmt.Errorf("%s", strings.Join(problems, "; "))
say("manifest", "INVALID: %v", err)
return Result{}, err
}
// The repository's list of predefined keys, at its root (novox/hq issue 630): read and judged here,
// where a changed list is first read, and handed on for the controller to keep. A preference that
// falls back to a key the list does not hold, or with another type, is refused here as at registration.
var predefined json.RawMessage
if raw, err := os.ReadFile(filepath.Join(tree, catalogue.PredefinedFile)); err == nil {
list, err := catalogue.ParsePredefined(raw)
if err != nil {
say("manifest", "INVALID: %v", err)
return Result{}, err
}
if problems := catalogue.PredefinedProblems(manifest, list); len(problems) > 0 {
err := fmt.Errorf("%s", strings.Join(problems, "; "))
say("manifest", "INVALID: %v", err)
return Result{}, err
}
predefined, _ = json.Marshal(list)
} else if !errors.Is(err, os.ErrNotExist) {
return Result{}, fmt.Errorf("%s's %s cannot be read: %w", repository, catalogue.PredefinedFile, err)
}
say("manifest", "%s v%s — %d artifact(s)", manifest.Module, manifest.Version, artifactCount(manifest))
// What it is made from, for its source fingerprint: the module's own tree first.
@@ -319,11 +268,6 @@ func build(ctx context.Context, run Runner, publish Publisher,
if src.tree, err = gitTree(ctx, run, tree, path); err != nil {
src.notPinned("its tree could not be named: " + err.Error())
}
// A module falling back to a predefined key was judged against the list, so a change to the list is a change
// to it: the planner rebuilds it, and its build publishes the list (ADR 0348 §7).
if predefined != nil && manifest.NamesPredefined() {
src.rootHas(catalogue.PredefinedFile)
}
// A build-time credential, written into the build context as .npmrc, but ONLY for a module that
// asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc.
@@ -364,9 +308,6 @@ func build(ctx context.Context, run Runner, publish Publisher,
var built []catalogue.Built
// stoodOn is every base the build was handed, as resolved — the edges the catalogue derives.
var stoodOn []string
// reused is the check build a walk took, and builtWhy why it built instead (novox/hq ADR 0350).
var reused *Reused
builtWhy := ""
if manifest.Build != nil {
// What this module said it stands on, answered with what this mesh actually holds. Done
// before anything is built, so a missing base is refused in front of the person who can
@@ -410,35 +351,7 @@ func build(ctx context.Context, run Runner, publish Publisher,
// Ordered, so two builds of one commit do the same work in the same sequence and their
// logs can be compared.
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
// **A walk takes the check's build of the same key** (novox/hq ADR 0350 §2): only a commit on its trunk, and
// only when asked — a check, a dry run, `build --ref` and `rebuild` never are. Its key is taken by the same
// steps a build takes, stopped before anything is built; what it takes is copied by digest.
if reuse != nil && reuse.keyFirst {
_, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
held, npmrcPath, npmrc, seatBases, src, say)
if err == nil && key.Key == "" {
err = fmt.Errorf("%w: %s", errNoKey, key.Why)
}
if err != nil {
say("reuse", "not built for the merge: %v", err)
return Result{Commit: commit, NoKey: err.Error()}, err
}
} else if reuse != nil && reuse.Scratch != nil {
taken, keyed, why := takeACheckBuild(ctx, run, publish, *reuse, manifest, within, workspace, commit,
credentials, artifacts, args, held, npmrcPath, npmrc, seatBases, src, onTrunk, say)
if taken != nil {
built = taken.Made
src = keyed
reused = taken
} else {
builtWhy = why
say("reuse", "built: %s", why)
}
}
for _, a := range artifacts {
if reused != nil {
break
}
say("artifact", "%s (%s%s) — starting", a.Name, a.Kind, langSuffix(a))
made, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held, npmrcPath, npmrc, seatBases, src, say)
if err != nil {
@@ -450,11 +363,6 @@ func build(ctx context.Context, run Runner, publish Publisher,
}
}
// Each template's text, read back from the archive the build made or took (novox/hq ADR 0349 §2).
if built, err = withTemplateTexts(manifest, within, built); err != nil {
say("templates", "FAILED: %v", err)
return Result{}, err
}
resolved, err := manifest.Resolve(built)
if err != nil {
say("resolve", "FAILED: %v", err)
@@ -479,123 +387,9 @@ func build(ctx context.Context, run Runner, publish Publisher,
say("source", "%d path(s) of %s", len(s.Paths), where)
}
}
// The reuse key (novox/hq ADR 0350 §1): what a check keeps its build under, and what a walk takes one by.
key := keyOf(src, manifest, Version)
return Result{Manifest: resolved, Commit: commit, Built: built,
Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint,
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources,
ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy,
Predefined: listed}, nil
}
// generationOf is how many commits the commit built stands on, itself included: of two commits on one trunk the
// later has more, so the controller keeps the list of the newer one whatever order it hears them in (ADR 0348
// §7). 0 when git could not say.
func generationOf(ctx context.Context, run Runner, tree, commit string) int {
out, err := run(ctx, tree, "git", "rev-list", "--count", commit)
if err != nil {
return 0
}
n, err := strconv.Atoi(strings.TrimSpace(out))
if err != nil || n < 0 {
return 0
}
return n
}
// takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by
// the steps a build takes, each artifact stopped before it is built; the build kept under it copied by digest into
// the module's namespace. Answers what it took and the source inputs its key was taken from, or why it builds. A
// copy that fails is said, and the walk builds (§3): it never fails the walk on its own.
func takeACheckBuild(ctx context.Context, run Runner, publish Publisher, reuse Reuse, manifest catalogue.Manifest,
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs, onTrunk bool,
say func(step, format string, args ...any)) (*Reused, *sourceInputs, string) {
if !onTrunk {
return nil, nil, "the commit is not on its trunk, and only a trunk commit's walk takes a check's build"
}
keyed, key, err := keyFor(ctx, run, publish, manifest, within, workspace, commit, credentials, artifacts, args,
held, npmrcPath, npmrc, seats, src, say)
if err != nil {
return nil, nil, err.Error()
}
if key.Key == "" {
return nil, nil, "it has no reuse key: " + key.Why
}
found, ok, err := reuse.Scratch.Find(ctx, manifest.Module, key.Key)
if err != nil {
return nil, nil, "the scratch namespace could not be read: " + err.Error()
}
if !ok {
return nil, nil, notKeptWhy(ctx, reuse, manifest.Module, key)
}
// **Only a build a verdict names** (ADR 0350 §2): the store takes writes from anything on the network, so a
// record is believed only when a check verdict of the walk's own pull requests names its key and its digests.
if why := unnamed(reuse.Named, manifest.Module, key.Key, found); why != "" {
return nil, nil, why
}
taken := &Reused{From: found}
for _, made := range found.Made {
copied, err := reuse.Scratch.Take(ctx, manifest.Module, made)
if err != nil {
why := fmt.Sprintf("copying %s's %s from %s failed (%v)", manifest.Module, made.Name, found.Said(), err)
say("reuse", "%s; built instead", why)
return nil, nil, why
}
taken.Made = append(taken.Made, copied)
}
// Every artifact the manifest declares, or none: a check build missing one is not this build.
if len(taken.Made) != len(artifacts) {
return nil, nil, fmt.Sprintf("%s holds %d artifact(s) of %s, which declares %d", found.Said(), len(taken.Made),
manifest.Module, len(artifacts))
}
say("reuse", "%s", taken.Words())
return taken, keyed, ""
}
// keyFor takes a build's reuse key by the steps a build takes, each artifact stopped before anything is built or
// published (ADR 0350 §2): its own copy of the source inputs, so the build's are untouched.
func keyFor(ctx context.Context, run Runner, publish Publisher, manifest catalogue.Manifest,
within, workspace, commit, credentials string, artifacts []catalogue.Artifact, args []string,
held map[string]string, npmrcPath string, npmrc Npmrc, seats map[string]string, src *sourceInputs,
say func(step, format string, args ...any)) (*sourceInputs, reuseKey, error) {
keyed := src.copy()
keyed.onlyKey = true
for _, a := range artifacts {
if _, err := one(ctx, run, publish, manifest.Module, within, workspace, commit, credentials, a, args, held,
npmrcPath, npmrc, seats, keyed, say); err != nil {
return nil, reuseKey{}, fmt.Errorf("its reuse key could not be taken: %w", err)
}
}
keyed.onlyKey = false
return keyed, keyOf(keyed, manifest, Version), nil
}
// notKeptWhy is why no check build has the key (§3): no check of the walk's pull requests built the module, the
// key differs — naming the part — or the build it made is gone.
func notKeptWhy(ctx context.Context, reuse Reuse, module string, key reuseKey) string {
if len(reuse.Pulls) == 0 {
return "no check built it with this key"
}
builds, err := reuse.Scratch.OfPulls(ctx, reuse.Pulls)
if err != nil {
return "no check build has this key, and which checks built it could not be read: " + err.Error()
}
for _, b := range builds {
if b.Module != module {
continue
}
if b.Key == key.Key {
return b.Said() + " built it with this key, and it was removed"
}
return "the key differs from " + b.Said() + "'s in " + strings.Join(differing(b.Parts, key.Parts), ", ")
}
var pulls []string
for _, p := range reuse.Pulls {
pulls = append(pulls, p.String())
}
return "no check of " + strings.Join(pulls, ", ") + " built it (a module its check did not move, or a check " +
"that did not finish)"
Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources, Predefined: predefined}, nil
}
// branchesHolding is every branch of a fresh clone's origin the commit is on, without `origin/`.
@@ -984,10 +778,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
switch a.Kind {
case catalogue.ArtifactUpstream:
// Its key is the image it copies, read from the manifest (keyOf): nothing to take here.
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
// Mirrored, not built: copied under a name of the mesh's own, so what a machine fetches is
// pinned by a digest this registry assigned rather than by a tag somebody else can move.
//
@@ -1116,10 +906,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
} else {
src.ownWhole()
}
// Taken for a reuse key (novox/hq ADR 0350 §2): everything its fingerprint reads is read; nothing is built.
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
invocation := append([]string{"build", "-f", recipePath, "-t", local}, args...)
if a.Target != "" {
invocation = append(invocation, "--target", a.Target)
@@ -1137,16 +923,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
}
say("image", "built, publishing")
// Its bases linked into its repository first, so the push needs none of their blobs from this
// machine's store, which may not hold them (novox/hq issue 694). Said and not fatal: a push
// from a store that holds them works without, and one that fails says which blob it lacked.
if lender, can := publish.(BaseLender); can {
if n, err := lender.LendBases(ctx, module+"/"+a.Name, buildArgValues(args)); err != nil {
say("image", "its bases could not all be linked into %s/%s before the push: %v", module, a.Name, err)
} else if n > 0 {
say("image", "%d blob(s) of its bases linked into %s/%s before the push", n, module, a.Name)
}
}
reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name)
if err != nil {
return catalogue.Built{}, err
@@ -1197,9 +973,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
src.notPinned(a.Name + " resolves packages of its own at build time")
}
}
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
// The module's own packages first, where the compiler and the bundler resolve them from
// (dependencies.go); nothing at all for a module whose package.json names only the SDK.
if err := installOwn(ctx, run, tree, chain, base, registry, say); err != nil {
@@ -1243,9 +1016,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
say("package", "building and publishing %s (%s)", a.Name, a.Language)
src.ownWhole()
src.notPinned(a.Name + " is a package, built from what the registry holds when it is built")
if src.keyOnly() {
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
reference, err := publishPackage(ctx, run, module, tree, a, npmrc, say)
if err != nil {
return catalogue.Built{}, fmt.Errorf("%s: publishing %s failed: %w", module, a.Name, err)
@@ -1254,12 +1024,6 @@ func one(ctx context.Context, run Runner, publish Publisher,
return catalogue.Built{Name: a.Name, Kind: a.Kind, Reference: reference}, nil
case catalogue.ArtifactArchive:
if src.keyOnly() {
if a.URL == "" {
src.ownHas(cleanEntry(a.From) + "/**")
}
return catalogue.Built{Name: a.Name, Kind: a.Kind}, nil
}
var body []byte
var err error
if a.URL != "" {
@@ -1498,19 +1262,6 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin
return args, resolved, nil
}
// buildArgValues is what each --build-arg of a build was handed: the bases, as this mesh holds them.
func buildArgValues(args []string) []string {
var values []string
for i := 0; i+1 < len(args); i += 2 {
if args[i] == "--build-arg" {
if _, value, has := strings.Cut(args[i+1], "="); has {
values = append(values, value)
}
}
}
return values
}
// compile runs a module's own code through its toolchain, and says where the result is.
//
// **In the module's own directory, under the path the toolchain expects.** A module is compiled
-2
View File
@@ -83,8 +83,6 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str
return r.tree + "\n", nil
case name == "git" && len(args) > 0 && args[0] == "rev-parse":
return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil
case name == "git" && len(args) > 1 && args[0] == "rev-list" && args[1] == "--count":
return "42\n", nil
}
return "", nil
}
-59
View File
@@ -98,28 +98,6 @@ type CheckSpec struct {
// and composed with it by the gate as one future state. The repository's own check is not run for a
// group: each member's pull request runs its own.
Group []GroupHead
// Builds are the modules the delivery plan moves, in the walk's order, and BuildWith what they are built with
// (novox/hq ADR 0350 §1): built from the merge the check judged into the scratch namespace once it has judged,
// for the walk to take. None for a group's composed check, whose members' own checks build.
Builds []CheckBuildSpec
BuildWith *CheckBuilding
// Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull
// request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read.
Tests string
StoreOnly bool
}
// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing
// for a pull request's check, which always runs whole.
func selection(spec CheckSpec) []string {
var env []string
if spec.Tests != "" {
env = append(env, EnvTests+"="+spec.Tests)
}
if spec.StoreOnly {
env = append(env, EnvStoreOnly+"=1")
}
return env
}
// GroupHead is one other head of a delivery group's composed check.
@@ -177,8 +155,6 @@ type CheckVerdict struct {
// Members are a delivery group's other heads as the composed check judged them, each with the trunk
// commit it was merged into (ADR 0323 §6).
Members []JudgedHead
// Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not.
Built []CheckBuilt
}
// Layer is one layer of a check, judged.
@@ -210,11 +186,6 @@ const (
// EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a
// pull request checked alone.
EnvGroup = "MESH_CHECK_GROUP"
// EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request
// asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's
// check. A script that reads neither runs whole.
EnvTests = "MESH_CHECK_TESTS"
EnvStoreOnly = "MESH_CHECK_STORE_ONLY"
)
// CheckTimeout bounds one check; a check that runs past it is an error, not a pass.
@@ -537,7 +508,6 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","),
EnvGroup + "=" + groupFile,
"GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")}
env = append(env, selection(spec)...)
v := CheckVerdict{}
modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...)
@@ -602,38 +572,9 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
v.Report, v.Took = redact.redact(withWhatFailed(out.String(), v.Repo)), time.Since(began)
say("check", "gate %s — %s; repository %s — %s (%s)", strings.ToUpper(v.Gate.Verdict), v.Gate.Summary,
strings.ToUpper(v.Repo.Verdict), v.Repo.Summary, v.Took.Round(time.Second))
// **What the merge would move, built once, here** (novox/hq ADR 0350 §1): after the judging, from the merge it
// judged, into the scratch namespace — so the walk of the merge commit takes it rather than building again.
if len(spec.Builds) > 0 && spec.BuildWith != nil && len(spec.Group) == 0 {
if why := skipsTheBuilds(v); why != "" {
say("check", "%s", why)
} else {
began := time.Now()
v.Built = buildForTheMerge(ctx, run, spec, *spec.BuildWith, workspace, tree, judged.Merge, judged, forge,
func(step, message string) {
if log != nil {
log(step, redact.redact(message))
}
}, say)
say("check", "built for the merge: %d of %d module(s) (%s)", builtCount(v.Built), len(v.Built),
time.Since(began).Round(time.Second))
}
}
return named(v), nil
}
// builtCount is how many of a check's builds for its merge were built.
func builtCount(built []CheckBuilt) int {
n := 0
for _, b := range built {
if b.NotBuilt == "" {
n++
}
}
return n
}
// ownCheck runs the repository's own check, part by part, each in its toolchain's own container: the
// layer's verdict. Nil when the check was ended from outside (not past its bound), which the caller
// answers as the context's error.
-151
View File
@@ -1,151 +0,0 @@
package builder
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// The builds a merge check makes for its merge (novox/hq ADR 0350 §1): every artifact of every module the delivery
// plan moves, built from the merged tree the check judged into the scratch namespace, kept under its reuse key, so
// the walk of the merge commit takes it instead of building again. Built after the check has judged, and only when
// it did not fail: a change that fails is not merged, and nothing it built would be taken.
// CheckBuildSpec is one module a check builds for its merge, in the walk's order (Tier).
type CheckBuildSpec struct {
Module string
// Repository, Path and Ref are where it is built from; Merged says from the merge the check made instead — the
// module is built from the repository checked, at the branch merged into.
Repository string
Path string
Ref string
Merged bool
Tier int
// Skip says why it cannot be built as the walk would build it: not built, and nothing after it.
Skip string
}
// CheckBuilding is what a check builds with: where it publishes and keeps its builds, and what a build is handed.
type CheckBuilding struct {
Publish Publisher
Scratch Scratch
Held map[string]string
Npmrc Npmrc
Seats map[string]string
}
// CheckBuilt is one module a check built for its merge, as its verdict lists it (§1): its key and its artifacts, or
// why it was not built.
type CheckBuilt struct {
Module string
Key string
Made []catalogue.Built
// NotBuilt says why it was not: no key, a build that failed, or a module after one of those.
NotBuilt string
Took time.Duration
}
// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
func skipsTheBuilds(v CheckVerdict) string {
for _, l := range []*Layer{v.Gate, v.Repo} {
if l != nil && (l.Verdict == "fail" || l.Verdict == "error") {
return "the check did not pass, so nothing is built for its merge"
}
}
return ""
}
// buildForTheMerge builds each module the plan moves, tier by tier, into the scratch namespace, and keeps what it
// built under its keys. A module standing after one not built is not built either: what it stands on in the walk
// is not what it would stand on here. tree is the check's clone and merge the merge commit it made there.
func buildForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with CheckBuilding, workspace, tree, merge string,
judged CheckVerdict, forge GitCredential, log Log, say func(step, format string, args ...any)) []CheckBuilt {
if len(spec.Builds) == 0 || with.Publish == nil || with.Scratch == nil {
return nil
}
ref := merge
if ref == "" {
ref = spec.Ref
}
held := map[string]string{}
for k, v := range with.Held {
held[k] = v
}
var out []CheckBuilt
var kept []CheckBuild
stopAt, stopWhy := -1, ""
for _, b := range spec.Builds {
began := time.Now()
if stopAt >= 0 && b.Tier > stopAt {
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: stopWhy})
continue
}
if b.Skip != "" {
say("build", "%s not built for the merge: %s", b.Module, b.Skip)
out = append(out, CheckBuilt{Module: b.Module, NotBuilt: b.Skip})
if stopAt < 0 || b.Tier < stopAt {
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
}
continue
}
repository, at := b.Repository, b.Ref
if b.Merged {
repository, at = tree, ref
}
say("build", "for the merge: %s (tier %d)", b.Module, b.Tier)
var mirrored []string
result, err := build(ctx, run, scratchPublisher{to: with.Publish}, repository, b.Path, at,
filepath.Join(workspace, "for-the-merge"), held, with.Npmrc, forge, log, &mirrored, &Reuse{keyFirst: true},
with.Seats)
built := CheckBuilt{Module: b.Module, Key: result.ReuseKey, Took: time.Since(began)}
switch {
case errors.Is(err, errNoKey):
built.NotBuilt = err.Error()
case err != nil:
built.NotBuilt = "its build failed: " + err.Error()
case result.ReuseKey == "":
built.NotBuilt = "it has no reuse key: " + result.NoKey
}
if built.NotBuilt != "" {
say("build", "%s not built for the merge: %s", b.Module, built.NotBuilt)
out = append(out, built)
if stopAt < 0 || b.Tier < stopAt {
stopAt, stopWhy = b.Tier, "it stands after "+b.Module+", which was not built for the merge"
}
continue
}
built.Made = result.Built
for _, m := range result.Built {
held[b.Module+"/"+m.Name] = m.Reference
}
say("build", "%s built for the merge under %s (%s)", b.Module, twelve(result.ReuseKey[3:]), built.Took.Round(time.Second))
out = append(out, built)
if len(result.Built) > 0 {
kept = append(kept, CheckBuild{Key: result.ReuseKey, Module: b.Module, Parts: result.ReuseParts,
Made: result.Built, Pull: PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number},
Head: judged.Head, Trunk: judged.Trunk, Check: spec.ID, At: time.Now().UTC()})
}
}
_ = os.RemoveAll(filepath.Join(workspace, "for-the-merge"))
if len(kept) > 0 {
if err := with.Scratch.Keep(ctx, PullRef{Owner: spec.Owner, Repo: spec.Repo, Number: spec.Number}, spec.ID,
time.Now().UTC(), kept); err != nil {
// Built and not findable: the walk builds, and says no check built it. Said, never the check's verdict.
say("build", "the builds for the merge could not be kept, and the walk will build: %v", err)
if err := with.Scratch.Discard(ctx, kept); err != nil {
say("build", "and what they made could not all be let go of: %v", err)
}
for i := range out {
if out[i].NotBuilt == "" {
out[i].NotBuilt = fmt.Sprintf("built, and could not be kept: %v", err)
}
}
}
}
return out
}
-12
View File
@@ -590,15 +590,3 @@ func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) {
t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer)
}
}
// A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a
// pull request's check is handed nothing, and runs whole.
func TestOnlyATrysSelectionReachesTheScript(t *testing.T) {
if env := selection(CheckSpec{Number: 7}); len(env) != 0 {
t.Fatalf("a pull request's check was handed a selection: %v", env)
}
env := selection(CheckSpec{Tests: "^TestX$", StoreOnly: true})
if len(env) != 2 || env[0] != "MESH_CHECK_TESTS=^TestX$" || env[1] != "MESH_CHECK_STORE_ONLY=1" {
t.Fatalf("a try's selection: %v", env)
}
}
-178
View File
@@ -1,178 +0,0 @@
package builder
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"runtime"
"strings"
)
// **A built image's bases are linked into its repository before it is pushed** (novox/hq issue 694).
//
// A push reads every blob of the image out of this machine's store, unless the repository it goes to
// already holds that blob: the runtime asks the registry first, blob by blob, and reads locally only
// what the registry lacks. A runtime keeping its images in containerd's store — the default for a
// fresh install from docker 29 on — does not always hold the compressed layers of the base an image
// was built FROM: a layer whose unpacked form it already had is never fetched. The push of such an
// image then fails on a layer this machine never had ("not all of them are available locally"), while
// the very bytes sit in this registry, under the repository the base was mirrored into.
//
// So the base's blobs are mounted into the image's repository first. A mount moves no bytes, and once
// the repository holds a blob the push asks this machine for nothing of it — whichever store the
// runtime keeps. A runtime that holds every layer (the classic store) is not changed by this: its push
// finds the same blobs already there.
// BaseLender links what a build's bases hold into the repository its image is about to be pushed to.
type BaseLender interface {
// LendBases mounts every blob of each base this registry holds — for the platform this machine
// builds — into `repository`, and answers how many it mounted.
LendBases(ctx context.Context, repository string, bases []string) (int, error)
}
// platformed is an index's entry: a manifest and the platform it is for.
type platformed struct {
descriptor
Platform *struct {
OS string `json:"os"`
Architecture string `json:"architecture"`
} `json:"platform"`
}
// LendBases mounts each base's blobs into the repository. A base this registry does not hold under its
// repository — named by tag, or held somewhere else — lends nothing and is no error: the push reads it
// from this machine's store as before.
func (r Registry) LendBases(ctx context.Context, repository string, bases []string) (int, error) {
mounted := 0
for _, base := range bases {
name, digest, pinned := strings.Cut(base, "@")
if !pinned || !strings.HasPrefix(digest, "sha256:") {
continue
}
where, err := parseReference(name + "@" + digest)
if err != nil {
continue
}
from := where.repository
if from == repository {
continue
}
blobs, held, err := r.blobsOf(ctx, from, digest)
if err != nil {
return mounted, fmt.Errorf("reading %s: %w", base, err)
}
if !held {
continue
}
for _, blob := range blobs {
did, err := r.mount(ctx, repository, from, blob)
if err != nil {
return mounted, fmt.Errorf("linking %s of %s into %s: %w", blob, base, repository, err)
}
if did {
mounted++
}
}
}
return mounted, nil
}
// blobsOf is the config and layers of the manifest under repository@digest — for an index, of each
// manifest it names for this machine's platform. False when the registry holds no such manifest there.
func (r Registry) blobsOf(ctx context.Context, repository, digest string) ([]string, bool, error) {
body, held, err := r.manifest(ctx, repository, digest)
if err != nil || !held {
return nil, held, err
}
var document struct {
Manifests []platformed `json:"manifests"`
Config *descriptor `json:"config"`
Layers []descriptor `json:"layers"`
}
if err := json.Unmarshal(body, &document); err != nil {
return nil, false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err)
}
var blobs []string
if document.Config != nil {
blobs = append(blobs, document.Config.Digest)
}
for _, l := range document.Layers {
blobs = append(blobs, l.Digest)
}
for _, m := range document.Manifests {
if m.Platform == nil || m.Platform.OS != runtime.GOOS || m.Platform.Architecture != runtime.GOARCH {
continue
}
more, held, err := r.blobsOf(ctx, repository, m.Digest)
if err != nil {
return nil, false, err
}
if held {
blobs = append(blobs, more...)
}
}
return blobs, true, nil
}
// manifest reads one manifest document from this registry; false when it is not there.
func (r Registry) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet,
"http://"+r.Address+"/v2/"+repository+"/manifests/"+digest, nil)
if err != nil {
return nil, false, err
}
request.Header.Set("Accept", manifestAccept)
response, err := r.client().Do(request)
if err != nil {
return nil, false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
}
defer response.Body.Close()
if response.StatusCode == http.StatusNotFound {
return nil, false, nil
}
if response.StatusCode != http.StatusOK {
return nil, false, fmt.Errorf("%s/%s@%s: %s", r.Address, repository, digest, response.Status)
}
body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20))
return body, err == nil, err
}
// mount links one blob from one repository into another; false when the repository already held it.
func (r Registry) mount(ctx context.Context, repository, from, digest string) (bool, error) {
base := "http://" + r.Address + "/v2/" + repository
if there, err := r.has(ctx, base+"/blobs/"+digest); err != nil {
return false, err
} else if there {
return false, nil
}
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
base+"/blobs/uploads/?mount="+digest+"&from="+from, nil)
if err != nil {
return false, err
}
begun, err := r.client().Do(start)
if err != nil {
return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err)
}
begun.Body.Close()
if begun.StatusCode == http.StatusCreated {
return true, nil
}
if begun.StatusCode == http.StatusAccepted {
// Not mounted, and an upload begun in its place: let go of it, so nothing is left half-open.
if location := begun.Header.Get("Location"); location != "" {
if strings.HasPrefix(location, "/") {
location = "http://" + r.Address + location
}
if cancel, err := http.NewRequestWithContext(ctx, http.MethodDelete, location, nil); err == nil {
if done, err := r.client().Do(cancel); err == nil {
done.Body.Close()
}
}
}
return false, fmt.Errorf("%s did not mount it from %s", r.Address, from)
}
return false, fmt.Errorf("%s answered %s when asked to mount it from %s", r.Address, begun.Status, from)
}
-140
View File
@@ -1,140 +0,0 @@
package builder
import (
"context"
"encoding/json"
"net/http/httptest"
"runtime"
"strings"
"testing"
)
// A built image's bases are linked into its repository before it is pushed (novox/hq issue 694): a
// runtime keeping containerd's image store may not hold a base layer it never had to fetch, and its push
// asks this machine only for what the repository lacks.
// aBaseForThisMachine puts an index into the registry under `repository`: one manifest for this
// machine's platform, one for another, and answers the index's digest and each manifest's blobs.
func aBaseForThisMachine(t *testing.T, dst *aRegistryOfRepositories, repository string) (string, []string, []string) {
t.Helper()
put := func(arch string) (string, []string) {
config := []byte(`{"architecture":"` + arch + `"}`)
layer := []byte(arch + " layer bytes")
for _, b := range [][]byte{config, layer} {
dst.blobs[digestOf(b)] = b
dst.link(repository, digestOf(b))
}
m, _ := json.Marshal(map[string]any{
"schemaVersion": 2, "mediaType": mediaManifestOCI,
"config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)},
"layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digestOf(layer), "size": len(layer)}},
})
dst.manifests[repository+"@"+digestOf(m)] = m
return digestOf(m), []string{digestOf(config), digestOf(layer)}
}
other := "s390x"
if runtime.GOARCH == other {
other = "amd64"
}
mine, mineBlobs := put(runtime.GOARCH)
theirs, theirBlobs := put(other)
index, _ := json.Marshal(map[string]any{
"schemaVersion": 2, "mediaType": mediaIndexOCI,
"manifests": []map[string]any{
{"mediaType": mediaManifestOCI, "digest": mine, "size": 1, "platform": map[string]string{"os": runtime.GOOS, "architecture": runtime.GOARCH}},
{"mediaType": mediaManifestOCI, "digest": theirs, "size": 1, "platform": map[string]string{"os": "linux", "architecture": other}},
},
})
dst.manifests[repository+"@"+digestOf(index)] = index
return digestOf(index), mineBlobs, theirBlobs
}
func TestABuildsBaseIsLinkedIntoItsRepositoryBeforeThePush(t *testing.T) {
dst := newRegistryOfRepositories()
server := httptest.NewServer(dst.handler())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
r := Registry{Address: address}
const from = "upstream/docker.io/library/golang"
index, mine, theirs := aBaseForThisMachine(t, dst, from)
bases := []string{
address + "/" + from + "@" + index,
// Not bases this registry holds: a package version, a tag, an image held nowhere here.
"@novox/mesh-sdk@1.2.3",
"golang:1-alpine",
"elsewhere.invalid/thing@sha256:" + strings.Repeat("e", 64),
}
n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases)
if err != nil {
t.Fatal(err)
}
// Every blob of this machine's platform is in the image's repository, so a push asks this machine's
// store for none of them; another platform's are not, since nothing built here stands on them.
for _, blob := range mine {
if !dst.links["mesh-tools-go/build"][blob] {
t.Fatalf("the base's blob %s is not in the image's repository", blob)
}
}
for _, blob := range theirs {
if dst.links["mesh-tools-go/build"][blob] {
t.Fatalf("another platform's blob %s was linked", blob)
}
}
if n != len(mine) || dst.mounts != len(mine) || dst.uploads != 0 {
t.Fatalf("%d linked, %d mounts, %d uploads; want %d mounts and no bytes moved", n, dst.mounts, dst.uploads, len(mine))
}
// Asked again, nothing is mounted twice.
if n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases); err != nil || n != 0 {
t.Fatalf("a second lending mounted %d: %v", n, err)
}
}
// A registry that will not mount is said, not hidden: the push is still tried, from the store.
func TestABaseTheRegistryWillNotMountIsAnError(t *testing.T) {
dst := newRegistryOfRepositories()
dst.noMount = true
server := httptest.NewServer(dst.handler())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
index, _, _ := aBaseForThisMachine(t, dst, "upstream/thing")
_, err := Registry{Address: address}.LendBases(context.Background(), "hello/server",
[]string{address + "/upstream/thing@" + index})
if err == nil || !strings.Contains(err.Error(), "did not mount") {
t.Fatalf("a refused mount passed silently: %v", err)
}
}
// lending records what a build lent, and when, against the push.
type lending struct {
*mirroring
order []string
}
func (l *lending) LendBases(_ context.Context, repository string, bases []string) (int, error) {
l.order = append(l.order, "lend "+repository+" "+strings.Join(bases, " "))
return len(bases), nil
}
func (l *lending) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
l.order = append(l.order, "push "+repository)
return l.mirroring.PublishImage(ctx, localTag, repository)
}
func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) {
r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"})
r.contents["modules/bus/module.json"] = anImage
l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}}
if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace,
nil, Npmrc{}, GitCredential{}, nil); err != nil {
t.Fatal(err)
}
want := []string{
"lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64),
"push bus/server",
}
if strings.Join(l.order, "\n") != strings.Join(want, "\n") {
t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n"))
}
}
-106
View File
@@ -1,106 +0,0 @@
package builder
import (
"context"
"slices"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A build reads the list of predefined keys at the root of the repository at the commit it builds (novox/hq ADR
// 0348 §7, design 57 package K2, issue 661): it refuses a list that is wrong, judges the manifest against it, and
// carries it with the commit's generation for the controller to publish.
const theFirstKeys = `{
"font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
"agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
}`
// fallingBack is a module of the catalogue whose font size falls back to the predefined key named.
func fallingBack(key, kind string) string {
def := "10"
if kind == "string" {
def = `"10"`
}
return `{"module":"dunst","version":"1",
"build":{"artifacts":[{"name":"look","kind":"archive","from":"files"}]},
"config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points",
"default":` + def + `,"predefined":"` + key + `"}},
"resources":[
{"id":"theme","type":"archive","path":"/opt/dunst","artifact":"look"},
{"id":"rc","type":"file","path":"/etc/xdg/dunst/dunstrc","content":"font = Inter ${config:font-size}\n"}]}`
}
func buildInTheCatalogue(t *testing.T, manifest string, list *string) (Result, error) {
t.Helper()
r := &recorded{contents: map[string]string{"modules/dunst/" + ManifestName: manifest,
"modules/dunst/files/theme.conf": "dark", "README.md": "the catalogue"}}
if list != nil {
r.contents[catalogue.PredefinedFile] = *list
}
return Build(context.Background(), onTrunk{recorded: r}.run, r, "https://forge.invalid/mesh-catalog.git",
"modules/dunst", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil)
}
func TestABuildNamingAPredefinedKeyIsJudgedAgainstItsTreesListAndCarriesIt(t *testing.T) {
list := theFirstKeys
got, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &list)
if err != nil {
t.Fatalf("a module falling back to font-size did not build: %v", err)
}
if got.Predefined == nil || got.Predefined.Generation != 42 {
t.Fatalf("the list was not carried with its generation: %+v", got.Predefined)
}
carried, err := catalogue.ParsePredefined(got.Predefined.Keys)
if err != nil || len(carried) != 2 || carried["font-size"].Type != catalogue.ConfigNumber {
t.Fatalf("the list carried is %s (%v)", got.Predefined.Keys, err)
}
// A change to the list is a change to the module that was judged against it.
if len(got.Sources) == 0 || !slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
t.Fatalf("its build source leaves the list out: %+v", got.Sources)
}
for _, c := range []struct{ manifest, says string }{
{fallingBack("font-scale", "number"), `"font-scale", which is not on the list`},
{fallingBack("font-size", "string"), "which is a number, and this value is a string"},
} {
if _, err := buildInTheCatalogue(t, c.manifest, &list); err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("wanted the build refused saying %q, got %v", c.says, err)
}
}
}
func TestABuildRefusesAWrongList(t *testing.T) {
wrong := `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`
if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &wrong); err == nil ||
!strings.Contains(err.Error(), `unknown field "unit"`) {
t.Fatalf("a wrong list was built with: %v", err)
}
}
// A repository without the list builds as before, carrying none; a value naming a key there is refused.
func TestABuildWithoutAListCarriesNone(t *testing.T) {
plain := strings.Replace(fallingBack("font-size", "number"), `,"predefined":"font-size"`, "", 1)
got, err := buildInTheCatalogue(t, plain, nil)
if err != nil {
t.Fatal(err)
}
if got.Predefined != nil {
t.Fatalf("a list was carried from a tree holding none: %+v", got.Predefined)
}
if slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
t.Fatalf("the build source names a list: %+v", got.Sources)
}
if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), nil); err == nil ||
!strings.Contains(err.Error(), "is not present") {
t.Fatalf("a key named where no list is: %v", err)
}
// A list present and a module naming no key: carried, and not the module's build source.
list := theFirstKeys
got, err = buildInTheCatalogue(t, plain, &list)
if err != nil || got.Predefined == nil || slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) {
t.Fatalf("a module naming no key: %+v %+v %v", got.Predefined, got.Sources, err)
}
}
-615
View File
@@ -1,615 +0,0 @@
package builder
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/novox/mesh-controller/internal/artifacts"
"github.com/novox/mesh-controller/internal/catalogue"
)
// One build per change (novox/hq ADR 0350).
//
// **The merge check builds what the merge would move** (§1): every artifact of every module the delivery plan
// moves, from the merged tree it checks, into the artifact store's scratch namespace — never a module's own —
// kept under a **reuse key**: the source fingerprint as the build takes it (the module's tree, its contexts' trees,
// its bases and toolchains by digest; issue 280) with the builder's own version and, for an upstream artifact, the
// image it copies. **The walk of a trunk commit takes that build** (§2): before building, it takes the reuse key
// of the trunk commit's tree the same way, and where a check build holds the same key it copies it by digest into
// the module's namespace and answers with that, building nothing. Otherwise it builds and says why (§3): no check
// built it, the key differs (naming the part), or the check build is gone; a copy that fails is said and built.
//
// **What the key names is what a fresh build could differ by.** The tree is every file the recipe and the
// compiler see; the contexts are the other repositories it reads, by tree; the bases and toolchains are what it
// stands on, by digest; the builder's version is how it is built. A build whose source does not pin it — one that
// resolves packages from the mesh's registry when it is built, or copies an upstream image named by tag — has no
// key, and is always built.
// Version is this builder's own version, as the reuse key takes it: the digest of the running executable, so any
// change to how it builds is a change of key (§1). Set once by the build seat's holder (SelfVersion); empty — no
// key, every build its own — when it cannot be said.
var Version string
// SelfVersion is the version the reuse key takes: the running executable's digest, or the version stamped at build
// when the executable cannot be read. Empty when neither says anything ("development" says nothing: every unstamped
// builder would read as one).
func SelfVersion(stamped string) string {
if exe, err := os.Executable(); err == nil {
if f, err := os.Open(exe); err == nil {
defer f.Close()
sum := sha256.New()
if _, err := io.Copy(sum, f); err == nil {
return "exe:sha256:" + hex.EncodeToString(sum.Sum(nil))
}
}
}
if stamped == "" || stamped == "development" {
return ""
}
return stamped
}
// ScratchPrefix is the namespace a check builds into (novox/hq ADR 0239 §9, ADR 0350 §1): `scratch/<module>/<artifact>`.
// Registration refuses a reference into it; only the walk's copy, in the module's own namespace, is registered.
const ScratchPrefix = "scratch/"
// checksRepository holds one document per check that built: which builds it made, of which pull request, when.
// What the holder lets go of by (§5).
const checksRepository = ScratchPrefix + "checks"
// reuseTagPrefix names a check build's record under its module's scratch repository: `r1-<hex>`.
const reuseTagPrefix = "r1-"
// CheckBuildsKept is how long a check build is kept in the scratch namespace (§5).
const CheckBuildsKept = 7 * 24 * time.Hour
// The parts of a reuse key, as the walk names the one that differs (§3).
const (
partModule = "the module"
partTree = "the build source"
partBases = "a base image"
partBuilder = "the builder's version"
partContext = "a context"
partTool = "a toolchain"
partUpstream = "an upstream image"
)
// reuseKey is a build's reuse key and the parts it was made from; Key empty, with why, when it has none.
type reuseKey struct {
Key string
Parts map[string]string
Why string
}
// keyOf is the reuse key of what src says the build was made from, built by version (ADR 0350 §1).
func keyOf(src *sourceInputs, manifest catalogue.Manifest, version string) reuseKey {
if src == nil || src.unpinned != "" || src.tree == "" {
why := "its source does not pin the build"
if src != nil && src.unpinned != "" {
why = src.unpinned
} else if src != nil && src.tree == "" {
why = "its tree could not be named"
}
return reuseKey{Why: why}
}
if version == "" {
return reuseKey{Why: "the builder's version cannot be said"}
}
parts := map[string]string{partModule: src.module, partTree: src.tree, partBuilder: version}
var bases []string
seen := map[string]bool{}
for _, b := range src.bases {
if d := referenceDigest(b); !seen[d] {
seen[d] = true
bases = append(bases, d)
}
}
sort.Strings(bases)
parts[partBases] = strings.Join(bases, " ")
for a, t := range src.contexts {
parts[partContext+" "+a] = t
}
for a, t := range src.toolchains {
parts[partTool+" "+a] = t
}
// An upstream artifact is copied, not built: what it copies is in module.json, and so in the tree — but a tag
// names whatever its registry says today, so only an image pinned by digest is the same image tomorrow.
if manifest.Build != nil {
for _, a := range manifest.Build.Artifacts {
if a.Kind != catalogue.ArtifactUpstream {
continue
}
if !strings.Contains(a.From, "@sha256:") {
return reuseKey{Why: fmt.Sprintf("%s copies %s, named by tag, which can move", a.Name, a.From)}
}
parts[partUpstream+" "+a.Name] = a.From
}
}
lines := make([]string, 0, len(parts))
for k, v := range parts {
lines = append(lines, k+"="+v)
}
sort.Strings(lines)
sum := sha256.Sum256([]byte(strings.Join(lines, "\n")))
return reuseKey{Key: "r1:" + hex.EncodeToString(sum[:]), Parts: parts}
}
// tagOf is the tag a key's record is kept under.
func tagOf(key string) string { return reuseTagPrefix + strings.TrimPrefix(key, "r1:") }
// differing names the parts of two keys that differ, in a fixed order: what the walk says when it builds (§3).
func differing(was, now map[string]string) []string {
names := map[string]bool{}
for k := range was {
names[k] = true
}
for k := range now {
names[k] = true
}
var out []string
for k := range names {
if was[k] != now[k] {
out = append(out, k)
}
}
sort.Strings(out)
return out
}
// PullRef names a pull request.
type PullRef struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
}
func (p PullRef) String() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) }
// slug is the pull request as a tag's first part: owner.repo.number, every other character a dash.
func (p PullRef) slug() string {
clean := func(s string) string {
return strings.Map(func(r rune) rune {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' {
return r
}
return '-'
}, s)
}
return clean(p.Owner) + "." + clean(p.Repo) + "." + strconv.Itoa(p.Number)
}
// CheckBuild is one module a check built into the scratch namespace, kept under its reuse key (§1).
type CheckBuild struct {
Key string `json:"key"`
Module string `json:"module"`
Parts map[string]string `json:"parts"`
// Made are its artifacts, in the scratch namespace.
Made []catalogue.Built `json:"made"`
// Pull is the pull request checked, Head its head and Trunk the trunk commit it was merged into; Check the
// check's ask; At when it was built.
Pull PullRef `json:"pull"`
Head string `json:"head,omitempty"`
Trunk string `json:"trunk,omitempty"`
Check string `json:"check"`
At time.Time `json:"at"`
}
// Said is the check build as the walk names it: "the check of owner/repo#n at head H (trunk T)".
func (c CheckBuild) Said() string {
s := "the check of " + c.Pull.String()
if c.Head != "" {
s += " at head " + twelve(c.Head)
}
if c.Trunk != "" {
s += " (merged into trunk " + twelve(c.Trunk) + ")"
}
return s
}
// checkIndex is one check's document under checksRepository: every build it made.
type checkIndex struct {
Pull PullRef `json:"pull"`
Check string `json:"check"`
At time.Time `json:"at"`
Builds []CheckBuild `json:"builds"`
}
// Scratch is where check builds are kept and taken from: the artifact store's scratch namespace (ADR 0350). The
// Registry is one; a test's is a map.
type Scratch interface {
// Keep records a check's builds: each under its module and key, and the check's own document.
Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error
// Find is the check build kept under a module's key; false when none is.
Find(ctx context.Context, module, key string) (CheckBuild, bool, error)
// OfPulls are the check builds kept for those pull requests, newest first: what the walk names a differing
// key from.
OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error)
// Take copies one artifact of a check build by digest into the module's own namespace, and answers it there.
Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error)
// Forget lets go of every check build of a pull request, and Expire of every check older than the bound;
// each answers how many checks it let go of.
Forget(ctx context.Context, pull PullRef) (int, error)
Expire(ctx context.Context, older time.Duration, now time.Time) (int, error)
// Discard lets go of what builds a check could not keep made: nothing would ever find them.
Discard(ctx context.Context, builds []CheckBuild) error
}
// scratchPublisher is a Publisher into the scratch namespace: what a check builds is never under a module's
// own name (ADR 0239 §9). A base the build stands on is copied where every build copies it.
type scratchPublisher struct{ to Publisher }
func (s scratchPublisher) PublishImage(ctx context.Context, localTag, repository string) (string, error) {
return s.to.PublishImage(ctx, localTag, ScratchPrefix+repository)
}
func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest)
}
func (s scratchPublisher) LendBases(ctx context.Context, repository string, bases []string) (int, error) {
if l, can := s.to.(BaseLender); can {
return l.LendBases(ctx, ScratchPrefix+repository, bases)
}
return 0, nil
}
func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) {
m, can := s.to.(Mirrorer)
if !can {
return "", errors.New("the artifact store here cannot copy an upstream image")
}
return m.MirrorImage(ctx, from, ScratchPrefix+repository)
}
func (s scratchPublisher) MirrorBase(ctx context.Context, from, repository, former string) (string, error) {
if m, can := s.to.(BaseMirrorer); can {
return m.MirrorBase(ctx, from, repository, former)
}
if m, can := s.to.(Mirrorer); can {
return m.MirrorImage(ctx, from, repository)
}
return "", errors.New("the artifact store here cannot copy a base")
}
// The registry as the scratch namespace.
func (r Registry) store() artifacts.Store {
return artifacts.Store{Address: r.Address, HTTP: r.client()}
}
const recordMedia = "application/vnd.novox.mesh.check-build.v1+json"
// Keep puts each build's record under its module's scratch repository, by key, and the check's document.
func (r Registry) Keep(ctx context.Context, pull PullRef, check string, at time.Time, builds []CheckBuild) error {
store := r.store()
for _, b := range builds {
body, err := json.Marshal(b)
if err != nil {
return err
}
if _, err := store.PutTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key), recordMedia, body); err != nil {
return fmt.Errorf("keeping %s's check build: %w", b.Module, err)
}
}
body, err := json.Marshal(checkIndex{Pull: pull, Check: check, At: at, Builds: builds})
if err != nil {
return err
}
if _, err := store.PutTagged(ctx, checksRepository, pull.slug()+"."+check, recordMedia, body); err != nil {
return fmt.Errorf("keeping which builds the check of %s made: %w", pull, err)
}
return nil
}
// Find reads the record kept under a module's key.
func (r Registry) Find(ctx context.Context, module, key string) (CheckBuild, bool, error) {
body, _, err := r.store().GetTagged(ctx, ScratchPrefix+module, tagOf(key))
if errors.Is(err, artifacts.ErrNoTag) {
return CheckBuild{}, false, nil
}
if err != nil {
return CheckBuild{}, false, err
}
var b CheckBuild
if err := json.Unmarshal(body, &b); err != nil {
return CheckBuild{}, false, fmt.Errorf("%s's check build under %s is not one: %w", module, tagOf(key), err)
}
if b.Key != key || b.Module != module {
return CheckBuild{}, false, nil
}
return b, true, nil
}
// checks are the check documents kept, by tag; only those of the pull requests named when any are.
func (r Registry) checks(ctx context.Context, pulls []PullRef) (map[string]checkIndex, error) {
store := r.store()
tags, err := store.Tags(ctx, checksRepository)
if err != nil {
return nil, err
}
wanted := map[string]PullRef{}
for _, p := range pulls {
wanted[p.slug()+"."] = p
}
out := map[string]checkIndex{}
for _, tag := range tags {
if len(pulls) > 0 {
match := false
for prefix := range wanted {
match = match || strings.HasPrefix(tag, prefix)
}
if !match {
continue
}
}
body, _, err := store.GetTagged(ctx, checksRepository, tag)
if errors.Is(err, artifacts.ErrNoTag) {
continue
}
if err != nil {
return nil, err
}
var c checkIndex
if err := json.Unmarshal(body, &c); err != nil {
continue // not a document this wrote: left to nobody, said by nothing — it names no build
}
if len(pulls) > 0 {
if p := wanted[c.Pull.slug()+"."]; p != c.Pull {
continue // a slug two pull requests share: only the one named
}
}
out[tag] = c
}
return out, nil
}
// OfPulls are the check builds of those pull requests, newest first.
func (r Registry) OfPulls(ctx context.Context, pulls []PullRef) ([]CheckBuild, error) {
if len(pulls) == 0 {
return nil, nil
}
checks, err := r.checks(ctx, pulls)
if err != nil {
return nil, err
}
var out []CheckBuild
for _, c := range checks {
out = append(out, c.Builds...)
}
sort.SliceStable(out, func(i, j int) bool { return out[i].At.After(out[j].At) })
return out, nil
}
// Take copies one artifact of a check build into the module's namespace by digest: an image's manifests as they
// are, so the digest registered is the digest the check built; an archive's blob mounted across, and held.
func (r Registry) Take(ctx context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
repository := module + "/" + made.Name
from := ScratchPrefix + repository
if made.Digest != "" || strings.Contains(made.Reference, "/blobs/") {
digest := made.Digest
if digest == "" {
_, digest, _ = strings.Cut(made.Reference, "/blobs/")
}
where := upstream{base: "http://" + r.Address, repository: from, reference: digest}
if err := r.copyBlob(ctx, &source{client: r.client(), mountFrom: from}, where, digest, repository); err != nil {
return catalogue.Built{}, err
}
size, err := r.blobSize(ctx, repository, digest)
if err != nil {
return catalogue.Built{}, err
}
if err := r.hold(ctx, repository, digest, int(size)); err != nil {
return catalogue.Built{}, err
}
return catalogue.Built{Name: made.Name, Kind: made.Kind, Digest: digest, Launchers: made.Launchers,
Reference: "http://" + r.Address + "/v2/" + repository + "/blobs/" + digest}, nil
}
_, digest, pinned := strings.Cut(made.Reference, "@")
if !pinned || !strings.HasPrefix(digest, "sha256:") {
return catalogue.Built{}, fmt.Errorf("%s's check build %s is not pinned by digest", module, made.Reference)
}
reference, err := r.MirrorBase(ctx, r.Address+"/"+from+"@"+digest, repository, from)
if err != nil {
return catalogue.Built{}, err
}
if !strings.HasSuffix(reference, "@"+digest) {
return catalogue.Built{}, fmt.Errorf("%s's check build was copied as %s, not as %s", module, reference, digest)
}
return catalogue.Built{Name: made.Name, Kind: made.Kind, Reference: reference, Launchers: made.Launchers}, nil
}
// blobSize is the size the store says a blob has.
func (r Registry) blobSize(ctx context.Context, repository, digest string) (int64, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+r.Address+"/v2/"+repository+"/blobs/"+digest, nil)
if err != nil {
return 0, err
}
response, err := r.client().Do(request)
if err != nil {
return 0, err
}
response.Body.Close()
if response.StatusCode != http.StatusOK || response.ContentLength < 0 {
return 0, fmt.Errorf("%s/blobs/%s: %s", repository, digest, response.Status)
}
return response.ContentLength, nil
}
// Forget lets go of every check build of a pull request (§5): what each check built in the scratch namespace, the
// record under its key when it is still that check's, and the check's document.
func (r Registry) Forget(ctx context.Context, pull PullRef) (int, error) {
all, err := r.checks(ctx, nil)
if err != nil {
return 0, err
}
mine := map[string]checkIndex{}
for tag, c := range all {
if c.Pull == pull {
mine[tag] = c
}
}
return r.letGo(ctx, mine, all)
}
// Expire lets go of every check older than the bound (§5).
func (r Registry) Expire(ctx context.Context, older time.Duration, now time.Time) (int, error) {
checks, err := r.checks(ctx, nil)
if err != nil {
return 0, err
}
old := map[string]checkIndex{}
for tag, c := range checks {
if now.Sub(c.At) > older {
old[tag] = c
}
}
return r.letGo(ctx, old, checks)
}
// Discard lets go of builds no check document names.
func (r Registry) Discard(ctx context.Context, builds []CheckBuild) error {
all, err := r.checks(ctx, nil)
if err != nil {
return err
}
_, err = r.letGo(ctx, map[string]checkIndex{"": {Builds: builds}}, all)
return err
}
// letGo lets go of each check's builds, then its document; a check whose builds could not all be let go of keeps
// its document, so the next sweep asks again. **A build another check still names stays**: deletion is by digest,
// and two checks that built one digest in one repository — a check asked again, a dependent two pull requests move —
// share it; it goes with the last check that names it. all are every check kept, the ones going among them.
func (r Registry) letGo(ctx context.Context, checks, all map[string]checkIndex) (int, error) {
store := r.store()
stays := map[string]bool{}
for tag, c := range all {
if _, going := checks[tag]; going {
continue
}
for _, b := range c.Builds {
for _, m := range b.Made {
stays[catalogue.Recorded(m.Reference)] = true
}
}
}
gone := 0
var failed []string
for tag, c := range checks {
ok := true
for _, b := range c.Builds {
for _, m := range b.Made {
reference := catalogue.Recorded(m.Reference)
if path, kept := catalogue.InArtifactStore(reference); !kept || !strings.HasPrefix(path, ScratchPrefix) {
continue // never anything outside the scratch namespace
}
if stays[reference] {
continue // another check still names it
}
if err := store.LetGo(ctx, reference); err != nil && !errors.Is(err, artifacts.Gone) {
ok = false
failed = append(failed, fmt.Sprintf("%s: %v", m.Reference, err))
}
}
// The record under the key, only while it is still this check's: a later check of the same key
// replaced it, and that one is its own check's to let go of.
if kept, found, err := r.Find(ctx, b.Module, b.Key); err == nil && found && kept.Check == c.Check {
if err := store.DeleteTagged(ctx, ScratchPrefix+b.Module, tagOf(b.Key)); err != nil && !errors.Is(err, artifacts.Gone) {
ok = false
failed = append(failed, fmt.Sprintf("%s's record: %v", b.Module, err))
}
}
}
if !ok {
continue
}
if tag == "" {
gone++
continue
}
if err := store.DeleteTagged(ctx, checksRepository, tag); err != nil && !errors.Is(err, artifacts.Gone) {
failed = append(failed, fmt.Sprintf("%s: %v", tag, err))
continue
}
gone++
}
if len(failed) > 0 {
sort.Strings(failed)
return gone, fmt.Errorf("not let go of: %s", strings.Join(failed, "; "))
}
return gone, nil
}
// Reused is what the walk took from a check instead of building (§2, §4).
type Reused struct {
From CheckBuild
// Made are the copies in the module's namespace: what is registered.
Made []catalogue.Built
}
// Words is the walk's word for a module it took: "reused from <check build>", with each digest (§4).
func (r Reused) Words() string {
var digests []string
for _, m := range r.Made {
d := m.Digest
if d == "" {
_, d, _ = strings.Cut(m.Reference, "@")
}
digests = append(digests, m.Name+" "+d)
}
return "reused from " + r.From.Said() + ": " + strings.Join(digests, ", ")
}
// NamedBuild is a build a check's verdict names (ADR 0350 §2): its module, key and each artifact's digest.
type NamedBuild struct {
Pull PullRef
Check string
Module string
Key string
Digests map[string]string
}
// DigestOf is the digest a reference names: an image's after `@`, an archive's after `/blobs/`.
func DigestOf(reference string) string {
if _, d, ok := strings.Cut(reference, "/blobs/"); ok {
return d
}
if _, d, ok := strings.Cut(reference, "@"); ok {
return d
}
return ""
}
// unnamed is why a record found under a key is not taken: no verdict of the walk's pull requests names that module,
// key and every one of its digests. Empty when one does.
func unnamed(named []NamedBuild, module, key string, found CheckBuild) string {
for _, n := range named {
if n.Module != module || n.Key != key || n.Check != found.Check || len(n.Digests) != len(found.Made) {
continue
}
same := true
for _, m := range found.Made {
d := m.Digest
if d == "" {
d = DigestOf(m.Reference)
}
same = same && d != "" && n.Digests[m.Name] == d
}
if same {
return ""
}
}
return "a build is kept under this key, and no check verdict of the walk's pull requests names it with its " +
"digests: not taken"
}
-639
View File
@@ -1,639 +0,0 @@
package builder
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"slices"
"strconv"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// One build per change (novox/hq ADR 0350): the merge check builds what the merge would move into the scratch
// namespace under its reuse key, and the walk of the trunk commit takes that build by digest instead of building.
// aScratch is the scratch namespace in memory: check builds by module and key, and what the walk took.
type aScratch struct {
builds map[string]CheckBuild
took []string
failTake bool
}
func newScratch() *aScratch { return &aScratch{builds: map[string]CheckBuild{}} }
func (s *aScratch) Keep(_ context.Context, _ PullRef, _ string, _ time.Time, builds []CheckBuild) error {
for _, b := range builds {
s.builds[b.Module+" "+b.Key] = b
}
return nil
}
func (s *aScratch) Find(_ context.Context, module, key string) (CheckBuild, bool, error) {
b, ok := s.builds[module+" "+key]
return b, ok, nil
}
func (s *aScratch) OfPulls(_ context.Context, pulls []PullRef) ([]CheckBuild, error) {
var out []CheckBuild
for _, b := range s.builds {
if slices.Contains(pulls, b.Pull) {
out = append(out, b)
}
}
return out, nil
}
func (s *aScratch) Take(_ context.Context, module string, made catalogue.Built) (catalogue.Built, error) {
if s.failTake {
return catalogue.Built{}, errors.New("the store refused the mount")
}
s.took = append(s.took, made.Reference)
made.Reference = strings.Replace(made.Reference, ScratchPrefix, "", 1)
return made, nil
}
func (s *aScratch) Forget(context.Context, PullRef) (int, error) { return 0, nil }
func (s *aScratch) Expire(context.Context, time.Duration, time.Time) (int, error) { return 0, nil }
func (s *aScratch) Discard(context.Context, []CheckBuild) error { return nil }
const reusedModule = `{"module":"meshboard","version":"1",
"build":{"artifacts":[
{"name":"server","kind":"image","from":"Dockerfile"},
{"name":"look","kind":"archive","from":"files"}]},
"resources":[
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
{"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}`
// aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk.
func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) {
t.Helper()
return aModuleWith(t, tree, map[string]string{"mod/" + ManifestName: reusedModule,
"mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"})
}
// aModuleWith is aModuleAt with the repository's files given.
func aModuleWith(t *testing.T, tree string, contents map[string]string) (*recorded, Runner, string) {
t.Helper()
r := &recorded{tree: tree, contents: contents}
run := func(ctx context.Context, dir, name string, args ...string) (string, error) {
if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" {
r.ran = append(r.ran, name+" "+strings.Join(args, " "))
return "origin/main\n", nil
}
return r.run(ctx, dir, name, args...)
}
return r, run, t.TempDir()
}
func dockerBuilt(r *recorded) bool {
for _, line := range r.ran {
if strings.HasPrefix(line, "docker build") {
return true
}
}
return false
}
var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7}
// aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace.
func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt {
t.Helper()
r, run, workspace := aModuleAt(t, tree)
return aCheckBuiltBy(t, scratch, r, run, workspace)
}
// aCheckBuiltBy is aCheckBuilt of the repository given.
func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt {
t.Helper()
spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number,
Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod",
Ref: "main"}}}
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "",
CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil))
if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" {
t.Fatalf("the check did not build the module for its merge: %+v", built)
}
if !dockerBuilt(r) {
t.Fatal("the check built no image")
}
for repository := range r.images {
if !strings.HasPrefix(repository, ScratchPrefix) {
t.Fatalf("the check published %s outside the scratch namespace", repository)
}
}
for repository := range r.archives {
if !strings.HasPrefix(repository, ScratchPrefix) {
t.Fatalf("the check published %s outside the scratch namespace", repository)
}
}
return built[0]
}
// named is the walk's leave naming what a check's verdict says it built, as the controller reads it.
func named(c CheckBuilt) []NamedBuild {
n := NamedBuild{Pull: aPull, Check: "build-1", Module: c.Module, Key: c.Key, Digests: map[string]string{}}
for _, m := range c.Made {
d := m.Digest
if d == "" {
d = DigestOf(m.Reference)
}
n.Digests[m.Name] = d
}
return []NamedBuild{n}
}
func withVersion(t *testing.T, v string) {
t.Helper()
was := Version
Version = v
t.Cleanup(func() { Version = was })
}
// A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest
// and registers that digest.
func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
checked := aCheckBuilt(t, scratch, "tree-one")
r, run, workspace := aModuleAt(t, "tree-one")
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 {
t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives)
}
if got.Reused == nil || got.Why != "" {
t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why)
}
if got.ReuseKey != checked.Key {
t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key)
}
if len(scratch.took) != 2 {
t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took)
}
for _, made := range checked.Made {
registered := strings.Replace(made.Reference, ScratchPrefix, "", 1)
found := false
for _, b := range got.Built {
found = found || (b.Reference == registered && b.Digest == made.Digest)
}
if !found {
t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference)
}
}
if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) {
t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image)
}
words := got.Reused.Words()
if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") ||
!strings.Contains(words, "sha256:") {
t.Fatalf("the walk says %q", words)
}
}
// A record no verdict names is refused and built instead, and the walk says so: the store takes writes from anything
// on its network, so a record planted under another merge's key is never registered.
func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
checked := aCheckBuilt(t, scratch, "tree-one")
for name, leave := range map[string][]NamedBuild{
"no verdict": nil,
"another digest": func() []NamedBuild {
n := named(checked)
n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64)
return n
}(),
"another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(),
} {
t.Run(name, func(t *testing.T) {
scratch.took = nil
r, run, workspace := aModuleAt(t, "tree-one")
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) {
t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took)
}
if !strings.Contains(got.Why, "no check verdict of the walk's pull requests names it") {
t.Fatalf("the walk says %q", got.Why)
}
})
}
}
// A differing key builds, and says which part differs.
func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
checked := aCheckBuilt(t, scratch, "tree-one")
r, run, workspace := aModuleAt(t, "tree-two")
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || got.Reused != nil {
t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused)
}
if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) {
t.Fatalf("the walk says %q, not which part of the key differs", got.Why)
}
// And a builder of another version builds too, saying so.
withVersion(t, "builder-2")
r, run, workspace = aModuleAt(t, "tree-one")
got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) {
t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why)
}
// No check of the walk's pull requests built it: said.
r, run, workspace = aModuleAt(t, "tree-one")
got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") {
t.Fatalf("the walk says %q", got.Why)
}
}
// A copy that fails is said and falls back to a build; it never fails the walk.
func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
checked := aCheckBuilt(t, scratch, "tree-one")
scratch.failTake = true
r, run, workspace := aModuleAt(t, "tree-one")
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatalf("a failed copy failed the walk: %v", err)
}
if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 {
t.Fatal("a failed copy did not fall back to a build")
}
if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") {
t.Fatalf("the walk says %q", got.Why)
}
for repository := range r.images {
if strings.HasPrefix(repository, ScratchPrefix) {
t.Fatalf("the walk's own build went into the scratch namespace: %s", repository)
}
}
}
// Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit
// off the trunk never takes one.
func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) {
withVersion(t, "builder-1")
scratch := newScratch()
checked := aCheckBuilt(t, scratch, "tree-one")
r, run, workspace := aModuleAt(t, "tree-one")
if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil,
Npmrc{}, GitCredential{}, nil); err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || len(scratch.took) > 0 {
t.Fatal("a build not asked to reuse took a check's build")
}
r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it
got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git",
"mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") {
t.Fatalf("a commit off the trunk took a check's build: %q", got.Why)
}
}
// A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing.
func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) {
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" {
t.Fatal("a check whose repository layer failed builds for its merge")
}
if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" {
t.Fatal("a check that passed with a warning builds nothing for its merge")
}
r, run, workspace := aModuleAt(t, "tree-one")
if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()},
workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 {
t.Fatalf("a check moving no module built %v, ran %v", built, r.ran)
}
}
// A module whose source does not pin its build has no key and is not built by the check; what stands after it
// is not built either.
func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) {
withVersion(t, "builder-1")
r, run, workspace := aModuleAt(t, "tree-one")
r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[
{"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}`
spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{
{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0},
{Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}}
built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "",
CheckVerdict{}, GitCredential{}, nil, logging(nil))
if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") ||
!strings.Contains(built[1].NotBuilt, "stands after meshboard") {
t.Fatalf("built %+v", built)
}
if len(r.images) > 0 {
t.Fatalf("a module without a key was published: %v", r.images)
}
}
// What the key names: every input a fresh build could differ by.
func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) {
src := newSourceInputs("m")
src.tree = "t1"
src.bases = []string{"store:5000/b/x@sha256:" + strings.Repeat("b", 64)}
src.contexts["server"] = "ctx1"
src.toolchains["tools"] = "go sha256:c 1234"
manifest := catalogue.Manifest{Module: "m"}
base := keyOf(src, manifest, "v1")
if base.Key == "" {
t.Fatalf("no key: %s", base.Why)
}
for name, change := range map[string]func(*sourceInputs) string{
partTree: func(s *sourceInputs) string { s.tree = "t2"; return "v1" },
partBases: func(s *sourceInputs) string {
s.bases = []string{"other@sha256:" + strings.Repeat("c", 64)}
return "v1"
},
partContext: func(s *sourceInputs) string { s.contexts["server"] = "ctx2"; return "v1" },
partTool: func(s *sourceInputs) string { s.toolchains["tools"] = "go sha256:d 1234"; return "v1" },
partBuilder: func(*sourceInputs) string { return "v2" },
} {
s := src.copy()
version := change(s)
if k := keyOf(s, manifest, version); k.Key == base.Key {
t.Errorf("a change of %s left the key the same", name)
}
}
// A base copied to another address is the same base: the key names it by digest.
moved := src.copy()
moved.bases = []string{"elsewhere:5000/scratch/b/x@sha256:" + strings.Repeat("b", 64)}
if keyOf(moved, manifest, "v1").Key != base.Key {
t.Error("the same base at another address changed the key")
}
if k := keyOf(src, manifest, ""); k.Key != "" {
t.Error("a builder whose version cannot be said keyed its build")
}
unpinned := src.copy()
unpinned.notPinned("it resolves packages from the mesh's registry at build time")
if k := keyOf(unpinned, manifest, "v1"); k.Key != "" || !strings.Contains(k.Why, "registry") {
t.Errorf("a build its source does not pin was keyed: %+v", k)
}
}
// aTaggingRegistry is the in-memory registry with tags and their listing, as the store's scratch namespace needs.
type aTaggingRegistry struct {
*aRegistryOfRepositories
tags map[string]map[string]string // repository → tag → manifest digest
}
func (m *aTaggingRegistry) serve() http.Handler {
under := m.handler()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if repository, ok := strings.CutSuffix(strings.TrimPrefix(r.URL.Path, "/v2/"), "/tags/list"); ok {
m.mu.Lock()
var tags []string
for tag := range m.tags[repository] {
tags = append(tags, tag)
}
m.mu.Unlock()
if tags == nil {
w.WriteHeader(http.StatusNotFound)
return
}
_ = json.NewEncoder(w).Encode(map[string]any{"name": repository, "tags": tags})
return
}
repository, kind, rest := splitPath(r.URL.Path)
if kind == "blobs" && r.Method == http.MethodHead {
m.mu.Lock()
body, linked := m.blobs[rest], m.links[repository][rest]
m.mu.Unlock()
if !linked {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
w.WriteHeader(http.StatusOK)
return
}
if kind != "manifests" {
under.ServeHTTP(w, r)
return
}
if !strings.HasPrefix(rest, "sha256:") {
m.mu.Lock()
switch r.Method {
case http.MethodPut:
body, _ := readAll(r)
if m.tags[repository] == nil {
m.tags[repository] = map[string]string{}
}
m.tags[repository][rest] = digestOf(body)
m.manifests[repository+"@"+digestOf(body)] = body
m.mu.Unlock()
w.WriteHeader(http.StatusCreated)
return
default:
digest, ok := m.tags[repository][rest]
body, held := m.manifests[repository+"@"+digest]
m.mu.Unlock()
if !ok || !held {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Docker-Content-Digest", digest)
w.Header().Set("Content-Type", mediaTypeOf(body))
if r.Method == http.MethodGet {
_, _ = w.Write(body)
return
}
w.WriteHeader(http.StatusOK)
return
}
}
if r.Method == http.MethodDelete {
m.mu.Lock()
for _, tags := range m.tags {
for tag, digest := range tags {
if digest == rest {
delete(tags, tag)
}
}
}
m.mu.Unlock()
}
under.ServeHTTP(w, r)
})
}
// A check's builds go after seven days, and when its pull request closes or is merged; the walk's copy in the
// module's own namespace stays.
func TestCheckBuildsExpireAfterSevenDaysAndGoWhenTheirPullRequestCloses(t *testing.T) {
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
server := httptest.NewServer(m.serve())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
registry := Registry{Address: address}
ctx := t.Context()
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
image := func(repository, body string) string {
digest := digestOf([]byte(body))
m.manifests[repository+"@"+digest] = []byte(body)
return address + "/" + repository + "@" + digest
}
check := func(pull PullRef, id string, at time.Time, key string) string {
reference := image("scratch/meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"`+id+`"}`)
build := CheckBuild{Key: key, Module: "meshboard", Made: []catalogue.Built{{Name: "server", Kind: "image",
Reference: reference}}, Pull: pull, Check: id, At: at}
if err := registry.Keep(ctx, pull, id, at, []CheckBuild{build}); err != nil {
t.Fatal(err)
}
return reference
}
held := func(reference string) bool {
_, digest, _ := strings.Cut(reference, "@")
_, ok := m.manifests["scratch/meshboard/server@"+digest]
return ok
}
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
old := check(aPull, "build-1", now.Add(-8*24*time.Hour), "r1:"+strings.Repeat("1", 64))
fresh := check(aPull, "build-2", now.Add(-time.Hour), "r1:"+strings.Repeat("2", 64))
others := check(other, "build-3", now.Add(-time.Hour), "r1:"+strings.Repeat("3", 64))
walked := image("meshboard/server", `{"mediaType":"`+mediaManifestOCI+`","layers":[],"n":"walk"}`)
if _, found, err := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("2", 64)); err != nil || !found {
t.Fatalf("a check build kept is not found by its key: %v %v", found, err)
}
n, err := registry.Expire(ctx, CheckBuildsKept, now)
if err != nil || n != 1 {
t.Fatalf("expired %d checks (%v), want the one eight days old", n, err)
}
if held(old) || !held(fresh) || !held(others) {
t.Fatalf("after the week's sweep: old %v, fresh %v, another's %v", held(old), held(fresh), held(others))
}
if _, found, _ := registry.Find(ctx, "meshboard", "r1:"+strings.Repeat("1", 64)); found {
t.Fatal("the expired check build's record is still found")
}
n, err = registry.Forget(ctx, aPull)
if err != nil || n != 1 {
t.Fatalf("forgot %d checks of %s (%v)", n, aPull, err)
}
if held(fresh) || !held(others) {
t.Fatalf("after %s closed: its build held %v, another pull request's %v", aPull, held(fresh), held(others))
}
if _, ok := m.manifests["meshboard/server@"+strings.SplitN(walked, "@", 2)[1]]; !ok {
t.Fatal("letting go of check builds took the walk's copy in the module's namespace")
}
if left, _ := registry.OfPulls(ctx, []PullRef{aPull}); len(left) != 0 {
t.Fatalf("%s still has check builds: %+v", aPull, left)
}
}
// The walk copies a check build by digest: the digest registered is the digest the check built, and the copy is
// in the module's own namespace.
func TestAChecksBuildIsCopiedByDigestIntoTheModulesNamespace(t *testing.T) {
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
server := httptest.NewServer(m.serve())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
registry := Registry{Address: address}
layer := []byte("layer")
m.blobs[digestOf(layer)] = layer
m.link("scratch/meshboard/server", digestOf(layer))
manifest := []byte(`{"mediaType":"` + mediaManifestOCI + `","layers":[{"digest":"` + digestOf(layer) + `","size":5}]}`)
m.manifests["scratch/meshboard/server@"+digestOf(manifest)] = manifest
archive := []byte("archive")
m.blobs[digestOf(archive)] = archive
m.link("scratch/meshboard/look", digestOf(archive))
got, err := registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "server", Kind: "image",
Reference: address + "/scratch/meshboard/server@" + digestOf(manifest)})
if err != nil {
t.Fatal(err)
}
if got.Reference != address+"/meshboard/server@"+digestOf(manifest) {
t.Fatalf("copied as %s", got.Reference)
}
if m.uploads != 0 || m.mounts != 1 {
t.Fatalf("the image's layer was moved (%d uploads, %d mounts), not mounted", m.uploads, m.mounts)
}
got, err = registry.Take(t.Context(), "meshboard", catalogue.Built{Name: "look", Kind: "archive",
Digest: digestOf(archive), Reference: "http://" + address + "/v2/scratch/meshboard/look/blobs/" + digestOf(archive)})
if err != nil {
t.Fatal(err)
}
if got.Digest != digestOf(archive) || got.Reference != "http://"+address+"/v2/meshboard/look/blobs/"+digestOf(archive) ||
!m.links["meshboard/look"][digestOf(archive)] {
t.Fatalf("the archive was copied as %+v", got)
}
}
// Two checks that built one digest in one repository share it: letting go of one check's builds keeps it while the
// other still names it, and the last to go takes it.
func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) {
m := &aTaggingRegistry{aRegistryOfRepositories: newRegistryOfRepositories(), tags: map[string]map[string]string{}}
server := httptest.NewServer(m.serve())
defer server.Close()
address := strings.TrimPrefix(server.URL, "http://")
registry := Registry{Address: address}
ctx := t.Context()
now := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC)
body := `{"mediaType":"` + mediaManifestOCI + `","layers":[]}`
digest := digestOf([]byte(body))
m.manifests["scratch/meshboard/server@"+digest] = []byte(body)
reference := address + "/scratch/meshboard/server@" + digest
other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8}
for i, pull := range []PullRef{aPull, other} {
id := "build-" + strconv.Itoa(i+1)
b := CheckBuild{Key: "r1:" + strings.Repeat(strconv.Itoa(i+1), 64), Module: "meshboard", Pull: pull, Check: id,
At: now, Made: []catalogue.Built{{Name: "server", Kind: "image", Reference: reference}}}
if err := registry.Keep(ctx, pull, id, now, []CheckBuild{b}); err != nil {
t.Fatal(err)
}
}
held := func() bool { _, ok := m.manifests["scratch/meshboard/server@"+digest]; return ok }
if n, err := registry.Forget(ctx, aPull); err != nil || n != 1 {
t.Fatalf("forgot %d (%v)", n, err)
}
if !held() {
t.Fatal("letting go of one check's builds took a build another check still names")
}
if n, err := registry.Forget(ctx, other); err != nil || n != 1 {
t.Fatalf("forgot %d (%v)", n, err)
}
if held() {
t.Fatal("the build stayed after the last check naming it went")
}
}
+1 -57
View File
@@ -58,55 +58,12 @@ type sourceInputs struct {
// whole when an artifact's is not known (novox/hq ADR 0267).
own map[string]bool
ownIsAll bool
// root is the module's build source in its own repository outside its directory, relative to the
// repository's root: the list of predefined keys a manifest naming one was judged against (ADR 0348 §7).
root map[string]bool
// read is, per context (repository and ref), the build source read there; nil for one read whole.
read map[string]map[string]bool
readAs map[string]catalogue.ArtifactContext
// contextsAtHead is false once a context was cloned at something other than its trunk's head: its
// closure is not the one the next merge there meets, and the build says no build source.
contextsAtHead bool
// onlyKey says the inputs are being taken for a reuse key (novox/hq ADR 0350 §2): each artifact goes as far
// as what its source fingerprint reads, and nothing is built or published.
onlyKey bool
}
// keyOnly is whether these inputs are only being taken for a reuse key.
func (s *sourceInputs) keyOnly() bool { return s != nil && s.onlyKey }
// copy is the inputs taken so far, apart: a walk's key is taken from them without touching the build's own.
func (s *sourceInputs) copy() *sourceInputs {
c := newSourceInputs(s.module)
c.tree, c.unpinned, c.prefix, c.ownIsAll, c.contextsAtHead = s.tree, s.unpinned, s.prefix, s.ownIsAll, s.contextsAtHead
c.bases = append([]string{}, s.bases...)
for k, v := range s.contexts {
c.contexts[k] = v
}
for k, v := range s.toolchains {
c.toolchains[k] = v
}
for k, v := range s.own {
c.own[k] = v
}
for k, v := range s.root {
c.root[k] = v
}
for k, v := range s.read {
if v == nil {
c.read[k] = nil
continue
}
set := map[string]bool{}
for e := range v {
set[e] = true
}
c.read[k] = set
}
for k, v := range s.readAs {
c.readAs[k] = v
}
return c
}
// contextOffHead says a context was not its trunk's head.
@@ -118,7 +75,7 @@ func (s *sourceInputs) contextOffHead() {
func newSourceInputs(module string) *sourceInputs {
return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{},
own: map[string]bool{}, root: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{},
contextsAtHead: true}
}
@@ -132,16 +89,6 @@ func (s *sourceInputs) ownHas(entries ...string) {
}
}
// rootHas adds entries, relative to the repository's root, to the module's build source in its own repository.
func (s *sourceInputs) rootHas(entries ...string) {
if s == nil {
return
}
for _, e := range entries {
s.root[e] = true
}
}
// ownWhole says an artifact's build source in the module's own repository is not known: the module's
// whole directory is its source, as it was before.
func (s *sourceInputs) ownWhole() {
@@ -194,9 +141,6 @@ func (s *sourceInputs) buildSources() []BuildSource {
for e := range s.own {
own = append(own, withPrefix(s.prefix, e))
}
for e := range s.root {
own = append(own, e)
}
sort.Strings(own)
out = append(out, BuildSource{Paths: compactSorted(own)})
}
-94
View File
@@ -1,94 +0,0 @@
package builder
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"path/filepath"
"github.com/novox/mesh-controller/internal/catalogue"
)
// withTemplateTexts is what was built with each template's text, for a module whose files name templates
// (novox/hq ADR 0349 §2): its templates archive gains the text of every file in it, read back from the packed
// bytes, by its path inside the archive. Resolve reads them into the files' content.
//
// **Read from the archive, proven by its digest.** The module's files/ is packed again, as the build packs it,
// and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive,
// or a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic,
// so one source packs to one digest; a tree that packs to another is not the archive, and its text is refused.
func withTemplateTexts(manifest catalogue.Manifest, within string, built []catalogue.Built) ([]catalogue.Built, error) {
if !manifest.Templated() {
return built, nil
}
name := catalogue.TemplatesArtifact(manifest)
at := -1
for i, b := range built {
if b.Name == name {
at = i
}
}
if name == "" || at < 0 {
return nil, fmt.Errorf("%s names templates and built no archive of %s/ to read them from",
manifest.Module, catalogue.TemplatesDir)
}
body, err := pack(filepath.Join(within, catalogue.TemplatesDir))
if err != nil {
return nil, fmt.Errorf("%s: packing %s/ to read its templates back failed: %w", manifest.Module,
catalogue.TemplatesDir, err)
}
sum := sha256.Sum256(body)
digest := "sha256:" + hex.EncodeToString(sum[:])
if built[at].Digest != digest {
return nil, fmt.Errorf("%s: %s/ packs to the digest %s, and the archive %q built or taken is %s: the "+
"templates are read only from the archive the mesh records", manifest.Module, catalogue.TemplatesDir,
digest, name, orNone(built[at].Digest))
}
texts, err := unpackTexts(body)
if err != nil {
return nil, fmt.Errorf("%s: reading the archive %q back failed: %w", manifest.Module, name, err)
}
out := append([]catalogue.Built(nil), built...)
out[at].Texts = texts
return out, nil
}
// unpackTexts is every file of a packed archive (pack), by its path inside it.
func unpackTexts(body []byte) (map[string]string, error) {
zipped, err := gzip.NewReader(bytes.NewReader(body))
if err != nil {
return nil, err
}
defer zipped.Close()
reader := tar.NewReader(zipped)
texts := map[string]string{}
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return texts, nil
}
if err != nil {
return nil, err
}
if header.Typeflag != tar.TypeReg {
continue
}
text, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
texts[header.Name] = string(text)
}
}
func orNone(digest string) string {
if digest == "" {
return "no digest"
}
return digest
}
-136
View File
@@ -1,136 +0,0 @@
package builder
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A module's files kept as template files in its repository (novox/hq ADR 0349 §2, design 57 package T1, issue
// 654): the build packs files/ into the module's templates archive and reads each template's text back from the
// packed bytes into the file's content, so the manifest the mesh holds is the one it would hold for the text inline.
const templatedModule = `{"module":"meshboard","version":"1",
"build":{"artifacts":[
{"name":"server","kind":"image","from":"Dockerfile"},
{"name":"files","kind":"archive","from":"files"}]},
"resources":[
{"id":"svc","type":"container","name":"meshboard","artifact":"server"},
{"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"files/meshboard.conf"},
{"id":"deeper","type":"file","path":"/etc/meshboard/deeper.conf","template":"files/a/deeper.conf"}]}`
var templatedFiles = map[string]string{
"Dockerfile": "FROM scratch", "files/meshboard.conf": "listen = 8080\n\"quoted\" é\n",
"files/a/deeper.conf": "deeper\n", "files/unused.conf": "packed, named by nothing",
}
func contentOf(t *testing.T, m catalogue.Manifest, id string) string {
t.Helper()
for _, r := range m.Resources {
if r["id"] == id {
if _, has := r[catalogue.TemplateField]; has {
t.Fatalf("%s still names its template in the built manifest: %v", id, r)
}
text, ok := r["content"].(string)
if !ok {
t.Fatalf("%s carries no content: %v", id, r)
}
return text
}
}
t.Fatalf("no resource %s in %v", id, m.Resources)
return ""
}
func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) {
r, workspace := aRepository(t, templatedModule, templatedFiles)
got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil,
Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
if r.archives["meshboard/files"] == "" {
t.Fatalf("the templates archive was not published: %v", r.archives)
}
if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" {
t.Fatal(problem)
}
}
func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) {
files := map[string]string{}
for k, v := range templatedFiles {
files[k] = v
}
delete(files, "files/a/deeper.conf")
r, workspace := aRepository(t, templatedModule, files)
_, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil,
Npmrc{}, GitCredential{}, nil)
if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") {
t.Fatalf("a build whose template is missing did not fail naming it: %v", err)
}
}
// A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back
// from the tree packed again, which is the archive only when it packs to the digest taken.
func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) {
withVersion(t, "builder-1")
contents := map[string]string{"mod/" + ManifestName: templatedModule}
for k, v := range templatedFiles {
contents["mod/"+k] = v
}
scratch := newScratch()
r, run, workspace := aModuleWith(t, "tree-one", contents)
checked := aCheckBuiltBy(t, scratch, r, run, workspace)
r, run, workspace = aModuleWith(t, "tree-one", contents)
got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)},
"https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil)
if err != nil {
t.Fatal(err)
}
if got.Reused == nil || len(r.archives) > 0 {
t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives)
}
if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] {
t.Fatalf("the template's text is %q, not the file's", text)
}
}
// The tree packed again must be the archive taken; text read from anything else is not the archive's.
func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) {
m, err := catalogue.ParseManifest([]byte(templatedModule))
if err != nil {
t.Fatal(err)
}
within := writeTree(t, t.TempDir(), templatedFiles)
built := []catalogue.Built{{Name: "files", Kind: catalogue.ArtifactArchive, Digest: "sha256:" + strings.Repeat("0", 64)}}
if _, err := withTemplateTexts(m, within, built); err == nil || !strings.Contains(err.Error(), "digest") {
t.Fatalf("text was read from a tree that does not pack to the archive taken: %v", err)
}
}
// writeTree writes files under dir and answers dir.
func writeTree(t *testing.T, dir string, files map[string]string) string {
t.Helper()
for name, body := range files {
full := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
return dir
}
-1
View File
@@ -127,7 +127,6 @@ func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node,
identity = pm.IdentityBoundOf(n.Name)
}
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
held.LeftOut = p.LeftOut
return held, true, ""
}
-24
View File
@@ -35,10 +35,6 @@ type Built struct {
// Launchers are, for a bundle in an interpreted language, the executable the build wrote beside
// each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it.
Launchers map[string]string
// Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the
// build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on
// the wire or kept: Resolve reads them into the files' content, which is what the mesh holds.
Texts map[string]string `json:"-"`
}
// Resolve fills a manifest's resources in from what was built.
@@ -75,26 +71,6 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) {
m.Module, strings.Join(missing, " and "), oneOrOther(len(missing)))
}
// **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into
// the file's content, so what the mesh holds and composes is what the same text inline would be.
if m.Templated() {
name := TemplatesArtifact(m)
texts := by[name].Texts
if name == "" || texts == nil {
return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+
"archive it packs %s/ into (%q): the builder that built it does not read templates",
m.Module, TemplatesDir, name)
}
filled, err := fillTemplates(m.Module, m.Resources, func(template string) (string, bool) {
t, ok := texts[strings.TrimPrefix(template, TemplatesDir+"/")]
return t, ok
}, fmt.Sprintf("the archive %q", name))
if err != nil {
return Manifest{}, err
}
m.Resources = filled
}
out := m
out.Build = nil
out.Resources = nil
-124
View File
@@ -1,124 +0,0 @@
package catalogue
import (
"fmt"
"sort"
"strings"
"time"
)
// A long-running container may say it is busy (novox/hq ADR 0335 §1).
//
// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check
// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle;
// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the
// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running
// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound,
// never the module's.
//
// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it
// does not know, whole, and to it the container is replaced at once, as before.
// BusyField is the resource field.
const BusyField = "busy"
// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author.
const (
BusyTimeoutDefault = 10 * time.Second
BusyTimeoutCeiling = time.Minute
)
// Busy is one container's busy, read.
type Busy struct {
Command string
Timeout time.Duration
}
// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none.
func ReadBusy(r map[string]any) (Busy, bool, []string) {
raw, present := r[BusyField]
if !present {
return Busy{}, false, nil
}
id := fmt.Sprint(r["id"])
fields, ok := raw.(map[string]any)
if !ok {
return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)}
}
var problems []string
keys := make([]string, 0, len(fields))
for k := range fields {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
if k != "command" && k != "timeout" {
problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k))
}
}
b := Busy{Timeout: BusyTimeoutDefault}
if v, has := fields["command"]; has {
s, ok := v.(string)
if !ok {
problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v))
}
b.Command = strings.TrimSpace(s)
}
if v, has := fields["timeout"]; has {
s, _ := v.(string)
d, err := time.ParseDuration(strings.TrimSpace(s))
switch {
case err != nil:
problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v))
case d <= 0 || d > BusyTimeoutCeiling:
problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s",
id, BusyTimeoutCeiling, d))
default:
b.Timeout = d
}
}
return b, true, problems
}
// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on
// one line, a timeout within its bounds, and only on a container that stays up.
func busyProblems(m Manifest) []string {
var problems []string
for _, r := range m.Resources {
b, has, read := ReadBusy(r)
if !has {
continue
}
where := m.Module + ": " + fmt.Sprint(r["id"])
for _, p := range read {
problems = append(problems, m.Module+": "+p)
}
if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) {
problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+
"asked of a running container before it is replaced (ADR 0335)", where))
}
if b.Command == "" {
problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+
"while it is busy")
}
if strings.ContainsAny(b.Command, "\n\r") {
problems = append(problems, where+"'s busy command is one line")
}
}
return problems
}
// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that
// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the
// container is replaced at once, as before.
func busyInto(resource map[string]any, with Rendering) {
if _, has := resource[BusyField]; !has {
return
}
if !with.ReadsBusy {
delete(resource, BusyField)
return
}
b, _, _ := ReadBusy(resource)
resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()}
}
-94
View File
@@ -1,94 +0,0 @@
package catalogue
import (
"encoding/json"
"strings"
"testing"
"github.com/novox/mesh-host/validate"
)
// A long-running container may say it is busy (novox/hq ADR 0335 §1): `module check` holds the field to its
// shape near its author, and it is sent only to a node-engine that reads it, in words the engine's own
// validator accepts.
func busyManifest(t *testing.T, on string, busy any) []byte {
t.Helper()
extra := map[string]any{"id": "agent", "type": "container", "name": "app-agent",
"image": "registry.example/app" + healthDigest}
var m map[string]any
if err := json.Unmarshal(healthManifest(t, "", nil, extra), &m); err != nil {
t.Fatal(err)
}
for _, r := range m["resources"].([]any) {
if r := r.(map[string]any); r["id"] == on {
r[BusyField] = busy
}
}
out, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return out
}
func TestABusyIsHeldToItsShape(t *testing.T) {
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"})); err != nil {
t.Fatalf("a well-formed busy was refused: %v", err)
}
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "x", "timeout": "30s"})); err != nil {
t.Fatalf("a busy with its timeout was refused: %v", err)
}
for name, c := range map[string]struct {
on string
busy any
says string
}{
"text": {"agent", "mesh-builder busy", "it is an object"},
"no command": {"agent", map[string]any{"timeout": "5s"}, "names no command"},
"two lines": {"agent", map[string]any{"command": "a\nb"}, "one line"},
"a long timeout": {"agent", map[string]any{"command": "x", "timeout": "5m"}, "at most 1m0s"},
"an unknown key": {"agent", map[string]any{"command": "x", "every": "1m"}, `says "every"`},
"on a step": {"seed", map[string]any{"command": "x"}, "not a container that stays up"},
"on a service": {"daemon", map[string]any{"command": "x"}, "not a container that stays up"},
} {
_, err := ParseManifest(busyManifest(t, c.on, c.busy))
if err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: want a refusal saying %q, got %v", name, c.says, err)
}
}
}
func TestABusyIsSentOnlyToAnEngineThatReadsIt(t *testing.T) {
m, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"}))
if err != nil {
t.Fatal(err)
}
compose := func(with Rendering) []map[string]any {
out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with)
if err != nil {
t.Fatal(err)
}
return out
}
older := compose(Rendering{})
if b, sent := older[indexOfID(older, "app.agent")][BusyField]; sent {
t.Fatalf("busy was sent to an engine that does not read it: %v", b)
}
newer := compose(Rendering{ReadsBusy: true})
got, _ := newer[indexOfID(newer, "app.agent")][BusyField].(map[string]any)
if got["command"] != "mesh-builder busy" || got["timeout"] != "10s" {
t.Fatalf("busy composed as %v", got)
}
// And the engine's own validator takes the container with it.
body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": newer})
if err != nil {
t.Fatal(err)
}
if problems := validate.Declaration(body); len(problems) > 0 {
t.Fatalf("the node-engine's validator refuses the composed busy: %v", problems)
}
if _, has := m.Resources[len(m.Resources)-1][BusyField].(map[string]any)["timeout"]; has {
t.Error("composing wrote the timeout into the catalogue's own manifest")
}
}
+3 -12
View File
@@ -51,15 +51,6 @@ func stepCA() Manifest {
}
}
// stepCAWithItsRoot is the authority whose root the operator supplies: the served key asks for it as
// `${setting:root}` (ADR 0155), the only way a setting reaches what a provider serves since a layer
// stopped replacing a served key (novox/hq issue 696; before it, the empty `root` was replaced whole).
func stepCAWithItsRoot() Manifest {
m := stepCA()
m.Serves["acme-ca"]["root"] = "${setting:root}"
return m
}
// routeProxy issues from that authority, so it must be told the root to verify it with.
func routeProxy() Manifest {
return Manifest{
@@ -87,7 +78,7 @@ func routeProxy() Manifest {
// The consequence is silent and total: route-proxy wrote an empty CA bundle, fell back to the
// system trust store, could not verify the internal authority, and no certificate was ever issued.
func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) {
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
r, err := Resolve(shelf(stepCA(), routeProxy()),
[]string{"step-ca", "route-proxy"}, reachable(), World{})
if err != nil {
t.Fatal(err)
@@ -115,7 +106,7 @@ func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) {
// And the binding file says the same thing, for a consumer that reads the binding rather than a
// substituted placeholder. The two are one fact and must not be able to disagree.
func TestACoLocatedConsumersBindingCarriesTheSettledValues(t *testing.T) {
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
r, err := Resolve(shelf(stepCA(), routeProxy()),
[]string{"step-ca", "route-proxy"}, reachable(), World{})
if err != nil {
t.Fatal(err)
@@ -161,7 +152,7 @@ func TestACoLocatedProviderIsSettledWhicheverWayItWasPulledIn(t *testing.T) {
{"step-ca", "route-proxy"},
{"route-proxy", "step-ca"},
} {
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), assigned, reachable(), World{})
r, err := Resolve(shelf(stepCA(), routeProxy()), assigned, reachable(), World{})
if err != nil {
t.Fatal(err)
}
-458
View File
@@ -1,458 +0,0 @@
package catalogue
import (
"encoding/json"
"fmt"
"regexp"
"slices"
"strings"
)
// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660).
//
// **One concept, two spellings for a while.** "Config" replaces "settings": the manifest says `config` for
// `settings`, `takes-config` for `takes-settings` and `${config:<key>}` for `${setting:<key>}`. The old names stay
// accepted as aliases until every module has moved (ADR 0348 §1), and a module uses one form or the other, never
// both. The config form is read into the very fields the settings form fills — Settings, a mergeable file's
// takes-settings and `${setting:<key>}` — so composition has one path and `${config:<key>}` fills exactly as
// `${setting:<key>}` does: a module rewritten from settings to config composes byte-identically. It is written
// back in the form it was read in, so what the mesh holds is what its author wrote.
//
// **A moved module is held to the whole declaration.** Under `config` every value has a type (number, string or
// boolean; an object or a list only when trusted), a description including its unit, its validation (enum,
// minimum and maximum, pattern), whether it is trusted, optionally the predefined key it falls back to, and either
// a default or `required`; it lands in a file or is read at runtime (`runtime: true`); every mergeable file names
// the keys it takes; a `reload-on` or `restart-on` names a file the module declares. A module not yet moved keeps
// today's form and today's rules (SettingProblems, TakenSettingsProblems) until the catalogue's list of such
// modules is empty (ADR 0348 §11).
//
// **A required value nothing sets** is the operator's own: it composes as an operator's setting nobody gave does
// today, refused by name at composition (UnsetSettingError). Who reports it once mesh-config answers values is the
// provisioner's (ADR 0349), not this file's.
// ConfigField is the manifest key a module declares its config values under.
const ConfigField = "config"
// TakesConfigField is the key a mergeable file names the config values it takes by.
const TakesConfigField = "takes-config"
// SettingsField is the old name of ConfigField, read as its alias.
const SettingsField = "settings"
// configRef is how a file asks for a config value: ${config:<key>}, the same keys as settingRef.
var configRef = regexp.MustCompile(`\$\{config:([a-z0-9][a-z0-9_.-]*)\}`)
// takesConfigKey and takesSettingsKey are the two spellings as a key of a JSON object. A string value is never
// followed by a colon, and a quote inside a string is escaped, so these match only keys.
var (
takesConfigKey = regexp.MustCompile(`"` + TakesConfigField + `"(\s*):`)
takesSettingsKey = regexp.MustCompile(`"` + TakesSettingsField + `"(\s*):`)
)
// The types a config value may have (ADR 0348 §4).
const (
ConfigNumber = "number"
ConfigString = "string"
ConfigBoolean = "boolean"
ConfigObject = "object"
ConfigList = "list"
)
// accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11).
var accountFacts = map[string]bool{"puid": true, "pgid": true}
// configForms is which of the two forms a manifest was written in, by the words of each it uses.
type configForms struct {
config, settings []string
}
// DeclaresConfig is whether the manifest is written in the config form: `config`, `takes-config` or
// `${config:<key>}` (ADR 0348 §1, §11).
func (m Manifest) DeclaresConfig() bool { return len(m.forms.config) > 0 }
// toSettingsForm is raw with the config form's placeholders and takes-config read as the settings form's, and
// which words of each form it used. `config` itself is a top-level key, renamed where the keys are read
// (UnmarshalJSON).
func toSettingsForm(raw []byte) ([]byte, configForms) {
var f configForms
if configRef.Match(raw) {
f.config = append(f.config, "${config:…}")
}
if takesConfigKey.Match(raw) {
f.config = append(f.config, TakesConfigField)
}
if settingRef.Match(raw) {
f.settings = append(f.settings, "${setting:…}")
}
if takesSettingsKey.Match(raw) {
f.settings = append(f.settings, TakesSettingsField)
}
if len(f.config) == 0 {
return raw, f
}
raw = configRef.ReplaceAllFunc(raw, func(ref []byte) []byte {
return []byte("${setting:" + string(configRef.FindSubmatch(ref)[1]) + "}")
})
raw = takesConfigKey.ReplaceAllFunc(raw, func(key []byte) []byte {
return []byte(`"` + TakesSettingsField + `"` + string(takesConfigKey.FindSubmatch(key)[1]) + ":")
})
return raw, f
}
// toConfigForm is a manifest written in the settings form's words, as JSON, in the config form's: what
// toSettingsForm and UnmarshalJSON read, written back.
func toConfigForm(raw []byte) ([]byte, error) {
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return nil, err
}
if settings, has := keys[SettingsField]; has {
keys[ConfigField] = settings
delete(keys, SettingsField)
var err error
if raw, err = json.Marshal(keys); err != nil {
return nil, err
}
}
raw = settingRef.ReplaceAllFunc(raw, func(ref []byte) []byte {
return []byte("${config:" + string(settingRef.FindSubmatch(ref)[1]) + "}")
})
raw = takesSettingsKey.ReplaceAllFunc(raw, func(key []byte) []byte {
return []byte(`"` + TakesConfigField + `"` + string(takesSettingsKey.FindSubmatch(key)[1]) + ":")
})
return raw, nil
}
// inConfigWords is a refusal said of a module in the config form in its words: `${config:<key>}` and
// takes-config, where the rules shared with the settings form say `${setting:<key>}` and takes-settings.
func inConfigWords(problem string) string {
problem = settingRef.ReplaceAllStringFunc(problem, func(ref string) string {
return "${config:" + settingRef.FindStringSubmatch(ref)[1] + "}"
})
return strings.ReplaceAll(problem, TakesSettingsField, TakesConfigField)
}
// MixedFormProblems is a manifest using both forms, said with the words of each (ADR 0348 §1).
func MixedFormProblems(m Manifest) []string {
if len(m.forms.config) == 0 || len(m.forms.settings) == 0 {
return nil
}
return []string{fmt.Sprintf("%s mixes the config form (%s) with the settings form (%s): a module uses one form "+
"or the other, never both; write %s, %s and ${config:<key>} throughout (novox/hq ADR 0348 §1)",
m.Module, strings.Join(m.forms.config, ", "), strings.Join(m.forms.settings, ", "),
ConfigField, TakesConfigField)}
}
// configDeclaration is the config form's declaration as written: nothing of the settings form's, and nothing
// unsaid.
type configDeclaration struct {
Type string `json:"type,omitempty"`
Description string `json:"description,omitempty"`
Enum []any `json:"enum,omitempty"`
Minimum *float64 `json:"minimum,omitempty"`
Maximum *float64 `json:"maximum,omitempty"`
Pattern string `json:"pattern,omitempty"`
Trusted bool `json:"trusted,omitempty"`
Predefined string `json:"predefined,omitempty"`
Default any `json:"default,omitempty"`
Required bool `json:"required,omitempty"`
Runtime bool `json:"runtime,omitempty"`
}
// MarshalJSON writes a settings declaration exactly as before, and a config declaration without the settings
// form's kind and why, and without a default it does not have.
func (d SettingDeclaration) MarshalJSON() ([]byte, error) {
type plain SettingDeclaration
if d.Kind != "" || d.Why != "" {
return json.Marshal(plain(d))
}
return json.Marshal(configDeclaration{Type: d.Type, Description: d.Description, Enum: d.Enum,
Minimum: d.Minimum, Maximum: d.Maximum, Pattern: d.Pattern, Trusted: d.Trusted, Predefined: d.Predefined,
Default: d.Default, Required: d.Required, Runtime: d.Runtime})
}
// configShaped is whether a declaration says anything of the config form's.
func configShaped(d SettingDeclaration) bool {
return d.Type != "" || d.Description != "" || len(d.Enum) > 0 || d.Minimum != nil || d.Maximum != nil ||
d.Pattern != "" || d.Trusted || d.Predefined != "" || d.Required || d.Runtime
}
// TypedSettingProblems is a declaration under `settings` that says what only `config` says: a value with a type,
// a description and its validation is declared in the config form (ADR 0348 §1, §4).
func TypedSettingProblems(m Manifest) []string {
var problems []string
for _, key := range sortedSettingKeys(m.Settings) {
if configShaped(m.Settings[key]) {
problems = append(problems, fmt.Sprintf("%s's setting %q says its type, description or validation "+
"under %s: a value declared with those is declared under config, with ${config:<key>} and "+
"takes-config, never under %s (novox/hq ADR 0348 §1, §4)", m.Module, key, SettingsField, SettingsField))
}
}
return problems
}
// ConfigProblems is every way a module in the config form declares its config wrongly, each said by name with
// what to do (ADR 0348 How it is checked, design 57 package K1).
func ConfigProblems(m Manifest) []string {
if !m.DeclaresConfig() {
return nil
}
var problems []string
placed := configPlaced(m)
for _, key := range sortedSettingKeys(m.Settings) {
d := m.Settings[key]
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("%s's config value %q: ", m.Module, key)+fmt.Sprintf(format, args...))
}
if !settingRef.MatchString("${setting:" + key + "}") {
say("not a usable key: lower-case letters, digits, dots, dashes and underscores")
continue
}
if meshWords[key] || accountFacts[key] {
say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key)
continue
}
if d.Kind != "" || d.Why != "" {
say("kind and why are the settings form's; under config a value says its type and its description " +
"(novox/hq ADR 0348 §1, §4)")
}
switch d.Type {
case ConfigNumber, ConfigString, ConfigBoolean:
case ConfigObject, ConfigList:
if !d.Trusted {
say("is %s and not trusted: an object or a list decides more than one value can say, so it is "+
"trusted, \"trusted\": true, or split into numbers, strings and booleans (novox/hq ADR 0348 §4)", d.Type)
}
case "":
say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)")
default:
say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", d.Type)
}
if strings.TrimSpace(d.Description) == "" {
say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §4)")
}
switch {
case d.Required && d.Default != nil:
say("both a default and required: a required value is the operator's own and has no default; give " +
"one or the other (novox/hq ADR 0348 §4)")
case !d.Required && d.Default == nil:
say("neither a default nor required: give a default, or say \"required\": true for a value that is " +
"the operator's own (novox/hq ADR 0348 §4)")
}
for _, wrong := range validationProblems(d) {
say("%s", wrong)
}
if word := operatorsWord(key); word != "" && d.Default != nil {
say("a key naming %q is the operator's value: declare it \"required\": true, with no default "+
"(novox/hq ADR 0348 §4, ADR 0112)", word)
}
if d.Predefined != "" {
p, listed := m.predefined[d.Predefined]
switch {
case m.predefined == nil:
say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+
"the catalogue's repository) is not present where it is judged: a value falls back only to a "+
"key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)",
d.Predefined, PredefinedFile)
case !listed:
say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+
"\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined)
default:
if p.Type != d.Type {
say("falls back to the predefined key %q, which is a %s, and this value is a %s: a value falls "+
"back only to a key of its own type (novox/hq ADR 0348 §4)", d.Predefined, p.Type, d.Type)
}
if p.Trusted && !d.Trusted {
say("falls back to the trusted predefined key %q and is not trusted itself: a value that falls "+
"back to a trusted key is trusted, \"trusted\": true (novox/hq ADR 0348 §8)", d.Predefined)
}
}
}
// A template's text is not here yet. Resolve fills it from the build, WithTemplatesFrom from the tree; neither
// judges again itself, but WithTemplatesFrom parses its result again, and a resolved build is parsed again
// where it is registered, and this is judged then.
if !placed[key] && !d.Runtime && !m.Templated() {
say("placed nowhere: no file says ${config:%s} or takes it under %s, and it is not marked "+
"\"runtime\": true for a value the module's code reads itself (novox/hq ADR 0348 §5)", key, TakesConfigField)
}
}
used := settingKeysUsedBy(m)
for _, key := range sortedKeys(used) {
if _, declared := m.Settings[key]; !declared {
problems = append(problems, fmt.Sprintf("%s says ${config:%s} and declares no such value under %s: "+
"declare %q with its type, description and a default or required (novox/hq ADR 0348 §4, §5)",
m.Module, key, ConfigField, key))
}
}
problems = append(problems, configTakesProblems(m)...)
problems = append(problems, reflectsProblems(m)...)
return problems
}
// configPlaced is every value a file, a unit, a directory's owner, a contribution or a served fact asks for, and
// every value a mergeable file takes.
func configPlaced(m Manifest) map[string]bool {
placed := settingKeysUsedBy(m)
for _, r := range m.Resources {
for _, k := range stringsOf(r[TakesSettingsField]) {
placed[k] = true
}
}
return placed
}
// validationProblems is what is wrong with a declaration's validation, and its default against it.
func validationProblems(d SettingDeclaration) []string {
var problems []string
isNumber := d.Type == ConfigNumber
if len(d.Enum) > 0 && d.Type != ConfigNumber && d.Type != ConfigString {
problems = append(problems, fmt.Sprintf("an enum is for a number or a string, and this is a %s", d.Type))
}
if (d.Minimum != nil || d.Maximum != nil) && !isNumber {
problems = append(problems, fmt.Sprintf("a minimum or maximum is for a number, and this is a %s", d.Type))
}
if d.Minimum != nil && d.Maximum != nil && *d.Minimum > *d.Maximum {
problems = append(problems, fmt.Sprintf("its minimum %v is above its maximum %v", *d.Minimum, *d.Maximum))
}
var pattern *regexp.Regexp
if d.Pattern != "" {
if d.Type != ConfigString {
problems = append(problems, fmt.Sprintf("a pattern is for a string, and this is a %s", d.Type))
}
var err error
if pattern, err = regexp.Compile(`^(?:` + d.Pattern + `)$`); err != nil {
problems = append(problems, fmt.Sprintf("its pattern %q is not a regular expression: %v", d.Pattern, err))
}
}
for _, v := range d.Enum {
if !ofType(v, d.Type) {
problems = append(problems, fmt.Sprintf("its enum holds %v, which is not a %s", v, d.Type))
}
}
if d.Default == nil {
return problems
}
if !ofType(d.Default, d.Type) {
if d.Type != "" {
problems = append(problems, fmt.Sprintf("its default %v is not a %s", d.Default, d.Type))
}
return problems
}
if len(d.Enum) > 0 && !slices.ContainsFunc(d.Enum, func(v any) bool { return v == d.Default }) {
problems = append(problems, fmt.Sprintf("its default %v is not one of %v", d.Default, d.Enum))
}
if n, ok := d.Default.(float64); ok {
if d.Minimum != nil && n < *d.Minimum {
problems = append(problems, fmt.Sprintf("its default %v is below its minimum %v", n, *d.Minimum))
}
if d.Maximum != nil && n > *d.Maximum {
problems = append(problems, fmt.Sprintf("its default %v is above its maximum %v", n, *d.Maximum))
}
}
if s, ok := d.Default.(string); ok && pattern != nil && !pattern.MatchString(s) {
problems = append(problems, fmt.Sprintf("its default %q does not match its pattern %q", s, d.Pattern))
}
return problems
}
// ofType is whether a value read from JSON is of a config type.
func ofType(v any, kind string) bool {
switch v.(type) {
case float64:
return kind == ConfigNumber
case string:
return kind == ConfigString
case bool:
return kind == ConfigBoolean
case map[string]any:
return kind == ConfigObject
case []any:
return kind == ConfigList
}
return false
}
// configTakesProblems is what ADR 0348 §11 adds for a module in the config form: an account fact in
// takes-config (the mesh's own words are refused for both forms by TakenSettingsProblems), and a mergeable file
// naming no keys where none of the module's does (where one does, TakenSettingsProblems names every one that
// does not).
func configTakesProblems(m Manifest) []string {
var problems, silent []string
naming := false
for _, r := range m.Resources {
if how, _ := r["merge"].(string); how == "" {
continue
}
if _, said := r[TakesSettingsField]; !said {
silent = append(silent, fmt.Sprint(r["id"]))
continue
}
naming = true
for _, k := range stringsOf(r[TakesSettingsField]) {
_, declared := m.Settings[k]
switch {
case accountFacts[k]:
problems = append(problems, fmt.Sprintf("%s's %v names %q in %s, and %q is an account fact the "+
"mesh places, never a config value (novox/hq ADR 0348 §11)", m.Module, r["id"], k,
TakesConfigField, k))
case meshWords[k]:
// Refused for both forms by TakenSettingsProblems, in its words.
case !declared:
problems = append(problems, fmt.Sprintf("%s's %v names %q in %s and declares no such value under "+
"%s: a value a file takes is declared like every other, with its type, description and a "+
"default or required, or the config verb could never set it (novox/hq ADR 0348 §4, §11)",
m.Module, r["id"], k, TakesConfigField, ConfigField))
}
}
}
if !naming {
for _, id := range silent {
problems = append(problems, fmt.Sprintf("%s's mergeable %s names no %s: a module that declares config "+
"names the keys each mergeable file takes, an empty list for none (novox/hq ADR 0348 §11)",
m.Module, id, TakesConfigField))
}
}
return problems
}
// meshMadeIDs are the resources the mesh composes into a module's own set, which its `restart-on` may name
// without declaring them; meshMadePrefixes are those named after what they are for.
var (
meshMadeIDs = []string{ComposedJailsID(), FilteringID(), CertificateID(), AuthorityID(), BusUsersID()}
meshMadePrefixes = []string{NeedID(""), SecretID(""), strings.TrimSuffix(GrantID("", ""), "-"), BoundID(""), ReceivedID(""), AccessID("/")}
)
// reflectsProblems is a `reload-on` or `restart-on` naming a file the module does not declare (ADR 0348 §6): not
// one of its own resources, not one the mesh composes for it, and not another module's, which is named
// `<module>.<id>`.
func reflectsProblems(m Manifest) []string {
own := map[string]bool{}
for _, r := range m.Resources {
own[fmt.Sprint(r["id"])] = true
}
meshMade := func(id string) bool {
if slices.Contains(meshMadeIDs, id) {
return true
}
for _, p := range meshMadePrefixes {
if strings.HasPrefix(id, p) && len(id) > len(p) {
return true
}
}
return false
}
var problems []string
for _, r := range m.Resources {
for _, field := range []string{"reload-on", "restart-on"} {
for _, id := range stringsOf(r[field]) {
if strings.Contains(id, ".") || own[id] || meshMade(id) {
continue
}
problems = append(problems, fmt.Sprintf("%s's %v names %q in its %s, and %s declares no such "+
"file: name a file of its own, or another module's as <module>.<id> (novox/hq ADR 0348 §6)",
m.Module, r["id"], id, field, m.Module))
}
}
}
return problems
}
-454
View File
@@ -1,454 +0,0 @@
package catalogue
import (
"bytes"
"encoding/json"
"errors"
"strings"
"testing"
)
// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660):
// one registration refusal per rule of design 57's K1, the old names read as aliases, and a module rewritten
// from settings to config composing byte-identically.
// inConfigForm is a module in the config form that registers: every value typed, described, defaulted or
// required, placed in a file or read at runtime, and its mergeable file naming its keys.
func inConfigForm() map[string]any {
return map[string]any{
"module": "notifier",
"version": "1",
"config": map[string]any{
"font-size": map[string]any{"type": "number", "description": "the notification's font size, in points",
"minimum": 6, "maximum": 40, "default": 10},
"width": map[string]any{"type": "number", "description": "the notification's width, in pixels",
"default": 250},
"position": map[string]any{"type": "string", "description": "the corner it shows in, a word",
"enum": []any{"top-right", "top-left"}, "default": "top-right"},
"theme": map[string]any{"type": "string", "description": "the colour theme, a lower-case name",
"pattern": "[a-z]+", "default": "dark"},
"history": map[string]any{"type": "number", "description": "how many notifications it keeps, a count",
"default": 20, "runtime": true},
"sender": map[string]any{"type": "string", "description": "the address it mails a digest from",
"required": true},
},
"resources": []any{
map[string]any{"id": "configuration", "type": "file", "path": "/x/notifierrc",
"content": "font = Inter ${config:font-size}\nwidth = ${config:width}\nfrom = ${config:sender}\n"},
map[string]any{"id": "rules", "type": "file", "path": "/x/rules.json", "merge": "json",
"takes-config": []any{"position", "theme"}, "content": "{\"position\": \"top-right\"}\n"},
map[string]any{"id": "notifier", "type": "service", "unit": "notifier.service",
"reload-on": []any{"configuration"}, "restart-on": []any{"rules"}},
},
}
}
// declaration is one value of inConfigForm's config, to change in place.
func declaration(m map[string]any, key string) map[string]any {
return m["config"].(map[string]any)[key].(map[string]any)
}
func resourceOf(m map[string]any, id string) map[string]any {
for _, r := range m["resources"].([]any) {
if r.(map[string]any)["id"] == id {
return r.(map[string]any)
}
}
return nil
}
func parseRaw(t *testing.T, m map[string]any) (Manifest, error) {
t.Helper()
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return ParseManifest(raw)
}
// refused is that the manifest is refused with every one of these words.
func refused(t *testing.T, m map[string]any, words ...string) {
t.Helper()
_, err := parseRaw(t, m)
if err == nil {
t.Fatalf("registered; wanted a refusal saying %q", words)
}
for _, w := range words {
if !strings.Contains(err.Error(), w) {
t.Fatalf("refused without %q:\n%v", w, err)
}
}
}
func TestAModuleInTheConfigFormRegisters(t *testing.T) {
m, err := parseRaw(t, inConfigForm())
if err != nil {
t.Fatal(err)
}
if !m.DeclaresConfig() {
t.Fatal("a module declaring config is not read as in the config form")
}
d := m.Settings["font-size"]
if d.Type != "number" || d.Description == "" || d.Minimum == nil || *d.Minimum != 6 || d.Default != float64(10) {
t.Fatalf("font-size read as %+v", d)
}
if s := m.Settings["sender"]; !s.Required || s.Default != nil {
t.Fatalf("sender read as %+v", s)
}
}
// Rule 1: a value without type or description.
func TestConfigRefusesAValueWithoutTypeOrDescription(t *testing.T) {
m := inConfigForm()
delete(declaration(m, "width"), "type")
refused(t, m, `notifier's config value "width"`, "no type")
m = inConfigForm()
declaration(m, "width")["type"] = "colour"
refused(t, m, `"width"`, `type "colour"`)
m = inConfigForm()
delete(declaration(m, "width"), "description")
refused(t, m, `"width"`, "no description")
}
// Rule 2: a value with neither a default nor required.
func TestConfigRefusesAValueWithNeitherDefaultNorRequired(t *testing.T) {
m := inConfigForm()
delete(declaration(m, "width"), "default")
refused(t, m, `"width"`, "neither a default nor required")
m = inConfigForm()
declaration(m, "sender")["default"] = "noreply@example.org"
refused(t, m, `"sender"`, "both a default and required")
}
// Rule 3: a default failing its validation.
func TestConfigRefusesADefaultFailingItsValidation(t *testing.T) {
for _, c := range []struct {
key string
value any
said string
}{
{"font-size", 4, "below its minimum"},
{"font-size", 41, "above its maximum"},
{"font-size", "10", "not a number"},
{"position", "bottom", "not one of"},
{"theme", "Dark", "does not match"},
} {
m := inConfigForm()
declaration(m, c.key)["default"] = c.value
refused(t, m, `"`+c.key+`"`, c.said)
}
}
// Rule 4: an object or a list not trusted.
func TestConfigRefusesAnObjectOrListNotTrusted(t *testing.T) {
for _, kind := range []string{"object", "list"} {
m := inConfigForm()
def := any(map[string]any{"a": 1})
if kind == "list" {
def = []any{"a"}
}
m["config"].(map[string]any)["rules"] = map[string]any{"type": kind, "description": "the rules, as " + kind,
"default": def, "runtime": true}
refused(t, m, `"rules"`, "is "+kind+" and not trusted")
m["config"].(map[string]any)["rules"].(map[string]any)["trusted"] = true
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("a trusted %s is refused: %v", kind, err)
}
}
}
// withPredefined is the list of predefined keys the mesh published, for the length of a test.
func withPredefined(t *testing.T, keys PredefinedList) {
t.Helper()
was := ReplacePublished(keys)
t.Cleanup(func() { ReplacePublished(was) })
}
// Rule 5: a predefined key not on the list, or of another type. Where no list is present, a value naming one is
// refused saying so.
func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) {
withPredefined(t, nil)
m := inConfigForm()
declaration(m, "font-size")["predefined"] = "font-size"
refused(t, m, `"font-size"`, "the list of predefined keys", "is not present")
withPredefined(t, map[string]PredefinedKey{"font-size": {Type: "number"}, "accent": {Type: "string"}})
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("a value falling back to a listed key of its type is refused: %v", err)
}
m = inConfigForm()
declaration(m, "font-size")["predefined"] = "font-scale"
refused(t, m, `"font-size"`, `"font-scale"`, "not on the list")
m = inConfigForm()
declaration(m, "font-size")["predefined"] = "accent"
refused(t, m, `"font-size"`, `"accent"`, "is a string")
}
// Rule 6: an untrusted value falling back to a trusted predefined key.
func TestConfigRefusesAnUntrustedValueFallingBackToATrustedKey(t *testing.T) {
withPredefined(t, nil)
m := inConfigForm()
declaration(m, "theme")["predefined"] = "theme"
refused(t, m, `"theme"`, "is not present")
withPredefined(t, map[string]PredefinedKey{"theme": {Type: "string", Trusted: true}})
refused(t, m, `"theme"`, "trusted", "is not")
declaration(m, "theme")["trusted"] = true
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("a trusted value falling back to a trusted key is refused: %v", err)
}
}
// Rule 7: a value neither placed in a template nor read at runtime.
func TestConfigRefusesAValueNeitherPlacedNorReadAtRuntime(t *testing.T) {
m := inConfigForm()
delete(declaration(m, "history"), "runtime")
refused(t, m, `"history"`, "${config:history}", `"runtime": true`)
// Taken by a mergeable file is placed.
m = inConfigForm()
if _, err := parseRaw(t, m); err != nil {
t.Fatal(err)
}
}
// Rule 8: a placeholder naming an undeclared value.
func TestConfigRefusesAPlaceholderNamingAnUndeclaredValue(t *testing.T) {
m := inConfigForm()
resourceOf(m, "configuration")["content"] = "font = ${config:font-size}\nwidth = ${config:width}\n" +
"from = ${config:sender}\ncolour = ${config:colour}\n"
refused(t, m, "${config:colour}", "declares no such value")
}
// Rule 9: a reload-on or restart-on naming a file the module does not declare.
func TestConfigRefusesAReloadOrRestartOnNamingAnUndeclaredFile(t *testing.T) {
m := inConfigForm()
resourceOf(m, "notifier")["reload-on"] = []any{"configuration", "dunstrc"}
refused(t, m, "reload-on", `"dunstrc"`, "declares no such")
m = inConfigForm()
resourceOf(m, "notifier")["restart-on"] = []any{"rule"}
refused(t, m, "restart-on", `"rule"`)
// Another module's file, and one the mesh makes for the module, are not the module's to declare.
m = inConfigForm()
resourceOf(m, "notifier")["restart-on"] = []any{"rules", "resolver.fact-zones", "secret-mail", "composed-jails"}
if _, err := parseRaw(t, m); err != nil {
t.Fatalf("refused: %v", err)
}
}
// Rule 10: a moved module's mergeable file not naming its keys under takes-config.
func TestConfigRefusesAMergeableFileNotNamingItsKeys(t *testing.T) {
m := inConfigForm()
delete(resourceOf(m, "rules"), "takes-config")
// position and theme are then placed nowhere; say they are read at runtime so only the file's rule speaks.
declaration(m, "position")["runtime"] = true
declaration(m, "theme")["runtime"] = true
refused(t, m, "rules", "names no takes-config")
}
// Rule 11: a mesh word, puid or pgid in takes-config.
func TestConfigRefusesAMeshWordOrAnAccountFactInTakesConfig(t *testing.T) {
for _, word := range []string{"ports", "expose", "reach", "endpoints", "places", "accesses", "networks", "puid", "pgid"} {
m := inConfigForm()
resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", word}
refused(t, m, `"`+word+`"`, "takes-config")
}
}
// A module uses one form or the other, never both.
func TestAManifestMixingTheConfigAndSettingsFormsIsRefused(t *testing.T) {
m := inConfigForm()
resourceOf(m, "configuration")["content"] = "font = ${config:font-size}\nwidth = ${setting:width}\nfrom = ${config:sender}\n"
refused(t, m, "mixes", "${setting:", "config")
m = inConfigForm()
delete(resourceOf(m, "rules"), "takes-config")
resourceOf(m, "rules")["takes-settings"] = []any{"position", "theme"}
refused(t, m, "mixes", "takes-settings")
m = inConfigForm()
m["settings"] = map[string]any{"colour": map[string]any{"kind": "preference", "default": "red", "why": "red"}}
refused(t, m, "mixes", "settings")
}
// The settings form keeps its own declaration; the config form's has no kind or why.
func TestEachFormKeepsItsOwnDeclaration(t *testing.T) {
m := inConfigForm()
declaration(m, "width")["why"] = "fits a title"
refused(t, m, `"width"`, "why")
err := parsed(t, Manifest{Module: "notifier",
Settings: map[string]SettingDeclaration{"width": {Kind: KindPreference, Default: float64(250),
Why: "fits a title", Type: "number"}},
Resources: []map[string]any{{"id": "c", "type": "file", "path": "/x", "content": "${setting:width}"}}})
if err == nil || !strings.Contains(err.Error(), "under config") {
t.Fatalf("a typed value under settings: %v", err)
}
}
// What the mesh holds is what was written: a module in the config form is written back in it, and read again
// the same.
func TestAConfigManifestIsWrittenBackInItsOwnForm(t *testing.T) {
m, err := parseRaw(t, inConfigForm())
if err != nil {
t.Fatal(err)
}
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
for _, said := range []string{`"config":`, `${config:font-size}`, `"takes-config":`, `"required":true`} {
if !strings.Contains(string(raw), said) {
t.Fatalf("written back without %s: %s", said, raw)
}
}
for _, unsaid := range []string{`"settings":`, `${setting:`, `"takes-settings"`, `"kind"`, `"why"`, `"default":null`} {
if strings.Contains(string(raw), unsaid) {
t.Fatalf("written back with %s: %s", unsaid, raw)
}
}
again, err := ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
twice, _ := json.Marshal(again)
if !bytes.Equal(raw, twice) {
t.Fatalf("read again it is written\n%s\nwhere it was\n%s", twice, raw)
}
}
// A module in the settings form is written back byte for byte as before.
func TestASettingsManifestIsWrittenBackAsBefore(t *testing.T) {
d := SettingDeclaration{Kind: KindPreference, Default: float64(10), Why: "readable"}
raw, err := json.Marshal(d)
if err != nil {
t.Fatal(err)
}
if string(raw) != `{"kind":"preference","default":10,"why":"readable"}` {
t.Fatalf("written as %s", raw)
}
}
// settingsDunst is the dunst module as it is written today; configDunst the same module rewritten from settings to
// config.
const settingsDunst = `{"module": "dunst", "version": "1",
"settings": {
"font-size": {"kind": "preference", "default": 10, "why": "10 points of Inter reads well on a screen of about 100 DPI"},
"width": {"kind": "preference", "default": 250, "why": "250 pixels fits a title of about forty characters at 10 points"}
},
"resources": [
{"id": "package", "type": "package", "package": "dunst"},
{"id": "configuration", "type": "file", "path": "/home/me/.config/dunst/dunstrc", "mode": "0644",
"content": "[global]\n font = Inter ${setting:font-size}\n width = ${setting:width}\n"},
{"id": "rules", "type": "file", "path": "/home/me/.config/dunst/rules.json", "merge": "json",
"takes-settings": [], "content": "{\"urgency\": \"low\"}\n"},
{"id": "notifier", "type": "service", "unit": "dunst.service", "reload-on": ["configuration"]}
]}`
const configDunst = `{"module": "dunst", "version": "1",
"config": {
"font-size": {"type": "number", "description": "the font size of a notification, in points", "minimum": 6,
"maximum": 40, "default": 10},
"width": {"type": "number", "description": "the width of a notification, in pixels", "minimum": 100,
"default": 250}
},
"resources": [
{"id": "package", "type": "package", "package": "dunst"},
{"id": "configuration", "type": "file", "path": "/home/me/.config/dunst/dunstrc", "mode": "0644",
"content": "[global]\n font = Inter ${config:font-size}\n width = ${config:width}\n"},
{"id": "rules", "type": "file", "path": "/home/me/.config/dunst/rules.json", "merge": "json",
"takes-config": [], "content": "{\"urgency\": \"low\"}\n"},
{"id": "notifier", "type": "service", "unit": "dunst.service", "reload-on": ["configuration"]}
]}`
// ${config:<key>} fills exactly as ${setting:<key>} does: the module rewritten from settings to config composes
// to the same bytes, with no layer, with the node's, and with the mesh's under the node's.
func TestAModuleRewrittenFromSettingsToConfigComposesByteIdentically(t *testing.T) {
composed := func(manifest string, layers []Layer) []byte {
t.Helper()
m, err := ParseManifest([]byte(manifest))
if err != nil {
t.Fatal(err)
}
r := anAdoptedAnchor()
r.Modules = append(r.Modules, m)
with := anchorRendering(false)
if layers != nil {
with.Settings["dunst"] = layers
}
c, err := r.Compose(with)
if err != nil {
t.Fatal(err)
}
if why, left := c.LeftOut["dunst"]; left {
t.Fatalf("dunst was left out: %s", why)
}
raw, err := json.Marshal(c)
if err != nil {
t.Fatal(err)
}
return raw
}
for _, layers := range [][]Layer{
nil,
{{From: "anchor", Values: map[string]any{"font-size": float64(13)}}},
{{From: MeshWideLayer, Values: map[string]any{"width": float64(300)}},
{From: "anchor", Values: map[string]any{"font-size": float64(12), "width": float64(320)}}},
} {
was, is := composed(settingsDunst, layers), composed(configDunst, layers)
if !bytes.Equal(was, is) {
t.Fatalf("composed differently with %v:\nsettings: %s\nconfig: %s", layers, was, is)
}
if !strings.Contains(string(is), "font = Inter 1") || strings.Contains(string(is), "${config:") {
t.Fatalf("not filled: %s", is)
}
}
}
// A key a mergeable file takes is a config value like every other: one not declared under config is refused by
// name (ADR 0348 §4, review of mesh-controller#320).
func TestConfigRefusesATakesConfigKeyNotDeclared(t *testing.T) {
m := inConfigForm()
resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", "urgency"}
refused(t, m, `"urgency"`, "takes-config", "declares no such value under config")
}
// A required value nothing sets is refused at composition in the config form's words: ${config:<key>} and the
// config verb, never `settings set` (review of mesh-controller#320). The settings form's words are unchanged.
func TestAnUnsetRequiredValueIsRefusedInTheConfigFormsWords(t *testing.T) {
m, err := parseRaw(t, inConfigForm())
if err != nil {
t.Fatal(err)
}
r := anAdoptedAnchor()
r.Modules = append(r.Modules, m)
composed, err := r.Compose(anchorRendering(false))
if err != nil {
t.Fatal(err)
}
why, left := composed.LeftOut["notifier"]
if !left {
t.Fatal("a module whose required value nothing sets was not left out")
}
for _, said := range []string{"${config:sender}", `required config value "sender"`, "config verb"} {
if !strings.Contains(why, said) {
t.Fatalf("left out without %q: %s", said, why)
}
}
for _, unsaid := range []string{"${setting:", "settings set"} {
if strings.Contains(why, unsaid) {
t.Fatalf("left out in the settings form's words (%q): %s", unsaid, why)
}
}
var unset *UnsetSettingError
if err := unsetInConfigWords(settingInto(map[string]any{"type": "file", "content": "${setting:sender}"}, nil,
"notifier"), m); !errors.As(err, &unset) || unset.Setting != "sender" {
t.Fatalf("not typed as a value nobody gave: %v", err)
}
settingsForm := notifier()
settingsForm.Resources[0]["content"] = "from = ${setting:domain}\n"
err = unsetInConfigWords(settingInto(settingsForm.Resources[0], nil, "notifier"), settingsForm)
if err == nil || !strings.Contains(err.Error(), "`settings set notifier <file>`") {
t.Fatalf("the settings form's refusal changed: %v", err)
}
}
@@ -135,9 +135,6 @@ func CheckServes(m Manifest) []string {
for _, key := range sortedAnyKeys(m.Serves[provision]) {
text, ok := m.Serves[provision][key].(string)
if !ok {
// A provider's own name is filled only in a value of its own (novox/hq ADR 0346): one
// nested in a list or an object would be served as the placeholder, unsaid.
problems = append(problems, nestedOwnNames(m, provision, key, m.Serves[provision][key])...)
continue
}
// A probe identity, because what is checked is the shape of the statement and not
@@ -146,8 +143,6 @@ func CheckServes(m Manifest) []string {
problems = append(problems, fmt.Sprintf(
"%s serves %s, and the value it serves as %q %s", m.Module, provision, key, err))
}
// And the provider's own names it may serve (novox/hq ADR 0346).
problems = append(problems, checkServedOwnNames(m, provision, key, text)...)
// A label longer than DNS keeps is not truncated here (asDNSLabel), so the offer's own
// bound has to keep the identity inside one (ADR 0225).
if strings.Contains(text, "${consumer:as:dns}") {
+1 -1
View File
@@ -173,7 +173,7 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
facts := map[string]string{"account-home": "/home/op"}
with := Rendering{Settings: SettingsBy{"media": {LayerOf("node", map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}})}}}
with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}}
backup, _, err := seatContributions([]Manifest{pg, agent, media}, Manifest{}, BackupSeat+":"+BackupKindBackup, with, facts, nil)
if err != nil {
t.Fatal(err)
+2 -15
View File
@@ -252,11 +252,6 @@ type Rendering struct {
// statement's contract is link.RootContract or later). To an older, strict engine the field is not
// sent, and the account it names is not judged — which the self-check says, as not judged.
JudgesRoot bool
// ReadsBusy says this machine's node-engine reads a container's `busy` (novox/hq ADR 0335: its statement's
// contract is link.BusyContract or later). To an older, strict engine the field is not sent, and the
// container is replaced at once, as before.
ReadsBusy bool
}
// RootField is a user resource's field saying the account must never become root without a person
@@ -1003,7 +998,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a
// definition may not carry because it is true of one installation only. Filled from
// the same layers a mergeable file takes, and refused when no layer set it.
if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil {
if err := settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module); err != nil {
return nil, err
}
// **Placed before anything reads a path.** A pathless directory receives the path
@@ -1067,8 +1062,6 @@ func (r Resolution) compose(with Rendering, owner map[string]string,
// How it is ready, in the node-engine's words: its endpoint as the port this machine
// published it on — or not sent at all to an engine older than the field (ADR 0240).
healthInto(copied, m, with)
// And whether it is busy (novox/hq ADR 0335): sent only to an engine that asks it.
busyInto(copied, with)
// And a user's `root` (novox/hq ADR 0266): sent only when it asserts something, to an engine
// that judges it.
rootInto(copied, with)
@@ -2046,13 +2039,7 @@ func (r Resolution) servedOnThisMachine(provision string, with Rendering) (map[s
if err != nil {
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
}
// And the provider's own names, as this machine composes them (novox/hq ADR 0346); one it does
// not compose leaves its key out, said by the plan on this machine (ServedLeftOut).
named, _, err := r.ServedOwnNames(m, provision, settled, WithDefaults(m, with.Settings[m.Module]))
if err != nil {
return nil, false, fmt.Errorf("%s serving %s: %w", m.Module, provision, err)
}
return named, true, nil
return settled, true, nil
}
return nil, false, nil
}
@@ -165,19 +165,16 @@ func TestTwoConsumersOfOneProviderGetTheirOwnDerivation(t *testing.T) {
}
}
// An operator may still set what the provider serves where its definition asks (${setting:…}), and
// the mesh still derives the rest: the setting is filled first, then the consumer's half. (Until
// novox/hq issue 696 the layer replaced the served bucket whole; a layer replaces no served key now.)
// An operator may still set what the provider serves, and the mesh still derives the rest: the
// setting is laid on first, then the consumer's half is filled.
func TestASettingComposesWithADerivedValue(t *testing.T) {
prefixed := store()
prefixed.Serves["s3-bucket"]["bucket"] = "${setting:bucket-prefix}${consumer:as:dns}"
r, err := Resolve(shelf(prefixed, files()), []string{"store", "files"}, reachable(), World{})
r, err := Resolve(shelf(store(), files()), []string{"store", "files"}, reachable(), World{})
if err != nil {
t.Fatal(err)
}
out, err := r.Declaration(Rendering{
Settings: SettingsBy{"store": {{From: "the operator",
Values: map[string]any{"bucket-prefix": "team-"}}}},
Values: map[string]any{"bucket": "team-${consumer:as:dns}"}}}},
Grants: []Grant{{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files",
Values: map[string]any{}, Sealed: "c2VhbGVk"}},
})
+8 -66
View File
@@ -530,10 +530,11 @@ type Manifest struct {
// stays refused by name until a layer sets it. The mesh's layer, then the node's, override it.
Settings map[string]SettingDeclaration `json:"settings,omitempty"`
// forms is which of the two forms its config is written in, `config` or its alias `settings` (config.go,
// novox/hq ADR 0348 §1): the config form is read into Settings, a mergeable file's takes-settings and
// `${setting:<key>}`, and written back as it was read.
forms configForms
// Preferences are the values this module reads that a machine may tune (novox/hq issue 630): each
// key's type, default and why, and the predefined key it falls back to, if any. The value resolves
// from the default, then mesh config and node config for the predefined key, then this module's own
// value on the node. `settings` entries of kind preference are read as preferences while they move.
Preferences map[string]Preference `json:"preferences,omitempty"`
// unknown is the first key this manifest has that this controller does not know, at any depth, when
// it was read from the store (novox/hq ADR 0262): a manifest a newer controller registered.
@@ -541,10 +542,6 @@ type Manifest struct {
// machine's declaration by name until the controller is updated (LeftOut).
unknown string
// predefined is the list of predefined keys it was judged against (predefined.go, novox/hq ADR 0348 §7):
// read again — its templates filled from the tree — it is judged against the same list.
predefined PredefinedList
// bestEffort marks the view of a left-out module that only its backup lines are made from
// (backupView, novox/hq ADR 0262): a line of it that cannot be placed is said in the plan's list of
// what could not be placed, never an error that would cost the whole machine its declaration.
@@ -1330,25 +1327,10 @@ type manifestFields Manifest
// strict read there failed the whole catalogue, and with it every plan and every send, the moment a
// controller was rolled back. UnknownField says what was set aside.
func (m *Manifest) UnmarshalJSON(raw []byte) error {
// The config form read into the settings form's fields, so both compose by one path (config.go).
raw, forms := toSettingsForm(raw)
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return err
}
if settings, has := keys[SettingsField]; has && string(settings) != "null" {
forms.settings = append(forms.settings, SettingsField)
}
if config, has := keys[ConfigField]; has {
if string(config) != "null" {
forms.config = append(forms.config, ConfigField)
}
if _, both := keys[SettingsField]; !both {
keys[SettingsField] = config
}
// Beside `settings` it is refused by name (MixedFormProblems), never merged into it.
delete(keys, ConfigField)
}
plain := map[string]string{}
many := map[string]map[string]string{}
if secrets, ok := keys["secrets"]; ok && string(secrets) != "null" {
@@ -1477,7 +1459,6 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
}
*m = Manifest(fields)
m.unknown = unknown
m.forms = forms
m.OptionalRequires = optionalRequires
if len(plain) > 0 {
m.Secrets = plain
@@ -1495,16 +1476,8 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error {
}
// MarshalJSON writes `secrets` and `contributes` back in the shape they were read: single values,
// and objects of local names; and its config in the form it was read in (config.go).
// and objects of local names.
func (m Manifest) MarshalJSON() ([]byte, error) {
raw, err := m.marshalFields()
if err != nil || !m.DeclaresConfig() {
return raw, err
}
return toConfigForm(raw)
}
func (m Manifest) marshalFields() ([]byte, error) {
raw, err := json.Marshal(manifestFields(m))
if err != nil {
return nil, err
@@ -1604,17 +1577,6 @@ func SecretLocal(to, local string) string {
func (m Manifest) UnknownField() string { return m.unknown }
func ParseManifest(raw []byte) (Manifest, error) {
return parseManifest(raw, PublishedPredefined())
}
// ParseManifestWith is ParseManifest judging every value that falls back to a predefined key against the list
// given — the one read from the tree the manifest is in (predefined.go, novox/hq ADR 0348 §7) — where
// ParseManifest judges against the list the mesh published. Nil is no list: a value naming a key is refused.
func ParseManifestWith(raw []byte, predefined PredefinedList) (Manifest, error) {
return parseManifest(raw, predefined)
}
func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
var m Manifest
// Strictly. **An unknown key is refused**, which is the discipline the host's declaration
// parser has and manifests lacked (novox/hq 04-ISSUES/003): a `scope:` key survived in five
@@ -1642,7 +1604,6 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
}
return Manifest{}, fmt.Errorf("this is not a module manifest: %w", err)
}
m.predefined = predefined
var problems []string
if !name.MatchString(m.Module) {
@@ -1828,14 +1789,8 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
problems = append(problems, StateProblems(m)...)
// And what it may call its logs, and how large it may ask them to grow (logs.go, novox/hq ADR 0297).
problems = append(problems, LogProblems(m)...)
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262), or, written in the
// config form, the whole declaration of each value (config.go, novox/hq ADR 0348).
if m.DeclaresConfig() {
problems = append(problems, ConfigProblems(m)...)
} else {
problems = append(problems, SettingProblems(m)...)
problems = append(problems, TypedSettingProblems(m)...)
}
// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
problems = append(problems, SettingProblems(m)...)
// And what it requires optionally, and the files it delivers only while bound (novox/hq ADR 0330).
problems = append(problems, OptionalProblems(m)...)
// And which setting keys each mergeable file takes (setting_takes.go, novox/hq ADR 0326).
@@ -1904,9 +1859,6 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
}
}
problems = append(problems, m.Build.problems(m.Module)...)
// A file's text in a template file of the module's repository (novox/hq ADR 0349 §2): only a file, never beside
// content, only under files/, and packed by the module's build.
problems = append(problems, templateProblems(m)...)
problems = append(problems, undeliveredBundles(m)...)
// **What provides the artifact store cannot be delivered through it** (novox/hq 04-ISSUES/029).
//
@@ -2390,16 +2342,6 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
// How each long-running resource is ready (novox/hq ADR 0240 rule 2): said near its author, in the
// words the node-engine would refuse it in far away.
problems = append(problems, healthProblems(m)...)
// And whether a container is busy (novox/hq ADR 0335), likewise.
problems = append(problems, busyProblems(m)...)
if m.DeclaresConfig() {
// Said in the words it was written in: the rules shared with the settings form say ${setting:<key>}
// and takes-settings.
for i, p := range problems {
problems[i] = inConfigWords(p)
}
problems = append(problems, MixedFormProblems(m)...)
}
if len(problems) > 0 {
sort.Strings(problems)
return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s",
+2 -65
View File
@@ -34,69 +34,6 @@ const PlacesSetting = "places"
// AccessesSetting is the settings key that says where a module's accesses are on this node, by id.
const AccessesSetting = "accesses"
// **Placement is not config** (novox/hq ADR 0348 clause 11, issue 651). `places`, `accesses` and
// `networks` are facts of an assignment the controller decides, held beside the assignment and apart
// from its settings: a Layer carries them in Placement, never in Values, and only Places, AccessPlaces
// and KeptNetworks read them. A person still says them as keys of a layer through `settings`; LayerOf
// splits such a layer, so what they type is routed to the placement and nothing they do changes.
// placementKeys are the keys of a layer that are placement, not config.
var placementKeys = map[string]bool{PlacesSetting: true, AccessesSetting: true, NetworksSetting: true}
// SplitPlacement divides a layer as a person says it into its config values and its placement. Each is
// nil when it holds nothing; the values themselves are not copied.
func SplitPlacement(values map[string]any) (config, placement map[string]any) {
for key, value := range values {
if placementKeys[key] {
if placement == nil {
placement = map[string]any{}
}
placement[key] = value
continue
}
if config == nil {
config = map[string]any{}
}
config[key] = value
}
return config, placement
}
// LayerOf is a layer as a person says it — config values and placement in one map — split into the two.
func LayerOf(from string, values map[string]any) Layer {
config, placement := SplitPlacement(values)
if config == nil {
config = map[string]any{}
}
return Layer{From: from, Values: config, Placement: placement}
}
// Whole is the layer as a person says it: its values and its placement in one map, a new map.
func (l Layer) Whole() map[string]any {
out := make(map[string]any, len(l.Values)+len(l.Placement))
for k, v := range l.Values {
out[k] = v
}
for k, v := range l.Placement {
out[k] = v
}
return out
}
// asSaid is the layers with placement folded back into the values, for what reads a layer whole as it
// did while placement was kept in it: today's mergeable file, which takes every key (ADR 0326 §3), and
// the judgement that no value holds a line end.
func asSaid(layers []Layer) []Layer {
out := make([]Layer, len(layers))
for i, l := range layers {
out[i] = l
if len(l.Placement) > 0 {
out[i] = Layer{From: l.From, Values: l.Whole(), Default: l.Default}
}
}
return out
}
// Placement is what an assignment says about one of a module's directories.
type Placement struct {
// Path is where the directory is on this machine. Absolute.
@@ -271,7 +208,7 @@ func Places(m Manifest, layers []Layer) (map[string]Placement, error) {
}
out := map[string]Placement{}
for _, layer := range layers {
raw, ok := layer.Placement[PlacesSetting]
raw, ok := layer.Values[PlacesSetting]
if !ok {
continue
}
@@ -341,7 +278,7 @@ func AccessPlaces(m Manifest, layers []Layer) (map[string]string, error) {
}
out := map[string]string{}
for _, layer := range layers {
raw, ok := layer.Placement[AccessesSetting]
raw, ok := layer.Values[AccessesSetting]
if !ok {
continue
}
+1 -41
View File
@@ -23,7 +23,7 @@ func placeable() Manifest {
}
func placedBy(values map[string]any) Rendering {
return Rendering{Settings: SettingsBy{"arr": {LayerOf("node anchor", values)}}}
return Rendering{Settings: SettingsBy{"arr": {{From: "node anchor", Values: values}}}}
}
func TestAnAssignmentPlacesDirectoriesAndAccesses(t *testing.T) {
@@ -179,43 +179,3 @@ func TestAnUnknownAccessReferenceIsRefusedAtParse(t *testing.T) {
t.Fatalf("an access with no id and no path was accepted: %v", err)
}
}
// **Placement is read from the placement, never from a layer's values** (novox/hq ADR 0348 clause 11, issue
// 651). A `places` or `accesses` among the config values places nothing; the same in Placement does. A
// mergeable file of today's form still takes every key of the layer as it is said, placement too, so it
// composes as it did while placement was kept in the layer.
func TestPlacementIsReadFromThePlacementOnly(t *testing.T) {
m := placeable()
said := map[string]any{
PlacesSetting: map[string]any{"config": "/services/arr/config"},
AccessesSetting: map[string]any{"series": "/storage/media/series", "spool": "/storage/downloads"},
"log": "warn",
}
among := []Layer{{From: "anchor", Values: said}}
if placed, err := Places(m, among); err != nil || placed != nil {
t.Fatalf("a place among the config values placed %v (%v)", placed, err)
}
if _, _, err := accessesFor(m, among); err == nil {
t.Fatal("an access among the config values placed it")
}
beside := []Layer{LayerOf("anchor", said)}
if beside[0].Values["log"] != "warn" || beside[0].Values[PlacesSetting] != nil || beside[0].Placement[PlacesSetting] == nil {
t.Fatalf("LayerOf split %+v", beside[0])
}
if placed, err := Places(m, beside); err != nil || placed["config"].Path != "/services/arr/config" {
t.Fatalf("the placement did not place: %v (%v)", placed, err)
}
file := map[string]any{"id": "runtime", "type": "file", "path": "/x", "content": `{"a":1}`, "merge": "json"}
whole, err := ApplySettings(file, among)
if err != nil {
t.Fatal(err)
}
split, err := ApplySettings(file, beside)
if err != nil {
t.Fatal(err)
}
if whole["content"] != split["content"] || !strings.Contains(split["content"].(string), `"places"`) {
t.Fatalf("today's mergeable file composes %q, was %q", split["content"], whole["content"])
}
}
-236
View File
@@ -1,236 +0,0 @@
package catalogue
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"sync"
)
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661).
//
// **A fixed, reviewable list, owned by the build.** Mesh config and node config hold values only for the keys
// in the catalogue's `predefined.json`, at the root of its repository: each key with its type, a description
// including its unit, its validation (enum, minimum and maximum, pattern) and whether it is trusted. A module's
// value may fall back to one of them, of the same type (ADR 0348 §2, §4).
//
// **Read where a manifest is judged, from the tree it is judged in.** A build reads the file at the root of the
// repository at the commit it builds, `module check` at the root of the repository above each manifest, the
// merge gate at the root of the tree under check; each refuses a file that is not exactly a list of keys, and
// judges every manifest it reads against it (ParseManifestWith). Everywhere else — registration, a stored build
// read again — a manifest is judged against the list the mesh published (PublishedPredefined): the newest trunk
// build's of the catalogue, which the controller keeps and loads with its store.
// PredefinedFile is the list's file at the root of the catalogue's repository.
const PredefinedFile = "predefined.json"
// CatalogueRepository is the name of the repository whose list the mesh publishes: the catalogue's, by the name
// a check finds it beside under (internal/builder's check, ADR 0121).
const CatalogueRepository = "mesh-catalog"
// PredefinedKey is one key of the list: its type, its description, its validation and whether it is trusted
// (ADR 0348 §7). It has no default: a module's value falling back to it has its own.
type PredefinedKey struct {
Type string `json:"type"`
Description string `json:"description"`
Enum []any `json:"enum,omitempty"`
Minimum *float64 `json:"minimum,omitempty"`
Maximum *float64 `json:"maximum,omitempty"`
Pattern string `json:"pattern,omitempty"`
Trusted bool `json:"trusted,omitempty"`
}
// PredefinedList is the list by key. Nil is no list at all, which is not the same as an empty one: a value
// naming a key is refused then saying the list is not present.
type PredefinedList map[string]PredefinedKey
// ParsePredefined reads predefined.json strictly: one JSON object of keys, each with exactly the fields of a
// PredefinedKey, and every key judged as a module's config value is (PredefinedProblems). Anything else is
// refused whole, naming each problem — a list half read would let a module fall back to a key nobody reviewed.
func ParsePredefined(raw []byte) (PredefinedList, error) {
var keys map[string]json.RawMessage
if err := json.Unmarshal(raw, &keys); err != nil {
return nil, fmt.Errorf("%s is not a JSON object of keys: %w", PredefinedFile, err)
}
if keys == nil {
return nil, fmt.Errorf("%s is not a JSON object of keys: it says null", PredefinedFile)
}
list := PredefinedList{}
var problems []string
for _, key := range sortedKeys(keys) {
decoder := json.NewDecoder(bytes.NewReader(keys[key]))
decoder.DisallowUnknownFields()
var k PredefinedKey
if err := decoder.Decode(&k); err != nil {
problems = append(problems, fmt.Sprintf("%q is not a predefined key's declaration (type, description, "+
"enum, minimum, maximum, pattern, trusted): %v", key, err))
continue
}
list[key] = k
}
problems = append(problems, PredefinedProblems(list)...)
if len(problems) > 0 {
return nil, fmt.Errorf("%s is refused: %s", PredefinedFile, strings.Join(problems, "; "))
}
return list, nil
}
// PredefinedProblems is every way a key of the list is declared wrongly, by the rules a module's config value
// is held to (ADR 0348 §4, §7, §11): a usable name that is not placement, a type, a description, validation
// that fits the type, and an object or a list only when trusted.
func PredefinedProblems(list PredefinedList) []string {
var problems []string
for _, key := range sortedKeys(list) {
k := list[key]
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("the predefined key %q: ", key)+fmt.Sprintf(format, args...))
}
if !settingRef.MatchString("${setting:" + key + "}") {
say("not a usable key: lower-case letters, digits, dots, dashes and underscores")
continue
}
if meshWords[key] || accountFacts[key] {
say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key)
continue
}
switch k.Type {
case ConfigNumber, ConfigString, ConfigBoolean:
case ConfigObject, ConfigList:
if !k.Trusted {
say("is %s and not trusted: an object or a list is trusted, \"trusted\": true (novox/hq ADR 0348 §4)", k.Type)
}
case "":
say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)")
default:
say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", k.Type)
}
if strings.TrimSpace(k.Description) == "" {
say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §7)")
}
for _, wrong := range validationProblems(SettingDeclaration{Type: k.Type, Enum: k.Enum, Minimum: k.Minimum,
Maximum: k.Maximum, Pattern: k.Pattern}) {
say("%s", wrong)
}
}
return problems
}
// ReadPredefined is the list in predefined.json at a repository's root, and nil with no error when the root
// holds none: a repository other than the catalogue, or the catalogue before the file.
func ReadPredefined(root string) (PredefinedList, error) {
path := filepath.Join(root, PredefinedFile)
info, err := os.Lstat(path)
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
if !info.Mode().IsRegular() {
return nil, fmt.Errorf("%s is %s, not a plain file", path, kindOf(info.Mode()))
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return ParsePredefined(raw)
}
// RepositoryRoot is the nearest directory at or above dir holding `.git`: the root of the repository a manifest
// is in, where `module check` reads the list. Empty when none is: a manifest outside a repository is judged
// against no list.
func RepositoryRoot(dir string) string {
at, err := filepath.Abs(dir)
if err != nil {
return ""
}
for {
if _, err := os.Lstat(filepath.Join(at, ".git")); err == nil {
return at
}
up := filepath.Dir(at)
if up == at {
return ""
}
at = up
}
}
// PredefinedAbove is the list at the root of the repository above dir (RepositoryRoot), nil when there is no
// repository or it holds no list.
func PredefinedAbove(dir string) (PredefinedList, error) {
root := RepositoryRoot(dir)
if root == "" {
return nil, nil
}
return ReadPredefined(root)
}
// Canonical is the list as JSON with its keys sorted: what a build carries and the controller keeps, so one list
// is one text.
func (l PredefinedList) Canonical() ([]byte, error) {
if l == nil {
return nil, nil
}
return json.Marshal(map[string]PredefinedKey(l))
}
// Keys is the list's keys, sorted.
func (l PredefinedList) Keys() []string {
out := make([]string, 0, len(l))
for k := range l {
out = append(out, k)
}
sort.Strings(out)
return out
}
// NamesPredefined is whether any value of the manifest falls back to a predefined key.
func (m Manifest) NamesPredefined() bool {
for _, d := range m.Settings {
if d.Predefined != "" {
return true
}
}
return false
}
// publishedPredefined is the list the mesh published, which a manifest is judged against where no tree is read.
var publishedPredefined struct {
sync.RWMutex
list PredefinedList
}
// UsePredefined makes list the one a manifest is judged against where no tree is read: what the controller
// loads from its store, and what it keeps when a newer trunk build of the catalogue publishes one. Nil leaves
// the list it has: a store that holds none yet never empties one already loaded.
func UsePredefined(list PredefinedList) {
if list == nil {
return
}
publishedPredefined.Lock()
defer publishedPredefined.Unlock()
publishedPredefined.list = list
}
// PublishedPredefined is the list a manifest is judged against where no tree is read; nil when none is loaded.
func PublishedPredefined() PredefinedList {
publishedPredefined.RLock()
defer publishedPredefined.RUnlock()
return publishedPredefined.list
}
// ReplacePublished sets the published list, nil included, and answers the one it replaced: for a test that
// judges against a list of its own and puts the mesh's back.
func ReplacePublished(list PredefinedList) (was PredefinedList) {
publishedPredefined.Lock()
defer publishedPredefined.Unlock()
was, publishedPredefined.list = publishedPredefined.list, list
return was
}
-175
View File
@@ -1,175 +0,0 @@
package catalogue
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): read strictly, and every
// manifest naming a key judged against the list of the tree it is read in.
// theFirstKeys is the catalogue's first list: font-size and agent-managed-settings.
const theFirstKeys = `{
"font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72},
"agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true}
}`
func TestTheFirstListIsRead(t *testing.T) {
list, err := ParsePredefined([]byte(theFirstKeys))
if err != nil {
t.Fatal(err)
}
if got := strings.Join(list.Keys(), ","); got != "agent-managed-settings,font-size" {
t.Fatalf("keys %s", got)
}
if k := list["font-size"]; k.Type != ConfigNumber || *k.Minimum != 6 || *k.Maximum != 72 || k.Trusted {
t.Fatalf("font-size read as %+v", k)
}
if k := list["agent-managed-settings"]; k.Type != ConfigObject || !k.Trusted {
t.Fatalf("agent-managed-settings read as %+v", k)
}
// One list is one text, however it was written.
a, _ := list.Canonical()
again, _ := ParsePredefined(a)
b, _ := again.Canonical()
if string(a) != string(b) {
t.Fatalf("canonical text moved:\n%s\n%s", a, b)
}
}
func TestAListIsRefusedWholeForAnyWrongKey(t *testing.T) {
for _, c := range []struct{ file, says string }{
{`[]`, "not a JSON object of keys"},
{`null`, "not a JSON object of keys"},
{`{"font-size": {"type": "number", "description": "points", "default": 10}}`, `unknown field "default"`},
{`{"font-size": {"description": "points"}}`, "no type"},
{`{"font-size": {"type": "decimal", "description": "points"}}`, `type "decimal"`},
{`{"font-size": {"type": "number"}}`, "no description"},
{`{"accent": {"type": "object", "description": "colours"}}`, "not trusted"},
{`{"font-size": {"type": "number", "description": "points", "minimum": 9, "maximum": 6}}`, "above its maximum"},
{`{"accent": {"type": "string", "description": "a name", "pattern": "("}}`, "not a regular expression"},
{`{"accent": {"type": "boolean", "description": "on or off", "minimum": 1}}`, "minimum or maximum is for a number"},
{`{"Font Size": {"type": "number", "description": "points"}}`, "not a usable key"},
{`{"places": {"type": "string", "description": "where"}}`, "placement"},
{`{"puid": {"type": "number", "description": "an id"}}`, "placement"},
} {
if _, err := ParsePredefined([]byte(c.file)); err == nil || !strings.Contains(err.Error(), c.says) {
t.Errorf("%s: wanted a refusal saying %q, got %v", c.file, c.says, err)
}
}
}
// fallingBackTo is inConfigForm with its font size falling back to a predefined key.
func fallingBackTo(t *testing.T, key string) []byte {
t.Helper()
m := inConfigForm()
declaration(m, "font-size")["predefined"] = key
raw, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return raw
}
// A manifest naming font-size registers against the list; one naming a key not on it, or of another type, is
// refused — judged against the list given, whatever the mesh published.
func TestAManifestIsJudgedAgainstTheListOfItsTree(t *testing.T) {
withPredefined(t, nil)
list, err := ParsePredefined([]byte(theFirstKeys))
if err != nil {
t.Fatal(err)
}
if _, err := ParseManifestWith(fallingBackTo(t, "font-size"), list); err != nil {
t.Fatalf("a value falling back to font-size is refused: %v", err)
}
if _, err := ParseManifestWith(fallingBackTo(t, "font-scale"), list); err == nil ||
!strings.Contains(err.Error(), `"font-scale", which is not on the list`) {
t.Fatalf("a key not on the list: %v", err)
}
m := inConfigForm()
declaration(m, "theme")["predefined"] = "font-size"
raw, _ := json.Marshal(m)
if _, err := ParseManifestWith(raw, list); err == nil || !strings.Contains(err.Error(), "which is a number") {
t.Fatalf("a key of another type: %v", err)
}
// Where no tree is read, the published list judges: none published refuses.
if _, err := ParseManifest(fallingBackTo(t, "font-size")); err == nil || !strings.Contains(err.Error(), "is not present") {
t.Fatalf("judged against no list: %v", err)
}
withPredefined(t, list)
if _, err := ParseManifest(fallingBackTo(t, "font-size")); err != nil {
t.Fatalf("judged against the published list: %v", err)
}
}
// A template's text filled from the tree keeps the list the manifest was judged against: read again, it is not
// judged against the published one.
func TestTemplatesFilledKeepTheListTheManifestWasJudgedAgainst(t *testing.T) {
withPredefined(t, nil)
list, _ := ParsePredefined([]byte(theFirstKeys))
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, TemplatesDir), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, TemplatesDir, "rc"), []byte("font = ${config:font-size}\n"), 0o644); err != nil {
t.Fatal(err)
}
m := inConfigForm()
declaration(m, "font-size")["predefined"] = "font-size"
r := resourceOf(m, "configuration")
delete(r, "content")
r["template"] = TemplatesDir + "/rc"
m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "files", "kind": "archive", "from": "files"}}}
declaration(m, "width")["runtime"] = true
declaration(m, "sender")["runtime"] = true
raw, _ := json.Marshal(m)
parsed, err := ParseManifestWith(raw, list)
if err != nil {
t.Fatal(err)
}
if _, err := WithTemplatesFrom(parsed, dir); err != nil {
t.Fatalf("filled from the tree, it was judged against another list: %v", err)
}
}
// module check reads the list at the root of the repository above the manifest; no repository, no list.
func TestTheListIsFoundAtTheRepositoryRootAboveAManifest(t *testing.T) {
root := t.TempDir()
module := filepath.Join(root, "modules", "dunst")
if err := os.MkdirAll(module, 0o755); err != nil {
t.Fatal(err)
}
if list, err := PredefinedAbove(module); err != nil || list != nil {
t.Fatalf("outside a repository: %v %v", list, err)
}
if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil {
t.Fatal(err)
}
if list, err := PredefinedAbove(module); err != nil || list != nil {
t.Fatalf("a repository without the file: %v %v", list, err)
}
if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(theFirstKeys), 0o644); err != nil {
t.Fatal(err)
}
list, err := PredefinedAbove(module)
if err != nil || len(list) != 2 {
t.Fatalf("the list above: %v %v", list, err)
}
if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(`{"font-size": {}}`), 0o644); err != nil {
t.Fatal(err)
}
if _, err := PredefinedAbove(module); err == nil {
t.Fatal("a wrong list above was read")
}
if err := os.Remove(filepath.Join(root, PredefinedFile)); err != nil {
t.Fatal(err)
}
if err := os.Symlink("/etc/hostname", filepath.Join(root, PredefinedFile)); err == nil {
if _, err := PredefinedAbove(module); err == nil || !strings.Contains(err.Error(), "not a plain file") {
t.Fatalf("a link read as the list: %v", err)
}
}
}
+270
View File
@@ -0,0 +1,270 @@
package catalogue
import (
"bytes"
"encoding/json"
"fmt"
"regexp"
"sort"
"strings"
)
// A config value resolves from four sources, the last set winning (novox/hq issue 630):
//
// 1. the default in the module's manifest;
// 2. mesh config: a mesh-wide value for a predefined key, such as `font-size`;
// 3. node config: a value for a predefined key for one node;
// 4. module-assignment config: that module's own value on that node, the most specific.
//
// An assignment is one module on one node: there is no module layer for the whole mesh. Mesh and node
// config hold values only for the keys on a fixed, reviewable list — the catalogue's `predefined.json` —
// that any module may fall back to, by naming one as its preference's `predefined`. A preference without
// `predefined` reads only its default and its module-assignment config. A preference is never trusted.
// Preference is one value a module reads that a machine may tune, as the manifest declares it under
// `preferences`.
type Preference struct {
// Type is "number", "string" or "boolean".
Type string `json:"type"`
// Default is the value when no source sets it, of Type.
Default any `json:"default"`
// Predefined names the key on the catalogue's list this preference falls back to: mesh config,
// then node config, for that key. Empty reads only the default and the module-assignment config.
Predefined string `json:"predefined,omitempty"`
// Why this default: one sentence for whoever wonders whether to change it.
Why string `json:"why"`
}
// The types a preference and a predefined key may have.
const (
TypeNumber = "number"
TypeString = "string"
TypeBoolean = "boolean"
)
// PredefinedFile is the catalogue repository's root file holding the predefined keys.
const PredefinedFile = "predefined.json"
// PredefinedKey is one key mesh and node config may hold a value for.
type PredefinedKey struct {
Type string `json:"type"`
Why string `json:"why"`
}
// Predefined is the list of predefined keys, by key.
type Predefined map[string]PredefinedKey
// Keys is the list's keys, sorted.
func (p Predefined) Keys() []string {
keys := make([]string, 0, len(p))
for k := range p {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// predefinedName is what a predefined key looks like: the shape of a setting key.
var predefinedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`)
// ParsePredefined reads `predefined.json` strictly, and refuses a key whose name, type or why is wrong.
func ParsePredefined(raw []byte) (Predefined, error) {
decoder := json.NewDecoder(bytes.NewReader(raw))
decoder.DisallowUnknownFields()
var list Predefined
if err := decoder.Decode(&list); err != nil {
return nil, fmt.Errorf("%s is not a list of predefined keys ({\"<key>\": {\"type\": …, \"why\": …}}): %w",
PredefinedFile, err)
}
var problems []string
for _, key := range list.Keys() {
k := list[key]
if !predefinedName.MatchString(key) {
problems = append(problems, fmt.Sprintf("%q is not a usable key: lower-case letters, digits, dots, dashes and underscores", key))
}
if meshWords[key] {
problems = append(problems, fmt.Sprintf("%q is a key the mesh reads itself", key))
}
if !knownType(k.Type) {
problems = append(problems, fmt.Sprintf("%q has the type %q; a type is %q, %q or %q", key, k.Type,
TypeNumber, TypeString, TypeBoolean))
}
if strings.TrimSpace(k.Why) == "" {
problems = append(problems, fmt.Sprintf("%q has no why: say in one sentence what the key is for", key))
}
}
if len(problems) > 0 {
return nil, fmt.Errorf("%s: %s", PredefinedFile, strings.Join(problems, "; "))
}
if list == nil {
list = Predefined{}
}
return list, nil
}
func knownType(t string) bool {
return t == TypeNumber || t == TypeString || t == TypeBoolean
}
// TypeOf is the type a JSON value has, as a preference names it, or "" for one that is none of them.
func TypeOf(v any) string {
switch v.(type) {
case float64, int, int64, json.Number:
return TypeNumber
case string:
return TypeString
case bool:
return TypeBoolean
}
return ""
}
// PreferencesOf is every preference a module declares: those under `preferences`, and — while the
// catalogue moves to them — every `settings` entry of kind preference, its type read from its default.
func PreferencesOf(m Manifest) map[string]Preference {
if len(m.Preferences) == 0 && len(m.Settings) == 0 {
return nil
}
out := make(map[string]Preference, len(m.Preferences)+len(m.Settings))
for key, d := range m.Settings {
out[key] = Preference{Type: TypeOf(d.Default), Default: d.Default, Why: d.Why}
}
for key, p := range m.Preferences {
out[key] = p
}
return out
}
// DeclaresPreference is whether a module declares a preference by this key.
func DeclaresPreference(m Manifest, key string) bool {
_, has := m.Preferences[key]
if !has {
_, has = m.Settings[key]
}
return has
}
// PredefinedProblems is every way a module's preferences disagree with the list of predefined keys: a
// `predefined` not on it, or a type other than the list's.
func PredefinedProblems(m Manifest, list Predefined) []string {
var problems []string
keys := make([]string, 0, len(m.Preferences))
for k := range m.Preferences {
keys = append(keys, k)
}
sort.Strings(keys)
for _, key := range keys {
p := m.Preferences[key]
if p.Predefined == "" {
continue
}
k, on := list[p.Predefined]
if !on {
listed := "the list holds none"
if len(list) > 0 {
listed = "the list holds " + strings.Join(list.Keys(), ", ")
}
problems = append(problems, fmt.Sprintf("%s's preference %q falls back to the predefined key %q, which "+
"is not on the catalogue's list of predefined keys (%s; %s)", m.Module, key, p.Predefined, PredefinedFile,
listed))
continue
}
if k.Type != p.Type {
problems = append(problems, fmt.Sprintf("%s's preference %q is a %s, and the predefined key %q it falls "+
"back to is a %s", m.Module, key, p.Type, p.Predefined, k.Type))
}
}
return problems
}
// Where a value from mesh or node config is said to come from.
const MeshConfigLayer = "mesh config"
// NodeConfigLayer is what the node config of one node is called where a value's source is said.
func NodeConfigLayer(node string) string { return "node config (" + node + ")" }
// ConfigLayers are the layers mesh config and node config make for one module: each of its preferences
// that names a predefined key, given that key's value. Laid between the module's defaults and its
// module-assignment config; like the defaults they fill `${setting:<key>}` only.
func ConfigLayers(m Manifest, mesh, node map[string]any, nodeName string) []Layer {
var out []Layer
for _, source := range []struct {
from string
values map[string]any
}{{MeshConfigLayer, mesh}, {NodeConfigLayer(nodeName), node}} {
values := map[string]any{}
for key, p := range m.Preferences {
if p.Predefined == "" {
continue
}
if v, set := source.values[p.Predefined]; set {
values[key] = v
}
}
if len(values) > 0 {
out = append(out, Layer{From: source.from, Values: values, Config: true})
}
}
return out
}
// ConfigValueProblem refuses a value of the wrong type for a predefined key or a preference.
func ConfigValueProblem(what string, want string, v any) error {
if got := TypeOf(v); got != want {
shown, _ := json.Marshal(v)
return fmt.Errorf("%s is a %s, and %s is %s", what, want, shown, aOrNothing(got))
}
return nil
}
func aOrNothing(t string) string {
if t == "" {
return "none of a number, a string or a boolean"
}
return "a " + t
}
// preferenceProblems is every way a module's `preferences` are wrong in themselves.
func preferenceProblems(m Manifest) []string {
var problems []string
keys := make([]string, 0, len(m.Preferences))
for k := range m.Preferences {
keys = append(keys, k)
}
sort.Strings(keys)
for _, key := range keys {
p := m.Preferences[key]
say := func(format string, args ...any) {
problems = append(problems, fmt.Sprintf("%s's preference %q: ", m.Module, key)+fmt.Sprintf(format, args...))
}
if _, twice := m.Settings[key]; twice {
say("declared under settings too; declare it once, under preferences")
}
if !knownType(p.Type) {
say("the type is %q; a type is %q, %q or %q", p.Type, TypeNumber, TypeString, TypeBoolean)
} else if p.Default == nil {
say("no default: a preference has one")
} else if got := TypeOf(p.Default); got != p.Type {
shown, _ := json.Marshal(p.Default)
say("it is a %s, and its default %s is %s", p.Type, shown, aOrNothing(got))
}
if p.Predefined != "" && !predefinedName.MatchString(p.Predefined) {
say("%q is not a predefined key's name", p.Predefined)
}
}
return problems
}
// PredefinedNamed is every predefined key a module's preferences fall back to, sorted.
func PredefinedNamed(m Manifest) []string {
seen := map[string]bool{}
var out []string
for _, p := range m.Preferences {
if p.Predefined != "" && !seen[p.Predefined] {
seen[p.Predefined] = true
out = append(out, p.Predefined)
}
}
sort.Strings(out)
return out
}
+117
View File
@@ -0,0 +1,117 @@
package catalogue
import (
"strings"
"testing"
)
// The list of predefined keys is read strictly: a type and a why for each key (novox/hq issue 630).
func TestThePredefinedListIsReadStrictly(t *testing.T) {
list, err := ParsePredefined([]byte(`{"font-size": {"type": "number", "why": "the size text is drawn at"},
"theme": {"type": "string", "why": "light or dark"}}`))
if err != nil || len(list) != 2 || list["font-size"].Type != TypeNumber {
t.Fatalf("a good list: %v %v", list, err)
}
for raw, want := range map[string]string{
`{"font-size": {"type": "integer", "why": "x"}}`: `the type "integer"`,
`{"font-size": {"type": "number"}}`: "no why",
`{"Font Size": {"type": "number", "why": "x"}}`: "not a usable key",
`{"font-size": {"type": "number", "why": "x", "a": 1}}`: "unknown field",
`{"ports": {"type": "number", "why": "x"}}`: "the mesh reads itself",
} {
if _, err := ParsePredefined([]byte(raw)); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%s: %v, want %q", raw, err, want)
}
}
}
// A manifest declares preferences with a type, a default of that type, a why and maybe a predefined key;
// a `settings` entry of kind preference is read as one, its type from its default.
func TestAManifestDeclaresPreferences(t *testing.T) {
file := `"resources": [{"id": "rc", "type": "file", "path": "/etc/rc", "mode": "0644", "trusted": false,
"content": "size = ${setting:font-size} width = ${setting:width}"}]`
m, err := ParseManifest([]byte(`{"module": "dunst", "version": "1",
"preferences": {"font-size": {"type": "number", "default": 10, "predefined": "font-size", "why": "100 DPI"}},
"settings": {"width": {"kind": "preference", "default": 250, "why": "forty characters"}}, ` + file + `}`))
if err != nil {
t.Fatal(err)
}
prefs := PreferencesOf(m)
if prefs["font-size"].Predefined != "font-size" || prefs["width"].Type != TypeNumber || prefs["width"].Default != float64(250) {
t.Fatalf("the preferences: %+v", prefs)
}
if got := PredefinedNamed(m); len(got) != 1 || got[0] != "font-size" {
t.Fatalf("the predefined keys named: %v", got)
}
for body, want := range map[string]string{
`"preferences": {"font-size": {"type": "number", "default": "10", "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: `its default "10" is a string`,
`"preferences": {"font-size": {"type": "float", "default": 10, "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: `the type is "float"`,
`"preferences": {"font-size": {"type": "number", "why": "x"}, "width": {"type": "number", "default": 1, "why": "x"}}`: "no default",
`"preferences": {"font-size": {"type": "number", "default": 10}, "width": {"type": "number", "default": 1, "why": "x"}}`: "no why",
} {
if _, err := ParseManifest([]byte(`{"module": "dunst", "version": "1", ` + body + `, ` + file + `}`)); err == nil ||
!strings.Contains(err.Error(), want) {
t.Errorf("%s: %v, want %q", body, err, want)
}
}
}
// A preference falls back only to a key on the list, with the list's type.
func TestAPreferencesPredefinedKeyIsOnTheListWithItsType(t *testing.T) {
m := Manifest{Module: "dunst", Preferences: map[string]Preference{
"font-size": {Type: TypeNumber, Default: float64(10), Predefined: "font-size", Why: "x"}}}
if p := PredefinedProblems(m, Predefined{"font-size": {Type: TypeNumber, Why: "x"}}); len(p) != 0 {
t.Fatalf("on the list: %v", p)
}
if p := PredefinedProblems(m, Predefined{"theme": {Type: TypeString, Why: "x"}}); len(p) != 1 ||
!strings.Contains(p[0], "not on the catalogue's list") || !strings.Contains(p[0], "theme") {
t.Fatalf("not on the list: %v", p)
}
if p := PredefinedProblems(m, Predefined{"font-size": {Type: TypeString, Why: "x"}}); len(p) != 1 ||
!strings.Contains(p[0], "is a string") {
t.Fatalf("another type: %v", p)
}
}
// Mesh and node config fill `${setting:<key>}` between the default and the module's own layer, and — like
// the default — never reach a mergeable file.
func TestConfigLayersFillAPlaceholderAndNoMergeableFile(t *testing.T) {
m := Manifest{Module: "dunst", Preferences: map[string]Preference{
"font-size": {Type: TypeNumber, Default: float64(10), Predefined: "font-size", Why: "x"},
"width": {Type: TypeNumber, Default: float64(250), Why: "x"}}}
fill := func(layers []Layer) string {
t.Helper()
r := map[string]any{"id": "rc", "type": "file", "content": "font ${setting:font-size} width ${setting:width}"}
if err := settingInto(r, WithDefaults(m, layers), m.Module); err != nil {
t.Fatal(err)
}
return r["content"].(string)
}
mesh := map[string]any{"font-size": float64(11), "width": float64(999)}
node := map[string]any{"font-size": float64(13)}
if got := fill(nil); got != "font 10 width 250" {
t.Fatalf("the defaults: %s", got)
}
if got := fill(ConfigLayers(m, mesh, nil, "g14")); got != "font 11 width 250" {
t.Fatalf("mesh config, which a preference without predefined does not read: %s", got)
}
both := ConfigLayers(m, mesh, node, "g14")
if got := fill(both); got != "font 13 width 250" {
t.Fatalf("node config over mesh config: %s", got)
}
own := append(both, Layer{From: "g14", Values: map[string]any{"font-size": float64(15)}})
if got := fill(own); got != "font 15 width 250" {
t.Fatalf("the module's own value over both: %s", got)
}
sources := map[string]string{}
for _, s := range Effective(m, both) {
sources[s.Key] = s.From
}
if sources["font-size"] != NodeConfigLayer("g14") || sources["width"] != DefaultLayer {
t.Fatalf("where each comes from: %v", sources)
}
merged, err := ApplySettings(map[string]any{"id": "conf", "type": "file", "merge": MergeJSON, "content": "{}"}, both)
if err != nil || merged["content"] != "{}\n" {
t.Fatalf("config reached a mergeable file: %v %v", merged["content"], err)
}
}
+1 -7
View File
@@ -111,9 +111,6 @@ type Provider struct {
// (novox/hq to-be 23), and the pair is what tells the holder of a seat apart from another module
// providing the same thing (ADR 0110).
Module string
// LeftOut says each served key left out of Serves because that node composes no name it asks
// for (novox/hq ADR 0346 §3): said under each consumer's need.
LeftOut []string
}
// Held is a claim somebody already has, used for the scopes wider than one node.
@@ -233,9 +230,6 @@ type Needed struct {
// state, not a consumer missing its key. Set by the plan, which is the only layer that knows a
// licence's manager; empty for every consumer.
Manager bool
// LeftOut says each served key the provider left out, because its machine composes no name the
// value asks for (novox/hq ADR 0346 §3).
LeftOut []string
// Module is the providing module on From, empty for a need answered by a record or by nothing
// that serves it. A provider is a (node, module) pair (novox/hq to-be 23), and a binding to data
// is a binding to the pair (ADR 0232).
@@ -588,7 +582,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world
bound = pm.IdentityBoundOf(want)
}
needs = append(needs, Needed{Name: want, From: p.Node, At: p.At, Module: p.Module,
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound, LeftOut: p.LeftOut})
Serves: p.Serves, For: because[want], SharedOwn: shared, Identity: bound})
}
switch {
case world.Unchecked:
+1 -13
View File
@@ -128,19 +128,7 @@ var defaultSeats = append([]Seat{
// The vault: the controller seals every minted credential with what it provides, which is the
// test for a seat of the mesh's own (novox/hq ADR 0161) — a second provider of `secret` is a
// second claimant, refused by name, rather than a candidate for a pin.
//
// Its one verb, `get`, hands a secret on request and reply, sealed to the key the asker sends and only
// to an asker the vault allows (novox/hq ADR 0349 §11, issue 674). Optional until the vault that runs
// serves it (design 33 §7).
{Name: "mesh-vault", Scope: ScopeMesh, Delivers: "secret", Decision: "novox/hq ADR 0161",
Serves: []Verb{
{Name: "get", Optional: true, Description: "One secret, sealed to the public key the asker sends (the " +
"sealed box of ADR 0206), with its generation; answered only to an asker the vault allows, as the " +
"bus proved it. Never the value in the clear, and never to a private key.",
Input: schema(map[string]string{"name": "the secret's name",
"public_key": "the asker's X25519 public key as SPKI PEM — never a private key"},
[]string{"name", "public_key"})},
}},
{Name: "mesh-vault", Scope: ScopeMesh, Delivers: "secret", Decision: "novox/hq ADR 0161"},
// Named for its scope since 2026-09-30 (novox/hq ADR 0156); `the-artifact-store` resolves to it as
// an alias on a mesh that predates the rename. It serves artifacts of every kind a build makes —
// images and archives, by digest — which is why the provision is the artifact store and not an

Some files were not shown because too many files have changed in this diff Show More