Compare commits
52
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e233e1d87 | ||
|
|
0d11b4fc93 | ||
|
|
d34b4f32e1 | ||
|
|
d941d798e9 | ||
|
|
b7f70bc4c0 | ||
|
|
8431ac9656 | ||
|
|
bc2a448512 | ||
|
|
7a41e12063 | ||
|
|
c7b534a08c | ||
|
|
ccc86d3594 | ||
|
|
6fe82c61bd | ||
|
|
8e95c1d9ed | ||
|
|
cb55a46400 | ||
|
|
37947ca4ee | ||
|
|
d8e785bb4b | ||
|
|
281837847e | ||
|
|
c7551162d6 | ||
|
|
2cffa33bcc | ||
|
|
2a2be687f3 | ||
|
|
303d4037e8 | ||
|
|
5c8e768ef5 | ||
|
|
aad33dfe5c | ||
|
|
ef26dba275 | ||
|
|
d12bba0109 | ||
|
|
c85c2804a0 | ||
|
|
2ff40109ac | ||
|
|
46c16e18fe | ||
|
|
f00f9c2a2f | ||
|
|
21cd036232 | ||
|
|
f0adb177f0 | ||
|
|
d2d9fbbd7d | ||
|
|
2026ef36f3 | ||
|
|
51e97a8c14 | ||
|
|
c7a4bf5c11 | ||
|
|
ec983bfc3d | ||
|
|
dd9ab7927d | ||
|
|
0818e4c29a | ||
|
|
41e640d668 | ||
|
|
a1f9856b39 | ||
|
|
74ed7cbe0a | ||
|
|
36704b5471 | ||
|
|
6e44a27a63 | ||
|
|
dce0eef1ca | ||
|
|
e3c3bb37d0 | ||
|
|
b92fb68116 | ||
|
|
3f7f797168 | ||
|
|
c433e76de2 | ||
|
|
f4efc3cad0 | ||
|
|
a76baa4b47 | ||
|
|
90ce9e9d9f | ||
|
|
df5de3bd9f | ||
|
|
a18d09db75 |
@@ -130,7 +130,7 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
|
||||
|
||||
The foundation's ports are this machine's, each checked free before anything is
|
||||
raised and kept as the node's setting for the module that binds it:
|
||||
--store-port 5432 --bus-port 5671 --amqp-port 5672 --management-port 15672
|
||||
--store-port 5432 --bus-port 5671
|
||||
--registry-port 5000 (follows --registry, and must agree with it)
|
||||
--packages-port 3000 --hub-port 51820/udp
|
||||
--overlay-range the private network's range (default 10.42.0.0/16); refused
|
||||
@@ -325,9 +325,7 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
|
||||
into *int
|
||||
}{
|
||||
{"store-port", "the store", &opts.Ports.Store},
|
||||
{"bus-port", "the bus (amqps)", &opts.Ports.Bus},
|
||||
{"amqp-port", "the broker's AMQP", &opts.Ports.AMQP},
|
||||
{"management-port", "the broker's management, on loopback", &opts.Ports.Management},
|
||||
{"bus-port", "the bus", &opts.Ports.Bus},
|
||||
{"registry-port", "the registry", &opts.Ports.Registry},
|
||||
{"packages-port", "the package registry", &opts.Ports.Packages},
|
||||
{"hub-port", "the private network's hub (udp)", &opts.Ports.Hub},
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// judgeDirectories judges every directory of the declaration this node keeps, read-only (novox/hq issue 358): which
|
||||
// the walk that follows no link another account could make would refuse. Nothing is made, chmodded or chowned.
|
||||
func judgeDirectories(statePath string) (string, error) {
|
||||
kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
|
||||
if errors.Is(err, store.ErrNothingDeclared) {
|
||||
return "", errors.New("this node keeps no declaration, so it is declared no directory")
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
d, err := declaration.Parse(kept.Declaration)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the declaration this node keeps does not read: %w", err)
|
||||
}
|
||||
return apply.JudgeDirectories(d), nil
|
||||
}
|
||||
|
||||
// judgeDirectoriesFrom is judgeDirectories as the controller asks it over the link.
|
||||
func judgeDirectoriesFrom(statePath string) link.JudgeDirectoriesFunc {
|
||||
return func(string) (string, error) { return judgeDirectories(statePath) }
|
||||
}
|
||||
|
||||
// judgeDirectoriesCommand is `nox-mesh-host judge-directories` at this machine, as root: the same judgement, for a
|
||||
// machine the controller cannot ask.
|
||||
func judgeDirectoriesCommand(opts options) error {
|
||||
said, err := judgeDirectories(opts.state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(opts.out, said)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A library is judged on every look (novox/hq ADR 0328 §8, §9).
|
||||
//
|
||||
// Each library this machine is reached by is said as a resource `library:<name>` of kind library, with no module:
|
||||
// healthy while it is kept (or, off its home, while its mount is there), waiting with an access wait of reason keep
|
||||
// or disk, unknown while its filesystem could not be named, unhealthy otherwise. The root is the target, never in a
|
||||
// reason nor in a wait's words: the controller says them on the operator's channel, which withholds a path (ADR
|
||||
// 0234 §6). An access is judged by nothing, as before ADR 0325.
|
||||
|
||||
// libraryJudge reads every library on every look. Safe for the apply and the looking loop at once.
|
||||
type libraryJudge struct {
|
||||
mu sync.Mutex
|
||||
libraries []declaration.Library
|
||||
declared *declaration.Declaration
|
||||
// was is each library's last verdict, with since when and how many looks in a row found it unhealthy.
|
||||
was map[string]link.ResourceHealth
|
||||
|
||||
// reading guards inFlight: every library whose look has not returned yet.
|
||||
reading sync.Mutex
|
||||
inFlight map[string]bool
|
||||
}
|
||||
|
||||
// accesses is the one library judge on this machine (its name from when it judged accesses).
|
||||
var accesses = newLibraryJudge()
|
||||
|
||||
func newLibraryJudge() *libraryJudge {
|
||||
return &libraryJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
|
||||
}
|
||||
|
||||
// set is what the declaration just applied declares: every library. One no longer declared is forgotten.
|
||||
func (j *libraryJudge) set(d *declaration.Declaration, _ map[string]bool) {
|
||||
var ls []declaration.Library
|
||||
if d != nil {
|
||||
ls = append(ls, d.Libraries...)
|
||||
}
|
||||
sort.Slice(ls, func(a, b int) bool { return ls[a].Library < ls[b].Library })
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
j.libraries, j.declared = ls, d
|
||||
declared := map[string]bool{}
|
||||
for _, l := range ls {
|
||||
declared[apply.LibraryAccess(l.Library)] = true
|
||||
}
|
||||
for id := range j.was {
|
||||
if !declared[id] {
|
||||
delete(j.was, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// declaration is the declaration last set.
|
||||
func (j *libraryJudge) declaration() *declaration.Declaration {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.declared
|
||||
}
|
||||
|
||||
// lookBound is how long a look waits for one library to answer. A hard network mount whose server is down blocks
|
||||
// a stat for ever, and one library must never stop every health statement this machine makes.
|
||||
var lookBound = 5 * time.Second
|
||||
|
||||
// verdict is one library's judgement in a look.
|
||||
type verdict struct {
|
||||
state, reason string
|
||||
waits []link.AccessWait
|
||||
act *link.AccessActRecord
|
||||
}
|
||||
|
||||
// judgeBounded judges one library within lookBound, outside any lock. A look that has not answered is said so;
|
||||
// while it still hangs, later looks start no other on the same library.
|
||||
func (j *libraryJudge) judgeBounded(l declaration.Library) (verdict, bool) {
|
||||
j.reading.Lock()
|
||||
if j.inFlight[l.Library] {
|
||||
j.reading.Unlock()
|
||||
return verdict{}, false
|
||||
}
|
||||
j.inFlight[l.Library] = true
|
||||
j.reading.Unlock()
|
||||
done := make(chan verdict, 1)
|
||||
go func() {
|
||||
v := judgeLibrary(l)
|
||||
j.reading.Lock()
|
||||
delete(j.inFlight, l.Library)
|
||||
j.reading.Unlock()
|
||||
done <- v
|
||||
}()
|
||||
select {
|
||||
case v := <-done:
|
||||
return v, true
|
||||
case <-time.After(lookBound):
|
||||
return verdict{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// judgeLibrary is one library's judgement now (ADR 0328 §7, §8, §9).
|
||||
func judgeLibrary(l declaration.Library) verdict {
|
||||
v := apply.JudgeLibrary(context.Background(), l)
|
||||
access := apply.LibraryAccess(l.Library)
|
||||
var out verdict
|
||||
if a := v.Act; a != nil {
|
||||
out.act = &link.AccessActRecord{Act: a.Act, Access: access, Warrant: a.Warrant, By: a.By, State: a.State,
|
||||
Started: a.Started, Ended: a.Ended, Folders: a.Folders, Files: a.Files, Kept: a.Kept, Links: a.Links,
|
||||
Other: a.Other, Mounts: a.Mounts, Shared: a.Shared, Linked: a.Linked, Stripped: a.Stripped, Why: a.Why}
|
||||
}
|
||||
switch v.Kind {
|
||||
case apply.VerdictWaitKeep:
|
||||
out.state, out.reason = link.StateWaiting, "its root holds data the mesh has not taken on the filesystem "+
|
||||
"it is on now, and waits for the operator's word (novox/hq ADR 0328 §7, §8)"
|
||||
out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
|
||||
What: apply.KeepWords(l.Library), Placed: v.Placed, Reason: link.WaitKeep}}
|
||||
case apply.VerdictWaitDisk:
|
||||
out.state, out.reason = link.StateWaiting, v.Why+" (novox/hq ADR 0328 §9)"
|
||||
out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
|
||||
What: apply.DiskWords(l.Library), Reason: link.WaitDisk}}
|
||||
case apply.VerdictUnknown:
|
||||
// Not a fault and not the operator's: said as not known, and judged again on the next look.
|
||||
out.state, out.reason = link.StateUnknown, v.Why+"; judged again on the next look"
|
||||
case apply.VerdictBroken:
|
||||
out.state, out.reason = link.StateUnhealthy, v.Why
|
||||
default:
|
||||
out.state = link.StateHealthy
|
||||
switch {
|
||||
case v.Act != nil && v.Act.State == link.ActRunning:
|
||||
out.state, out.reason = link.StateStarting, "the take act is setting its group under its root"
|
||||
case v.Act != nil && (v.Act.State == link.ActStopped || v.Act.State == link.ActFailed):
|
||||
out.state, out.reason = link.StateStarting, "the take act goes on from where it stopped: "+v.Act.Why
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// look reads every library now. Whether any verdict changed since the last look, for the loop to say it.
|
||||
func (j *libraryJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
|
||||
j.mu.Lock()
|
||||
ls := append([]declaration.Library(nil), j.libraries...)
|
||||
j.mu.Unlock()
|
||||
|
||||
read := make([]verdict, len(ls))
|
||||
for i, l := range ls {
|
||||
v, answered := j.judgeBounded(l)
|
||||
if !answered {
|
||||
v = verdict{state: link.StateUnknown, reason: "its root does not answer: judging it did not return " +
|
||||
"within " + lookBound.String()}
|
||||
}
|
||||
read[i] = v
|
||||
}
|
||||
|
||||
// And every tool runner's account the last apply could not put in a group, as a resource of its own (ADR 0334
|
||||
// §2): unhealthy, never failing the apply.
|
||||
warnings := apply.RunnerWarnings()
|
||||
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
changed := false
|
||||
out := make([]link.ResourceHealth, 0, len(ls)+len(warnings))
|
||||
for _, w := range warnings {
|
||||
v := link.ResourceHealth{Module: w.Module, Resource: w.Resource, Kind: link.KindRunner, State: link.StateUnhealthy,
|
||||
Reason: "the tool runner's account joins no group: " + w.Why, Since: now.UTC(), Streak: 1}
|
||||
if was, had := j.was[w.Resource]; had && was.Reason == v.Reason {
|
||||
v.Since, v.Streak = was.Since, was.Streak+1
|
||||
} else {
|
||||
changed = true
|
||||
}
|
||||
j.was[w.Resource] = v
|
||||
out = append(out, v)
|
||||
}
|
||||
for i, l := range ls {
|
||||
r := read[i]
|
||||
id := apply.LibraryAccess(l.Library)
|
||||
v := link.ResourceHealth{Resource: id, Kind: link.KindLibrary, Target: l.Root, State: r.state,
|
||||
Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act}
|
||||
if was, had := j.was[id]; had && was.State == r.state && was.Reason == r.reason {
|
||||
v.Since = was.Since
|
||||
v.Streak = was.Streak
|
||||
} else {
|
||||
changed = true
|
||||
}
|
||||
if r.state == link.StateUnhealthy {
|
||||
v.Streak++
|
||||
}
|
||||
j.was[id] = v
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, changed
|
||||
}
|
||||
|
||||
// withAccesses adds to a statement every library, as a resource of kind library.
|
||||
func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
|
||||
h.Resources = append(h.Resources, vs...)
|
||||
return h
|
||||
}
|
||||
|
||||
// accessesHealthy says no library in vs is anything but healthy.
|
||||
func accessesHealthy(vs []link.ResourceHealth) bool {
|
||||
for _, v := range vs {
|
||||
if v.State != link.StateHealthy {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// keeper goes on with take acts stopped at their bound, beside the applies, one at a time (ADR 0328 §7).
|
||||
type keeper struct {
|
||||
running atomic.Bool
|
||||
say func(string)
|
||||
}
|
||||
|
||||
// keeping is the one keeper on this machine.
|
||||
var keeping = &keeper{say: func(string) {}}
|
||||
|
||||
// poke goes on with every stopped take act of the declaration last applied, unless that is running already.
|
||||
func (k *keeper) poke(ctx context.Context) {
|
||||
d := accesses.declaration()
|
||||
if d == nil || len(d.Libraries) == 0 || !k.running.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer k.running.Store(false)
|
||||
for _, line := range apply.ReconcileLibraries(ctx, d) {
|
||||
k.say(line)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// startKeeping has the keeper say what it does.
|
||||
func startKeeping(ctx context.Context, say func(string)) {
|
||||
keeping.say = say
|
||||
keeping.poke(ctx)
|
||||
}
|
||||
|
||||
// accessActFrom is the take act as the controller asks it over the link (ADR 0328 §7): judged against the
|
||||
// declaration this node keeps, and run beside the applies.
|
||||
func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc {
|
||||
return func(act link.AccessAct) (string, error) {
|
||||
kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
|
||||
if errors.Is(err, store.ErrNothingDeclared) {
|
||||
return "", errors.New("this node keeps no declaration, so it is declared no library")
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
d, err := declaration.Parse(kept.Declaration)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the declaration this node keeps does not read: %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
_, said, err := apply.TakeLibrary(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err)
|
||||
return "", err
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
}
|
||||
|
||||
// withWaiting is held with every id of waiting beside it: what liveness does not judge.
|
||||
func withWaiting(held, waiting map[string]bool) map[string]bool {
|
||||
if len(waiting) == 0 {
|
||||
return held
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for id := range held {
|
||||
out[id] = true
|
||||
}
|
||||
for id := range waiting {
|
||||
out[id] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// A library whose root holds data never taken is said waiting with one access wait: the access `library:<name>`, the
|
||||
// part its name, words with no path, the filesystem's name, and reason keep (novox/hq ADR 0328 §8); one whose root
|
||||
// is absent waits with reason disk (§9). Neither carries a module.
|
||||
func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
||||
was := apply.Libraries
|
||||
apply.Libraries = apply.NewLibraryMemory()
|
||||
defer func() { apply.Libraries = was }()
|
||||
dir := t.TempDir()
|
||||
root := filepath.Join(dir, "media")
|
||||
if err := os.MkdirAll(filepath.Join(root, "a film"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"groups":[{"name":"mesh-media","gid":62000},` +
|
||||
`{"name":"mesh-books","gid":62001}],"libraries":[` +
|
||||
`{"library":"media","root":"` + root + `","group":"mesh-media","gid":62000,"home":true},` +
|
||||
`{"library":"books","root":"` + filepath.Join(dir, "books") + `","group":"mesh-books","gid":62001,"home":true}],` +
|
||||
`"owns_nothing":true,"resources":[]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := newLibraryJudge()
|
||||
j.set(d, nil)
|
||||
vs, _ := j.look(time.Now())
|
||||
if len(vs) != 2 {
|
||||
t.Fatalf("the libraries are not said: %+v", vs)
|
||||
}
|
||||
books, media := vs[0], vs[1]
|
||||
if media.Kind != link.KindLibrary || media.Resource != "library:media" || media.Module != "" ||
|
||||
media.State != link.StateWaiting || len(media.AccessWaits) != 1 {
|
||||
t.Fatalf("a library holding data is not said waiting: %+v", media)
|
||||
}
|
||||
raw, _ := json.Marshal(media.AccessWaits[0])
|
||||
want := `{"kind":"access","access":"library:media","part":"media","what":"let the mesh keep the library media",` +
|
||||
`"placed":"` + media.AccessWaits[0].Placed + `","reason":"keep"}`
|
||||
if string(raw) != want || !strings.HasPrefix(media.AccessWaits[0].Placed, "fs-") {
|
||||
t.Fatalf("the wait reads %s", raw)
|
||||
}
|
||||
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
||||
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 160: a machine nobody pushed to still says what it is made of once a day, so the
|
||||
// newest of it the mesh holds is no older than a day. Before, the reconcile said it only with something
|
||||
// else (a hold, a firewall, an outward link changed), and a converged machine's picture was as old as the
|
||||
// last delivery to it.
|
||||
func TestAMachineNobodyPushedToSaysWhatItIsOnceADay(t *testing.T) {
|
||||
at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
|
||||
w := &adoptionWatch{now: func() time.Time { return at }}
|
||||
quiet := link.Report{Outward: []string{"wlan0"},
|
||||
Profile: map[string]any{"machine": map[string]any{"observed_at": at}}}
|
||||
|
||||
if !w.news(quiet) {
|
||||
t.Fatal("the first reconcile after the engine started did not say what the machine is")
|
||||
}
|
||||
w.said(quiet)
|
||||
|
||||
for _, after := range []time.Duration{5 * time.Minute, 12 * time.Hour, 22*time.Hour + 55*time.Minute} {
|
||||
at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(after)
|
||||
if w.news(quiet) {
|
||||
t.Errorf("%s after the mesh heard it, an unchanged reconcile was said again", after)
|
||||
}
|
||||
}
|
||||
// The reconcile that falls past 23 hours says it, so it reaches the mesh inside the day.
|
||||
at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(23*time.Hour + 5*time.Minute)
|
||||
if !w.news(quiet) {
|
||||
t.Fatal("a day on, the reconcile did not say what the machine is")
|
||||
}
|
||||
w.said(quiet)
|
||||
at = at.Add(5 * time.Minute)
|
||||
if w.news(quiet) {
|
||||
t.Error("once said, it was said again at the next reconcile")
|
||||
}
|
||||
}
|
||||
|
||||
// A delivery's account carries the profile too: heard, it starts the day again.
|
||||
func TestADeliveryHeardStartsTheDayAgain(t *testing.T) {
|
||||
at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
|
||||
w := &adoptionWatch{now: func() time.Time { return at }}
|
||||
report := link.Report{Outward: []string{"eth0"}, Profile: map[string]any{"capabilities": []any{}}}
|
||||
w.said(report)
|
||||
at = at.Add(20 * time.Hour)
|
||||
w.said(report) // a delivery's apply
|
||||
at = at.Add(20 * time.Hour)
|
||||
if w.news(report) {
|
||||
t.Error("said again unasked 20 hours after a delivery told the mesh")
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal carries nothing about the machine, so it is never said for being a day old.
|
||||
func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) {
|
||||
at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC)
|
||||
w := &adoptionWatch{now: func() time.Time { return at }}
|
||||
if w.machineDue(link.Report{Refused: "no"}) {
|
||||
t.Error("a refusal was due for what the machine is made of")
|
||||
}
|
||||
}
|
||||
|
||||
// A laptop that slept still says what it is within a day of wall time (review of mesh-host#92). Go's monotonic
|
||||
// clock stops while a machine is suspended, and Sub between two times that both carry a monotonic reading uses
|
||||
// it, so a watch that kept one would count a day asleep as almost nothing. Simulated with a clock that first
|
||||
// reads now (a monotonic reading), and after the sleep reads the wall time a day on: the watch must keep no
|
||||
// monotonic reading, and judge the day as the wall clock says it.
|
||||
func TestALaptopThatSleptStillSaysWhatItIsWithinADay(t *testing.T) {
|
||||
before := time.Now() // carries a monotonic reading
|
||||
asleep := before.Add(3 * time.Second)
|
||||
if asleep.Sub(before) != 3*time.Second {
|
||||
t.Fatal("the monotonic reading is not what this test simulates with")
|
||||
}
|
||||
calls := 0
|
||||
w := &adoptionWatch{now: func() time.Time {
|
||||
calls++
|
||||
if calls == 1 {
|
||||
return before
|
||||
}
|
||||
// The wall clock a day on, as a machine's clock reads after waking.
|
||||
return asleep.Round(0).Add(24 * time.Hour)
|
||||
}}
|
||||
report := link.Report{Outward: []string{"wlan0"}, Profile: map[string]any{"capabilities": []any{}}}
|
||||
w.said(report)
|
||||
if strings.Contains(w.machineSaid.String(), "m=") {
|
||||
t.Fatalf("the watch kept a monotonic reading (%s): a sleep would not count toward the day", w.machineSaid)
|
||||
}
|
||||
if !w.news(report) {
|
||||
t.Error("a day of wall time on, a laptop that slept did not say what it is")
|
||||
}
|
||||
}
|
||||
|
||||
// And the watch's own clock never carries a monotonic reading.
|
||||
func TestTheWatchClockIsWallTime(t *testing.T) {
|
||||
w := &adoptionWatch{}
|
||||
if s := w.clock().String(); strings.Contains(s, "m=") {
|
||||
t.Errorf("the watch's clock carries a monotonic reading: %s", s)
|
||||
}
|
||||
w = &adoptionWatch{now: time.Now}
|
||||
if s := w.clock().String(); strings.Contains(s, "m=") {
|
||||
t.Errorf("a given clock's time kept its monotonic reading: %s", s)
|
||||
}
|
||||
}
|
||||
+175
-31
@@ -116,6 +116,8 @@ reach, run as root at the machine.
|
||||
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared
|
||||
owner and mode. The operator's line is nox node hand-over <node> <dir> on the
|
||||
control-node; this is the same act for a machine with no mesh to reach
|
||||
judge-directories which directories this node is declared the walk that follows no link another
|
||||
account could make would refuse, read-only (novox/hq issue 358); run as root
|
||||
version
|
||||
|
||||
--json machine-readable output
|
||||
@@ -239,11 +241,15 @@ func run(ctx context.Context, command string, opts options) error {
|
||||
// found (novox/hq issue 339).
|
||||
handedOverFrom(opts.state)
|
||||
foundState.Store(opts.state)
|
||||
// And what it knows of the libraries it keeps is beside its state (novox/hq ADR 0328 §6).
|
||||
apply.Libraries = store.NewLibraryFile(opts.state)
|
||||
// And the guard asks the running container runtimes where they keep their data (novox/hq issue 339).
|
||||
apply.AskRuntimes = apply.ExecRunner
|
||||
switch command {
|
||||
case "hand-over":
|
||||
return handOverCommand(opts)
|
||||
case "judge-directories":
|
||||
return judgeDirectoriesCommand(opts)
|
||||
case "key":
|
||||
return keyCommand(opts)
|
||||
|
||||
@@ -632,21 +638,31 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
|
||||
return err
|
||||
}
|
||||
// And prove this is the machine the host was built for. Installing the arch host on Alpine
|
||||
// must say so once, at the start, rather than failing later inside pacman.
|
||||
if err := sys.Confirm(ctx, apply.ExecRunner); err != nil {
|
||||
// must say so once, at the start, rather than failing later inside pacman. A package manager that
|
||||
// does not answer fails only what needs it (novox/hq issue 233, probe.go).
|
||||
probe, err := probePackageManager(ctx, sys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if probe != nil {
|
||||
fmt.Fprintf(out, "\n%s\n", probe.said())
|
||||
}
|
||||
|
||||
fmt.Fprintf(out, "\napplying:\n")
|
||||
// Minding the windows the daemon's scheduler has open: this runs as its own process, and a
|
||||
// `reconcile` by hand at 03:31 is exactly the apply that must not restart a server a collector
|
||||
// is holding still (novox/hq issue 224).
|
||||
report, updated, applyErr := apply.ApplyMindingWindows(ctx, sys, d, known, origin,
|
||||
apply.ExecRunner, func(line string) {
|
||||
if !opts.json {
|
||||
fmt.Fprintln(out, line)
|
||||
}
|
||||
}, sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
|
||||
// Once more when the package manager did not answer and this apply wrote something: what it wrote may be
|
||||
// what makes it answer, and then the packages go in now rather than in five minutes (issue 233, probe.go).
|
||||
tell := func(line string) {
|
||||
if !opts.json {
|
||||
fmt.Fprintln(out, line)
|
||||
}
|
||||
}
|
||||
report, updated, applyErr, probe, _ := applyMindingProbe(ctx, sys, probe, known, func(known store.State) (apply.Report, store.State, error) {
|
||||
return apply.ApplyMindingWindows(ctx, sys, d, known, origin, apply.ExecRunner, tell, sealOpener(opts.state),
|
||||
apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
|
||||
}, tell)
|
||||
|
||||
// What this apply settles about the node, whichever way it went. The mode is what the
|
||||
// declaration said and the check above agreed with; the bundle, once applied, is consumed.
|
||||
@@ -667,6 +683,9 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
|
||||
if applyErr != nil {
|
||||
return applyErr
|
||||
}
|
||||
if probe != nil {
|
||||
return probe.partial()
|
||||
}
|
||||
|
||||
// Only now, and only after a clean apply: this version got as far as a completed
|
||||
// reconcile, which is the whole of what "known good" claims (novox/hq ADR 0005). Not
|
||||
@@ -1104,6 +1123,8 @@ func runLink(ctx context.Context, opts options) error {
|
||||
// this process's, a `reconcile` run by hand, the installer's — reads it (novox/hq issue 224).
|
||||
sched.RecordWindowsIn(apply.WindowsIn(filepath.Dir(opts.state)))
|
||||
go sched.Run(ctx)
|
||||
// The take act's walks that stopped at their bound go on beside the applies (novox/hq ADR 0328 §7).
|
||||
startKeeping(ctx, func(line string) { say(line) })
|
||||
|
||||
// The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every
|
||||
// report this host makes carries what it says.
|
||||
@@ -1192,7 +1213,7 @@ func runLink(ctx context.Context, opts options) error {
|
||||
Membership: membership,
|
||||
Apply: applier,
|
||||
Reconcile: reconcileKept(opts, mine.Membership.Signer, sched, say),
|
||||
News: func(r link.Report) bool { return worthSaying(r) && watch.differs(r) },
|
||||
News: watch.news,
|
||||
// Counted as said only once the broker has taken it: queued and lost — the link down, the
|
||||
// publish refused — the change would never be said again (novox/hq ADR 0100).
|
||||
// And the first account under this build is what proves it to the launcher (to-be 45 §8).
|
||||
@@ -1212,8 +1233,13 @@ func runLink(ctx context.Context, opts options) error {
|
||||
}
|
||||
return setuidSearch.SearchNow(by), nil
|
||||
},
|
||||
Say: say,
|
||||
Timeout: opts.timeout,
|
||||
// A read-only judgement of the directories this node is declared, asked by the controller (novox/hq
|
||||
// issue 358): which the walk that follows no link another account could make would refuse.
|
||||
JudgeDirectories: judgeDirectoriesFrom(opts.state),
|
||||
// The take act on a library, on the operator's warrant (novox/hq ADR 0328 §7).
|
||||
AccessAct: accessActFrom(opts.state, opts.timeout),
|
||||
Say: say,
|
||||
Timeout: opts.timeout,
|
||||
}
|
||||
worker := make(chan struct{})
|
||||
go func() {
|
||||
@@ -1330,6 +1356,34 @@ func (n numbersFile) Save(reportSequence, refusedOlder int64) error {
|
||||
type adoptionWatch struct {
|
||||
mu sync.Mutex
|
||||
last string
|
||||
// faulty is whether the last report the mesh heard said an apply refused or failed (novox/hq issue 233).
|
||||
faulty bool
|
||||
// machineSaid is when the mesh last heard what this machine is made of (novox/hq issue 160), and now the
|
||||
// clock it is judged by (time.Now when nil).
|
||||
machineSaid time.Time
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// machineEvery is how old what a machine said of itself may grow before a reconcile says it again unasked
|
||||
// (novox/hq issue 160): a machine nobody pushed to still says it once a day, so no fact the mesh holds of it
|
||||
// is older than a day. Under a day by more than one reconcile interval, so the reconcile that says it falls
|
||||
// inside the day rather than just past it.
|
||||
const machineEvery = 23 * time.Hour
|
||||
|
||||
// clock is the wall time, its monotonic reading stripped (Round(0)). **Wall, because a laptop sleeps**: Go's
|
||||
// monotonic clock does not advance while the machine is suspended, so a day judged by it on a laptop that slept
|
||||
// for half of it is a day and a half of wall time, and what the mesh holds of the machine is older than a day.
|
||||
func (w *adoptionWatch) clock() time.Time {
|
||||
if w.now != nil {
|
||||
return w.now().Round(0)
|
||||
}
|
||||
return time.Now().Round(0)
|
||||
}
|
||||
|
||||
// machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago
|
||||
// or more. A refusal carries none, so it never is.
|
||||
func (w *adoptionWatch) machineDue(r link.Report) bool {
|
||||
return r.Refused == "" && len(r.Profile) > 0 && w.clock().Sub(w.machineSaid) >= machineEvery
|
||||
}
|
||||
|
||||
// fingerprint is what a report says about adoption: each hold and whether it changed, the
|
||||
@@ -1356,6 +1410,12 @@ func adoptionFingerprint(r link.Report) string {
|
||||
for _, v := range r.Rollbacks {
|
||||
parts = append(parts, fmt.Sprintf("rollback %s %s %s %s", v.Component, v.From, v.To, v.Outcome))
|
||||
}
|
||||
// **And how the apply went** (novox/hq issue 233): refused, clean, or which resources it did not apply. A
|
||||
// reconcile is the only apply a machine makes between deliveries, so a machine whose package manager stopped
|
||||
// answering, or answers again, says so at the next reconcile — otherwise the mesh's record of what it did
|
||||
// stays whatever the last delivery left, and a fault that began or ended between deliveries is never seen.
|
||||
// Not the words: an error that carries a time would read as new on every reconcile.
|
||||
parts = append(parts, applyState(r))
|
||||
if r.FoundFirewall != nil {
|
||||
parts = append(parts, fmt.Sprintf("found-firewall %s active=%v retired-by=%s", r.FoundFirewall.Kind,
|
||||
r.FoundFirewall.Active, r.FoundFirewall.RetiredBy))
|
||||
@@ -1368,6 +1428,45 @@ func adoptionFingerprint(r link.Report) string {
|
||||
return strings.Join(parts, "\n")
|
||||
}
|
||||
|
||||
// applyState is how an apply went, as the fingerprint keeps it: refused, clean, or failed with what it did not
|
||||
// apply and which keys its failure is said under.
|
||||
func applyState(r link.Report) string {
|
||||
switch {
|
||||
case r.Refused != "":
|
||||
return "apply refused"
|
||||
case len(r.Failed) == 0:
|
||||
return "apply clean"
|
||||
}
|
||||
keys := make([]string, 0, len(r.Failed))
|
||||
for k := range r.Failed {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return "apply failed " + strings.Join(keys, ",") + " unapplied " + strings.Join(r.Unapplied, ",")
|
||||
}
|
||||
|
||||
// faulty is whether a report says an apply refused or failed.
|
||||
func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 }
|
||||
|
||||
// news is whether a reconcile's report is worth saying unasked: what the machine is made of last reached the
|
||||
// mesh a day ago or more (novox/hq issue 160); or it says something the last one the mesh heard
|
||||
// did not, and it is about the machine (worthSaying) — or about how the apply went, refused or failed now, or
|
||||
// clean after the last one the mesh heard was not (novox/hq issue 233). A refusal says nothing about the
|
||||
// machine, but that the machine refuses is what the mesh raises a condition from, and a clean apply after it is
|
||||
// what clears one.
|
||||
func (w *adoptionWatch) news(r link.Report) bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
// What the machine is made of, once a day whatever else changed (novox/hq issue 160).
|
||||
if w.machineDue(r) {
|
||||
return true
|
||||
}
|
||||
if adoptionFingerprint(r) == w.last {
|
||||
return false
|
||||
}
|
||||
return worthSaying(r) || faulty(r) || w.faulty
|
||||
}
|
||||
|
||||
// differs says whether a report says anything the last one that went out did not. It records
|
||||
// nothing: what was said is what reached the mesh, not what was written down to send.
|
||||
func (w *adoptionWatch) differs(r link.Report) bool {
|
||||
@@ -1381,6 +1480,10 @@ func (w *adoptionWatch) said(r link.Report) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.last = adoptionFingerprint(r)
|
||||
w.faulty = faulty(r)
|
||||
if len(r.Profile) > 0 {
|
||||
w.machineSaid = w.clock()
|
||||
}
|
||||
}
|
||||
|
||||
// changed is differs and said together, for a report published as it is made.
|
||||
@@ -1533,7 +1636,7 @@ const (
|
||||
// judgeWhatRuns looks at every long-running resource every liveness.LookEvery and says the statement on
|
||||
// the bus when a state changed, again every minute while one is not healthy, and every five minutes
|
||||
// anyway. A statement the bus did not take is said at the next look. It reads; it never acts (ADR 0240
|
||||
// rule 6).
|
||||
// rule 6) — but for going on with a take act stopped at its bound (ADR 0328 §7), which runs beside it.
|
||||
func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, say link.Announce) {
|
||||
ticker := time.NewTicker(liveness.LookEvery)
|
||||
defer ticker.Stop()
|
||||
@@ -1572,6 +1675,18 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every library, judged on every look (novox/hq ADR 0328 §8, §9); and a take act stopped at its bound
|
||||
// goes on, beside.
|
||||
accessSt, accessesChanged := accesses.look(time.Now())
|
||||
keeping.poke(ctx)
|
||||
owed = owed || accessesChanged
|
||||
if accessesChanged {
|
||||
for _, v := range accessSt {
|
||||
if v.State != link.StateHealthy {
|
||||
say(fmt.Sprintf("%s (access %s) is %s: %s", v.Resource, v.Target, v.State, v.Reason))
|
||||
}
|
||||
}
|
||||
}
|
||||
var netSt *network.Statement
|
||||
if n := netJudge.get(); n != nil {
|
||||
ns, netChanged := n.Look(ctx)
|
||||
@@ -1597,20 +1712,20 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
found := foundNow()
|
||||
since := time.Since(lastSaid)
|
||||
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
|
||||
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0
|
||||
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
|
||||
accessesHealthy(accessSt)
|
||||
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
|
||||
continue
|
||||
}
|
||||
if changed {
|
||||
for _, r := range st.Resources {
|
||||
if r.State == liveness.Unhealthy {
|
||||
say(fmt.Sprintf("%s (%s %s) is unhealthy: %s, %d restart(s) counted", r.ID, r.Kind, r.Target,
|
||||
r.Reason, r.Restarts))
|
||||
say(unhealthyWords(r))
|
||||
}
|
||||
}
|
||||
}
|
||||
if queue.SayHealth(ctx, *withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
|
||||
accountSt), found)) {
|
||||
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
|
||||
unitSt), accountSt), found), accessSt)) {
|
||||
lastSaid, owed = time.Now(), false
|
||||
}
|
||||
}
|
||||
@@ -1618,10 +1733,10 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
|
||||
|
||||
// healthAsReported is a statement as the report and the event carry it.
|
||||
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
|
||||
// RootContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), and
|
||||
// judges a user's declared `root` (novox/hq ADR 0266), which is what tells the controller it may be sent
|
||||
// either field.
|
||||
h := &link.Health{Contract: link.RootContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
||||
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
|
||||
// user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries
|
||||
// (novox/hq ADR 0328), which is what tells the controller it may be sent any of those fields.
|
||||
h := &link.Health{Contract: link.OwnGroupContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
|
||||
for _, r := range st.Resources {
|
||||
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
|
||||
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
|
||||
@@ -1761,8 +1876,10 @@ func holdTheMachine(ctx context.Context, queue *link.Queue) {
|
||||
// node when it can say which links face outside (ADR 0140) — a converged node holds nothing and found
|
||||
// no firewall, so without that it could speak only in reply to a declaration, while the mesh composes
|
||||
// no declaration for a node that has not said which links face outside. Whether a report is news is
|
||||
// the queue's News (worthSaying and the watch), so an unchanged answer costs one comparison every
|
||||
// reconcile and nothing on the bus.
|
||||
// the queue's News (the watch's news), so an unchanged answer costs one comparison every reconcile and
|
||||
// nothing on the bus. And on any node when how the apply went changed — refused, clean, or which resources it
|
||||
// did not apply — since that is what the mesh raises a machine that does not apply from, and clears it by
|
||||
// (novox/hq issue 233).
|
||||
func reconcileKept(opts options, signer ed25519.PublicKey, sched *apply.Scheduler,
|
||||
say link.Announce) link.Reconcile {
|
||||
return func(ctx context.Context) (link.Report, bool) {
|
||||
@@ -1926,9 +2043,15 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
|
||||
if err := built.Confirm(ctx, apply.ExecRunner); err != nil {
|
||||
// A package manager that does not answer fails only what needs it, and the report says so (novox/hq
|
||||
// issue 233, probe.go).
|
||||
probe, err := probePackageManager(ctx, built)
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
if probe != nil {
|
||||
announceOr(say)(probe.said())
|
||||
}
|
||||
|
||||
// Declared, not carried. A declaration from the mesh removes only what the mesh previously
|
||||
// declared — never what this machine raised for itself from its bundle (04-ISSUES/010).
|
||||
@@ -1942,9 +2065,13 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
//
|
||||
// `say` already reaches stdout, and the launcher's unit sends that to the journal, so this needs
|
||||
// no new mechanism — only for the argument to be passed.
|
||||
outcome, updated, applyErr := apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared,
|
||||
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
||||
apply.WindowsIn(filepath.Dir(opts.state)))
|
||||
// Once more when the package manager did not answer and this apply wrote something (issue 233, probe.go).
|
||||
outcome, updated, applyErr, probe, firstPass := applyMindingProbe(ctx, built, probe, known,
|
||||
func(known store.State) (apply.Report, store.State, error) {
|
||||
return apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared,
|
||||
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
|
||||
apply.WindowsIn(filepath.Dir(opts.state)))
|
||||
}, announceOr(say))
|
||||
|
||||
// The mode the mesh said, recorded whichever way the apply went: the declaration is kept
|
||||
// either way, and the node is held to it from the next reconcile (novox/hq ADR 0100).
|
||||
@@ -1955,7 +2082,7 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
// Root acted on what an agent account's judgement reads — an account, a package, an action, sudo's
|
||||
// rules: the search for setuid programs runs again (novox/hq issue 361). A failed apply counts what it
|
||||
// did before it stopped.
|
||||
moved := outcome.Outcomes
|
||||
moved := append(append([]apply.Outcome(nil), firstPass...), outcome.Outcomes...)
|
||||
var partial *apply.Error
|
||||
if errors.As(applyErr, &partial) {
|
||||
moved = append(append([]apply.Outcome(nil), moved...), partial.Done.Outcomes...)
|
||||
@@ -1990,7 +2117,9 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
// What runs here for a module, and how each is (novox/hq ADR 0240): judged from what was just
|
||||
// applied, so every resource this apply started says `starting` and the gate waits for it.
|
||||
if j := judging; j != nil {
|
||||
j.Set(liveness.LongRunning(declared, held))
|
||||
// A container that waits for a library whose disk or mount is not there was not created: the library says
|
||||
// why, and liveness judges it neither alive nor dead (novox/hq ADR 0328 §9).
|
||||
j.Set(liveness.LongRunning(declared, withWaiting(held, apply.WaitingOnLibrary(declared))))
|
||||
st, _ := j.Look(ctx)
|
||||
netJudge.declare(declaredResolvConf(declared))
|
||||
var netSt *network.Statement
|
||||
@@ -2012,8 +2141,12 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
as, _ := a.Look(ctx)
|
||||
accountSt = &as
|
||||
}
|
||||
report.Health = withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt), accountSt),
|
||||
foundDirectories(updated))
|
||||
accesses.set(declared, held)
|
||||
// A take act stopped at its bound goes on, beside (ADR 0328 §7).
|
||||
keeping.poke(ctx)
|
||||
accessSt, _ := accesses.look(time.Now())
|
||||
report.Health = withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), unitSt),
|
||||
accountSt), foundDirectories(updated)), accessSt)
|
||||
}
|
||||
// Which of this machine's links face outside, for the filter the mesh writes around them
|
||||
// (novox/hq ADR 0140). Reported whatever the node's mode: a converged node's filter needs it,
|
||||
@@ -2096,6 +2229,9 @@ func applyAndKeepHeld(ctx context.Context, opts options, raw []byte, signed *sto
|
||||
if applyErr != nil {
|
||||
report.Failed = map[string]string{"apply": applyErr.Error()}
|
||||
}
|
||||
report.Failed = withUnanswered(report.Failed, probe)
|
||||
// Which resources did not apply, for whether a reconcile's account is news (issue 233): not said on the wire.
|
||||
report.Unapplied = unapplied(declared, outcome, applyErr)
|
||||
return report
|
||||
}
|
||||
|
||||
@@ -2338,3 +2474,11 @@ func factsAsReported(f *store.Facts) map[string]any {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unhealthyWords is what the engine says of an unhealthy resource. Its restart count is the judge's own
|
||||
// (kept.Counted): automatic restarts since it began watching the resource, kept across recreates and starts
|
||||
// by hand, so it is said as that and never read as since the state began (novox/hq issue 458).
|
||||
func unhealthyWords(r liveness.State) string {
|
||||
return fmt.Sprintf("%s (%s %s) is unhealthy: %s, %d automatic restart(s) since this engine began watching it",
|
||||
r.ID, r.Kind, r.Target, r.Reason, r.Restarts)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os/exec"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// packageManagers are the tools a system's probe asks, for a system whose probe is its package manager.
|
||||
var packageManagers = map[string]string{"arch": "pacman", "alpine": "apk"}
|
||||
|
||||
// unanswered is a probe of the package manager that failed before an apply: which tool, and its words.
|
||||
type unanswered struct {
|
||||
tool string
|
||||
words string
|
||||
}
|
||||
|
||||
// failedKey is how a report names the probe among what failed: one key, the same on every report, so the
|
||||
// mesh counts a package manager that keeps not answering as one failure that goes on.
|
||||
func (u *unanswered) failedKey() string { return "the package manager (" + u.tool + ")" }
|
||||
|
||||
// said is the line the apply says before it begins.
|
||||
func (u *unanswered) said() string {
|
||||
return u.tool + " does not answer here, so every package in this declaration fails and the rest of it is " +
|
||||
"applied (novox/hq issue 233): " + u.words
|
||||
}
|
||||
|
||||
// partial is the failure of an apply that applied everything else, when no package was declared to fail.
|
||||
func (u *unanswered) partial() error {
|
||||
return errors.New("applied only in part: " + u.words + "\n\nevery resource that does not need " + u.tool +
|
||||
" was attempted; none here needed it, and " + u.tool + " must answer before this machine applies whole " +
|
||||
"(novox/hq issue 233)")
|
||||
}
|
||||
|
||||
// probePackageManager proves the machine is the system the host was built for, before anything is applied.
|
||||
//
|
||||
// **A package manager that does not answer fails only what needs it** (novox/hq issue 233). The probe was a
|
||||
// refusal of the whole declaration — right for a host installed on a machine it was not built for, and a
|
||||
// wedge when what the probe reads is the mesh's own: a declaration that left the package manager's module
|
||||
// out removed its configuration file, and from then on every declaration was refused, the one that would
|
||||
// write the file back and the node-engine's own upgrade included. A person on the machine was the only way
|
||||
// back. Now the rest applies — files, units, accounts, the node-engine itself — and each package fails with
|
||||
// the probe's words (the system's PackageInstalled asks the same probe first), so a later declaration can
|
||||
// undo what an earlier one did. Before the rest is applied, a declaration composed from an older view of the
|
||||
// assignments than this machine applied has already been refused (issue 234's generation).
|
||||
//
|
||||
// **A package manager that is not there at all still refuses whole.** That is the host installed on a
|
||||
// machine it was not built for — the arch host on Alpine — which is what the probe exists to catch, and no
|
||||
// declaration the mesh sends removes a package manager's program. Only one that is there and errors (a
|
||||
// configuration file gone, a database locked) is the mesh's own fault and applies in part.
|
||||
//
|
||||
// A system with no package manager (android) still refuses whole: what its probe asks is the system itself,
|
||||
// which no resource the mesh declares can remove, and nothing else stands between it and the wrong machine.
|
||||
func probePackageManager(ctx context.Context, sys system.System) (*unanswered, error) {
|
||||
err := sys.Confirm(ctx, apply.ExecRunner)
|
||||
if err == nil {
|
||||
return nil, nil
|
||||
}
|
||||
tool, ok := packageManagers[sys.Name()]
|
||||
if !ok || errors.Is(err, exec.ErrNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
return &unanswered{tool: tool, words: err.Error()}, nil
|
||||
}
|
||||
|
||||
// applyPass is one apply of the declaration over what the node's state says.
|
||||
type applyPass func(known store.State) (apply.Report, store.State, error)
|
||||
|
||||
// applyMindingProbe applies, and **once more when the package manager did not answer and this apply wrote
|
||||
// something** (novox/hq issue 233). What it wrote may be what the package manager needed — the declaration
|
||||
// that names its module again writes its configuration file back — and then the packages go in now rather
|
||||
// than at the next reconcile. Asked again only then: an apply that wrote nothing changed nothing the package
|
||||
// manager reads. The second pass is the same declaration over the state the first left, so what the first
|
||||
// applied reads unchanged and only what needs the package manager moves.
|
||||
//
|
||||
// It returns the last pass's account, the probe as it stands after it (nil once the package manager
|
||||
// answers), and the first pass's outcomes when a second ran, for whoever counts what an apply moved.
|
||||
func applyMindingProbe(ctx context.Context, sys system.System, probe *unanswered, known store.State,
|
||||
pass applyPass, say func(string)) (apply.Report, store.State, error, *unanswered, []apply.Outcome) {
|
||||
report, updated, err := pass(known)
|
||||
if probe == nil || !wrote(report, err) {
|
||||
return report, updated, err, probe, nil
|
||||
}
|
||||
again, refused := probePackageManager(ctx, sys)
|
||||
if refused != nil || again != nil {
|
||||
return report, updated, err, probe, nil
|
||||
}
|
||||
say(probe.tool + " answers now that this apply wrote what it reads; applying again, so its packages are " +
|
||||
"installed in this apply (novox/hq issue 233)")
|
||||
first := append([]apply.Outcome(nil), report.Outcomes...)
|
||||
var partial *apply.Error
|
||||
if errors.As(err, &partial) {
|
||||
first = append(first, partial.Done.Outcomes...)
|
||||
}
|
||||
report, updated, err = pass(updated)
|
||||
return report, updated, err, nil, first
|
||||
}
|
||||
|
||||
// wrote is whether an apply moved anything on the machine, failed or not.
|
||||
func wrote(report apply.Report, err error) bool {
|
||||
if report.Changed() {
|
||||
return true
|
||||
}
|
||||
var partial *apply.Error
|
||||
return errors.As(err, &partial) && partial.Done.Changed()
|
||||
}
|
||||
|
||||
// unapplied is which declared resources an apply did not apply, failed or not attempted, sorted; nil for a
|
||||
// clean one.
|
||||
func unapplied(d *declaration.Declaration, report apply.Report, err error) []string {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
done := map[string]bool{}
|
||||
for _, o := range report.Outcomes {
|
||||
done[o.ID] = o.Action != "skipped"
|
||||
}
|
||||
var partial *apply.Error
|
||||
if errors.As(err, &partial) {
|
||||
for _, o := range partial.Done.Outcomes {
|
||||
done[o.ID] = done[o.ID] || o.Action != "skipped"
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, r := range d.Resources {
|
||||
if !done[r.Identity()] {
|
||||
out = append(out, r.Identity())
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
// Failed, and every resource has an account: what failed is no resource (the firewall found, a removal).
|
||||
out = []string{"(apply)"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withUnanswered adds the probe to what a report says failed.
|
||||
func withUnanswered(failed map[string]string, u *unanswered) map[string]string {
|
||||
if u == nil {
|
||||
return failed
|
||||
}
|
||||
if failed == nil {
|
||||
failed = map[string]string{}
|
||||
}
|
||||
failed[u.failedKey()] = u.said()
|
||||
return failed
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 233: a package manager that does not answer is said even when nothing declared
|
||||
// needs it — the apply is partial, never "applied" — and the report names it under one key, so the mesh
|
||||
// counts it as one failure that goes on.
|
||||
func TestAPackageManagerThatDoesNotAnswerIsSaidWithNoPackageDeclared(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[{"id":"pacman.conf","type":"file","path":"` + conf +
|
||||
`","content":"[options]\n"}]}`)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = runApply(context.Background(), opts, d, raw, fromFile)
|
||||
if err == nil || !strings.Contains(err.Error(), "applied only in part") || !strings.Contains(err.Error(), "could not be read") {
|
||||
t.Fatalf("an apply whose package manager did not answer was not said to be partial: %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(conf); statErr != nil {
|
||||
t.Fatalf("the file was not written: %v", statErr)
|
||||
}
|
||||
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
if report.Refused != "" {
|
||||
t.Fatalf("refused whole: %s", report.Refused)
|
||||
}
|
||||
why, ok := report.Failed["the package manager (pacman)"]
|
||||
if !ok || !strings.Contains(why, "could not be read") {
|
||||
t.Errorf("the report does not name the package manager with the probe's words: %v", report.Failed)
|
||||
}
|
||||
}
|
||||
|
||||
// A system whose probe asks the system itself, not a tool a resource needs, still refuses whole: nothing
|
||||
// the mesh declares can remove it, and nothing else stands between that host and the wrong machine.
|
||||
func TestASystemWithNoPackageManagerStillRefusesWhole(t *testing.T) {
|
||||
t.Setenv("PATH", t.TempDir())
|
||||
sys, err := system.For("android")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe, err := probePackageManager(context.Background(), sys)
|
||||
if err == nil || probe != nil {
|
||||
t.Fatalf("the android host's failed probe did not refuse: %v, %+v", err, probe)
|
||||
}
|
||||
}
|
||||
|
||||
// pacmanAnsweringOnce is a pacman that does not answer until conf exists, as the workstation's did until its
|
||||
// configuration file was written back, and installs into a directory of its own once it does.
|
||||
func pacmanAnsweringOnce(t *testing.T, conf string) {
|
||||
t.Helper()
|
||||
bin, installed := t.TempDir(), t.TempDir()
|
||||
script := "#!/bin/sh\n" +
|
||||
"[ -f '" + conf + "' ] || { echo \"error: config file " + conf + " could not be read: No such file or directory\" >&2; exit 1; }\n" +
|
||||
"for a; do last=$a; done\n" +
|
||||
"case \"$1\" in\n" +
|
||||
"-Q) [ \"$2\" = pacman ] && exit 0; [ -f '" + installed + "'/\"$2\" ] && exit 0; exit 1;;\n" +
|
||||
"-S) touch '" + installed + "'/\"$last\"; exit 0;;\n" +
|
||||
"esac\nexit 1\n"
|
||||
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
}
|
||||
|
||||
// Defends the review of novox/hq issue 233: **the apply that writes the package manager's configuration back
|
||||
// installs its packages too**. The probe is asked again once the partial apply wrote something, and when it
|
||||
// answers the declaration is applied again over what the first pass left — the packages in this apply, not five
|
||||
// minutes later, and the report clean.
|
||||
func TestThePackagesGoInWhenTheApplyThatRestoresThePackageManagerWroteIt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
pacmanAnsweringOnce(t, conf)
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
// The package first: within one pass it is asked about before the file is written, which is the order that
|
||||
// leaves it to a second pass.
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"pacman.base","type":"package","package":"base"},` +
|
||||
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"}]}`)
|
||||
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
|
||||
if report.Refused != "" || len(report.Failed) != 0 {
|
||||
t.Fatalf("the apply that made the package manager answer did not install its packages: refused %q, failed %v",
|
||||
report.Refused, report.Failed)
|
||||
}
|
||||
if !slices.Contains(report.Applied, "pacman.base") || !slices.Contains(report.Applied, "pacman.conf") {
|
||||
t.Errorf("the report does not account for both: %v", report.Applied)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends decision (1) of the review of novox/hq issue 233: **a package manager that is not there at all still
|
||||
// refuses whole**. That is the host installed on the wrong machine, which the probe exists to catch.
|
||||
func TestAPackageManagerThatIsNotThereRefusesWhole(t *testing.T) {
|
||||
t.Setenv("PATH", t.TempDir())
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[{"id":"pacman.conf","type":"file","path":"` + conf +
|
||||
`","content":"[options]\n"}]}`)
|
||||
|
||||
report := applyAndKeep(context.Background(), options{state: filepath.Join(dir, "state.json")}, raw, nil, nil, nil)
|
||||
|
||||
if !strings.Contains(report.Refused, "pacman does not answer") {
|
||||
t.Fatalf("a machine with no pacman at all was not refused whole: %+v", report)
|
||||
}
|
||||
if _, err := os.Stat(conf); err == nil {
|
||||
t.Error("the refused declaration touched the machine")
|
||||
}
|
||||
}
|
||||
|
||||
// Defends the review of novox/hq issue 233: **a re-apply whose result changed is said unasked**. A machine
|
||||
// reconciles every five minutes and speaks only when its account is news; how the apply went was not part of
|
||||
// it, so a package manager that stopped answering between deliveries was never said, and one that answered
|
||||
// again never cleared. Now a failure is said once, the same failure again is not, and a clean re-apply after it
|
||||
// is — which is what clears the mesh's condition.
|
||||
func TestAReapplyWhoseResultChangedIsSaidUnasked(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "elsewhere", "pacman.conf") // not declared: the apply cannot make it answer
|
||||
pacmanAnsweringOnce(t, conf)
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"motd","type":"file","path":"` + filepath.Join(dir, "motd") + `","content":"hello\n"},` +
|
||||
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||
w := &adoptionWatch{}
|
||||
|
||||
// Clean once, as it was before: base installed while pacman answered.
|
||||
if err := os.MkdirAll(filepath.Dir(conf), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
clean := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
if faulty(clean) {
|
||||
t.Fatalf("the first apply was not clean: %+v", clean)
|
||||
}
|
||||
w.said(clean)
|
||||
if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) {
|
||||
t.Error("a clean re-apply after a clean one was said, so every machine speaks every five minutes")
|
||||
}
|
||||
|
||||
// The configuration goes: the re-apply fails, and that is news once.
|
||||
if err := os.Remove(conf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failed := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
if !faulty(failed) {
|
||||
t.Fatalf("the re-apply did not fail: %+v", failed)
|
||||
}
|
||||
if !w.news(failed) {
|
||||
t.Fatal("a re-apply that failed after a clean one was not said, so the mesh never hears the machine fails")
|
||||
}
|
||||
w.said(failed)
|
||||
if w.news(applyAndKeep(context.Background(), opts, raw, nil, nil, nil)) {
|
||||
t.Error("the same failure was said again on the next re-apply")
|
||||
}
|
||||
|
||||
// It comes back: the clean re-apply is news, and is what clears the mesh's condition.
|
||||
if err := os.WriteFile(conf, []byte("[options]\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
if faulty(again) {
|
||||
t.Fatalf("the re-apply did not come back clean: %+v", again)
|
||||
}
|
||||
if !w.news(again) {
|
||||
t.Fatal("a clean re-apply after a failed one was not said, so the mesh's condition never clears")
|
||||
}
|
||||
}
|
||||
|
||||
// A refusal is news too, once, and a reconcile with nothing to say after a clean one stays silent.
|
||||
func TestARefusalIsSaidOnceAndNothingIsNotSaid(t *testing.T) {
|
||||
w := &adoptionWatch{}
|
||||
w.said(link.Report{})
|
||||
if w.news(link.Report{}) {
|
||||
t.Error("nothing was said again")
|
||||
}
|
||||
refused := link.Report{Refused: "pacman does not answer here"}
|
||||
if !w.news(refused) {
|
||||
t.Fatal("a refusal was not said")
|
||||
}
|
||||
w.said(refused)
|
||||
if w.news(link.Report{Refused: "pacman does not answer here, at another time"}) {
|
||||
t.Error("the same refusal in other words was said again")
|
||||
}
|
||||
if !w.news(link.Report{}) {
|
||||
t.Error("a clean apply after a refusal was not said")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// unansweringPacman puts a pacman on PATH that answers as one whose configuration file is gone, as the
|
||||
// workstation's did on 2026-10-04 (novox/hq issue 233), and makes this an arch host.
|
||||
func unansweringPacman(t *testing.T) {
|
||||
t.Helper()
|
||||
bin := t.TempDir()
|
||||
script := "#!/bin/sh\necho 'error: config file /etc/pacman.conf could not be read: No such file or directory' >&2\nexit 1\n"
|
||||
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
}
|
||||
|
||||
// TestReplay233 replays novox/hq issue 233. The package manager's module wrote pacman's configuration
|
||||
// file; a declaration that left the module out removed it; from then on pacman did not answer, and the
|
||||
// node-engine refused every declaration whole — the one that named the module again and would have
|
||||
// written the file back included. A person restored the file by hand an hour and a half later.
|
||||
//
|
||||
// Held to: a declaration that restores the file is applied while pacman does not answer. Only what needs
|
||||
// pacman fails, with the probe's words; the file is written and recorded. Written only with what the
|
||||
// node-engine had before the fix, so it lays over the commit before it, where it fails.
|
||||
func TestReplay233(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
|
||||
if report.Refused != "" {
|
||||
t.Fatalf("the declaration that restores the package manager's configuration was refused whole: %s", report.Refused)
|
||||
}
|
||||
if got, err := os.ReadFile(conf); err != nil || string(got) != "[options]\n" {
|
||||
t.Fatalf("the file that restores the package manager was not written: %q, %v", got, err)
|
||||
}
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := known.Find("pacman.conf"); !ok {
|
||||
t.Error("the file written was not recorded, so the next apply would not know it is the mesh's")
|
||||
}
|
||||
words := strings.Join(failedWords(report.Failed), "\n")
|
||||
if !strings.Contains(words, "pacman does not answer") || !strings.Contains(words, "could not be read") {
|
||||
t.Errorf("the report does not say the package manager did not answer, in the probe's words: %q", words)
|
||||
}
|
||||
if !strings.Contains(words, "pacman.base") {
|
||||
t.Errorf("the report does not name the package that was not applied: %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
func failedWords(failed map[string]string) []string {
|
||||
var out []string
|
||||
for id, why := range failed {
|
||||
out = append(out, id+": "+why)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 233 at the machine: `mesh-host apply` applies what does not need the package
|
||||
// manager, and says the apply was partial with the probe's words — a non-zero exit, never "applied".
|
||||
func TestApplyByHandAppliesWhatDoesNotNeedAPackageManagerThatDoesNotAnswer(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
dir := t.TempDir()
|
||||
out := &bytes.Buffer{}
|
||||
opts := options{state: filepath.Join(dir, "state.json"), out: out}
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = runApply(context.Background(), opts, d, raw, fromFile)
|
||||
|
||||
if _, statErr := os.Stat(conf); statErr != nil {
|
||||
t.Fatalf("the file was not written: %v (the apply said %v)", statErr, err)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("an apply whose package manager did not answer said it applied")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "pacman does not answer") || !strings.Contains(err.Error(), "pacman.base") {
|
||||
t.Errorf("the failure does not name the package or the probe's words: %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "pacman does not answer") {
|
||||
t.Errorf("what was said before applying does not say the package manager does not answer:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
)
|
||||
|
||||
// The restart count the engine says is its own since it began watching the resource, never since the state
|
||||
// began (novox/hq issue 458).
|
||||
func TestAnUnhealthyResourceSaysWhatItsRestartCountCounts(t *testing.T) {
|
||||
r := liveness.State{Resource: liveness.Resource{ID: "messenger.own-runtime", Kind: "process",
|
||||
Target: "messenger-runtime.service"}, State: liveness.Unhealthy, Reason: liveness.ReasonRestarting, Restarts: 3890}
|
||||
said := unhealthyWords(r)
|
||||
if !strings.Contains(said, "3890 automatic restart(s) since this engine began watching it") ||
|
||||
strings.Contains(said, "counted") {
|
||||
t.Fatalf("the restart count is not said as what it counts: %s", said)
|
||||
}
|
||||
}
|
||||
@@ -87,7 +87,10 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
||||
Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
|
||||
}}
|
||||
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
|
||||
if h.Contract != link.RootContract || link.RootContract != 3 {
|
||||
// At RootContract or later: an engine that also reads an identity's runner says OwnGroupContract (novox/hq ADR
|
||||
// 0334), one above ADR 0328's LibraryContract.
|
||||
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.OwnGroupContract ||
|
||||
link.OwnGroupContract != link.LibraryContract+1 {
|
||||
t.Fatalf("contract %d", h.Contract)
|
||||
}
|
||||
got := map[string]string{}
|
||||
|
||||
+26
-33
@@ -2,51 +2,44 @@
|
||||
|
||||
## `foundation-first-node.lock`
|
||||
|
||||
What a machine must be before a mesh exists — the bootstrap in
|
||||
[novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole:
|
||||
The one genesis bundle (novox/hq ADR 0331): what a machine must be before a mesh exists — the
|
||||
bootstrap in [novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole, on the bus the
|
||||
mesh runs on:
|
||||
|
||||
```
|
||||
0 a container runtime
|
||||
0 a container runtime, and a filter before anything listens
|
||||
1 the store runs
|
||||
2 a database per context `inventory` and `identity`
|
||||
2 a database per context `inventory`, `identity` and `licences`
|
||||
3 those contexts' schemas mesh-controller migrate
|
||||
4 the broker runs with a certificate it generated itself
|
||||
5 the control plane runs mesh-controller serve
|
||||
4 the bus's certificate mesh-controller broker certificate
|
||||
5 the bus runs NATS as mesh-broker-nats, where the nats module keeps it
|
||||
(ADR 0333), with the controller's account as its first user
|
||||
6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS)
|
||||
```
|
||||
|
||||
**A machine that applies this is a mesh** — one node, with nothing joined to it yet, which is
|
||||
exactly what the first node is (novox/hq ADR 0004). From here it hands out tokens and everything
|
||||
else joins the ordinary way.
|
||||
|
||||
This file said it stopped at step 4 for longer than that was true, which is its own small lesson:
|
||||
a comment about what something does not do is a comment nobody updates.
|
||||
It is a **template**: `mesh-bootstrap` reads it, makes the store's superuser and the controller's
|
||||
account on the bus, and writes them where the template says `bootstrap` (internal/bootstrap,
|
||||
rootsecrets.go). A template whose bus connection it does not recognise is refused at preflight,
|
||||
naming the placeholder it looked for. The bundle that raised the previous broker was deleted with
|
||||
ADR 0331; there is one bundle, and every lab bed names it.
|
||||
|
||||
Build a host carrying it:
|
||||
**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container
|
||||
`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in
|
||||
`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus
|
||||
port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once,
|
||||
on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest.
|
||||
|
||||
Build an installer carrying a builder, and point it at this:
|
||||
|
||||
```
|
||||
make host SYSTEM=arch BUNDLE=examples/foundation-first-node.lock
|
||||
make bootstrap IMAGE=<builder image>
|
||||
mesh-bootstrap --bundle examples/foundation-first-node.lock …
|
||||
```
|
||||
|
||||
**The registry address and digests have to be replaced before this is useful.** They are written
|
||||
as `192.0.2.250:5000/…@sha256:…` because a digest belongs to whatever registry serves it — here,
|
||||
one a lab scenario raises, which reports its digests when it comes up. That is not a placeholder
|
||||
to be tidied away: a bundle is built *for a target*, and which registry that target pulls from is
|
||||
part of the target.
|
||||
|
||||
### What was verified, and how
|
||||
|
||||
On a lab machine confirmed sealed — `curl https://example.com` times out, the lab registry answers
|
||||
200 — the whole bundle applied from bare: eight resources, `inventory` created and `mesh` nowhere,
|
||||
the `node` table present with its indexes, the migration recorded, and LavinMQ answering
|
||||
`lavinmqctl status` with AMQP listening on 5672.
|
||||
|
||||
Three consecutive reconciles after that: **already matches — 8 resource(s) checked**, each time.
|
||||
|
||||
Then the machine was **rebooted**, and everything came back: docker from `boot: enabled`, both
|
||||
containers because the host creates every container `--restart unless-stopped`
|
||||
(`internal/apply/apply.go`), the schema intact in its named volume, and reconcile still finding
|
||||
nothing to do.
|
||||
|
||||
The reboot is worth doing rather than assuming. Nothing in the declaration asks for a container to
|
||||
return, so that it does is a property of the host, and the only way to know it holds is to take
|
||||
the machine away and give it back.
|
||||
**The registry address is the lab's.** The images are written as `192.0.2.250:5000/…@sha256:…`; the
|
||||
lab rewrites the third-party ones to upstream references by the same digest, and the installer
|
||||
replaces the control plane's with the image it built.
|
||||
|
||||
+31
-1
@@ -7,6 +7,7 @@ package examples
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -48,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin
|
||||
}
|
||||
sort.Strings(images)
|
||||
|
||||
want := []string{"lavinmq", "mesh-controller", "postgres"}
|
||||
want := []string{"mesh-controller", "nats", "postgres"}
|
||||
if strings.Join(images, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+
|
||||
"Adding one is a change to what every first node becomes, and to ADR 0006's "+
|
||||
@@ -67,3 +68,32 @@ func TestTheControlPlaneIsCarriedToo(t *testing.T) {
|
||||
t.Fatal("nothing in the bundle starts the control plane, so the machine would raise a " +
|
||||
"foundation and stop")
|
||||
}
|
||||
|
||||
// One genesis bundle, on the bus the mesh runs on (novox/hq ADR 0331 §2). Two bundles let every bed
|
||||
// and README name the one that could not raise a mesh, for weeks (issue 146).
|
||||
func TestThereIsOneGenesisBundle(t *testing.T) {
|
||||
bundles, err := filepath.Glob("*.lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(bundles) != 1 || bundles[0] != "foundation-first-node.lock" {
|
||||
t.Fatalf("examples/ holds %v; it holds one genesis bundle, foundation-first-node.lock", bundles)
|
||||
}
|
||||
}
|
||||
|
||||
// The control plane the bundle raises is given the bus, with the account the installer fills and
|
||||
// the certificate it pins (ADR 0331 §4); without it, it crash-loops for ever (issue 146).
|
||||
func TestTheControlPlaneIsGivenTheBus(t *testing.T) {
|
||||
for _, r := range bundle(t).Resources {
|
||||
c, ok := r.(*declaration.Container)
|
||||
if !ok || !strings.Contains(c.Image, "mesh-controller") {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(c.Env["MESH_BUS_NATS"], "nats://controller:bootstrap@") || c.Env["MESH_BROKER_CERTIFICATE"] == "" {
|
||||
t.Fatalf("the control plane is given the bus %q and the certificate %q",
|
||||
c.Env["MESH_BUS_NATS"], c.Env["MESH_BROKER_CERTIFICATE"])
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("nothing in the bundle starts the control plane")
|
||||
}
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being
|
||||
// built (novox/hq ADR 0106, design 25).
|
||||
//
|
||||
// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh
|
||||
// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the
|
||||
// first one — the controller's own account, at a well-known bootstrap password, exactly as the store
|
||||
// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and
|
||||
// from the controller's first composition onward the file is the controller's to write.
|
||||
//
|
||||
// The accounts file is its own file beside the server's configuration, because the server's own
|
||||
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
||||
// one directory, of necessity: an include path is resolved relative to the including file's own
|
||||
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
||||
// start.
|
||||
//
|
||||
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
||||
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
||||
// authenticates with a password (ADR 0004, design 25 §4).
|
||||
|
||||
{
|
||||
"declaration": 1,
|
||||
"resources": [
|
||||
{
|
||||
"id": "container-runtime",
|
||||
"type": "package",
|
||||
"package": "docker"
|
||||
},
|
||||
{
|
||||
"id": "container-runtime-running",
|
||||
"type": "service",
|
||||
"unit": "docker.service",
|
||||
"state": "running",
|
||||
"boot": "enabled"
|
||||
},
|
||||
// **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the
|
||||
// broker are adopted as modules later and so bind to every interface from the moment they
|
||||
// start; the packet filter that governs who may reach them is a module too, installed a
|
||||
// dozen steps later. Between the two, a control-node facing the network had its store and
|
||||
// its bus open to anyone who could reach the machine. So the foundation carries a filter of
|
||||
// its own — the same table the filter module will replace wholesale once it can derive one:
|
||||
// drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node
|
||||
// enrols over, the registry a node pulls from), and let the container runtime's own
|
||||
// networks through the forward chain so containers keep working. A published container port
|
||||
// is forwarded, never input (issue 047), which is why the forward chain is where the store's
|
||||
// and broker's ports are refused from outside — and a container on this machine dialling a
|
||||
// port this machine publishes reaches it through the runtime's proxy, which IS input, which
|
||||
// is why the bus and the registry are opened in both chains, exactly as the derived ruleset
|
||||
// does.
|
||||
{
|
||||
"id": "base-filter-package",
|
||||
"type": "package",
|
||||
"package": "nftables"
|
||||
},
|
||||
{
|
||||
"id": "base-filter",
|
||||
"type": "file",
|
||||
"path": "/etc/nftables.conf",
|
||||
"mode": "0644",
|
||||
"content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t\t# the mesh's own, from anywhere: the bus a node enrols over and a container on this machine reaches through the proxy, the registry a node pulls from\n\t\ttcp dport 5671 accept\n\t\ttcp dport 5000 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n"
|
||||
},
|
||||
{
|
||||
"id": "base-filter-loaded",
|
||||
"type": "service",
|
||||
"unit": "nftables.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": ["base-filter"]
|
||||
},
|
||||
{
|
||||
"id": "store",
|
||||
"type": "container",
|
||||
"name": "mesh-store",
|
||||
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
||||
"env": {
|
||||
"POSTGRES_PASSWORD": "bootstrap",
|
||||
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
||||
},
|
||||
"ports": ["5432:5432"],
|
||||
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
||||
},
|
||||
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
||||
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
||||
// exits happy, and the verify a moment later lands in the gap and fails. The action and its
|
||||
// verify must ask the same question, or the action can succeed into a state verify rejects.
|
||||
{
|
||||
"id": "store-ready",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
||||
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
||||
},
|
||||
{
|
||||
"id": "inventory-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
||||
},
|
||||
{
|
||||
"id": "identity-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
||||
},
|
||||
// Each context owns its own database (novox/hq ADR 0008). A third one is a third database,
|
||||
// created the same way and named the same way — which is the whole of adding a context to the
|
||||
// bootstrap, and is why the count is not something the foundation has an opinion about.
|
||||
{
|
||||
"id": "licences-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
||||
},
|
||||
{
|
||||
"id": "context-schemas",
|
||||
"type": "action",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
||||
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"migrate"],
|
||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
},
|
||||
{
|
||||
"id": "bus-certificate",
|
||||
"type": "action",
|
||||
// **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
|
||||
// broker's image while the broker was one that carried it; the bus that replaced it has a
|
||||
// shell and no openssl, and no other image the bundle names has one either. So the program
|
||||
// that needs the certificate writes it — already on this machine, since the schema step ran
|
||||
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
||||
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
||||
// to ask an authority of.
|
||||
// `--user 0:0` because the volume is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh volume.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--into", "/tls"],
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--check", "--into", "/tls"]
|
||||
},
|
||||
{
|
||||
"id": "bus-conf-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-bus-conf",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-conf",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/nats.conf",
|
||||
"mode": "0644",
|
||||
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
},
|
||||
{
|
||||
"id": "bus-accounts",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/accounts.conf",
|
||||
"mode": "0600",
|
||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||
},
|
||||
{
|
||||
"id": "broker",
|
||||
"type": "container",
|
||||
"name": "mesh-broker",
|
||||
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
|
||||
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||
},
|
||||
{
|
||||
"id": "broker-ready",
|
||||
"type": "action",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
|
||||
},
|
||||
{
|
||||
"id": "control-plane",
|
||||
"type": "container",
|
||||
"name": "mesh-controller",
|
||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"network": "host",
|
||||
"args": ["serve"],
|
||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
||||
"env": {
|
||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
||||
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
||||
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
||||
}
|
||||
}
|
||||
|
||||
]
|
||||
}
|
||||
@@ -1,36 +1,39 @@
|
||||
// foundation-first-node.lock — what a machine must be before a mesh exists.
|
||||
// foundation-first-node.lock — what a machine must be before a mesh exists: the one genesis bundle
|
||||
// (novox/hq ADR 0331 §2), on the bus the mesh runs on (ADR 0106, ADR 0131, design 25).
|
||||
//
|
||||
// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a
|
||||
// store, a database per context, those contexts' schemas, the broker, and the control plane
|
||||
// running on top of them.
|
||||
// filter, a store, a database per context, those contexts' schemas, the bus with its certificate
|
||||
// and its first user list, and the control plane running on top of them.
|
||||
//
|
||||
// It stopped before the control plane once, and this comment said so for longer than it was true.
|
||||
// **The mesh writes its own user list, and at genesis there is no mesh yet to write it.** So this
|
||||
// carries the first one — the controller's own account — and the control plane's address names the
|
||||
// same account. Both say `bootstrap`, and both are placeholders: the installer makes the account's
|
||||
// password, puts it in the address and its hash in the list (mesh-host internal/bootstrap
|
||||
// rootsecrets.go), and refuses a bundle whose bus connection it does not recognise. Until the first
|
||||
// node has enrolled the installer places the list the controller composes; from the bus's adoption
|
||||
// as the `nats` module on, the module receives it like any module (ADR 0331 §1, §5).
|
||||
//
|
||||
// The broker generates its OWN certificate, in its own image, into a volume it then mounts read
|
||||
// only. Self-signed, because at this moment there is no mesh to issue one and no public name to
|
||||
// obtain one for -- and it does not matter, because what a joining node checks is the fingerprint
|
||||
// pinned in its token, not a chain or a name (novox/hq ADR 0004). The subject is decoration.
|
||||
// The accounts file is its own file beside the server's configuration, because the server's own
|
||||
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
||||
// one directory, of necessity: an include path is resolved relative to the including file's own
|
||||
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
||||
// start.
|
||||
//
|
||||
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
||||
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
||||
// authenticates with a password (ADR 0004, design 25 §4). The control plane pins it too, through
|
||||
// MESH_BROKER_CERTIFICATE, and dials the bus on the machine's own loopback.
|
||||
//
|
||||
// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different
|
||||
// image, and this file is applied on a machine with no mesh to ask about anything. These digests
|
||||
// belong to the registry the lab raises, which is what a real node pulls from anyway — what is
|
||||
// required is a reference that is exact and cannot move (novox/hq ADR 0006).
|
||||
//
|
||||
// The store waits up to three minutes rather than one. A machine that has just pulled the
|
||||
// image and is running initdb for the first time can take longer than sixty seconds, and it
|
||||
// failed that way three times in the lab -- a flaky bootstrap that a second run always fixed,
|
||||
// which is the worst kind because it teaches people to run things twice.
|
||||
//
|
||||
// It failed a fourth time on 2026-08-30, on a loaded machine, and the report was `docker exited
|
||||
// 1:` with nothing after the colon. Raising the timeout again would treat the symptom; what makes
|
||||
// a retry the only available response is a timeout that reports nothing. So the wait now says
|
||||
// what it saw before giving up.
|
||||
// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
|
||||
// registry prefix is the lab's; the digests are upstream's.
|
||||
//
|
||||
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
|
||||
// creates is owned by root, and the database runs as somebody else inside the container — so it
|
||||
// could not write, and the container crash-looped. A named volume lets the image set up its own
|
||||
// ownership, and outlives the container, which is what you want for the thing holding the mesh's
|
||||
// state.
|
||||
|
||||
{
|
||||
"declaration": 1,
|
||||
"resources": [
|
||||
@@ -138,31 +141,94 @@
|
||||
"migrate"],
|
||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
},
|
||||
// **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
|
||||
// `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
|
||||
// /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the
|
||||
// catalogue's module declares. So adopting it as that module at the broker step keeps it: the
|
||||
// module's built image takes the place of the upstream one below, once, on the same streams, users
|
||||
// and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other
|
||||
// against the module's own manifest (busLayout), at the broker step and in its tests.
|
||||
//
|
||||
// Directories on the machine, not volumes the runtime names: the module mounts these paths, and the
|
||||
// control plane, once it runs as the mesh's process, reads the certificate at its path on the machine.
|
||||
// The certificate's directory is readable by everyone because the certificate is public and the control
|
||||
// plane runs as nobody; its key is written 0600 by the program that makes it.
|
||||
{
|
||||
"id": "broker-certificate",
|
||||
"type": "action",
|
||||
"command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
||||
"-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
|
||||
"verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
||||
"-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
|
||||
"id": "bus-tls-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-tls",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "bus-data-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-nats",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-certificate",
|
||||
"type": "action",
|
||||
// **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
|
||||
// broker's image while the broker was one that carried it; the bus that replaced it has a
|
||||
// shell and no openssl, and no other image the bundle names has one either. So the program
|
||||
// that needs the certificate writes it — already on this machine, since the schema step ran
|
||||
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
||||
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
||||
// to ask an authority of.
|
||||
// `--user 0:0` because the directory is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh directory.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--into", "/tls"],
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--check", "--into", "/tls"]
|
||||
},
|
||||
// **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are
|
||||
// the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer
|
||||
// run again after the adoption applies this bundle again, and an ordinary file would put the
|
||||
// bundle's one user back over every user the mesh composed, and leave the node unable to reach its
|
||||
// bus. Until the adoption the installer places the composed list over the seed (enrol.go).
|
||||
{
|
||||
"id": "bus-conf-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/nats-module/conf",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-conf",
|
||||
"type": "file",
|
||||
"path": "/var/lib/nats-module/conf/nats.conf",
|
||||
"mode": "0644",
|
||||
"create-once": true,
|
||||
"content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
},
|
||||
{
|
||||
"id": "bus-accounts",
|
||||
"type": "file",
|
||||
"path": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"mode": "0600",
|
||||
"create-once": true,
|
||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||
},
|
||||
// The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
|
||||
// mesh, takes its place there once, on these same directories (ADR 0333 §3). The published port is the
|
||||
// node's bus port, 5671 unless given, which every enrolment token names; the installer gives the
|
||||
// module the setting that maps its declared 4222 to it (ADR 0333 §2).
|
||||
{
|
||||
"id": "broker",
|
||||
"type": "container",
|
||||
"name": "mesh-broker",
|
||||
"image": "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
||||
"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"],
|
||||
"volumes": ["mesh-broker-data:/var/lib/lavinmq", "mesh-broker-tls:/tls:ro"],
|
||||
"args": ["--amqps-port=5671", "--cert=/tls/tls.crt", "--key=/tls/tls.key"]
|
||||
"name": "mesh-broker-nats",
|
||||
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"],
|
||||
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||
},
|
||||
{
|
||||
"id": "broker-ready",
|
||||
"type": "action",
|
||||
"in": "mesh-broker",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do lavinmqctl status >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["lavinmqctl", "status"]
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
|
||||
},
|
||||
{
|
||||
"id": "control-plane",
|
||||
@@ -171,13 +237,12 @@
|
||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"network": "host",
|
||||
"args": ["serve"],
|
||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
||||
"volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"],
|
||||
"env": {
|
||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"MESH_BROKER_AMQP": "amqp://guest:guest@127.0.0.1:5672/",
|
||||
"MESH_BROKER_MANAGEMENT": "http://guest:guest@127.0.0.1:15672",
|
||||
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
||||
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
||||
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
||||
}
|
||||
|
||||
@@ -65,9 +65,11 @@ const ReasonRoot = "can become root without a person"
|
||||
// RootGroups are the groups whose members become root by membership alone, without a sudo rule naming
|
||||
// them: the administrators' groups a distribution's own rules or polkit treat as root, the container
|
||||
// runtime's socket (a container with the host's root mounted), the raw disks, and the virtualisation
|
||||
// daemons that start a guest with the host's devices. A closed list, so what the judge calls "grants root"
|
||||
// is written down and reviewable rather than guessed per machine.
|
||||
var RootGroups = []string{"root", "wheel", "sudo", "admin", "docker", "disk", "lxd", "incus-admin", "libvirt"}
|
||||
// daemons that start a guest with the host's devices — and the kernel's memory (kmem) and the password hashes
|
||||
// (shadow), either of which is root a step away (novox/hq ADR 0312 §4 review). A closed list, so what the judge
|
||||
// calls "grants root" is written down and reviewable rather than guessed per machine.
|
||||
var RootGroups = []string{"root", "wheel", "sudo", "admin", "docker", "disk", "lxd", "incus-admin", "libvirt",
|
||||
"kmem", "shadow"}
|
||||
|
||||
// Account is one user resource of a module that declares groups.
|
||||
type Account struct {
|
||||
|
||||
@@ -285,3 +285,32 @@ func (e Exec) heldBy(pid string) (map[string]bool, error) {
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// IDs is `getent passwd`: an account's uid and primary gid, and whether the user database names it at all. A
|
||||
// database that does not answer is an error, never "no such account".
|
||||
func (e Exec) IDs(ctx context.Context, account string) (uid, gid int, found bool, err error) {
|
||||
out, err := e.Run(ctx, "getent", "passwd", account)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exited 2") || strings.Contains(err.Error(), "exit status 2") {
|
||||
return 0, 0, false, nil
|
||||
}
|
||||
return 0, 0, false, fmt.Errorf("the user database did not answer about %q: %w", account, err)
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(out), ":")
|
||||
if len(fields) < 4 {
|
||||
return 0, 0, false, fmt.Errorf("the user database gave %q for %q, which is not a passwd entry",
|
||||
strings.TrimSpace(out), account)
|
||||
}
|
||||
// getent reads an all-digit key as a uid: "1000" answers with whoever has uid 1000. Only the account named is
|
||||
// the one asked for.
|
||||
if fields[0] != account {
|
||||
return 0, 0, false, fmt.Errorf("the user database answered %q with the account %q: an account is named, "+
|
||||
"never numbered", account, fields[0])
|
||||
}
|
||||
uid, uerr := strconv.Atoi(fields[2])
|
||||
gid, gerr := strconv.Atoi(fields[3])
|
||||
if uerr != nil || gerr != nil {
|
||||
return 0, 0, false, fmt.Errorf("the user database gave no numbers for %q", account)
|
||||
}
|
||||
return uid, gid, true, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package accounts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An account is read by its name (novox/hq ADR 0325 §4): getent reads an all-digit key as a uid, so "1000" would
|
||||
// answer with whoever has uid 1000 — refused, never taken for the account named.
|
||||
func TestAnAccountIsReadByItsNameNeverItsNumber(t *testing.T) {
|
||||
db := map[string]string{
|
||||
"nox": "nox:x:1003:1003::/home/nox:/bin/sh",
|
||||
"1000": "jochen:x:1000:1000::/home/jochen:/bin/zsh", // what getent answers for a uid
|
||||
}
|
||||
e := Exec{Run: func(_ context.Context, name string, args ...string) (string, error) {
|
||||
if line, ok := db[args[len(args)-1]]; ok {
|
||||
return line + "\n", nil
|
||||
}
|
||||
return "", errors.New("getent exited 2")
|
||||
}}
|
||||
if uid, gid, found, err := e.IDs(context.Background(), "nox"); err != nil || !found || uid != 1003 || gid != 1003 {
|
||||
t.Fatalf("nox: %d %d %v %v", uid, gid, found, err)
|
||||
}
|
||||
if _, _, found, err := e.IDs(context.Background(), "1000"); found || err == nil || !strings.Contains(err.Error(), "jochen") {
|
||||
t.Fatalf("an all-digit name was read as a uid: found=%v err=%v", found, err)
|
||||
}
|
||||
if _, _, found, err := e.IDs(context.Background(), "ghost"); found || err != nil {
|
||||
t.Fatalf("an absent account: found=%v err=%v", found, err)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package apply
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
@@ -32,28 +31,6 @@ func TestAnAccessPresentIsConfirmedAndNothingIsChanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Absent is refused, not created. A bind mount whose source does not exist is made by the
|
||||
// container runtime as root, with whatever mode it picks — the silent wrong-ownership
|
||||
// 04-ISSUES/026 records. So the host says plainly that the operator must provide the path, rather
|
||||
// than conjuring a directory it does not own.
|
||||
func TestAnAccessThatIsAbsentIsRefusedClearlyAndNotCreated(t *testing.T) {
|
||||
missing := t.TempDir() + "/media/library" // named, never created
|
||||
d := declare(t, `{"id":"lib","type":"access","path":"`+missing+`"}`)
|
||||
|
||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{},
|
||||
store.OriginDeclared, noServices, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("an absent operator-owned path was accepted, and would be created as root by the runtime")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "the operator must provide") ||
|
||||
!strings.Contains(err.Error(), "does not own") {
|
||||
t.Fatalf("the refusal does not say whose the path is: %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(missing); statErr == nil {
|
||||
t.Fatal("the host created the path it does not own")
|
||||
}
|
||||
}
|
||||
|
||||
// Undeclaring an access never removes the path. Unassigning the module that reached the media
|
||||
// library must not delete the library — that is the data loss ADR 0030 exists to prevent, on a
|
||||
// directory the mesh never made. The record is dropped; the operator's data is left exactly as it
|
||||
|
||||
+312
-51
@@ -41,6 +41,9 @@ type Outcome struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Target string `json:"target"`
|
||||
// changes is what a tool runner leaving a group changes (`library:<name>`, `identity:<module>`): what the runner's
|
||||
// unit names under restart-on. Never said.
|
||||
changes string
|
||||
// Action is created · updated · unchanged · corrected · removed — and held-still, a container
|
||||
// a maintenance window is holding stopped, left for the next apply after it (novox/hq issue 224).
|
||||
//
|
||||
@@ -262,6 +265,12 @@ func ApplyMindingWindows(
|
||||
}
|
||||
report = Report{}
|
||||
|
||||
// **No module identity is root** (novox/hq ADR 0328 §15): an identity or a container told a root number refuses
|
||||
// the declaration whole, before anything of it is made or started.
|
||||
if err := refuseRootIdentities(d); err != nil {
|
||||
return report, known, &Error{Resource: "the declaration's identities", Err: err, Done: report}
|
||||
}
|
||||
|
||||
declared := map[string]bool{}
|
||||
for _, r := range d.Resources {
|
||||
declared[r.Identity()] = true
|
||||
@@ -315,6 +324,8 @@ func ApplyMindingWindows(
|
||||
// (novox/hq ADR 0252), on the unit's rule above: two modules may each want the account in one group,
|
||||
// and the one going takes nothing the other still needs.
|
||||
wanted := groupsWanted(d.Resources)
|
||||
// And what a record going hands to a resource that still asks for its group (novox/hq ADR 0252 §3).
|
||||
handed := Handed{}
|
||||
|
||||
removeOrphan := func(orphan store.Applied) error {
|
||||
var action, detail string
|
||||
@@ -339,7 +350,23 @@ func ApplyMindingWindows(
|
||||
case declaration.TypeOpening:
|
||||
action, detail, err = removeOpening(ctx, orphan, run, known.Firewall)
|
||||
case declaration.TypeUser:
|
||||
action, detail, err = removeUser(ctx, sys, orphan, run, wanted)
|
||||
action, detail, err = removeUser(ctx, sys, orphan, run, wanted, handed)
|
||||
case declaration.TypeAccess:
|
||||
// The path is left as it is (novox/hq ADR 0051); what ADR 0312's grant put an account in is taken back.
|
||||
action, detail = "forgotten", "an operator-owned path is never the host's to remove"
|
||||
gave, left, said := takeBackGranted(ctx, sys, orphan, run, wanted, handed)
|
||||
if said != "" {
|
||||
detail = joinDetail(detail, said)
|
||||
if gave {
|
||||
action = "restored"
|
||||
}
|
||||
}
|
||||
if len(left) > 0 {
|
||||
// Kept recorded with what is still to take back, and tried again (ADR 0177's waiting removal).
|
||||
orphan.Groups = left
|
||||
known.Record(orphan)
|
||||
err = fmt.Errorf("%w: %s", errRemovalWaits, said)
|
||||
}
|
||||
default:
|
||||
action, detail, err = remove(ctx, sys, orphan, run, made)
|
||||
}
|
||||
@@ -506,7 +533,7 @@ func ApplyMindingWindows(
|
||||
// And one patience with the artifact store for the whole apply (novox/hq issue 291): a fetch it
|
||||
// does not answer during a maintenance window waits for the window instead of failing.
|
||||
in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
|
||||
groups: wanted}
|
||||
groups: wanted, handed: handed}
|
||||
// Where nothing is placed or mounted from, judged before anything is applied (novox/hq issue 339), and
|
||||
// which declared directories were already here: one this apply makes is the mesh's, one found may not be.
|
||||
refreshRuntimeRoots(ctx)
|
||||
@@ -552,6 +579,135 @@ func ApplyMindingWindows(
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
// **Every library's group, then every module's identity, is made before anything of it is applied** (novox/hq
|
||||
// ADR 0328 §3, §4): a directory a module owns is given to it, and its user is in its libraries' groups. One that
|
||||
// cannot be made — its name or numbers another's here — fails, naming whose, and what needs it is not
|
||||
// attempted; the rest of the machine is.
|
||||
var identityFailures []*Error
|
||||
// warnings is every tool runner's account that joins no group this apply: said, never failing it.
|
||||
var warnings []RunnerWarning
|
||||
brokenGroups := map[int]bool{}
|
||||
if len(d.Groups) > 0 || len(d.Identities) > 0 {
|
||||
tool, toolErr := toolFor(sys)
|
||||
for _, g := range d.Groups {
|
||||
var outcome Outcome
|
||||
err := toolErr
|
||||
if err == nil {
|
||||
outcome, err = applyGroup(ctx, tool, g, &known, run)
|
||||
}
|
||||
if err != nil {
|
||||
identityFailures = append(identityFailures, &Error{Resource: "group " + g.Name, Err: err})
|
||||
log(fmt.Sprintf(" failed group %s: %v", g.Name, err))
|
||||
brokenGroups[g.GID] = true
|
||||
continue
|
||||
}
|
||||
report.Outcomes = append(report.Outcomes, outcome)
|
||||
if outcome.Action != "unchanged" {
|
||||
log(fmt.Sprintf(" %s %s: %s", outcome.Action, outcome.ID, outcome.Detail))
|
||||
}
|
||||
}
|
||||
names := map[int]string{}
|
||||
for _, g := range d.Groups {
|
||||
names[g.GID] = g.Name
|
||||
}
|
||||
for _, id := range d.Identities {
|
||||
var outcome Outcome
|
||||
err := toolErr
|
||||
for _, gid := range append([]int{id.GID}, id.Groups...) {
|
||||
if brokenGroups[gid] && err == nil {
|
||||
err = fmt.Errorf("its library group %s could not be made; nothing of the module was attempted",
|
||||
names[gid])
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
outcome, err = applyIdentity(ctx, tool, id, names, &known, run)
|
||||
}
|
||||
if err != nil {
|
||||
// Counted with the others once the resources are through: here, before the guard, it would read
|
||||
// as the guard not being up.
|
||||
identityFailures = append(identityFailures, &Error{Resource: "identity " + id.Name, Err: err})
|
||||
log(fmt.Sprintf(" failed identity %s: %v", id.Name, err))
|
||||
gated[id.Module] = true
|
||||
gatedWhy[id.Module] = "its identity " + id.Name + " could not be made, so nothing of this module " +
|
||||
"was attempted"
|
||||
continue
|
||||
}
|
||||
// The tool runner joins the module's own group, so its tools read what the module keeps (ADR 0334
|
||||
// §2); a change of `identity:<module>` restarts it once, through its unit's restart-on.
|
||||
if id.Runner != "" {
|
||||
moved, said, err := joinGroup(ctx, sys, id.Runner, id.Name, store.Membership{Module: id.Module}, &known, run)
|
||||
if err != nil {
|
||||
// A warning, never a failure of the apply or of the module: the module runs; its tools may not
|
||||
// read its directories until the account is right (said as runner:<module>).
|
||||
warnings = append(warnings, RunnerWarning{Resource: "runner:" + id.Module, Module: id.Module,
|
||||
Why: err.Error()})
|
||||
outcome.Detail = joinDetail(outcome.Detail, "its tool runner joins no group: "+err.Error())
|
||||
log(fmt.Sprintf(" warning identity %s's runner: %v", id.Name, err))
|
||||
} else if moved {
|
||||
changed[IdentityChange(id.Module)] = true
|
||||
outcome.Detail = joinDetail(outcome.Detail, said)
|
||||
if outcome.Action == "unchanged" {
|
||||
outcome.Action = "updated"
|
||||
}
|
||||
}
|
||||
}
|
||||
report.Outcomes = append(report.Outcomes, outcome)
|
||||
if outcome.Action != "unchanged" {
|
||||
log(fmt.Sprintf(" %s %s: %s", outcome.Action, outcome.ID, outcome.Detail))
|
||||
}
|
||||
}
|
||||
}
|
||||
// **Every library this machine is reached by, before anything mounts it** (ADR 0328 §6, §9, §11): kept on its
|
||||
// home, its tool runner's account in its group, and the containers that mount one whose disk or mount is not
|
||||
// there held. A tool runner that joined a group is a change of `library:<name>`, which the tool runner's unit
|
||||
// names under restart-on to be restarted once.
|
||||
verdicts := map[string]libraryVerdict{}
|
||||
for _, l := range d.Libraries {
|
||||
outcome, v, err := applyLibrary(ctx, l)
|
||||
verdicts[l.Library] = v
|
||||
if err == nil {
|
||||
var moved bool
|
||||
var said string
|
||||
var refused error
|
||||
moved, said, refused = joinRunner(ctx, sys, l, &known, run)
|
||||
if refused != nil {
|
||||
// A warning, never a failure: the library is kept as it is, and said as runner:library:<name>.
|
||||
warnings = append(warnings, RunnerWarning{Resource: "runner:" + LibraryAccess(l.Library),
|
||||
Why: refused.Error()})
|
||||
outcome.Detail = joinDetail(outcome.Detail, "its tool runner joins no group: "+refused.Error())
|
||||
log(fmt.Sprintf(" warning %s's runner: %v", LibraryAccess(l.Library), refused))
|
||||
}
|
||||
if moved {
|
||||
changed[LibraryAccess(l.Library)] = true
|
||||
outcome.Detail = joinDetail(outcome.Detail, said)
|
||||
if outcome.Action == "unchanged" {
|
||||
outcome.Action = "updated"
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
identityFailures = append(identityFailures, &Error{Resource: LibraryAccess(l.Library), Err: err})
|
||||
log(fmt.Sprintf(" failed %s (%s): %v", LibraryAccess(l.Library), l.Root, err))
|
||||
continue
|
||||
}
|
||||
report.Outcomes = append(report.Outcomes, outcome)
|
||||
if outcome.Action != "unchanged" {
|
||||
log(fmt.Sprintf(" %s %s (%s): %s", outcome.Action, outcome.ID, outcome.Target, outcome.Detail))
|
||||
}
|
||||
}
|
||||
in.held = heldRoots(verdicts, d.Libraries)
|
||||
// A tool runner no longer put in a group leaves it before anything is applied, so its unit, which names the
|
||||
// change under restart-on, is restarted in this same apply to lose it (ADR 0328 §11, ADR 0334 §2, ADR 0252 §3).
|
||||
if origin == store.OriginDeclared {
|
||||
for _, o := range leaveRunners(ctx, sys, d.Libraries, d.Identities, &known, run, wanted) {
|
||||
if o.Action == "restored" {
|
||||
changed[o.changes] = true
|
||||
}
|
||||
report.Outcomes = append(report.Outcomes, o)
|
||||
log(fmt.Sprintf(" %s %s: %s", o.Action, o.ID, o.Detail))
|
||||
}
|
||||
}
|
||||
setRunnerWarnings(warnings)
|
||||
for i, resource := range ordered {
|
||||
if !orphansRemoved && i == guardFirst {
|
||||
// **Only a guard that is up may let the filter go.** Removing the derived filter's
|
||||
@@ -717,6 +873,13 @@ func ApplyMindingWindows(
|
||||
// The file guard (refusedPlaces) refuses what is below the directory whatever its module; this
|
||||
// keeps the module from going on as if its directory were there. Only the guard's refusals: a
|
||||
// directory that failed to be made is a failure like any other.
|
||||
// A directory refused at the moment of use (novox/hq issue 358: a link another account could make
|
||||
// found in its path by the walk) is a refusal like the guard's, from here on: it gates its module, and
|
||||
// what is below it is refused whatever its module (applyOne's guard reads in.refused as it grows).
|
||||
var refusal *PlacementRefusedError
|
||||
if dir, ok := resource.(*declaration.Directory); ok && errors.As(err, &refusal) && in.refused[filepath.Clean(dir.Path)] == nil {
|
||||
in.refused[filepath.Clean(dir.Path)] = err
|
||||
}
|
||||
if dir, ok := resource.(*declaration.Directory); ok && in.refused[filepath.Clean(dir.Path)] != nil {
|
||||
if module, ours := moduleOf(resource.Identity()); ours {
|
||||
gated[module] = true
|
||||
@@ -907,6 +1070,8 @@ func ApplyMindingWindows(
|
||||
}
|
||||
}
|
||||
|
||||
failures = append(failures, identityFailures...)
|
||||
|
||||
// A replacement that did not apply — failed, skipped behind its module's step, held — leaves
|
||||
// what it replaces running and recorded, said, for the next apply to hand over.
|
||||
unhanded := make([]string, 0, len(handover))
|
||||
@@ -927,6 +1092,22 @@ func ApplyMindingWindows(
|
||||
}
|
||||
}
|
||||
|
||||
// Only the mesh says what is assigned: a carried bundle's silence undeclares no identity and no folder.
|
||||
if origin == store.OriginDeclared {
|
||||
// And an identity, then a library group, no longer declared goes, once what ran as it went above (ADR 0328
|
||||
// §3, §4).
|
||||
if tool, err := toolFor(sys); err == nil {
|
||||
for _, o := range removeIdentities(ctx, tool, d.Identities, &known, run) {
|
||||
report.Outcomes = append(report.Outcomes, o)
|
||||
log(fmt.Sprintf(" %s %s: %s", o.Action, o.ID, o.Detail))
|
||||
}
|
||||
for _, o := range removeGroups(ctx, tool, d.Groups, &known, run) {
|
||||
report.Outcomes = append(report.Outcomes, o)
|
||||
log(fmt.Sprintf(" %s %s: %s", o.Action, o.ID, o.Detail))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A converged node whose found firewall was in force retires it only now, once everything —
|
||||
// the mesh's derived filter among it — applied cleanly (novox/hq ADR 0100), and on every
|
||||
// converged apply, not once (ADR 0168). Skipped, it is said: a step that does nothing is never
|
||||
@@ -1043,10 +1224,26 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
|
||||
if err := in.refused[filepath.Clean(r.Target())]; err != nil {
|
||||
return begin(r), err
|
||||
}
|
||||
// And below a directory refused during this apply (novox/hq issue 358), which the guard could not see
|
||||
// before it began.
|
||||
if _, isDir := r.(*declaration.Directory); !isDir {
|
||||
if err := belowRefused(r.Target(), in.refused); err != nil {
|
||||
return begin(r), err
|
||||
}
|
||||
}
|
||||
case *declaration.Container:
|
||||
if err := refuseMounts(res, in.refused); err != nil {
|
||||
return begin(r), err
|
||||
}
|
||||
// Not created over a library whose disk or mount is not there: the runtime would make the root as root,
|
||||
// and the container's writes would land on the filesystem below (novox/hq ADR 0328 §9).
|
||||
if name := mountsHeld(res, in.held); name != "" {
|
||||
out := begin(r)
|
||||
out.Action = "waiting"
|
||||
out.Detail = "not created: it mounts the library " + name + ", whose disk or mount is not there; " +
|
||||
"created once it is (novox/hq ADR 0328 §9)"
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
// Nothing below a home is touched through a link an account put there (novox/hq ADR 0266).
|
||||
// And nothing is unpacked or written into below an agent's home (agent_home_guard.go).
|
||||
@@ -1071,7 +1268,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
|
||||
case *declaration.Container:
|
||||
return applyContainer(ctx, res, run, changed, in, previous)
|
||||
case *declaration.User:
|
||||
return applyUser(ctx, sys, res, run, previous, in.groups)
|
||||
return applyUser(ctx, sys, res, run, previous, in.groups, in.handed)
|
||||
case *declaration.Archive:
|
||||
return applyArchive(ctx, res, previous, in.away)
|
||||
case *declaration.Process:
|
||||
@@ -1081,7 +1278,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
|
||||
case *declaration.Network:
|
||||
return applyNetwork(ctx, res, run)
|
||||
case *declaration.Access:
|
||||
return applyAccess(res)
|
||||
return applyAccess(ctx, sys, res, run, previous, in)
|
||||
default:
|
||||
// Unreachable: the declaration refused this already. Present because "unreachable"
|
||||
// stops being true the moment someone adds a kind and forgets this switch.
|
||||
@@ -1095,6 +1292,44 @@ func begin(r declaration.Resource) Outcome {
|
||||
return Outcome{ID: r.Identity(), Type: string(r.Kind()), Target: r.Target()}
|
||||
}
|
||||
|
||||
// dirModeOf is a directory's mode as declared: its permission bits, and the setgid bit when it is 2xxx (novox/hq ADR
|
||||
// 0334 §1), as os.ModeSetgid.
|
||||
func dirModeOf(spec string, fallback os.FileMode) (os.FileMode, error) {
|
||||
mode, err := modeOf(spec, fallback)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
out := mode & os.ModePerm
|
||||
if mode&0o2000 != 0 {
|
||||
out |= os.ModeSetgid
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// dirBits is what of a directory's mode the mesh keeps: its permission bits and its setgid bit.
|
||||
func dirBits(m os.FileMode) os.FileMode { return m & (os.ModePerm | os.ModeSetgid) }
|
||||
|
||||
// unixDirMode is dirBits as a chmod takes it.
|
||||
func unixDirMode(m os.FileMode) uint32 {
|
||||
out := uint32(m.Perm())
|
||||
if m&os.ModeSetgid != 0 {
|
||||
out |= 0o2000
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// statDirMode is a directory's mode as the kernel says it, in dirBits' terms.
|
||||
func statDirMode(raw uint32) os.FileMode {
|
||||
out := os.FileMode(raw & 0o777)
|
||||
if raw&0o2000 != 0 {
|
||||
out |= os.ModeSetgid
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// modeText is a directory's mode as a person writes it: 0750, 2750.
|
||||
func modeText(m os.FileMode) string { return fmt.Sprintf("%04o", unixDirMode(m)) }
|
||||
|
||||
func modeOf(spec string, fallback os.FileMode) (os.FileMode, error) {
|
||||
if spec == "" {
|
||||
return fallback, nil
|
||||
@@ -1119,7 +1354,7 @@ func modeOf(spec string, fallback os.FileMode) (os.FileMode, error) {
|
||||
// already has the declared owner and mode, so a person who sets them by hand at the machine hands it over.
|
||||
func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore bool) (Outcome, error) {
|
||||
out := begin(r)
|
||||
mode, err := modeOf(r.Mode, 0o755)
|
||||
mode, err := dirModeOf(r.Mode, 0o755)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -1136,6 +1371,18 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
|
||||
if existed && !before.IsDir() {
|
||||
return out, fmt.Errorf("%s exists and is not a directory", r.Path)
|
||||
}
|
||||
// Outside every home, nothing more is done — not even used as found, which lets what is declared below it go
|
||||
// on — when the walk finds a link another account could make in its path now (novox/hq issue 358). A handed-over
|
||||
// directory has its own, stricter walk below.
|
||||
if meshDirsByDescriptor && existed && homeAbove(r.Path) == "" && !HandedOver()[handOverKey(r.ID, r.Path)] {
|
||||
if err := judgeMeshDir(r.Path); err != nil {
|
||||
var link *LinkInPathError
|
||||
if errors.As(err, &link) {
|
||||
return out, &PlacementRefusedError{Path: r.Path, Why: "it " + link.why()}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
// Handed to the mesh by the operator (`nox node hand-over`, or `hand-over` at the machine): given the declared
|
||||
// owner and mode through a descriptor that followed no link (giveHandedOver) — **on every apply, not only the
|
||||
// first** (review of issue 356). After the first, the record is no longer used as found and would read as the
|
||||
@@ -1151,25 +1398,30 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !owned || before.Mode().Perm() != mode.Perm() {
|
||||
if !owned || dirBits(before.Mode()) != dirBits(mode) {
|
||||
out.Action, out.asFound = "unchanged", true
|
||||
owner := r.Owner
|
||||
if owner == "" {
|
||||
owner = "root"
|
||||
}
|
||||
out.asFoundDeclared = fmt.Sprintf("%s, mode %o", owner, mode.Perm())
|
||||
out.asFoundDeclared = fmt.Sprintf("%s, mode %s", owner, modeText(mode))
|
||||
out.asFoundAt = time.Now().UTC()
|
||||
if previous.AsFound && filepath.Clean(previous.Target) == filepath.Clean(r.Path) && !previous.FoundAt.IsZero() {
|
||||
out.asFoundAt = previous.FoundAt
|
||||
}
|
||||
out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %o are "+
|
||||
out.Detail = fmt.Sprintf("found here before the mesh and used as found: its owner %s and mode %s are "+
|
||||
"left, though %s were declared (novox/hq issue 339); `nox node hand-over <node> %s` on the "+
|
||||
"control-node hands it to the mesh", ownerText(before), before.Mode().Perm(), out.asFoundDeclared, r.Path)
|
||||
"control-node hands it to the mesh", ownerText(before), modeText(dirBits(before.Mode())), out.asFoundDeclared, r.Path)
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Outside every home, made and given its mode and owner only through a descriptor opened from /, following
|
||||
// no link another account could have made (novox/hq issue 358, mesh_dirs.go).
|
||||
if meshDirsByDescriptor && homeAbove(r.Path) == "" {
|
||||
return applyMeshDirectory(r, out, mode, existed)
|
||||
}
|
||||
if !existed {
|
||||
if err := makeDirs(r.Path, mode, r.Owner); err != nil {
|
||||
return out, err
|
||||
@@ -1190,8 +1442,8 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
|
||||
if !after.IsDir() {
|
||||
return out, fmt.Errorf("%s is not a directory after applying", r.Path)
|
||||
}
|
||||
if after.Mode().Perm() != mode.Perm() {
|
||||
return out, fmt.Errorf("%s is mode %o after setting %o", r.Path, after.Mode().Perm(), mode.Perm())
|
||||
if dirBits(after.Mode()) != dirBits(mode) {
|
||||
return out, fmt.Errorf("%s is mode %s after setting %s", r.Path, modeText(dirBits(after.Mode())), modeText(mode))
|
||||
}
|
||||
|
||||
ownedAlready, err := ownedBy(r.Path, r.Owner)
|
||||
@@ -1209,9 +1461,9 @@ func applyDirectory(r *declaration.Directory, previous store.Applied, wasBefore
|
||||
out.Action = "created"
|
||||
} else if !ownedAlready {
|
||||
out.Action = "updated"
|
||||
} else if before.Mode().Perm() != mode.Perm() {
|
||||
} else if dirBits(before.Mode()) != dirBits(mode) {
|
||||
out.Action = "updated"
|
||||
out.Detail = fmt.Sprintf("mode %o to %o", before.Mode().Perm(), mode.Perm())
|
||||
out.Detail = fmt.Sprintf("mode %s to %s", modeText(dirBits(before.Mode())), modeText(mode))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1247,10 +1499,10 @@ func applyHandedOver(r *declaration.Directory, out Outcome, mode os.FileMode) (O
|
||||
return out, err
|
||||
}
|
||||
out.Action = "unchanged"
|
||||
if before.Perm() != mode.Perm() || (chown && (beforeUID != uid || beforeGID != gid)) {
|
||||
if dirBits(before) != dirBits(mode) || (chown && (beforeUID != uid || beforeGID != gid)) {
|
||||
out.Action = "updated"
|
||||
out.Detail = fmt.Sprintf("handed to the mesh: owner %d:%d, mode %o, given what was declared", beforeUID, beforeGID,
|
||||
before.Perm())
|
||||
out.Detail = fmt.Sprintf("handed to the mesh: owner %d:%d, mode %s, given what was declared", beforeUID, beforeGID,
|
||||
modeText(dirBits(before)))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1262,39 +1514,6 @@ func applyHandedOver(r *declaration.Directory, out Outcome, mode os.FileMode) (O
|
||||
// library, a download spool that several modules share — and the host's only job is to be sure it
|
||||
// is there before anything mounts it.
|
||||
//
|
||||
// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by
|
||||
// the container runtime, as root, with whatever mode it picks — which is exactly the silent
|
||||
// wrong-ownership 04-ISSUES/026 records. So the host checks first and says plainly that the
|
||||
// operator must provide the path, rather than conjuring a directory it does not own and cannot
|
||||
// give the right owner. Nothing is written, so this never reports a change: the machine did not
|
||||
// move, the host merely confirmed a fact about it.
|
||||
func applyAccess(r *declaration.Access) (Outcome, error) {
|
||||
out := begin(r)
|
||||
info, err := os.Stat(r.Path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return out, fmt.Errorf(
|
||||
"%s is not there, and the mesh does not own it — the operator must provide it. It is "+
|
||||
"shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+
|
||||
"nothing, so a missing one is said here rather than made as root by the container "+
|
||||
"runtime", r.Path)
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return out, fmt.Errorf(
|
||||
"%s is not a directory, and an access is a shared directory the operator provides",
|
||||
r.Path)
|
||||
}
|
||||
mode := declaration.AccessRead
|
||||
if r.Mode != "" {
|
||||
mode = r.Mode
|
||||
}
|
||||
out.Action = "unchanged"
|
||||
out.Detail = "operator-owned; present, " + mode + ", nothing managed"
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keepFound, when not nil, is where the original of a file this host has no record of is kept
|
||||
// before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome.
|
||||
func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) {
|
||||
@@ -1953,7 +2172,7 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
|
||||
case declaration.TypeUser:
|
||||
// Kept, with its shell given back when that is safe (novox/hq ADR 0176 §2, issue 228).
|
||||
return removeUser(ctx, sys, a, run, nil)
|
||||
return removeUser(ctx, sys, a, run, nil, nil)
|
||||
|
||||
case declaration.TypeNetwork:
|
||||
// **The reason this is a shape at all** (novox/hq ADR 0029). Orphans are removed in
|
||||
@@ -2112,6 +2331,11 @@ type inputs struct {
|
||||
// dirsBefore is every declared directory that was on the machine when this apply began (novox/hq issue
|
||||
// 339): one the apply itself made — a file's parent — is the mesh's, one that was there may not be.
|
||||
dirsBefore map[string]bool
|
||||
// held is the root of every library whose disk or mount is not there, with its name: no container that mounts
|
||||
// it is created (novox/hq ADR 0328 §9).
|
||||
held map[string]string
|
||||
// handed is every group a record that went this apply handed to a resource still asking for it.
|
||||
handed Handed
|
||||
}
|
||||
|
||||
// fileDigest is what a file the container reads holds, by digest.
|
||||
@@ -2271,6 +2495,12 @@ func containerSpecReading(r *declaration.Container, declares, reads map[string]s
|
||||
if h := r.Health; h.RunByRuntime() {
|
||||
b.WriteString("health " + strings.Join(runtimeCheckArgs(h), " ") + "\n")
|
||||
}
|
||||
// And who it runs as (novox/hq ADR 0325 §2, §3): fixed when the container is created, so an identity told
|
||||
// another way, or renumbered, is a container recreated with it. Only when there is one, so no container that
|
||||
// runs as its image says has its digest moved.
|
||||
if args := identityArgs(r); len(args) > 0 {
|
||||
b.WriteString("identity " + strings.Join(args, " ") + "\n")
|
||||
}
|
||||
// **What this container reads is part of what it is.**
|
||||
//
|
||||
// A container takes its environment and its mounted files once, at start, and never looks
|
||||
@@ -2514,6 +2744,7 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
||||
for _, v := range r.Volumes {
|
||||
args = append(args, "--volume", v)
|
||||
}
|
||||
args = append(args, identityArgs(r)...)
|
||||
// **The one check the container carries is the one the module declared** (ADR 0240 rule 3): an
|
||||
// exec command, or the image's own adopted by name, with the declared timing. Nothing else on the
|
||||
// machine sets a container's check.
|
||||
@@ -2570,6 +2801,9 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
|
||||
}
|
||||
case len(reasons) > 0:
|
||||
out.Detail = "recreated to pick up " + strings.Join(reasons, ", ")
|
||||
case r.RunAs != nil && before.Spec == specWithoutIdentity(r, in.declares, reads):
|
||||
// It was created before it was told its module's identity (novox/hq ADR 0325 §3).
|
||||
out.Detail = "recreated to run as its module's identity (novox/hq ADR 0325 §3)"
|
||||
default:
|
||||
out.Detail = "replaced; a container's configuration is fixed when it is created"
|
||||
}
|
||||
@@ -2609,7 +2843,8 @@ func runtimeCheckArgs(h *declaration.Health) []string {
|
||||
// - It writes a seed only if it is absent and never reconciles it, so what a running program
|
||||
// grows in that seed afterward is never wiped (04-ISSUES/035). The host's marker keeps the
|
||||
// step from re-running; the step's own code keeps it from clobbering on the pass it does run.
|
||||
func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string, previous store.Applied) (Outcome, error) {
|
||||
func applyRunOnce(ctx context.Context, r *declaration.Container, run Runner, cri, want string,
|
||||
previous store.Applied) (Outcome, error) {
|
||||
out := begin(r)
|
||||
|
||||
// Already completed for this exact declaration. The marker is the store, because a run-once
|
||||
@@ -2665,6 +2900,7 @@ func foregroundRunArgs(r *declaration.Container, want string) []string {
|
||||
for _, v := range r.Volumes {
|
||||
args = append(args, "--volume", v)
|
||||
}
|
||||
args = append(args, identityArgs(r)...)
|
||||
for _, h := range r.Hosts {
|
||||
args = append(args, "--add-host", h)
|
||||
}
|
||||
@@ -2673,6 +2909,31 @@ func foregroundRunArgs(r *declaration.Container, want string) []string {
|
||||
return args
|
||||
}
|
||||
|
||||
// specWithoutIdentity is the spec r had before it was told who to run as.
|
||||
func specWithoutIdentity(r *declaration.Container, declares, reads map[string]string) string {
|
||||
plain := *r
|
||||
plain.RunAs = nil
|
||||
return containerSpecReading(&plain, declares, reads)
|
||||
}
|
||||
|
||||
// identityArgs is the runtime's arguments that run a container as its module's identity (novox/hq ADR 0325 §2):
|
||||
// the image told by the variables it reads, or by the runtime's own user option, by number — a container's own
|
||||
// account database is not the machine's. None for a container that runs as its image says.
|
||||
func identityArgs(r *declaration.Container) []string {
|
||||
id := r.RunAs
|
||||
if id == nil {
|
||||
return nil
|
||||
}
|
||||
if id.Mode == declaration.IdentityByRun {
|
||||
return []string{"--user", strconv.Itoa(id.UID) + ":" + strconv.Itoa(id.GID)}
|
||||
}
|
||||
args := []string{"--env", id.UserEnv + "=" + strconv.Itoa(id.UID)}
|
||||
if id.GroupEnv != "" {
|
||||
args = append(args, "--env", id.GroupEnv+"="+strconv.Itoa(id.GID))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// ensureImage makes the pinned image present on the node without running anything.
|
||||
//
|
||||
// A service or a run-once container gets its image as a side effect of `docker run` — the first run
|
||||
|
||||
@@ -63,7 +63,7 @@ func (w Wanted) othersAsk(account, group, except string) []string {
|
||||
// applyGroups makes the account be in every group the resource declares, and takes it out of every group
|
||||
// this resource put it in and no longer asks for. What the mesh put the account in is recorded on out.
|
||||
func applyGroups(ctx context.Context, sys system.System, r *declaration.User, run Runner,
|
||||
previous store.Applied, wanted Wanted, out *Outcome) error {
|
||||
previous store.Applied, wanted Wanted, handed Handed, out *Outcome) error {
|
||||
// What this resource put the account in before, for this account only: a declaration that renamed its
|
||||
// user says nothing about the new one's groups.
|
||||
var added []string
|
||||
@@ -83,6 +83,10 @@ func applyGroups(ctx context.Context, sys system.System, r *declaration.User, ru
|
||||
var put []string
|
||||
for _, want := range r.Groups {
|
||||
if already[want] {
|
||||
// A record that went handed it here: the mesh's, and this resource's to take back (novox/hq ADR 0312 §5).
|
||||
if handed.has(r.Name, want) && !contains(added, want) {
|
||||
added = append(added, want)
|
||||
}
|
||||
continue
|
||||
}
|
||||
exists, err := system.GroupExists(ctx, system.Runner(run), want)
|
||||
@@ -139,7 +143,11 @@ func applyGroups(ctx context.Context, sys system.System, r *declaration.User, ru
|
||||
// giveGroupsBack is removeUser's groups: every group the mesh put the account in, taken back unless
|
||||
// another declared resource still asks for it. Never fatal, for the shell's reason: a removal that failed
|
||||
// would stay recorded and fail the same way on every apply after. Empty when there was nothing to say.
|
||||
func giveGroupsBack(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted) (bool, string) {
|
||||
//
|
||||
// A group left because another declared resource still asks for it is handed to that resource (handed), which
|
||||
// records it as the mesh's, so it is taken back once nobody asks (novox/hq ADR 0312 §5).
|
||||
func giveGroupsBack(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted,
|
||||
handed Handed) (bool, string) {
|
||||
if len(a.Groups) == 0 {
|
||||
return false, ""
|
||||
}
|
||||
@@ -157,6 +165,7 @@ func giveGroupsBack(ctx context.Context, sys system.System, a store.Applied, run
|
||||
}
|
||||
if others := wanted.othersAsk(a.Target, g, a.ID); len(others) > 0 {
|
||||
said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g, strings.Join(others, ", ")))
|
||||
handed.hand(a.Target, g)
|
||||
continue
|
||||
}
|
||||
gone, why := leaveGroup(ctx, sys, a.Target, g, run)
|
||||
|
||||
@@ -137,7 +137,7 @@ func chmodUnderAs(home, path string, mode os.FileMode, want int) error {
|
||||
return err
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
if err := unix.Fchmod(fd, uint32(mode.Perm())); err != nil {
|
||||
if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
|
||||
return &os.PathError{Op: "chmod", Path: path, Err: err}
|
||||
}
|
||||
return nil
|
||||
@@ -256,8 +256,8 @@ func giveHandedOver(path string, mode os.FileMode, uid, gid int, chown bool) (be
|
||||
if err := unix.Fstat(fd, &st); err != nil {
|
||||
return 0, 0, 0, &os.PathError{Op: "fstat", Path: path, Err: err}
|
||||
}
|
||||
before, beforeUID, beforeGID = os.FileMode(st.Mode&0o777), int(st.Uid), int(st.Gid)
|
||||
if err := unix.Fchmod(fd, uint32(mode.Perm())); err != nil {
|
||||
before, beforeUID, beforeGID = statDirMode(st.Mode), int(st.Uid), int(st.Gid)
|
||||
if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
|
||||
return 0, 0, 0, &os.PathError{Op: "chmod", Path: path, Err: err}
|
||||
}
|
||||
if chown && (beforeUID != uid || beforeGID != gid) {
|
||||
|
||||
@@ -8,6 +8,8 @@ package apply
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
func chmodUnder(home, path string, mode os.FileMode) error {
|
||||
@@ -64,3 +66,19 @@ func giveHandedOver(path string, _ os.FileMode, _, _ int, _ bool) (os.FileMode,
|
||||
return 0, 0, 0, &PlacementRefusedError{Path: path, Why: "a handed-over directory is changed only through a " +
|
||||
"descriptor that followed no link, which this system cannot open"}
|
||||
}
|
||||
|
||||
// judgeMeshDir refuses where there is no openat with O_NOFOLLOW (novox/hq issue 358). The node-engine runs on
|
||||
// Linux.
|
||||
func judgeMeshDir(path string) error {
|
||||
return &PlacementRefusedError{Path: path, Why: "a mesh directory is judged only by a walk that follows no link " +
|
||||
"another account could make, which this system cannot do"}
|
||||
}
|
||||
|
||||
// meshDirsByDescriptor is false where there is no openat with O_NOFOLLOW: a mesh directory is changed by path, as
|
||||
// before novox/hq issue 358. The node-engine runs on Linux.
|
||||
const meshDirsByDescriptor = false
|
||||
|
||||
func applyMeshDirectory(r *declaration.Directory, out Outcome, _ os.FileMode, _ bool) (Outcome, error) {
|
||||
return out, &PlacementRefusedError{Path: r.Path, Why: "a mesh directory is changed only through a descriptor " +
|
||||
"that followed no link another account could make, which this system cannot open"}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// Every module is given an identity of its own (novox/hq ADR 0328 §3, §4, §15, issue 567).
|
||||
//
|
||||
// The controller allocates each module a user and a group number once, from a range of its own, the same on every
|
||||
// machine, and declares them as `mesh-<module>`. The engine makes the group and the account with exactly those
|
||||
// numbers — a system account: no login shell, no home, no password — before anything of the declaration is
|
||||
// applied, so a directory the module owns can be given to it and a container told to run as it.
|
||||
//
|
||||
// **Never root** (§11): a number that is 0, or a gid a root group has here (by its number and by its name), is
|
||||
// refused for the whole declaration before anything is made or started, naming why — for an account and for an
|
||||
// image alike. **Never another's**: a name that exists with other numbers, or numbers another name holds, is
|
||||
// refused for that identity alone, naming whose, and its module is not attempted; the rest of the machine is.
|
||||
//
|
||||
// **What it made, and only that, goes** when the identity is no longer declared: the record says whether the
|
||||
// engine made the account, the group or both, and an account or a group that was there before is never removed.
|
||||
|
||||
// accountsTool is how the engine reads and changes the machine's accounts: the system's own tools through the
|
||||
// runner, which is what a test stands in for.
|
||||
type accountsTool interface {
|
||||
CreateSystemAccount(ctx context.Context, run system.Runner, name string, uid, gid int, groups []string) error
|
||||
ModifyAccount(ctx context.Context, run system.Runner, name string, gid int, groups []string) error
|
||||
CreateSystemGroup(ctx context.Context, run system.Runner, name string, gid int) error
|
||||
RemoveAccount(ctx context.Context, run system.Runner, name string) error
|
||||
RemoveGroup(ctx context.Context, run system.Runner, name string) error
|
||||
}
|
||||
|
||||
// entry is one line the user or group database gives for a key: its name and its numbers.
|
||||
type entry struct {
|
||||
name string
|
||||
id, prim int
|
||||
}
|
||||
|
||||
// lookUp asks the database (passwd or group) for key, a name or a number. Absent is (entry{}, false, nil); a
|
||||
// database that does not answer is an error, never read as absent (system.LookUpUser's rule).
|
||||
func lookUp(ctx context.Context, run Runner, database, key string) (entry, bool, error) {
|
||||
out, err := run(ctx, "getent", database, key)
|
||||
if err != nil {
|
||||
if code, ok := system.ExitCode(err); ok && code == 2 {
|
||||
return entry{}, false, nil
|
||||
}
|
||||
return entry{}, false, fmt.Errorf("the %s database did not answer about %s: %w", database, key, err)
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(strings.SplitN(out, "\n", 2)[0]), ":")
|
||||
if len(fields) < 3 {
|
||||
return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
|
||||
}
|
||||
e := entry{name: fields[0], prim: -1}
|
||||
if e.id, err = strconv.Atoi(fields[2]); err != nil {
|
||||
return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
|
||||
}
|
||||
if database == "passwd" && len(fields) >= 4 {
|
||||
if e.prim, err = strconv.Atoi(fields[3]); err != nil {
|
||||
e.prim = -1
|
||||
}
|
||||
}
|
||||
return e, true, nil
|
||||
}
|
||||
|
||||
// refuseRootIdentities is every identity and every container's run-as that is root's (§11), judged before
|
||||
// anything of the declaration is applied: one is enough to refuse it whole, so nothing is made or started.
|
||||
func refuseRootIdentities(d *declaration.Declaration) error {
|
||||
var problems []string
|
||||
for _, id := range d.Identities {
|
||||
// Its user's number, its own group's (the same number), its primary group and every added group.
|
||||
if err := rootIdentity(id.UID, id.UID); err != nil {
|
||||
problems = append(problems, fmt.Sprintf("the identity %s: %v", id.Name, err))
|
||||
}
|
||||
for _, gid := range append([]int{id.GID}, id.Groups...) {
|
||||
if why := rootGroup(gid); why != "" {
|
||||
problems = append(problems, fmt.Sprintf("the identity %s is given gid %s, and a module's identity "+
|
||||
"is never in a root group (novox/hq ADR 0328 §15)", id.Name, why))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, g := range d.Groups {
|
||||
if why := rootGroup(g.GID); why != "" {
|
||||
problems = append(problems, fmt.Sprintf("the library group %s is gid %s, and a library's group is never a "+
|
||||
"root group (novox/hq ADR 0328 §15)", g.Name, why))
|
||||
}
|
||||
}
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok && c.RunAs != nil {
|
||||
if err := rootIdentity(c.RunAs.UID, c.RunAs.GID); err != nil {
|
||||
problems = append(problems, fmt.Sprintf("the container %s is told to run as uid %d, gid %d: %v",
|
||||
c.ID, c.RunAs.UID, c.RunAs.GID, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(problems)
|
||||
return fmt.Errorf("%s; nothing of this declaration was made or started", strings.Join(problems, "; "))
|
||||
}
|
||||
|
||||
// applyIdentity makes one identity as declared (ADR 0328 §4): its own group `mesh-<module>` with the number of its
|
||||
// uid, and its user with that uid, its primary group gid (its own, or its first library's) and its other libraries'
|
||||
// groups as added groups — or refuses it naming whose the name or the numbers are. A user the engine made is given
|
||||
// a changed primary or added groups; one that was there before is never changed, and is refused if it is not as
|
||||
// declared. What it made is recorded on known; an identity already right is unchanged.
|
||||
func applyIdentity(ctx context.Context, tool accountsTool, id declaration.Identity, names map[int]string,
|
||||
known *store.State, run Runner) (Outcome, error) {
|
||||
out := Outcome{ID: "identity " + id.Name, Type: "identity", Target: id.Name, Action: "unchanged"}
|
||||
user, hasUser, err := lookUp(ctx, run, "passwd", id.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
group, hasGroup, err := lookUp(ctx, run, "group", id.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
byUID, uidUsed, err := lookUp(ctx, run, "passwd", strconv.Itoa(id.UID))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(id.UID))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
rec := madeIdentity(known, id.Name)
|
||||
const never = "; nothing was made, and nothing of the module was attempted (novox/hq ADR 0328 §4)"
|
||||
switch {
|
||||
case hasUser && user.id != id.UID:
|
||||
return out, fmt.Errorf("the account %s is here with uid %d, and the mesh allocated uid %d%s", id.Name,
|
||||
user.id, id.UID, never)
|
||||
case hasUser && user.prim != id.GID && !rec.MadeUser:
|
||||
return out, fmt.Errorf("the account %s is here with group %d, was not made by the mesh, and the mesh "+
|
||||
"gives it group %d%s", id.Name, user.prim, id.GID, never)
|
||||
case hasGroup && group.id != id.UID:
|
||||
return out, fmt.Errorf("the group %s is here with gid %d, and a module's own group has its uid, %d%s",
|
||||
id.Name, group.id, id.UID, never)
|
||||
case uidUsed && byUID.name != id.Name:
|
||||
return out, fmt.Errorf("uid %d is the account %s's here, and the mesh allocated it to %s%s", id.UID,
|
||||
byUID.name, id.Name, never)
|
||||
case gidUsed && byGID.name != id.Name:
|
||||
return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s%s", id.UID,
|
||||
byGID.name, id.Name, never)
|
||||
}
|
||||
added := make([]string, 0, len(id.Groups))
|
||||
for _, gid := range id.Groups {
|
||||
added = append(added, names[gid])
|
||||
}
|
||||
sort.Strings(added)
|
||||
rec.Module, rec.Name, rec.UID, rec.GID = id.Module, id.Name, id.UID, id.GID
|
||||
var made []string
|
||||
if !hasGroup {
|
||||
if err := tool.CreateSystemGroup(ctx, run, id.Name, id.UID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
rec.MadeGroup = true
|
||||
made = append(made, fmt.Sprintf("its own group (gid %d)", id.UID))
|
||||
}
|
||||
if !hasUser {
|
||||
if err := tool.CreateSystemAccount(ctx, run, id.Name, id.UID, id.GID, added); err != nil {
|
||||
recordIdentity(known, rec)
|
||||
return out, err
|
||||
}
|
||||
rec.MadeUser = true
|
||||
made = append(made, fmt.Sprintf("the account (uid %d, group %d%s), with no login, no home and no password",
|
||||
id.UID, id.GID, alsoIn(added)))
|
||||
} else if rec.MadeUser {
|
||||
// The mesh's own account: its primary and added groups follow what is declared.
|
||||
in, err := system.GroupsOf(ctx, system.Runner(run), id.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, g := range in {
|
||||
if g != id.Name && g != names[user.prim] {
|
||||
have[g] = true
|
||||
}
|
||||
}
|
||||
same := user.prim == id.GID && len(have) == len(added)
|
||||
for _, g := range added {
|
||||
same = same && have[g]
|
||||
}
|
||||
if !same {
|
||||
if err := tool.ModifyAccount(ctx, run, id.Name, id.GID, added); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Action = "updated"
|
||||
out.Detail = fmt.Sprintf("gave the account group %d%s", id.GID, alsoIn(added))
|
||||
}
|
||||
}
|
||||
if rec.At.IsZero() {
|
||||
rec.At = time.Now().UTC()
|
||||
}
|
||||
recordIdentity(known, rec)
|
||||
if len(made) > 0 {
|
||||
out.Action, out.Detail = "created", "made "+strings.Join(made, " and ")
|
||||
} else if out.Detail == "" {
|
||||
out.Detail = "here with its numbers"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func alsoIn(groups []string) string {
|
||||
if len(groups) == 0 {
|
||||
return ""
|
||||
}
|
||||
return ", also in " + strings.Join(groups, ", ")
|
||||
}
|
||||
|
||||
// applyGroup makes one library's group with the number the controller allocated (ADR 0328 §3), or refuses it naming
|
||||
// whose the name or the number is, or that the number is a root group's (§15). What it made is recorded on known.
|
||||
func applyGroup(ctx context.Context, tool accountsTool, g declaration.Group, known *store.State,
|
||||
run Runner) (Outcome, error) {
|
||||
out := Outcome{ID: "group " + g.Name, Type: "group", Target: g.Name, Action: "unchanged"}
|
||||
if why := rootGroup(g.GID); why != "" {
|
||||
return out, fmt.Errorf("its gid is %s, and a library's group is never a root group (novox/hq ADR 0328 §15); "+
|
||||
"nothing was made", why)
|
||||
}
|
||||
byName, hasName, err := lookUp(ctx, run, "group", g.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(g.GID))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
switch {
|
||||
case hasName && byName.id != g.GID:
|
||||
return out, fmt.Errorf("the group %s is here with gid %d, and the mesh allocated gid %d; nothing was made",
|
||||
g.Name, byName.id, g.GID)
|
||||
case gidUsed && byGID.name != g.Name:
|
||||
return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s; nothing was made",
|
||||
g.GID, byGID.name, g.Name)
|
||||
case hasName:
|
||||
out.Detail = "here with its number"
|
||||
return out, nil
|
||||
}
|
||||
if err := tool.CreateSystemGroup(ctx, run, g.Name, g.GID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
known.Groups = append(known.Groups, store.MadeGroup{Name: g.Name, GID: g.GID, At: time.Now().UTC()})
|
||||
out.Action, out.Detail = "created", fmt.Sprintf("made the library's group (gid %d)", g.GID)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// removeGroups removes every library group the engine made that is no longer declared, while it still has the
|
||||
// number the engine gave it. Never fatal: one that could not be removed (still a user's primary group) stays
|
||||
// recorded, said, for the next apply.
|
||||
func removeGroups(ctx context.Context, tool accountsTool, declared []declaration.Group, known *store.State,
|
||||
run Runner) []Outcome {
|
||||
wanted := map[string]bool{}
|
||||
for _, g := range declared {
|
||||
wanted[g.Name] = true
|
||||
}
|
||||
var out []Outcome
|
||||
var keep []store.MadeGroup
|
||||
for _, m := range known.Groups {
|
||||
if wanted[m.Name] {
|
||||
keep = append(keep, m)
|
||||
continue
|
||||
}
|
||||
o := Outcome{ID: "group " + m.Name, Type: "group", Target: m.Name, Action: "removed",
|
||||
Detail: "no longer declared: removed the group"}
|
||||
g, has, err := lookUp(ctx, run, "group", m.Name)
|
||||
switch {
|
||||
case err != nil || (has && g.id != m.GID):
|
||||
o.Action, o.Detail = "forgotten", "no longer declared: the group was left, it no longer has the gid the "+
|
||||
"mesh gave it, or the database did not answer"
|
||||
case has:
|
||||
if err := tool.RemoveGroup(ctx, run, m.Name); err != nil {
|
||||
o.Action, o.Detail = "waiting", "no longer declared: the group could not be removed: "+err.Error()
|
||||
keep = append(keep, m)
|
||||
}
|
||||
default:
|
||||
o.Action, o.Detail = "forgotten", "no longer declared, and no longer here"
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
known.Groups = keep
|
||||
return out
|
||||
}
|
||||
|
||||
// madeIdentity is what the engine recorded of name, or an empty record.
|
||||
func madeIdentity(known *store.State, name string) store.MadeIdentity {
|
||||
for _, m := range known.Identities {
|
||||
if m.Name == name {
|
||||
return m
|
||||
}
|
||||
}
|
||||
return store.MadeIdentity{}
|
||||
}
|
||||
|
||||
// recordIdentity records what was made of one identity, in place of any record before.
|
||||
func recordIdentity(known *store.State, rec store.MadeIdentity) {
|
||||
for i, m := range known.Identities {
|
||||
if m.Name == rec.Name {
|
||||
known.Identities[i] = rec
|
||||
return
|
||||
}
|
||||
}
|
||||
if rec.MadeUser || rec.MadeGroup {
|
||||
known.Identities = append(known.Identities, rec)
|
||||
}
|
||||
}
|
||||
|
||||
// removeIdentities removes what the engine made of every identity no longer declared — the account, then the
|
||||
// group — and only while it still has the numbers the engine gave it. Never fatal: what could not be removed
|
||||
// stays recorded, said, for the next apply.
|
||||
func removeIdentities(ctx context.Context, tool accountsTool, declared []declaration.Identity, known *store.State,
|
||||
run Runner) []Outcome {
|
||||
wanted := map[string]bool{}
|
||||
for _, id := range declared {
|
||||
wanted[id.Name] = true
|
||||
}
|
||||
var out []Outcome
|
||||
var keep []store.MadeIdentity
|
||||
for _, m := range known.Identities {
|
||||
if wanted[m.Name] {
|
||||
keep = append(keep, m)
|
||||
continue
|
||||
}
|
||||
o := Outcome{ID: "identity " + m.Name, Type: "identity", Target: m.Name, Action: "removed"}
|
||||
var said []string
|
||||
if m.MadeUser {
|
||||
if u, has, err := lookUp(ctx, run, "passwd", m.Name); err != nil || (has && u.id != m.UID) {
|
||||
said = append(said, "the account was left: it no longer has the uid the mesh gave it, or the "+
|
||||
"database did not answer")
|
||||
m.MadeUser = false
|
||||
} else if has {
|
||||
if err := tool.RemoveAccount(ctx, run, m.Name); err != nil {
|
||||
said = append(said, "the account could not be removed: "+err.Error())
|
||||
} else {
|
||||
m.MadeUser = false
|
||||
said = append(said, "removed the account")
|
||||
}
|
||||
} else {
|
||||
m.MadeUser = false
|
||||
}
|
||||
}
|
||||
if m.MadeGroup && !m.MadeUser {
|
||||
if g, has, err := lookUp(ctx, run, "group", m.Name); err != nil || (has && g.id != m.GID) {
|
||||
said = append(said, "the group was left: it no longer has the gid the mesh gave it, or the "+
|
||||
"database did not answer")
|
||||
m.MadeGroup = false
|
||||
} else if has {
|
||||
if err := tool.RemoveGroup(ctx, run, m.Name); err != nil {
|
||||
said = append(said, "the group could not be removed: "+err.Error())
|
||||
} else {
|
||||
m.MadeGroup = false
|
||||
said = append(said, "removed the group")
|
||||
}
|
||||
} else {
|
||||
m.MadeGroup = false
|
||||
}
|
||||
}
|
||||
if m.MadeUser || m.MadeGroup {
|
||||
o.Action = "waiting"
|
||||
keep = append(keep, m)
|
||||
}
|
||||
o.Detail = "no longer declared: " + strings.Join(said, "; ")
|
||||
out = append(out, o)
|
||||
}
|
||||
known.Identities = keep
|
||||
return out
|
||||
}
|
||||
|
||||
// toolFor is the accounts tool of a system, or why it has none.
|
||||
func toolFor(sys system.System) (accountsTool, error) {
|
||||
if t, ok := sys.(accountsTool); ok {
|
||||
return t, nil
|
||||
}
|
||||
return nil, errors.New("this system makes no module identity: it has no way to make a system account")
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// The engine's half of ADR 0325 §3 and §11 (issue 567): every module given an account and a group of its own with
|
||||
// the numbers the controller allocated, a container told to run as it, and never a root number.
|
||||
|
||||
// accountsMachine is a user and group database the runner answers from, and every command that changed it.
|
||||
type accountsMachine struct {
|
||||
users, groups map[string][2]int // name → id, primary gid (a group's second is unused)
|
||||
// in is each account's added groups, by name; homes each account's home, "/" when not set.
|
||||
in map[string][]string
|
||||
homes map[string]string
|
||||
ran []string
|
||||
}
|
||||
|
||||
// groupNamed is the name of gid in m, or the number.
|
||||
func (m *accountsMachine) groupNamed(gid int) string {
|
||||
for n, ids := range m.groups {
|
||||
if ids[0] == gid {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return strconv.Itoa(gid)
|
||||
}
|
||||
|
||||
func (m *accountsMachine) run(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
switch name {
|
||||
case "getent":
|
||||
db, key := args[0], args[1]
|
||||
table := m.users
|
||||
if db == "group" {
|
||||
table = m.groups
|
||||
}
|
||||
for n, ids := range table {
|
||||
if n == key || strconv.Itoa(ids[0]) == key {
|
||||
if db == "group" {
|
||||
return n + ":x:" + strconv.Itoa(ids[0]) + ":\n", nil
|
||||
}
|
||||
home := m.homes[n]
|
||||
if home == "" {
|
||||
home = "/"
|
||||
}
|
||||
return n + ":x:" + strconv.Itoa(ids[0]) + ":" + strconv.Itoa(ids[1]) + "::" + home + ":/usr/bin/nologin\n", nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("getent exited 2")
|
||||
case "id":
|
||||
name := args[len(args)-1]
|
||||
u, ok := m.users[name]
|
||||
if !ok {
|
||||
return "", errors.New("id exited 1")
|
||||
}
|
||||
return strings.Join(append([]string{m.groupNamed(u[1])}, m.in[name]...), " ") + "\n", nil
|
||||
case "groupadd":
|
||||
m.groups[args[len(args)-1]] = [2]int{mustAtoi(args[2]), 0}
|
||||
case "useradd":
|
||||
name := args[len(args)-1]
|
||||
m.users[name] = [2]int{mustAtoi(args[2]), mustAtoi(args[4])}
|
||||
if args[5] == "--groups" {
|
||||
m.setIn(name, strings.Split(args[6], ","))
|
||||
}
|
||||
if m.homes == nil {
|
||||
m.homes = map[string]string{}
|
||||
}
|
||||
m.homes[name] = "/nonexistent"
|
||||
case "usermod":
|
||||
name := args[len(args)-1]
|
||||
if args[0] == "--append" {
|
||||
m.setIn(name, append(m.in[name], args[2]))
|
||||
} else if args[0] == "--home" {
|
||||
if m.homes == nil {
|
||||
m.homes = map[string]string{}
|
||||
}
|
||||
m.homes[name] = args[1]
|
||||
} else {
|
||||
u := m.users[name]
|
||||
m.users[name] = [2]int{u[0], mustAtoi(args[1])}
|
||||
m.setIn(name, strings.Split(args[3], ","))
|
||||
}
|
||||
case "gpasswd":
|
||||
var kept []string
|
||||
for _, g := range m.in[args[1]] {
|
||||
if g != args[2] {
|
||||
kept = append(kept, g)
|
||||
}
|
||||
}
|
||||
m.setIn(args[1], kept)
|
||||
case "userdel":
|
||||
delete(m.users, args[0])
|
||||
case "groupdel":
|
||||
delete(m.groups, args[0])
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
m.ran = append(m.ran, line)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (m *accountsMachine) setIn(name string, groups []string) {
|
||||
if m.in == nil {
|
||||
m.in = map[string][]string{}
|
||||
}
|
||||
var out []string
|
||||
for _, g := range groups {
|
||||
if g != "" {
|
||||
out = append(out, g)
|
||||
}
|
||||
}
|
||||
m.in[name] = out
|
||||
}
|
||||
|
||||
func mustAtoi(s string) int {
|
||||
n, _ := strconv.Atoi(s)
|
||||
return n
|
||||
}
|
||||
|
||||
// withGroups is a declaration with library groups, identities and resources.
|
||||
func withGroups(t *testing.T, groups, identities, resources string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
nothing := ""
|
||||
if resources == "" {
|
||||
nothing = `"owns_nothing":true,`
|
||||
}
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"groups":[` + groups + `],"identities":[` + identities +
|
||||
`],` + nothing + `"resources":[` + resources + `]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func withIdentities(t *testing.T, identities, resources string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"identities":[` + identities + `],"resources":[` +
|
||||
resources + `]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Row 3: the account made as a system account with exactly its numbers, recorded, and removed — only what was
|
||||
// made — when no longer declared.
|
||||
func TestAModulesIdentityIsMadeWithItsNumbersAndRemovedWhenUndeclared(t *testing.T) {
|
||||
m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
|
||||
dir := t.TempDir()
|
||||
d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`)
|
||||
report, known, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"groupadd --system --gid 61001 mesh-radarr",
|
||||
"useradd --system --uid 61001 --gid 61001 --no-create-home --home-dir /nonexistent --shell /usr/bin/nologin mesh-radarr",
|
||||
}
|
||||
if strings.Join(m.ran, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("made with:\n%s\nwant:\n%s", strings.Join(m.ran, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if o := outcomeOf(report, "identity mesh-radarr"); o.Action != "created" {
|
||||
t.Errorf("the identity's outcome: %+v", o)
|
||||
}
|
||||
if len(known.Identities) != 1 || !known.Identities[0].MadeUser || !known.Identities[0].MadeGroup {
|
||||
t.Fatalf("what was made is not recorded: %+v", known.Identities)
|
||||
}
|
||||
|
||||
// Applied again: nothing is made twice.
|
||||
m.ran = nil
|
||||
if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.ran) != 0 {
|
||||
t.Fatalf("an identity already here was made again: %v", m.ran)
|
||||
}
|
||||
|
||||
// Undeclared, with a group the machine had before for another identity: only what was made goes.
|
||||
m.groups["mesh-sonarr"] = [2]int{61002, 0}
|
||||
other := withIdentities(t, `{"module":"sonarr","name":"mesh-sonarr","uid":61002,"gid":61002}`,
|
||||
`{"id":"x.dir","type":"directory","path":"`+dir+`/x"}`)
|
||||
if _, known, err = Apply(context.Background(), archHost(t), other, known, store.OriginDeclared, m.run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, has := m.users["mesh-radarr"]; has {
|
||||
t.Error("an identity no longer declared kept its account")
|
||||
}
|
||||
if _, has := m.groups["mesh-radarr"]; has {
|
||||
t.Error("an identity no longer declared kept its group")
|
||||
}
|
||||
if rec := madeIdentity(&known, "mesh-sonarr"); rec.MadeGroup {
|
||||
t.Error("a group that was there before is recorded as the mesh's")
|
||||
}
|
||||
}
|
||||
|
||||
// Row 3: a name that exists with other numbers, or numbers another name holds, is refused naming whose, and the
|
||||
// module is not attempted; the rest of the machine is.
|
||||
func TestAnIdentityWhoseNameOrNumbersAreAnothersIsRefused(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, tc := range []struct {
|
||||
users, groups map[string][2]int
|
||||
says string
|
||||
}{
|
||||
{users: map[string][2]int{"mesh-radarr": {1500, 1500}}, groups: map[string][2]int{}, says: "uid 1500"},
|
||||
{users: map[string][2]int{"someone": {61001, 100}}, groups: map[string][2]int{}, says: "someone"},
|
||||
{users: map[string][2]int{}, groups: map[string][2]int{"media": {61001, 0}}, says: "media"},
|
||||
} {
|
||||
m := &accountsMachine{users: tc.users, groups: tc.groups}
|
||||
d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/radarr"},`+
|
||||
`{"id":"plex.config","type":"directory","path":"`+dir+`/plex"}`)
|
||||
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.says) {
|
||||
t.Errorf("not refused naming %s: %v", tc.says, err)
|
||||
}
|
||||
if len(m.ran) != 0 {
|
||||
t.Errorf("something was made for an identity that is another's: %v", m.ran)
|
||||
}
|
||||
if o := outcomeOf(report, "radarr.config"); o.Action != "skipped" {
|
||||
t.Errorf("the module of a refused identity was attempted: %+v", o)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "plex")); err != nil {
|
||||
t.Errorf("the rest of the machine was not applied: %v", err)
|
||||
}
|
||||
os.RemoveAll(filepath.Join(dir, "plex"))
|
||||
}
|
||||
}
|
||||
|
||||
// Row 11: a declaration naming a root group for an account or an image is refused with its reason, and nothing is
|
||||
// made or started. A number 0 is refused on arrival, by the parser.
|
||||
func TestNoModuleIdentityIsRoot(t *testing.T) {
|
||||
was := RootGroupIDs
|
||||
RootGroupIDs = func() map[int]string { return map[int]string{10: "wheel"} }
|
||||
defer func() { RootGroupIDs = was }()
|
||||
dir := t.TempDir()
|
||||
for _, d := range []*declaration.Declaration{
|
||||
withGroups(t, `{"name":"mesh-media","gid":10}`, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":10}`,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`),
|
||||
withGroups(t, `{"name":"mesh-media","gid":10}`, ``,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`),
|
||||
withIdentities(t, ``, `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"},`+
|
||||
`{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`",`+
|
||||
`"identity":{"mode":"run","uid":61001,"gid":10}}`),
|
||||
} {
|
||||
m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
|
||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "wheel") || !strings.Contains(err.Error(), "never") {
|
||||
t.Errorf("a root group was not refused naming why: %v", err)
|
||||
}
|
||||
if len(m.ran) != 0 {
|
||||
t.Errorf("something was made or started: %v", m.ran)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "config")); err == nil {
|
||||
t.Error("a resource of a declaration naming a root group was applied")
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{
|
||||
`{"declaration":1,"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]}`,
|
||||
`{"declaration":1,"resources":[{"id":"m.app","type":"container","name":"m","image":"` + pinned + `",` +
|
||||
`"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":0}}]}`,
|
||||
} {
|
||||
if _, err := declaration.Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "never root") {
|
||||
t.Errorf("a number 0 was not refused on arrival: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Row 3: a container declaring identity is told it — by the variables its image reads, or the runtime's user
|
||||
// option — and a changed identity is a container recreated.
|
||||
func TestAContainerIsToldItsModulesIdentity(t *testing.T) {
|
||||
env := &declaration.Container{ID: "radarr.app", Name: "radarr", Image: pinned, RunAs: &declaration.ContainerIdentity{
|
||||
Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", UID: 61001, GID: 61001}}
|
||||
run := &declaration.Container{ID: "plex.app", Name: "plex", Image: pinned, RunAs: &declaration.ContainerIdentity{
|
||||
Mode: declaration.IdentityByRun, UID: 61002, GID: 61002}}
|
||||
if got := strings.Join(foregroundRunArgs(env, "x"), " "); !strings.Contains(got, "--env PUID=61001 --env PGID=61001") {
|
||||
t.Errorf("an image told by environment was not: %s", got)
|
||||
}
|
||||
if got := strings.Join(foregroundRunArgs(run, "x"), " "); !strings.Contains(got, "--user 61002:61002") {
|
||||
t.Errorf("an image told by the runtime's user option was not: %s", got)
|
||||
}
|
||||
plain := *env
|
||||
plain.RunAs = nil
|
||||
moved := *env
|
||||
moved.RunAs = &declaration.ContainerIdentity{Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID",
|
||||
UID: 61009, GID: 61009}
|
||||
if containerSpec(env, inputs{}) == containerSpec(&plain, inputs{}) ||
|
||||
containerSpec(env, inputs{}) == containerSpec(&moved, inputs{}) {
|
||||
t.Error("an identity told, or renumbered, does not recreate the container")
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0325 §5 supersedes ADR 0312's group grant: what a grant put an account in is taken back on the first apply of
|
||||
// the access, and nothing is granted again.
|
||||
func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
groups := "nox media"
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
switch name {
|
||||
case "id":
|
||||
return groups + "\n", nil
|
||||
case "gpasswd":
|
||||
groups = "nox"
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
|
||||
Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
|
||||
d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write","grant":{"account":"nox"}}`)
|
||||
report, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if groups != "nox" || !strings.Contains(strings.Join(ran, "\n"), "gpasswd --delete nox media") {
|
||||
t.Fatalf("the grant's group was not taken back: %v", ran)
|
||||
}
|
||||
if o := outcomeOf(report, "importer.books"); o.Action != "updated" || !strings.Contains(o.Detail, "taken back") {
|
||||
t.Errorf("the take-back is not said: %+v", o)
|
||||
}
|
||||
if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 {
|
||||
t.Errorf("the record still holds the grant's groups: %+v", rec)
|
||||
}
|
||||
}
|
||||
|
||||
// Review L7: a group the grant put an account in that could not be left stays recorded, and is tried again.
|
||||
func TestAGrantsGroupThatCouldNotBeLeftIsTriedAgain(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
refuse := true
|
||||
groups := "nox media"
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
switch name {
|
||||
case "id":
|
||||
return groups + "\n", nil
|
||||
case "gpasswd":
|
||||
if refuse {
|
||||
return "", errors.New("gpasswd exited 1: locked")
|
||||
}
|
||||
groups = "nox"
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
|
||||
Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
|
||||
d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write"}`)
|
||||
_, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec, _ := known.Find("importer.books"); rec.User != "nox" || len(rec.Groups) != 1 {
|
||||
t.Fatalf("a group that could not be left was forgotten: %+v", rec)
|
||||
}
|
||||
refuse = false
|
||||
if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 || groups != "nox" {
|
||||
t.Fatalf("the group was not taken back on the next apply: %+v (%s)", rec, groups)
|
||||
}
|
||||
}
|
||||
|
||||
// Every system the engine runs on that makes accounts makes module identities: arch and alpine alike.
|
||||
func TestArchAndAlpineMakeModuleIdentities(t *testing.T) {
|
||||
for _, name := range []string{"arch", "alpine"} {
|
||||
sys, err := system.For(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := toolFor(sys); err != nil {
|
||||
t.Errorf("%s makes no module identity: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The read-only judgement of a node's directories (novox/hq issue 358): one line each — placed, would be refused
|
||||
// naming the link and who could replace it, or not there yet (placed: an apply makes it) — and nothing made or
|
||||
// changed.
|
||||
func TestJudgeDirectoriesSaysWhatTheWalkWouldRefuseAndChangesNothing(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
plain := filepath.Join(base, "plain")
|
||||
if err := os.Mkdir(plain, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
elsewhere := filepath.Join(base, "elsewhere", "state")
|
||||
if err := os.MkdirAll(elsewhere, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join(base, "elsewhere"), filepath.Join(base, "linked")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
missing := filepath.Join(base, "missing", "deeper")
|
||||
d := declare(t, `{"id":"a.plain","type":"directory","path":"`+plain+`","mode":"0700"},`+
|
||||
`{"id":"a.linked","type":"directory","path":"`+filepath.Join(base, "linked", "state")+`","mode":"0700"},`+
|
||||
`{"id":"a.missing","type":"directory","path":"`+missing+`","mode":"0700"}`)
|
||||
|
||||
said := JudgeDirectories(d)
|
||||
lines := strings.Split(said, "\n")
|
||||
if len(lines) != 4 || !strings.HasPrefix(lines[0], "3 directories declared: 2 placed, 1 would be refused") {
|
||||
t.Fatalf("the judgement: %q", said)
|
||||
}
|
||||
if lines[1] != plain+": placed" || lines[3] != missing+": placed" {
|
||||
t.Errorf("a directory the walk reaches is not said placed: %q", said)
|
||||
}
|
||||
if !strings.Contains(lines[2], "would be refused") || !strings.Contains(lines[2], filepath.Join(base, "linked")) ||
|
||||
!strings.Contains(lines[2], "every account") {
|
||||
t.Errorf("the linked directory is not said refused naming the link and who: %q", lines[2])
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(base, "missing")); !os.IsNotExist(err) {
|
||||
t.Error("the judgement made a directory")
|
||||
}
|
||||
for _, p := range []string{plain, elsewhere} {
|
||||
if info, _ := os.Stat(p); info.Mode().Perm() != 0o755 {
|
||||
t.Errorf("the judgement changed %s to %o", p, info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A link only root could have made is followed by the judgement as by the apply.
|
||||
func TestJudgeDirectoriesFollowsARootOnlyLink(t *testing.T) {
|
||||
wasDirs, wasLinks := couldReplace, linkReplacer
|
||||
couldReplace = func(string, uint32, uint32, uint32) string { return "" }
|
||||
linkReplacer = func(string, uint32) string { return "" }
|
||||
t.Cleanup(func() { couldReplace, linkReplacer = wasDirs, wasLinks })
|
||||
base := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(base, "real", "state"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("real", filepath.Join(base, "run")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(base, "run", "state")
|
||||
if said := JudgeDirectories(declare(t, `{"id":"a.s","type":"directory","path":"`+path+`"}`)); !strings.Contains(said, path+": placed") {
|
||||
t.Fatalf("a directory behind a root-only link is not said placed: %q", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// An access is an operator's path a module mounts and the mesh does not own (novox/hq ADR 0051): mounted when
|
||||
// present, refused when absent, nothing changed on it. ADR 0325 kept a shared folder through ACL entries; ADR 0328
|
||||
// supersedes it with libraries (library.go), and an access is again exactly what it was before: an access's `users`
|
||||
// and `taken` are accepted and not acted on.
|
||||
|
||||
// applyAccess is one access, applied as ADR 0051 has it; what ADR 0312's grant put an account in is taken back
|
||||
// first, so no account is in a group for data but a library's (ADR 0328 §11).
|
||||
func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
|
||||
in inputs) (Outcome, error) {
|
||||
gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed)
|
||||
out, err := legacyAccess(r)
|
||||
if len(left) > 0 {
|
||||
out.user, out.groups = previous.User, left
|
||||
}
|
||||
if said != "" {
|
||||
out.Detail = joinDetail(out.Detail, said)
|
||||
if gave && out.Action == "unchanged" {
|
||||
// Updated, so what names the access under restart-on — the tool runner — holds the groups it has now.
|
||||
out.Action = "updated"
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes
|
||||
// back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group.
|
||||
// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next
|
||||
// apply tries again — and what was said; nothing for an access that granted nothing.
|
||||
func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted,
|
||||
handed Handed) (bool, []string, string) {
|
||||
if previous.User == "" || len(previous.Groups) == 0 {
|
||||
return false, nil, ""
|
||||
}
|
||||
account := previous.User
|
||||
in, err := system.GroupsOf(ctx, system.Runner(run), account)
|
||||
if err != nil {
|
||||
return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+
|
||||
"back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err)
|
||||
}
|
||||
already := setOf(in)
|
||||
gave := false
|
||||
var left, said []string
|
||||
for _, g := range previous.Groups {
|
||||
switch {
|
||||
case !already[g]:
|
||||
continue
|
||||
case len(wanted.othersAsk(account, g, previous.ID)) > 0:
|
||||
said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g,
|
||||
strings.Join(wanted.othersAsk(account, g, previous.ID), ", ")))
|
||||
handed.hand(account, g)
|
||||
continue
|
||||
}
|
||||
gone, why := leaveGroup(ctx, sys, account, g, run)
|
||||
if !gone {
|
||||
left = append(left, g)
|
||||
why += "; tried again on the next apply"
|
||||
}
|
||||
gave = gave || gone
|
||||
said = append(said, why)
|
||||
}
|
||||
return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// legacyAccess is an access from a controller older than ADR 0325, applied exactly as before it (ADR 0051 §7).
|
||||
//
|
||||
// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by the container
|
||||
// runtime, as root, with whatever mode it picks — which is exactly the silent wrong-ownership 04-ISSUES/026
|
||||
// records. So the host checks first and says plainly that the operator must provide the path, rather than
|
||||
// conjuring a directory it does not own and cannot give the right owner. Nothing is written, so this never reports
|
||||
// a change: the machine did not move, the host merely confirmed a fact about it.
|
||||
func legacyAccess(r *declaration.Access) (Outcome, error) {
|
||||
out := begin(r)
|
||||
info, err := os.Stat(r.Path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return out, fmt.Errorf(
|
||||
"%s is not there, and the mesh does not own it — the operator must provide it. It is "+
|
||||
"shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+
|
||||
"nothing, so a missing one is said here rather than made as root by the container "+
|
||||
"runtime", r.Path)
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return out, fmt.Errorf(
|
||||
"%s is not a directory, and an access is a shared directory the operator provides",
|
||||
r.Path)
|
||||
}
|
||||
mode := declaration.AccessRead
|
||||
if r.Mode != "" {
|
||||
mode = r.Mode
|
||||
}
|
||||
out.Action = "unchanged"
|
||||
out.Detail = "operator-owned; present, " + mode + ", nothing managed"
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the
|
||||
// folder's filesystem is not the one a warrant was given for. Empty when the filesystem cannot be named, and a folder
|
||||
// on it is then never kept (it fails closed, and waits).
|
||||
func PlacedID(mount string) string { return placedFrom(mount, filesystemKey(mount)) }
|
||||
|
||||
// placedFrom is PlacedID from a filesystem key already read.
|
||||
func placedFrom(mount, key string) string {
|
||||
if mount == "" || key == "" {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256([]byte(mount + "\x00" + key))
|
||||
return "fs-" + hex.EncodeToString(sum[:6])
|
||||
}
|
||||
|
||||
// filesystemKey is what names the filesystem at mount across reboots, in this order:
|
||||
//
|
||||
// - a filesystem on a block device: its UUID, found by matching the mount's device against the devices in
|
||||
// /dev/disk/by-uuid (never blkid). xfs's statfs id follows the device number, so the UUID comes first;
|
||||
// - else the kernel's filesystem id (statfs f_fsid) when it is not zero: ZFS's from its dataset, btrfs's from its
|
||||
// UUID;
|
||||
// - else mountinfo's source with the filesystem's type: an NFS export's server:/export, a FUSE filesystem's own
|
||||
// source string (both report f_fsid 0).
|
||||
//
|
||||
// Never the mount point and type alone, and never a device's name, which a reboot may renumber. Empty when none of
|
||||
// these answers within statBound: then nothing is kept there. Read once per mount at a time.
|
||||
func filesystemKey(mount string) string {
|
||||
if mount == "" {
|
||||
return ""
|
||||
}
|
||||
v, ok := boundedOnce("fs "+mount, func() any { return readFilesystemKey(mount) })
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return v.(string)
|
||||
}
|
||||
|
||||
func readFilesystemKey(mount string) string {
|
||||
if u := MountUUID(mount); u != "" {
|
||||
return "uuid:" + u
|
||||
}
|
||||
if id := MountFsid(mount); strings.Trim(id, "0") != "" {
|
||||
return "fsid:" + id
|
||||
}
|
||||
if source, fstype := mountSourceOf(mount); source != "" && source != "none" {
|
||||
return "source:" + fstype + ":" + source
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
|
||||
// server is down blocks a stat for ever, and one access must never hold the apply.
|
||||
var statBound = 5 * time.Second
|
||||
|
||||
// errNoAnswer is a folder whose read did not return within statBound.
|
||||
var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")
|
||||
|
||||
// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
|
||||
|
||||
func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) }
|
||||
func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) }
|
||||
|
||||
// flight is one read in flight, shared by everyone who asks while it is: a read that hangs on a dead mount holds
|
||||
// one goroutine per path, never one per apply or look.
|
||||
type flight struct {
|
||||
done chan struct{}
|
||||
v any
|
||||
}
|
||||
|
||||
var inFlight = struct {
|
||||
sync.Mutex
|
||||
reads map[string]*flight
|
||||
}{reads: map[string]*flight{}}
|
||||
|
||||
// boundedOnce is read's answer within statBound, sharing a read of key already in flight; false when it did not
|
||||
// answer in time.
|
||||
func boundedOnce(key string, read func() any) (any, bool) {
|
||||
inFlight.Lock()
|
||||
f, running := inFlight.reads[key]
|
||||
if !running {
|
||||
f = &flight{done: make(chan struct{})}
|
||||
inFlight.reads[key] = f
|
||||
go func() {
|
||||
f.v = read()
|
||||
inFlight.Lock()
|
||||
delete(inFlight.reads, key)
|
||||
inFlight.Unlock()
|
||||
close(f.done)
|
||||
}()
|
||||
}
|
||||
inFlight.Unlock()
|
||||
select {
|
||||
case <-f.done:
|
||||
return f.v, true
|
||||
case <-time.After(statBound):
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
|
||||
type read struct {
|
||||
info os.FileInfo
|
||||
err error
|
||||
}
|
||||
v, ok := boundedOnce(op+" "+path, func() any {
|
||||
info, err := stat(path)
|
||||
return read{info, err}
|
||||
})
|
||||
if !ok {
|
||||
return nil, errNoAnswer
|
||||
}
|
||||
r := v.(read)
|
||||
return r.info, r.err
|
||||
}
|
||||
|
||||
// absPath is an absolute path in words: what pathless takes out.
|
||||
var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`)
|
||||
|
||||
// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other
|
||||
// path said as "a path".
|
||||
func Pathless(err error) string { return pathless(err) }
|
||||
|
||||
func pathless(err error) string {
|
||||
var pe *os.PathError
|
||||
if errors.As(err, &pe) {
|
||||
return pe.Op + ": " + pe.Err.Error()
|
||||
}
|
||||
return absPath.ReplaceAllString(err.Error(), "${1}a path")
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// A library is kept usable by its group (novox/hq ADR 0328, issue 567).
|
||||
//
|
||||
// A library is a named tree of shared data with one home (§1). On its home the engine keeps its root and every
|
||||
// sub-path its modules ask for — those folders themselves, never what is in them — in the library's group with
|
||||
// group read, write and enter and the setgid bit, so what is made beneath keeps the group (§6); a missing sub-path
|
||||
// is made with the root's owner and the library's group (§9). The owner, the owner's bits and others' bits are
|
||||
// never changed. What the root already holds joins only on the operator's word (§7): until then the library waits
|
||||
// with reason keep and nothing under it is walked. A root that is absent, or whose disk is not there, makes
|
||||
// nothing, holds every container that mounts the library, and waits with reason disk (§9). Off its home a library is
|
||||
// a mount the mounts module makes: nothing is kept there, and a mount that is not there holds the containers too.
|
||||
//
|
||||
// **When a root is the mesh's to keep without asking.** When it holds nothing yet, or is already in the library's
|
||||
// group with group write and setgid (kept so before, by hand or by the mesh). Every other root waits for the take
|
||||
// act, which walks it whole (TakeLibrary). Kept, it stays kept on the filesystem it was kept on — named by UUID,
|
||||
// filesystem id or source (PlacedID) — and another filesystem at its root is judged afresh.
|
||||
|
||||
// libraryRecords is where the engine keeps what it knows of each library it is home to: the file beside the state
|
||||
// in the engine, memory in a test.
|
||||
type libraryRecords interface {
|
||||
All() (map[string]store.LibraryRecord, error)
|
||||
Update(name string, change func(r *store.LibraryRecord, had bool) bool) error
|
||||
}
|
||||
|
||||
// Libraries is the engine's record of the libraries it is home to. The engine sets the file beside its state.
|
||||
var Libraries libraryRecords = NewLibraryMemory()
|
||||
|
||||
type libraryMemory struct {
|
||||
mu sync.Mutex
|
||||
m map[string]store.LibraryRecord
|
||||
}
|
||||
|
||||
// NewLibraryMemory is an empty record held in memory: a test's, or a one-shot apply's.
|
||||
func NewLibraryMemory() libraryRecords { return &libraryMemory{m: map[string]store.LibraryRecord{}} }
|
||||
|
||||
func (k *libraryMemory) All() (map[string]store.LibraryRecord, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
out := map[string]store.LibraryRecord{}
|
||||
for n, r := range k.m {
|
||||
out[n] = r
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *libraryMemory) Update(name string, change func(r *store.LibraryRecord, had bool) bool) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
r, had := k.m[name]
|
||||
r.Library = name
|
||||
if change(&r, had) {
|
||||
k.m[name] = r
|
||||
} else {
|
||||
delete(k.m, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// libraryRecord is the record of one library, and whether there is one. A record that cannot be read is none: the
|
||||
// library is judged afresh, which waits rather than keeps.
|
||||
func libraryRecord(name string) (store.LibraryRecord, bool) {
|
||||
all, err := Libraries.All()
|
||||
if err != nil {
|
||||
return store.LibraryRecord{}, false
|
||||
}
|
||||
r, had := all[name]
|
||||
return r, had
|
||||
}
|
||||
|
||||
// libraryLocks is one lock per library, held by every change to its root and every walk under it.
|
||||
var libraryLocks sync.Map
|
||||
|
||||
func libraryLock(name string) *sync.Mutex {
|
||||
l, _ := libraryLocks.LoadOrStore(name, &sync.Mutex{})
|
||||
return l.(*sync.Mutex)
|
||||
}
|
||||
|
||||
// libraryWalks is every library a take walk is running under now, with the filesystem it walks: set once the walk
|
||||
// has been judged and started, cleared when it ends.
|
||||
var libraryWalks sync.Map
|
||||
|
||||
// LibraryAccess is the access a library's wait and its take act name.
|
||||
func LibraryAccess(name string) string { return declaration.LibraryAccessPrefix + name }
|
||||
|
||||
// KeepWords and DiskWords are what a library's wait says the operator gives (ADR 0328 §8, §9): never a path.
|
||||
func KeepWords(name string) string { return "let the mesh keep the library " + name }
|
||||
func DiskWords(name string) string { return "mount the disk that holds the library " + name }
|
||||
|
||||
// The judgements of a library.
|
||||
const (
|
||||
// libraryKeep is a library the mesh keeps: taken, empty when first seen, or found in shape.
|
||||
libraryKeep = "keep"
|
||||
// libraryWaitKeep is a root that holds data never taken: it waits for the operator's word.
|
||||
libraryWaitKeep = "wait-keep"
|
||||
// libraryWaitDisk is a root that is absent, or whose disk (or, off its home, whose mount) is not there.
|
||||
libraryWaitDisk = "wait-disk"
|
||||
// libraryUnknown is a root whose filesystem could not be named now: left as it is until it is.
|
||||
libraryUnknown = "unknown"
|
||||
// libraryMounted is a library off its home whose mount is there: nothing to keep.
|
||||
libraryMounted = "mounted"
|
||||
)
|
||||
|
||||
// libraryVerdict is the judgement of one library, with what it read.
|
||||
type libraryVerdict struct {
|
||||
kind string
|
||||
// how is why a kept library is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptInShape.
|
||||
how string
|
||||
// mount is the mount point that holds the root; placed and fs name its filesystem.
|
||||
mount, placed, fs string
|
||||
// moved is that another filesystem than the one the library was kept on is at its root.
|
||||
moved bool
|
||||
why string
|
||||
err error
|
||||
}
|
||||
|
||||
// assessLibrary judges a library as an apply, a look and the take act all do (ADR 0328 §7, §9, §15): its group
|
||||
// first, then its disk, then whether the mesh may keep it. Its words carry no path: they reach the operator's
|
||||
// channel. Reads only.
|
||||
func assessLibrary(ctx context.Context, l declaration.Library, rec store.LibraryRecord) libraryVerdict {
|
||||
if why := rootGroup(l.GID); why != "" {
|
||||
return libraryVerdict{err: fmt.Errorf("its group is %s, and a library's group is never a root group "+
|
||||
"(novox/hq ADR 0328 §15); nothing was made or changed", why)}
|
||||
}
|
||||
// Nowhere the machine's own trees are, as for an access (novox/hq issue 339): a library at /etc would be
|
||||
// the machine's configuration given a group.
|
||||
if err := refusePath(l.Root, placingAccess, "", ""); err != nil {
|
||||
return libraryVerdict{err: errors.New(pathless(err) + "; nothing was made or changed")}
|
||||
}
|
||||
if !l.Home {
|
||||
// Off its home the mounts module mounts it: there when its root is a mount point now.
|
||||
for _, p := range MountPoints() {
|
||||
if p == l.Root {
|
||||
return libraryVerdict{kind: libraryMounted, mount: p}
|
||||
}
|
||||
}
|
||||
return libraryVerdict{kind: libraryWaitDisk, why: "its mount is not there"}
|
||||
}
|
||||
info, err := lstatWithin(l.Root)
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
return libraryVerdict{kind: libraryWaitDisk, why: "its root is not there"}
|
||||
case errors.Is(err, errNoAnswer):
|
||||
return libraryVerdict{kind: libraryUnknown, why: "its root does not answer: a read of it did not return " +
|
||||
"within its bound"}
|
||||
case err != nil:
|
||||
return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
|
||||
case !info.IsDir():
|
||||
return libraryVerdict{err: errors.New("its root is not a folder (a file, or a link), and a library's root " +
|
||||
"is a folder the engine reaches through no link")}
|
||||
}
|
||||
pl := placedOn(ctx, l.Root, l.Root, rec.Mount, recordedLibraryMounts(l.Library))
|
||||
if pl.Away {
|
||||
return libraryVerdict{kind: libraryWaitDisk, mount: pl.Mount, why: "the disk that holds its root is not there"}
|
||||
}
|
||||
fs := filesystemKey(pl.Mount)
|
||||
here := placedFrom(pl.Mount, fs)
|
||||
// **A filesystem that cannot be named now is unknown, never a move**: the library holds what it is — nothing
|
||||
// walked, nothing newly kept, nothing reset — and is judged again when it is named. One never kept is so never
|
||||
// kept: it fails closed.
|
||||
if here == "" {
|
||||
return libraryVerdict{kind: libraryUnknown, mount: pl.Mount,
|
||||
why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"}
|
||||
}
|
||||
moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here)
|
||||
v := libraryVerdict{mount: pl.Mount, placed: here, fs: fs, moved: moved}
|
||||
// The controller's word that the library was taken holds for the filesystem the warrant was carried out on.
|
||||
takenHere := l.Taken && (rec.KeptOn == here || (rec.KeptOn == "" && !moved))
|
||||
switch {
|
||||
case rec.Kept && rec.KeptOn == here && !moved && rec.How == store.KeptByWarrant, takenHere:
|
||||
// Taken whole on the operator's word: every sub-path is the library's.
|
||||
v.kind, v.how = libraryKeep, store.KeptByWarrant
|
||||
return v
|
||||
case rec.Kept && rec.KeptOn == here && !moved:
|
||||
// Kept without a warrant (empty, or in shape): a sub-path asked for since that holds data not in the group
|
||||
// waits for the operator's word as the root would (§7).
|
||||
v.kind, v.how = libraryKeep, rec.How
|
||||
if !pathsInShape(l, rec.Paths) {
|
||||
v.kind, v.how = libraryWaitKeep, ""
|
||||
}
|
||||
return v
|
||||
}
|
||||
facts, err := readFolder(l.Root)
|
||||
if err != nil {
|
||||
return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
|
||||
}
|
||||
switch {
|
||||
case facts.Empty:
|
||||
// Nothing there, so no sub-path either: every one is made.
|
||||
v.kind, v.how = libraryKeep, store.KeptEmpty
|
||||
case int(facts.GID) == l.GID && facts.Mode&0o2070 == 0o2070 && sampleInShape(l.Root, l.GID) &&
|
||||
pathsInShape(l, nil):
|
||||
// The root and a bounded sample beneath it already in the group with group write and setgid: data that
|
||||
// is the library's already, kept without asking. One entry of the sample out of shape asks (§7).
|
||||
v.kind, v.how = libraryKeep, store.KeptInShape
|
||||
default:
|
||||
// What is there holds data the mesh never took — a disk mounted over a root kept while it was empty below
|
||||
// it included (a move): it waits for the operator's word, and nothing under it is walked.
|
||||
v.kind = libraryWaitKeep
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// recordedLibraryMounts is every mount point recorded for another library: where a disk was once seen.
|
||||
func recordedLibraryMounts(except string) []string {
|
||||
all, err := Libraries.All()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for name, r := range all {
|
||||
if name != except && r.Mount != "" && r.Mount != "/" {
|
||||
out = append(out, r.Mount)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// applyLibrary is one library, applied (ADR 0328 §6, §7, §9): kept on its home when the mesh may keep it, waiting
|
||||
// otherwise, and failing for a root group. Nothing is changed on the root but the group, group rwx and setgid of
|
||||
// the root and its requested sub-paths, and the sub-paths made.
|
||||
func applyLibrary(ctx context.Context, l declaration.Library) (Outcome, libraryVerdict, error) {
|
||||
out := Outcome{ID: LibraryAccess(l.Library), Type: "library", Target: l.Root}
|
||||
lock := libraryLock(l.Library)
|
||||
if !lock.TryLock() {
|
||||
// **A take walk is running (up to its bound, half an hour): the apply does not wait for it.** The library is
|
||||
// the walk's until it is through; it keeps it, and its containers are not held for it. Only a walk that
|
||||
// has started, on the filesystem it judged: anything else holding the lock (a take act about to be
|
||||
// refused, say) is short, and the apply waits for it and judges the library as always.
|
||||
if on, walking := libraryWalks.Load(l.Library); walking {
|
||||
out.Action = "unchanged"
|
||||
out.Detail = "the take act is setting its group under its root now; this apply leaves the library to it"
|
||||
return out, libraryVerdict{kind: libraryKeep, how: store.KeptByWarrant, placed: on.(string)}, nil
|
||||
}
|
||||
lock.Lock()
|
||||
}
|
||||
defer lock.Unlock()
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
v := assessLibrary(ctx, l, rec)
|
||||
if v.err != nil {
|
||||
return out, v, v.err
|
||||
}
|
||||
switch v.kind {
|
||||
case libraryMounted:
|
||||
out.Action, out.Detail = "unchanged", "not its home: mounted, and nothing is kept here"
|
||||
return out, v, nil
|
||||
case libraryWaitDisk:
|
||||
out.Action = "waiting"
|
||||
out.Detail = v.why + ": nothing is made, and no container that mounts it is created until it is there " +
|
||||
"(novox/hq ADR 0328 §9); waits for the operator to " + DiskWords(l.Library)
|
||||
return out, v, nil
|
||||
case libraryUnknown:
|
||||
out.Action = "waiting"
|
||||
out.Detail = v.why + ": the library is left exactly as it is, and judged again on the next apply"
|
||||
return out, v, nil
|
||||
case libraryWaitKeep:
|
||||
_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, had bool) bool {
|
||||
if !had || v.moved {
|
||||
r.Root, r.Mount, r.Placed, r.Fs = l.Root, v.mount, v.placed, v.fs
|
||||
}
|
||||
if v.moved {
|
||||
// Kept on another filesystem: what was kept there no longer holds here.
|
||||
r.Kept, r.How, r.Cursor = false, "", nil
|
||||
}
|
||||
return true
|
||||
})
|
||||
out.Action = "waiting"
|
||||
out.Detail = "its root holds data the mesh has not taken: nothing is changed under it until the operator's " +
|
||||
"warrant (novox/hq ADR 0328 §7); waits for the operator to " + KeepWords(l.Library)
|
||||
return out, v, nil
|
||||
}
|
||||
changed, made, err := keepFolders(l.Root, l.Paths, l.GID)
|
||||
if err != nil {
|
||||
return out, v, err
|
||||
}
|
||||
_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, _ bool) bool {
|
||||
if !r.Kept || r.KeptOn != v.placed {
|
||||
r.How, r.At = v.how, time.Now().UTC()
|
||||
}
|
||||
r.Root, r.Mount, r.Placed, r.Fs, r.GID = l.Root, v.mount, v.placed, v.fs, l.GID
|
||||
r.Kept, r.KeptOn = true, v.placed
|
||||
r.Paths = unionPaths(r.Paths, l.Paths)
|
||||
return true
|
||||
})
|
||||
out.Action = "unchanged"
|
||||
if changed+made > 0 {
|
||||
out.Action = "updated"
|
||||
}
|
||||
out.Detail = fmt.Sprintf("kept in %s: its root and %d sub-path(s) with group rwx and setgid, %d made; owners "+
|
||||
"and everyone else's rights unchanged (novox/hq ADR 0328 §6)", l.Group, len(l.Paths), made)
|
||||
return out, v, nil
|
||||
}
|
||||
|
||||
// heldRoots is every library root whose containers are held: its disk, or its mount, is not there (§9).
|
||||
func heldRoots(verdicts map[string]libraryVerdict, libs []declaration.Library) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, l := range libs {
|
||||
if v, ok := verdicts[l.Library]; ok && v.kind == libraryWaitDisk {
|
||||
out[l.Root] = l.Library
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mountsHeld is the library a container waits for: one whose root it mounts, or mounts below, is held.
|
||||
func mountsHeld(c *declaration.Container, held map[string]string) string {
|
||||
for _, v := range c.Volumes {
|
||||
src := mountSource(v)
|
||||
if !strings.HasPrefix(src, "/") {
|
||||
continue
|
||||
}
|
||||
for root, name := range held {
|
||||
if atOrBelow(filepath.Clean(src), root) {
|
||||
return name
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// WaitingOnLibrary is every container of the declaration that waits for a library whose disk or mount is not
|
||||
// there, read now: not created by the apply, so not judged alive or dead — the library says why.
|
||||
func WaitingOnLibrary(d *declaration.Declaration) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if d == nil || len(d.Libraries) == 0 {
|
||||
return out
|
||||
}
|
||||
verdicts := map[string]libraryVerdict{}
|
||||
for _, l := range d.Libraries {
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
verdicts[l.Library] = assessLibrary(context.Background(), l, rec)
|
||||
}
|
||||
held := heldRoots(verdicts, d.Libraries)
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok && mountsHeld(c, held) != "" {
|
||||
out[c.ID] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The judgements of a library as a look says them.
|
||||
const (
|
||||
VerdictKeep = "keep"
|
||||
VerdictMounted = "mounted"
|
||||
VerdictWaitKeep = "wait-keep"
|
||||
VerdictWaitDisk = "wait-disk"
|
||||
VerdictUnknown = "unknown"
|
||||
VerdictBroken = "broken"
|
||||
)
|
||||
|
||||
// LibraryJudgement is a look's judgement of a library, and why, in words with no path; Placed names its root's
|
||||
// filesystem for a keep wait, with no path (PlacedID).
|
||||
type LibraryJudgement struct {
|
||||
Kind, Why, Placed string
|
||||
// Act is the last walk under its root.
|
||||
Act *store.KeptAct
|
||||
}
|
||||
|
||||
// JudgeLibrary judges a library for a look, read only.
|
||||
func JudgeLibrary(ctx context.Context, l declaration.Library) LibraryJudgement {
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
v := assessLibrary(ctx, l, rec)
|
||||
j := LibraryJudgement{Act: rec.Act}
|
||||
switch {
|
||||
case v.err != nil:
|
||||
j.Kind, j.Why = VerdictBroken, pathless(v.err)
|
||||
case v.kind == libraryWaitKeep:
|
||||
j.Kind, j.Placed = VerdictWaitKeep, v.placed
|
||||
case v.kind == libraryWaitDisk:
|
||||
j.Kind, j.Why = VerdictWaitDisk, v.why
|
||||
case v.kind == libraryUnknown:
|
||||
j.Kind, j.Why = VerdictUnknown, v.why
|
||||
case v.kind == libraryMounted:
|
||||
j.Kind = VerdictMounted
|
||||
default:
|
||||
j.Kind = VerdictKeep
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
// joinRunner puts a library's tool runner's account in its group (ADR 0328 §11).
|
||||
func joinRunner(ctx context.Context, sys system.System, l declaration.Library, known *store.State,
|
||||
run Runner) (bool, string, error) {
|
||||
if l.Runner == "" {
|
||||
return false, "", nil
|
||||
}
|
||||
return joinGroup(ctx, sys, l.Runner, l.Group, store.Membership{Library: l.Library}, known, run)
|
||||
}
|
||||
|
||||
// IdentityChange is what a tool runner joining a module's own group changes (ADR 0334 §2): `identity:<module>`,
|
||||
// which the tool runner's unit names under restart-on, as it names `library:<name>`.
|
||||
func IdentityChange(module string) string { return "identity:" + module }
|
||||
|
||||
// joinGroup puts a tool runner's account in a group (ADR 0328 §11, ADR 0334 §2), as ADR 0252 puts an account in a
|
||||
// group: recorded only when the mesh put it there, so one it was in before is never taken back. The account must be
|
||||
// here and never root's: uid 0, or a root group as its primary (§15). Whether it moved, and what was said.
|
||||
func joinGroup(ctx context.Context, sys system.System, account, group string, of store.Membership, known *store.State,
|
||||
run Runner) (bool, string, error) {
|
||||
who, found, err := lookUp(ctx, run, "passwd", account)
|
||||
switch {
|
||||
case err != nil:
|
||||
return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
|
||||
case !found:
|
||||
return false, "", fmt.Errorf("the tool runner's account %s is not on this machine, so it joins no group", account)
|
||||
}
|
||||
if err := rootIdentity(who.id, who.prim); err != nil {
|
||||
return false, "", fmt.Errorf("the tool runner's account %s joins no group: %w", account, err)
|
||||
}
|
||||
in, err := system.GroupsOf(ctx, system.Runner(run), account)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
|
||||
}
|
||||
if setOf(in)[group] {
|
||||
return false, "", nil
|
||||
}
|
||||
if err := sys.AddUserToGroup(ctx, system.Runner(run), account, group); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
for _, m := range known.Members {
|
||||
if m.Account == account && m.Group == group {
|
||||
return true, fmt.Sprintf("put the tool runner's %s back in %s", account, group), nil
|
||||
}
|
||||
}
|
||||
of.Account, of.Group = account, group
|
||||
known.Members = append(known.Members, of)
|
||||
return true, fmt.Sprintf("put the tool runner's %s in %s; it holds it once restarted", account, group), nil
|
||||
}
|
||||
|
||||
// leaveRunners takes every tool runner's account the mesh put in a library's group, or a module's own group, out of
|
||||
// it once no declared library or identity puts it there and no declared user resource asks for it (ADR 0252 §3). Never fatal: what could not be
|
||||
// left stays recorded, said, for the next apply.
|
||||
func leaveRunners(ctx context.Context, sys system.System, libs []declaration.Library, ids []declaration.Identity,
|
||||
known *store.State, run Runner, wanted Wanted) []Outcome {
|
||||
declared := map[string]bool{}
|
||||
for _, l := range libs {
|
||||
if l.Runner != "" {
|
||||
declared[l.Runner+" "+l.Group] = true
|
||||
}
|
||||
}
|
||||
for _, id := range ids {
|
||||
if id.Runner != "" {
|
||||
declared[id.Runner+" "+id.Name] = true
|
||||
}
|
||||
}
|
||||
var out []Outcome
|
||||
var keep []store.Membership
|
||||
for _, m := range known.Members {
|
||||
if declared[m.Account+" "+m.Group] {
|
||||
keep = append(keep, m)
|
||||
continue
|
||||
}
|
||||
o := Outcome{ID: LibraryAccess(m.Library) + " runner", Type: "library", Target: m.Account,
|
||||
changes: LibraryAccess(m.Library)}
|
||||
if m.Module != "" {
|
||||
o.ID, o.Type = "identity "+declaration.IdentityPrefix+m.Module+" runner", "identity"
|
||||
o.changes = IdentityChange(m.Module)
|
||||
}
|
||||
if others := wanted.othersAsk(m.Account, m.Group, ""); len(others) > 0 {
|
||||
o.Action, o.Detail = "forgotten", fmt.Sprintf("left %s in %s, which %s asks for", m.Account, m.Group,
|
||||
strings.Join(others, ", "))
|
||||
out = append(out, o)
|
||||
continue
|
||||
}
|
||||
gone, why := leaveGroup(ctx, sys, m.Account, m.Group, run)
|
||||
o.Action, o.Detail = "restored", m.Account+" "+why
|
||||
if !gone {
|
||||
o.Action = "waiting"
|
||||
keep = append(keep, m)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
known.Members = keep
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out
|
||||
}
|
||||
|
||||
// pathsInShape is whether every requested sub-path not kept before is absent (it is made), empty, or already in the
|
||||
// group with group write and setgid, it and a bounded sample beneath it: what may be kept without the operator's
|
||||
// word (§7). One that cannot be read is not.
|
||||
func pathsInShape(l declaration.Library, kept []string) bool {
|
||||
done := map[string]bool{}
|
||||
for _, p := range kept {
|
||||
done[p] = true
|
||||
}
|
||||
for _, p := range l.Paths {
|
||||
if done[p] {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(l.Root, p)
|
||||
if _, err := lstatWithin(path); errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
facts, err := readFolder(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if facts.Empty {
|
||||
continue
|
||||
}
|
||||
if int(facts.GID) != l.GID || facts.Mode&0o2070 != 0o2070 || !sampleInShape(path, l.GID) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// unionPaths is a with every path of b not in it.
|
||||
func unionPaths(a, b []string) []string {
|
||||
have := map[string]bool{}
|
||||
for _, p := range a {
|
||||
have[p] = true
|
||||
}
|
||||
for _, p := range b {
|
||||
if !have[p] {
|
||||
a = append(a, p)
|
||||
have[p] = true
|
||||
}
|
||||
}
|
||||
sort.Strings(a)
|
||||
return a
|
||||
}
|
||||
|
||||
// RunnerWarning is a tool runner's account that joins no group (missing, or root's): said as a resource of its own,
|
||||
// `runner:<module>` or `runner:library:<name>`, unhealthy with words with no path — never a failure of the apply.
|
||||
type RunnerWarning struct {
|
||||
Resource, Module, Why string
|
||||
}
|
||||
|
||||
var runnerWarnings struct {
|
||||
sync.Mutex
|
||||
list []RunnerWarning
|
||||
}
|
||||
|
||||
// setRunnerWarnings is the warnings the last apply found, in place of the ones before.
|
||||
func setRunnerWarnings(ws []RunnerWarning) {
|
||||
runnerWarnings.Lock()
|
||||
defer runnerWarnings.Unlock()
|
||||
runnerWarnings.list = append([]RunnerWarning(nil), ws...)
|
||||
}
|
||||
|
||||
// RunnerWarnings is every tool runner's account the last apply could not put in a group.
|
||||
func RunnerWarnings() []RunnerWarning {
|
||||
runnerWarnings.Lock()
|
||||
defer runnerWarnings.Unlock()
|
||||
return append([]RunnerWarning(nil), runnerWarnings.list...)
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The take act on a library, and the walk that goes on with one stopped at its bound (novox/hq ADR 0328 §7).
|
||||
|
||||
// actRecord is a walk's counts as the record keeps them.
|
||||
func actRecord(warrant, by, on string, started time.Time, w *treeWalk, err error) *store.KeptAct {
|
||||
c := w.counts
|
||||
rec := &store.KeptAct{Act: "take", Warrant: warrant, By: by, On: on, State: "done", Started: started,
|
||||
Ended: time.Now().UTC(), Folders: c.Folders, Files: c.Files, Kept: c.Kept, Links: c.Links, Other: c.Other,
|
||||
Mounts: c.Mounts, Shared: c.Shared, Linked: c.Linked, Stripped: c.Stripped}
|
||||
if c.Shared+c.Linked+c.Stripped > 0 {
|
||||
rec.Why = fmt.Sprintf("%d file(s) with another name given the group, %d root's left as they are, %d whose "+
|
||||
"setuid or setgid bit the group change took", c.Linked, c.Shared, c.Stripped)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, ErrWalkBound):
|
||||
rec.State, rec.Why = "stopped", "it met its bound of entries or time; it goes on from where it stopped"
|
||||
case err != nil:
|
||||
rec.State, rec.Why = "failed", err.Error()
|
||||
case w.Failed != "":
|
||||
rec.State = "failed"
|
||||
rec.Why = fmt.Sprintf("%d entries could not be given the group: %s; walked again on the next look", c.Failed,
|
||||
w.Failed)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
// TakeLibrary is the take act (ADR 0328 §7): on the operator's warrant, the library's group, group read and write,
|
||||
// and on folders group enter and setgid, set on its root and everything under it — no link followed, no other
|
||||
// filesystem entered, the owner and the owner's and others' bits unchanged, a file with another name elsewhere left
|
||||
// as it is. It judges the ask against the declaration last applied and the root as it is, and starts the walk: what
|
||||
// it answers at once, and a channel closed when the walk is through or stopped at its bound. Refused, nothing is
|
||||
// changed.
|
||||
func TakeLibrary(ctx context.Context, d *declaration.Declaration, access, path, warrant, by string) (<-chan struct{},
|
||||
string, error) {
|
||||
if d == nil {
|
||||
return nil, "", errors.New("this engine has applied no declaration, and takes no library it was not told of")
|
||||
}
|
||||
var lib *declaration.Library
|
||||
for i := range d.Libraries {
|
||||
if LibraryAccess(d.Libraries[i].Library) == access {
|
||||
lib = &d.Libraries[i]
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case lib == nil:
|
||||
return nil, "", fmt.Errorf("the access %s names no library declared on this machine", access)
|
||||
case !lib.Home:
|
||||
return nil, "", fmt.Errorf("this machine is not the home of the library %s, and a library is taken on its "+
|
||||
"home", lib.Library)
|
||||
case lib.Root != path:
|
||||
return nil, "", fmt.Errorf("the library %s is declared at another root than the one asked", lib.Library)
|
||||
}
|
||||
lock := libraryLock(lib.Library)
|
||||
if !lock.TryLock() {
|
||||
return nil, "", errors.New("a walk of the library is running; asked again once it is through, the act goes on")
|
||||
}
|
||||
rec, _ := libraryRecord(lib.Library)
|
||||
v := assessLibrary(ctx, *lib, rec)
|
||||
var refused error
|
||||
switch {
|
||||
case v.err != nil:
|
||||
refused = v.err
|
||||
case v.kind == libraryWaitDisk:
|
||||
refused = errors.New(v.why)
|
||||
case v.kind == libraryUnknown:
|
||||
refused = errors.New(v.why + ", so a warrant could name no filesystem it was given for")
|
||||
}
|
||||
if refused != nil {
|
||||
lock.Unlock()
|
||||
return nil, "", refused
|
||||
}
|
||||
if _, _, err := keepFolders(lib.Root, lib.Paths, lib.GID); err != nil {
|
||||
lock.Unlock()
|
||||
return nil, "", err
|
||||
}
|
||||
started := time.Now().UTC()
|
||||
_ = Libraries.Update(lib.Library, func(r *store.LibraryRecord, _ bool) bool {
|
||||
r.Root, r.Mount, r.Placed, r.Fs, r.GID = lib.Root, v.mount, v.placed, v.fs, lib.GID
|
||||
r.Kept, r.KeptOn, r.How, r.At = true, v.placed, store.KeptByWarrant, started
|
||||
r.Warrant, r.By, r.Cursor = warrant, by, nil
|
||||
r.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, On: v.placed, State: "running", Started: started}
|
||||
return true
|
||||
})
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer lock.Unlock()
|
||||
walkLibrary(lib.Library, lib.Root, lib.GID, warrant, by, v.placed, nil)
|
||||
}()
|
||||
return done, fmt.Sprintf("the take act started under the warrant %s: %s's group, group read and write, and "+
|
||||
"setgid on its folders are set on everything under its root, following no link and entering no other "+
|
||||
"filesystem", warrant, lib.Library), nil
|
||||
}
|
||||
|
||||
// walkLibrary walks a library's root from resume with its group, and records what it did: a walk stopped at its
|
||||
// bound keeps its cursor for the next. The caller holds the library's lock.
|
||||
func walkLibrary(name, root string, gid int, warrant, by, on string, resume []string) {
|
||||
libraryWalks.Store(name, on)
|
||||
defer libraryWalks.Delete(name)
|
||||
started := time.Now().UTC()
|
||||
w, err := walkRoot(root, ActLimits, machineStats, resume, groupVisit(gid))
|
||||
rec := actRecord(warrant, by, on, started, w, err)
|
||||
_ = Libraries.Update(name, func(r *store.LibraryRecord, _ bool) bool {
|
||||
r.Act = rec
|
||||
r.Cursor = nil
|
||||
if errors.Is(err, ErrWalkBound) {
|
||||
r.Cursor = w.Cursor
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// ReconcileLibraries goes on with every take act that stopped at its bound or left entries it could not change,
|
||||
// from where it stopped — only while the library is still kept on the filesystem it was taken on (judged as an apply
|
||||
// judges it). What it did, in words, for the engine to say.
|
||||
func ReconcileLibraries(ctx context.Context, d *declaration.Declaration) []string {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
var said []string
|
||||
for _, l := range d.Libraries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if !l.Home {
|
||||
continue
|
||||
}
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
if rec.Act == nil || (rec.Act.State != "stopped" && rec.Act.State != "failed") || rec.How != store.KeptByWarrant {
|
||||
continue
|
||||
}
|
||||
lock := libraryLock(l.Library)
|
||||
if !lock.TryLock() {
|
||||
continue
|
||||
}
|
||||
v := assessLibrary(ctx, l, rec)
|
||||
if v.err != nil || v.kind != libraryKeep || v.placed != rec.KeptOn || v.moved {
|
||||
lock.Unlock()
|
||||
continue
|
||||
}
|
||||
walkLibrary(l.Library, l.Root, l.GID, rec.Warrant, rec.By, rec.KeptOn, rec.Cursor)
|
||||
lock.Unlock()
|
||||
after, _ := libraryRecord(l.Library)
|
||||
said = append(said, fmt.Sprintf("the library %s: the take act went on, %s (%d folders and %d files changed)",
|
||||
l.Library, after.Act.State, after.Act.Folders, after.Act.Files))
|
||||
}
|
||||
return said
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,125 @@
|
||||
package apply
|
||||
|
||||
// A mesh directory outside a home is given its mode and owner only through a descriptor (novox/hq issue 358).
|
||||
//
|
||||
// Below a home every component is opened without following a link (ADR 0266, home_links.go), and a directory the
|
||||
// operator handed to the mesh is changed through one descriptor opened from / with no link followed (issue 356).
|
||||
// Every other directory the mesh owns — a module's state under /var/lib, a drop-in directory under /etc — was
|
||||
// given its mode and owner by path, with chmod and chown, which follow a link anywhere in the path: whoever could
|
||||
// turn a parent into a link between two applies directed root's chmod and chown at the link's target.
|
||||
//
|
||||
// The danger is a link another account can make or replace, not a link as such: /var/run is a link to /run on
|
||||
// every node, and a disk is sometimes reached through one. So (the decision recorded on issue 358, 2026-10-11):
|
||||
//
|
||||
// - **the directory is opened component by component from /**, each with O_DIRECTORY|O_NOFOLLOW, and its mode
|
||||
// and owner are changed with fchmod and fchown on that descriptor; a directory the walk makes is made in its
|
||||
// parent's descriptor. A link put in place after any check is refused by the kernel at the moment of use;
|
||||
// - **a link in a parent is followed only when root alone could have put it there**: the link is root's, and
|
||||
// every directory from / down to the one holding it is root's and writable by no other account (no write for
|
||||
// others, no write for a group other than root's). Its target is then walked the same way, from /;
|
||||
// - **any other link refuses that directory**, naming the link and the account that could replace it; nothing
|
||||
// is changed there, and the rest of the declaration applies;
|
||||
// - **the directory itself is never a link** (issue 339, applyDirectory).
|
||||
//
|
||||
// A directory below a home keeps the walk of ADR 0266, which follows no link below the home at all.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
osuser "os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// LinkInPathError is a mesh directory refused because a link in its path is one another account could replace.
|
||||
type LinkInPathError struct {
|
||||
// Path is the directory declared, Link the link in its path, Who the account that could replace it.
|
||||
Path, Link, Who string
|
||||
}
|
||||
|
||||
func (e *LinkInPathError) Error() string { return e.Path + " " + e.why() }
|
||||
|
||||
// why is the refusal without the path it refuses, as a PlacementRefusedError says it.
|
||||
func (e *LinkInPathError) why() string {
|
||||
return fmt.Sprintf("is reached through the link %s, which %s could replace: the node-engine runs as root and "+
|
||||
"follows a link to a directory it gives a mode and owner only when root alone could have made it (novox/hq "+
|
||||
"issue 358). Make %s a directory, or make it and every directory above it root's and writable by no other "+
|
||||
"account", e.Link, e.Who, e.Link)
|
||||
}
|
||||
|
||||
// couldReplace says who besides root could replace an entry of a directory with this owner, group and mode: ""
|
||||
// when nobody can. A variable so a test, which runs below a directory every account may write (/tmp), can name
|
||||
// which of its own directories count as root's alone.
|
||||
var couldReplace = func(path string, uid, gid, mode uint32) string {
|
||||
if uid != 0 {
|
||||
return "the account " + accountNamed(uid) + " (it owns " + path + ")"
|
||||
}
|
||||
if mode&0o002 != 0 {
|
||||
return "every account (" + path + " is writable by all)"
|
||||
}
|
||||
if mode&0o020 != 0 && gid != 0 {
|
||||
return "the group " + groupNamed(gid) + " (it may write " + path + ")"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// linkReplacer says who besides root owns a link: "" when root does. A variable for the same reason.
|
||||
var linkReplacer = func(link string, uid uint32) string {
|
||||
if uid != 0 {
|
||||
return "the account " + accountNamed(uid) + " (it owns the link)"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func accountNamed(uid uint32) string {
|
||||
if u, err := osuser.LookupId(strconv.FormatUint(uint64(uid), 10)); err == nil {
|
||||
return u.Username
|
||||
}
|
||||
return "with uid " + strconv.FormatUint(uint64(uid), 10)
|
||||
}
|
||||
|
||||
func groupNamed(gid uint32) string {
|
||||
if g, err := osuser.LookupGroupId(strconv.FormatUint(uint64(gid), 10)); err == nil {
|
||||
return g.Name
|
||||
}
|
||||
return "with gid " + strconv.FormatUint(uint64(gid), 10)
|
||||
}
|
||||
|
||||
// JudgeDirectories is the walk's judgement over every directory a declaration names, read-only (novox/hq issue
|
||||
// 358): nothing is made, chmodded or chowned. One line per directory, in the declaration's order, after one line
|
||||
// that counts them: placed, would be refused and why, refused already by the placement guard (issue 339), or
|
||||
// below a home, where ADR 0266's walk judges it as before.
|
||||
func JudgeDirectories(d *declaration.Declaration) string {
|
||||
var lines []string
|
||||
var placed, refused, guarded, homed int
|
||||
for _, r := range d.Resources {
|
||||
dir, ok := r.(*declaration.Directory)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
path := filepath.Clean(dir.Path)
|
||||
if err := refusePath(path, placingDirectory, moduleOfID(dir.ID), dir.Owner); err != nil {
|
||||
guarded++
|
||||
lines = append(lines, path+": refused already by the placement guard: "+err.Error())
|
||||
continue
|
||||
}
|
||||
if home := homeAbove(path); home != "" {
|
||||
homed++
|
||||
lines = append(lines, path+": below the home "+home+", walked without following any link as before (ADR 0266)")
|
||||
continue
|
||||
}
|
||||
if err := judgeMeshDir(path); err != nil {
|
||||
refused++
|
||||
lines = append(lines, path+": would be refused: "+err.Error())
|
||||
continue
|
||||
}
|
||||
placed++
|
||||
lines = append(lines, path+": placed")
|
||||
}
|
||||
head := fmt.Sprintf("%d directories declared: %d placed, %d would be refused, %d refused already by the "+
|
||||
"placement guard, %d below a home (novox/hq issue 358; judged read-only, nothing changed)",
|
||||
placed+refused+guarded+homed, placed, refused, guarded, homed)
|
||||
return strings.Join(append([]string{head}, lines...), "\n")
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
// The apply half of mesh_dirs.go (novox/hq issue 358): a mesh directory outside every home made and given its mode
|
||||
// and owner through the walk's descriptor.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// meshDirsByDescriptor says this system can open a directory component by component without following a link.
|
||||
const meshDirsByDescriptor = true
|
||||
|
||||
// openMeshDir opens the directory at path, from /, one component at a time with O_DIRECTORY|O_NOFOLLOW; a missing
|
||||
// component is made in its parent's descriptor with mode, and made is called with each one made and its descriptor.
|
||||
// A link in a parent is followed only when couldReplace and linkReplacer find that root alone could have made it,
|
||||
// and its target is walked again from /; any other link is a LinkInPathError, and the last component being one is
|
||||
// refused as not placed. Answers the directory's descriptor, which the caller closes, with the walk: what it made
|
||||
// and the path it reached the directory at.
|
||||
func openMeshDir(path string, mode os.FileMode, made func(dir string, fd int) error) (int, *meshWalk, error) {
|
||||
w := &meshWalk{want: filepath.Clean(path), create: true, mode: mode, made: made}
|
||||
fd, err := w.open()
|
||||
return fd, w, err
|
||||
}
|
||||
|
||||
// applyMeshDirectory gives a mesh directory outside every home its declared mode and owner through a descriptor
|
||||
// (novox/hq issue 358): opened by openMeshDir, made there when missing, judged a directory by fstat on it, changed
|
||||
// with fchmod and fchown on it, and read back from it. The parents it makes inside the owner's home are the owner's,
|
||||
// as makeDirs gives them (novox/hq ADR 0182).
|
||||
func applyMeshDirectory(r *declaration.Directory, out Outcome, mode os.FileMode, existed bool) (Outcome, error) {
|
||||
uid, gid, chown := 0, 0, r.Owner != ""
|
||||
var err error
|
||||
if chown {
|
||||
if uid, gid, err = idsOf(r.Owner); err != nil {
|
||||
return out, fmt.Errorf("%s should belong to %q: %w", r.Path, r.Owner, err)
|
||||
}
|
||||
}
|
||||
clean := filepath.Clean(r.Path)
|
||||
home := ""
|
||||
if chown {
|
||||
if h, herr := homeOf(r.Owner); herr == nil && h != "" {
|
||||
home = filepath.Clean(h)
|
||||
}
|
||||
}
|
||||
giveMade := func(dir string, fd int) error {
|
||||
if dir == clean || home == "" || (dir != home && !strings.HasPrefix(dir, home+string(os.PathSeparator))) {
|
||||
return nil
|
||||
}
|
||||
if err := unix.Fchown(fd, uid, gid); err != nil {
|
||||
return fmt.Errorf("cannot give %s to %q: %w", dir, r.Owner, &os.PathError{Op: "chown", Path: dir, Err: err})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fd, walk, err := openMeshDir(clean, mode, giveMade)
|
||||
if err != nil {
|
||||
var link *LinkInPathError
|
||||
if errors.As(err, &link) {
|
||||
return out, &PlacementRefusedError{Path: r.Path, Why: "it " + link.why()}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
// Made by this walk when its last component is: at the path the walk reached it at, which root-only links in its
|
||||
// path may have moved from the path declared.
|
||||
created := len(walk.Made) > 0 && walk.Made[len(walk.Made)-1] == walk.At
|
||||
|
||||
var st unix.Stat_t
|
||||
if err := unix.Fstat(fd, &st); err != nil {
|
||||
return out, &os.PathError{Op: "fstat", Path: r.Path, Err: err}
|
||||
}
|
||||
beforeMode, beforeUID, beforeGID := statDirMode(st.Mode), int(st.Uid), int(st.Gid)
|
||||
// Set explicitly even when it existed: a mode given at creation is not a mode maintained — its setgid bit
|
||||
// with it (novox/hq ADR 0334 §1).
|
||||
if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
|
||||
return out, &os.PathError{Op: "chmod", Path: r.Path, Err: err}
|
||||
}
|
||||
ownedAlready := !chown || (beforeUID == uid && beforeGID == gid)
|
||||
if !ownedAlready {
|
||||
if err := unix.Fchown(fd, uid, gid); err != nil {
|
||||
return out, fmt.Errorf("cannot give %s to %q: %w", r.Path, r.Owner, &os.PathError{Op: "chown", Path: r.Path, Err: err})
|
||||
}
|
||||
// A chown may take the setgid bit: the mode is set again after it.
|
||||
if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
|
||||
return out, &os.PathError{Op: "chmod", Path: r.Path, Err: err}
|
||||
}
|
||||
}
|
||||
|
||||
// Read back, from the descriptor that was changed.
|
||||
if err := unix.Fstat(fd, &st); err != nil {
|
||||
return out, fmt.Errorf("made %s and cannot stat it: %w", r.Path, err)
|
||||
}
|
||||
if st.Mode&unix.S_IFMT != unix.S_IFDIR {
|
||||
return out, fmt.Errorf("%s is not a directory after applying", r.Path)
|
||||
}
|
||||
if got := statDirMode(st.Mode); got != dirBits(mode) {
|
||||
return out, fmt.Errorf("%s is mode %s after setting %s", r.Path, modeText(got), modeText(mode))
|
||||
}
|
||||
|
||||
out.Action = "unchanged"
|
||||
switch {
|
||||
case created || !existed:
|
||||
out.Action = "created"
|
||||
case !ownedAlready:
|
||||
out.Action = "updated"
|
||||
case beforeMode != dirBits(mode):
|
||||
out.Action = "updated"
|
||||
out.Detail = fmt.Sprintf("mode %s to %s", modeText(beforeMode), modeText(mode))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Review 47 of novox/hq issue 358: a directory the walk refuses gates its module, and nothing below it is written,
|
||||
// whether the link was there when the apply began or appeared while it ran.
|
||||
|
||||
// aLinkedModule is a module's directory at <base>/parent/state, its parent a link to <base>/elsewhere, and a file of
|
||||
// the module below the directory; and another module's file elsewhere.
|
||||
func aLinkedModule(t *testing.T) (target, other, decl string) {
|
||||
t.Helper()
|
||||
base := t.TempDir()
|
||||
target = filepath.Join(base, "elsewhere", "state")
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join(base, "elsewhere"), filepath.Join(base, "parent")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Join(base, "parent", "state")
|
||||
other = filepath.Join(base, "other.conf")
|
||||
decl = `{"id":"mod.state","type":"directory","path":"` + dir + `","mode":"0700"},` +
|
||||
`{"id":"mod.conf","type":"file","path":"` + dir + `/settings.conf","content":"a=1\n"},` +
|
||||
`{"id":"other.conf","type":"file","path":"` + other + `","content":"kept apart\n"}`
|
||||
return target, other, decl
|
||||
}
|
||||
|
||||
func holdsTheGate(t *testing.T, target, other string, report Report, err error) {
|
||||
t.Helper()
|
||||
var refusal *PlacementRefusedError
|
||||
if !errors.As(err, &refusal) {
|
||||
t.Fatalf("the directory behind the link was not refused: %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(target, "settings.conf")); statErr == nil {
|
||||
t.Error("a file was written into the link's target, below the directory the engine refused")
|
||||
}
|
||||
if info, _ := os.Stat(target); info.Mode().Perm() != 0o755 {
|
||||
t.Errorf("the link's target was given mode %o (was 755)", info.Mode().Perm())
|
||||
}
|
||||
if o := outcomeOf(report, "mod.conf"); o.Action != "skipped" {
|
||||
t.Errorf("the file below the refused directory is not reported skipped: %+v", o)
|
||||
}
|
||||
if strings.Contains(err.Error(), "everything was attempted") || !strings.Contains(err.Error(), "the rest of mod was not attempted") {
|
||||
t.Errorf("the report does not say the module was gated: %v", err)
|
||||
}
|
||||
if got, readErr := os.ReadFile(other); readErr != nil || string(got) != "kept apart\n" {
|
||||
t.Errorf("another module's file was not applied: %q, %v", got, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
// The link there when the apply begins: refused by the guard before anything is applied.
|
||||
func TestADirectoryBehindALinkGatesItsModuleThroughApply(t *testing.T) {
|
||||
target, other, decl := aLinkedModule(t)
|
||||
report, _, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared,
|
||||
noServices, nil, nil)
|
||||
holdsTheGate(t, target, other, report, err)
|
||||
}
|
||||
|
||||
// The link found only at the moment of use: refused by the walk during the apply, and still a refusal that gates its
|
||||
// module and what is below it. Simulated by trusting / for the guard's walk (two walks of /: one to the link, one
|
||||
// from its target) and not for the apply's.
|
||||
func TestADirectoryRefusedDuringTheApplyGatesItsModule(t *testing.T) {
|
||||
target, other, decl := aLinkedModule(t)
|
||||
wasDirs, wasLinks := couldReplace, linkReplacer
|
||||
walksOfRoot := 0
|
||||
couldReplace = func(path string, _, _, _ uint32) string {
|
||||
if path == "/" {
|
||||
walksOfRoot++
|
||||
if walksOfRoot > 2 {
|
||||
return "every account (a test's /)"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
linkReplacer = func(string, uint32) string { return "" }
|
||||
t.Cleanup(func() { couldReplace, linkReplacer = wasDirs, wasLinks })
|
||||
|
||||
report, _, err := Apply(context.Background(), archHost(t), declare(t, decl), store.State{}, store.OriginDeclared,
|
||||
noServices, nil, nil)
|
||||
if walksOfRoot < 3 {
|
||||
t.Fatalf("the apply did not walk again at the moment of use (%d walks)", walksOfRoot)
|
||||
}
|
||||
holdsTheGate(t, target, other, report, err)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
// The descriptor half of mesh_dirs.go (novox/hq issue 358).
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// maxLinksFollowed bounds the root-only links one walk follows, as the kernel bounds a lookup's (MAXSYMLINKS).
|
||||
const maxLinksFollowed = 40
|
||||
|
||||
// errNotThereYet is a read-only walk meeting a component that does not exist: an apply would make it there.
|
||||
var errNotThereYet = errors.New("not there yet")
|
||||
|
||||
// meshWalk is one walk of a mesh directory's path from /.
|
||||
type meshWalk struct {
|
||||
// want is the directory as declared.
|
||||
want string
|
||||
// create makes a missing component, with mode, in its parent's descriptor; without it the walk stops there
|
||||
// with errNotThereYet and makes nothing.
|
||||
create bool
|
||||
mode os.FileMode
|
||||
// made is called with each directory the walk made and its descriptor.
|
||||
made func(dir string, fd int) error
|
||||
// Made is every directory the walk made, outermost first, and At the path the directory was reached at:
|
||||
// want, or where the root-only links in its path led.
|
||||
Made []string
|
||||
At string
|
||||
}
|
||||
|
||||
// judgeMeshDir is openMeshDir's judgement and nothing else (novox/hq issue 358): the same walk, making nothing
|
||||
// and changing nothing; nil when an apply would give the directory its mode and owner, also when a part of its
|
||||
// path is not there yet (an apply makes it, as root, in a directory it reached).
|
||||
func judgeMeshDir(path string) error {
|
||||
w := &meshWalk{want: filepath.Clean(path)}
|
||||
fd, err := w.open()
|
||||
if errors.Is(err, errNotThereYet) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
unix.Close(fd)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *meshWalk) open() (int, error) {
|
||||
if !filepath.IsAbs(w.want) {
|
||||
return -1, fmt.Errorf("%s is not an absolute path", w.want)
|
||||
}
|
||||
p := w.want
|
||||
for followed := 0; ; followed++ {
|
||||
if followed > maxLinksFollowed {
|
||||
return -1, &os.PathError{Op: "open", Path: w.want, Err: unix.ELOOP}
|
||||
}
|
||||
fd, next, err := w.fromRoot(p)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if next == "" {
|
||||
w.At = p
|
||||
return fd, nil
|
||||
}
|
||||
p = next
|
||||
}
|
||||
}
|
||||
|
||||
// fromRoot is one walk down p: the directory's descriptor, or the path to walk next when a root-only link was
|
||||
// met (with -1), or the refusal.
|
||||
func (w *meshWalk) fromRoot(p string) (int, string, error) {
|
||||
fd, err := unix.Open("/", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return -1, "", &os.PathError{Op: "open", Path: "/", Err: err}
|
||||
}
|
||||
// Who could replace an entry of the directories walked so far: once one could, every link below it is
|
||||
// theirs to make, whoever owns the link.
|
||||
who, err := replacerOf(fd, "/")
|
||||
if err != nil {
|
||||
unix.Close(fd)
|
||||
return -1, "", err
|
||||
}
|
||||
at := "/"
|
||||
var parts []string
|
||||
if p != "/" {
|
||||
parts = strings.Split(strings.TrimPrefix(p, "/"), "/")
|
||||
}
|
||||
const dirFlags = unix.O_RDONLY | unix.O_DIRECTORY | unix.O_NOFOLLOW | unix.O_CLOEXEC
|
||||
for i, part := range parts {
|
||||
here := filepath.Join(at, part)
|
||||
last := i == len(parts)-1
|
||||
next, err := unix.Openat(fd, part, dirFlags, 0)
|
||||
if errors.Is(err, unix.ENOENT) {
|
||||
if !w.create {
|
||||
unix.Close(fd)
|
||||
return -1, "", errNotThereYet
|
||||
}
|
||||
mkErr := unix.Mkdirat(fd, part, uint32(w.mode.Perm()))
|
||||
if mkErr != nil && !errors.Is(mkErr, unix.EEXIST) {
|
||||
unix.Close(fd)
|
||||
return -1, "", &os.PathError{Op: "mkdir", Path: here, Err: mkErr}
|
||||
}
|
||||
next, err = unix.Openat(fd, part, dirFlags, 0)
|
||||
if mkErr == nil && err == nil {
|
||||
w.Made = append(w.Made, here)
|
||||
if w.made != nil {
|
||||
if gErr := w.made(here, next); gErr != nil {
|
||||
unix.Close(next)
|
||||
unix.Close(fd)
|
||||
return -1, "", gErr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if errors.Is(err, unix.ELOOP) || errors.Is(err, unix.ENOTDIR) {
|
||||
var st unix.Stat_t
|
||||
if lerr := unix.Fstatat(fd, part, &st, unix.AT_SYMLINK_NOFOLLOW); lerr == nil && st.Mode&unix.S_IFMT == unix.S_IFLNK {
|
||||
to, rerr := w.followable(fd, part, here, st, who, last)
|
||||
unix.Close(fd)
|
||||
if rerr != nil {
|
||||
return -1, "", rerr
|
||||
}
|
||||
if !filepath.IsAbs(to) {
|
||||
to = filepath.Join(at, to)
|
||||
}
|
||||
return -1, filepath.Join("/", to, filepath.Join(parts[i+1:]...)), nil
|
||||
}
|
||||
}
|
||||
unix.Close(fd)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, unix.ENOTDIR) && last:
|
||||
return -1, "", fmt.Errorf("%s exists and is not a directory", here)
|
||||
case errors.Is(err, unix.ENOTDIR):
|
||||
return -1, "", fmt.Errorf("%s is not a directory, and %s lies below it", here, w.want)
|
||||
}
|
||||
return -1, "", &os.PathError{Op: "open", Path: here, Err: err}
|
||||
}
|
||||
fd = next
|
||||
at = here
|
||||
if who == "" {
|
||||
if who, err = replacerOf(fd, here); err != nil {
|
||||
unix.Close(fd)
|
||||
return -1, "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
return fd, "", nil
|
||||
}
|
||||
|
||||
// replacerOf is couldReplace for the directory open on fd.
|
||||
func replacerOf(fd int, path string) (string, error) {
|
||||
var st unix.Stat_t
|
||||
if err := unix.Fstat(fd, &st); err != nil {
|
||||
return "", &os.PathError{Op: "fstat", Path: path, Err: err}
|
||||
}
|
||||
return couldReplace(path, st.Uid, st.Gid, st.Mode&0o7777), nil
|
||||
}
|
||||
|
||||
// followable judges the link at here, held by the directory open on fd: its target when root alone could have
|
||||
// made it, else the refusal. who is who could replace an entry of a directory above it ("" when nobody).
|
||||
func (w *meshWalk) followable(fd int, name, here string, st unix.Stat_t, who string, last bool) (string, error) {
|
||||
if last {
|
||||
return "", &PlacementRefusedError{Path: w.want, Why: "it is a symbolic link, and the node-engine follows no " +
|
||||
"link to a directory it owns"}
|
||||
}
|
||||
if who == "" {
|
||||
who = linkReplacer(here, st.Uid)
|
||||
}
|
||||
if who != "" {
|
||||
return "", &LinkInPathError{Path: w.want, Link: here, Who: who}
|
||||
}
|
||||
buf := make([]byte, unix.PathMax)
|
||||
n, err := unix.Readlinkat(fd, name, buf)
|
||||
if err != nil {
|
||||
return "", &os.PathError{Op: "readlink", Path: here, Err: err}
|
||||
}
|
||||
return string(buf[:n]), nil
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 358: a mesh directory outside every home is given its mode and owner only through a
|
||||
// descriptor opened from /, so a parent swapped for a link between two applies cannot carry root's chmod and
|
||||
// chown to the link's target — unless root alone could have made that link (/var/run is one on every node).
|
||||
|
||||
// A link only root could have made — itself root's, in directories no other account may write — is followed, as
|
||||
// /var/run to /run is on every node; relative or absolute.
|
||||
func TestALinkOnlyRootCouldMakeIsFollowed(t *testing.T) {
|
||||
for name, to := range map[string]func(base string) string{
|
||||
"absolute": func(base string) string { return filepath.Join(base, "elsewhere") },
|
||||
"relative": func(string) string { return "elsewhere" },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
rootAlone(t, "")
|
||||
base, r, previous := aMeshDirectory(t)
|
||||
target := swapParentForLink(t, base, to(base))
|
||||
|
||||
out, err := applyDirectory(r, previous, true)
|
||||
if err != nil {
|
||||
t.Fatalf("a directory behind a root-only link was refused: %v", err)
|
||||
}
|
||||
if got := permOf(t, target); got != 0o700 {
|
||||
t.Errorf("the directory behind a root-only link is mode %o, want 700", got)
|
||||
}
|
||||
if out.Action != "updated" {
|
||||
t.Errorf("the apply said %q, want updated", out.Action)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A link another account owns refuses the directory even in directories root alone may write, naming the account.
|
||||
func TestALinkAnotherAccountOwnsIsRefused(t *testing.T) {
|
||||
rootAlone(t, "the account mallory (it owns the link)")
|
||||
base, r, previous := aMeshDirectory(t)
|
||||
target := swapParentForLink(t, base, filepath.Join(base, "elsewhere"))
|
||||
|
||||
_, err := applyDirectory(r, previous, true)
|
||||
if err == nil || !strings.Contains(err.Error(), "mallory") {
|
||||
t.Fatalf("a link another account owns was not refused naming it: %v", err)
|
||||
}
|
||||
if got := permOf(t, target); got != 0o755 {
|
||||
t.Errorf("the link's target was given mode %o (was 755)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A directory that is missing is made through the walk, below a link root alone could make.
|
||||
func TestAMissingDirectoryIsMadeBehindARootOnlyLink(t *testing.T) {
|
||||
rootAlone(t, "")
|
||||
base := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(base, "real"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("real", filepath.Join(base, "link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := &declaration.Directory{ID: "mod.state", Path: filepath.Join(base, "link", "a", "b"), Mode: "0750"}
|
||||
out, err := applyDirectory(r, store.Applied{}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Action != "created" {
|
||||
t.Errorf("the apply said %q, want created", out.Action)
|
||||
}
|
||||
if got := permOf(t, filepath.Join(base, "real", "a", "b")); got != 0o750 {
|
||||
t.Errorf("the made directory is mode %o, want 750", got)
|
||||
}
|
||||
}
|
||||
|
||||
// rootAlone has every directory count as writable by root alone, as /, /var and /run are on a node, and a link's
|
||||
// owner be linkOwner ("" for root): a test runs below /tmp, which every account may write.
|
||||
func rootAlone(t *testing.T, linkOwner string) {
|
||||
t.Helper()
|
||||
wasDirs, wasLinks := couldReplace, linkReplacer
|
||||
couldReplace = func(string, uint32, uint32, uint32) string { return "" }
|
||||
linkReplacer = func(string, uint32) string { return linkOwner }
|
||||
t.Cleanup(func() { couldReplace, linkReplacer = wasDirs, wasLinks })
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Whether a library's disk is there (novox/hq ADR 0328 §9; first ADR 0325 §9).
|
||||
//
|
||||
// **The rule the engine keeps.** A folder is placed on the filesystem it is first seen on: the mount point that
|
||||
// holds it — the longest mount point in /proc/self/mountinfo that is the folder or above it — is recorded beside the
|
||||
// kept folder. On every later apply the folder (or, when it is absent, its nearest existing parent) is judged
|
||||
// against that record: held by another mount point than the one recorded while the recorded one is not mounted is
|
||||
// the disk not there. A recorded mount point that is mounted while another holds the folder is a **move**: what is
|
||||
// at the path now is not what was kept (a disk mounted over a folder first seen empty below it, a mount made
|
||||
// deeper), so the folder is judged afresh — kept again only if it holds nothing, else it waits for the operator's
|
||||
// word as any folder holding data does (ADR 0328 §7).
|
||||
//
|
||||
// **On first sight, with no record,** the engine knows a disk is missing only from where disks are meant to be:
|
||||
// every mount target in /etc/fstab, every `Where=` of a systemd .mount unit, every mount point a ZFS dataset names
|
||||
// (`zfs list`, when the machine has zfs), every mount point this engine has seen mounted since it started, and every
|
||||
// mount point recorded for another kept folder. A target that is the folder or above it, lies below the mount point
|
||||
// that holds what is there now, and is not mounted, is the disk not there. Otherwise the filesystem that holds what
|
||||
// is there is the placed one. A mount nothing names in those places (a pool not imported, whose datasets `zfs list`
|
||||
// does not show; a hand `mount` never seen) is not known to be missing on first sight: then a folder first seen
|
||||
// empty below it is kept as empty, and the move above takes it back the moment the disk is there.
|
||||
|
||||
// MountPoints is every mount point this machine has mounted now, from /proc/self/mountinfo. A variable so a test
|
||||
// can stand in for mounts it cannot make.
|
||||
var MountPoints = func() []string {
|
||||
return readMountPoints()
|
||||
}
|
||||
|
||||
// mountHistory is every mount point this engine has seen mounted since it started (§9's "mountinfo history").
|
||||
var mountHistory = struct {
|
||||
sync.Mutex
|
||||
seen map[string]bool
|
||||
}{seen: map[string]bool{}}
|
||||
|
||||
// sawMounted adds points to the history, and is the history: every mount point seen, never "/".
|
||||
func sawMounted(points []string) []string {
|
||||
mountHistory.Lock()
|
||||
defer mountHistory.Unlock()
|
||||
for _, p := range points {
|
||||
if p != "/" {
|
||||
mountHistory.seen[p] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(mountHistory.seen))
|
||||
for p := range mountHistory.seen {
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func readMountPoints() []string {
|
||||
raw, err := os.ReadFile("/proc/self/mountinfo")
|
||||
if err != nil {
|
||||
return []string{"/"}
|
||||
}
|
||||
return mountinfoPoints(string(raw))
|
||||
}
|
||||
|
||||
// MountTargets is every place a disk is meant to be mounted: /etc/fstab's targets and ZFS datasets' mount points,
|
||||
// read at most once a minute — every look judges every access by them. A variable so a test can stand in.
|
||||
var MountTargets = func(ctx context.Context) []string {
|
||||
targets.Lock()
|
||||
defer targets.Unlock()
|
||||
if targets.read.IsZero() || time.Since(targets.read) > time.Minute {
|
||||
targets.list, targets.read = readMountTargets(ctx), time.Now()
|
||||
}
|
||||
return append([]string(nil), targets.list...)
|
||||
}
|
||||
|
||||
var targets struct {
|
||||
sync.Mutex
|
||||
list []string
|
||||
read time.Time
|
||||
}
|
||||
|
||||
func readMountTargets(ctx context.Context) []string {
|
||||
var out []string
|
||||
if raw, err := os.ReadFile("/etc/fstab"); err == nil {
|
||||
out = append(out, fstabTargets(string(raw))...)
|
||||
}
|
||||
out = append(out, mountUnitTargets(systemdUnitDirs)...)
|
||||
if _, err := exec.LookPath("zfs"); err == nil {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if raw, err := exec.CommandContext(ctx, "zfs", "list", "-H", "-o", "mountpoint").Output(); err == nil {
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if p := strings.TrimSpace(line); strings.HasPrefix(p, "/") {
|
||||
out = append(out, filepath.Clean(p))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// systemdUnitDirs are where systemd keeps the units a machine mounts by.
|
||||
var systemdUnitDirs = []string{"/etc/systemd/system", "/run/systemd/system", "/run/systemd/generator",
|
||||
"/usr/lib/systemd/system", "/lib/systemd/system"}
|
||||
|
||||
// mountUnitTargets reads the `Where=` of every .mount unit in dirs.
|
||||
func mountUnitTargets(dirs []string) []string {
|
||||
var out []string
|
||||
for _, dir := range dirs {
|
||||
names, err := filepath.Glob(filepath.Join(dir, "*.mount"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, name := range names {
|
||||
raw, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "Where="); ok && strings.HasPrefix(v, "/") {
|
||||
out = append(out, filepath.Clean(strings.TrimSpace(v)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mountinfoPoints reads the mount point of each line of mountinfo, its fifth field, with the kernel's octal escapes
|
||||
// undone.
|
||||
func mountinfoPoints(raw string) []string {
|
||||
var out []string
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 5 {
|
||||
continue
|
||||
}
|
||||
out = append(out, filepath.Clean(unescapeMount(fields[4])))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mountSourceOf is the source and filesystem type of the mount at point, from mountinfo: for a ZFS dataset, its
|
||||
// name. Empty when the machine says nothing of it.
|
||||
func mountSourceOf(point string) (source, fstype string) { return MountSource(point) }
|
||||
|
||||
// MountSource is mountSourceOf as the machine answers it. A variable so a test can stand in for another filesystem
|
||||
// mounted at the same point.
|
||||
var MountSource = func(point string) (source, fstype string) {
|
||||
raw, err := os.ReadFile("/proc/self/mountinfo")
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
before, after, ok := strings.Cut(line, " - ")
|
||||
fields, rest := strings.Fields(before), strings.Fields(after)
|
||||
if !ok || len(fields) < 5 || len(rest) < 2 {
|
||||
continue
|
||||
}
|
||||
if filepath.Clean(unescapeMount(fields[4])) == point {
|
||||
source, fstype = unescapeMount(rest[1]), rest[0]
|
||||
}
|
||||
}
|
||||
return source, fstype
|
||||
}
|
||||
|
||||
// fstabTargets reads the second field of each fstab line that mounts somewhere: not swap, not a comment.
|
||||
func fstabTargets(raw string) []string {
|
||||
var out []string
|
||||
sc := bufio.NewScanner(strings.NewReader(raw))
|
||||
for sc.Scan() {
|
||||
fields := strings.Fields(sc.Text())
|
||||
if len(fields) < 2 || strings.HasPrefix(fields[0], "#") || !strings.HasPrefix(fields[1], "/") {
|
||||
continue
|
||||
}
|
||||
out = append(out, filepath.Clean(unescapeMount(fields[1])))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unescapeMount undoes the kernel's and fstab's octal escapes: \040 is a space.
|
||||
func unescapeMount(s string) string {
|
||||
if !strings.Contains(s, `\`) {
|
||||
return s
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] == '\\' && i+4 <= len(s) {
|
||||
if v, ok := octal3(s[i+1 : i+4]); ok {
|
||||
b.WriteByte(v)
|
||||
i += 3
|
||||
continue
|
||||
}
|
||||
}
|
||||
b.WriteByte(s[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func octal3(s string) (byte, bool) {
|
||||
if len(s) != 3 {
|
||||
return 0, false
|
||||
}
|
||||
v := 0
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '7' {
|
||||
return 0, false
|
||||
}
|
||||
v = v*8 + int(c-'0')
|
||||
}
|
||||
return byte(v), v < 256
|
||||
}
|
||||
|
||||
// mountHolding is the longest of points that is path or above it: the mount point that holds it.
|
||||
func mountHolding(path string, points []string) string {
|
||||
best := "/"
|
||||
for _, p := range points {
|
||||
if atOrBelow(path, p) && len(p) > len(best) {
|
||||
best = p
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// placement is the judgement of where a folder's data lives.
|
||||
type placement struct {
|
||||
// Mount is the mount point that holds what is there: the folder, or its nearest existing parent.
|
||||
Mount string
|
||||
// Away is that the disk the folder was placed on is not there.
|
||||
Away bool
|
||||
// Moved is that the recorded mount point is mounted and another holds the folder: what is there is not what
|
||||
// was kept.
|
||||
Moved bool
|
||||
}
|
||||
|
||||
// placedOn judges path, of which existing is what is there (the folder itself, or its nearest existing parent),
|
||||
// against what was recorded and where disks are meant to be.
|
||||
func placedOn(ctx context.Context, path, existing, recorded string, others []string) placement {
|
||||
points := MountPoints()
|
||||
mounted := map[string]bool{}
|
||||
for _, p := range points {
|
||||
mounted[p] = true
|
||||
}
|
||||
have := mountHolding(existing, points)
|
||||
seen := sawMounted(points)
|
||||
if recorded != "" {
|
||||
return placement{Mount: have, Away: have != recorded && !mounted[recorded],
|
||||
Moved: have != recorded && mounted[recorded]}
|
||||
}
|
||||
for _, t := range append(append(MountTargets(ctx), others...), seen...) {
|
||||
if atOrBelow(path, t) && t != have && atOrBelow(t, have) && !mounted[t] {
|
||||
return placement{Mount: have, Away: true}
|
||||
}
|
||||
}
|
||||
return placement{Mount: have}
|
||||
}
|
||||
@@ -523,6 +523,20 @@ func refusedPlaces(resources []declaration.Resource) map[string]error {
|
||||
switch res := r.(type) {
|
||||
case *declaration.Directory:
|
||||
err = refusePath(res.Path, placingDirectory, moduleOfID(res.ID), res.Owner)
|
||||
// And outside every home, the walk's judgement before anything is applied (novox/hq issue 358): a
|
||||
// link another account could make in its path refuses it, and with it what is below it, whatever
|
||||
// order they are declared in. The apply walks again at the moment of use. A directory handed over
|
||||
// keeps its own, stricter walk (issue 356), which refuses any link at all.
|
||||
if err == nil && meshDirsByDescriptor && homeAbove(res.Path) == "" &&
|
||||
!HandedOver()[handOverKey(res.ID, res.Path)] {
|
||||
if judged := judgeMeshDir(res.Path); judged != nil {
|
||||
err = judged
|
||||
var link *LinkInPathError
|
||||
if errors.As(judged, &link) {
|
||||
err = &PlacementRefusedError{Path: res.Path, Why: "it " + link.why()}
|
||||
}
|
||||
}
|
||||
}
|
||||
case *declaration.Access:
|
||||
err = refusePath(res.Path, placingAccess, moduleOfID(res.ID), "")
|
||||
default:
|
||||
|
||||
@@ -111,6 +111,11 @@ func Plan(d *declaration.Declaration, known store.State, origin string) []Step {
|
||||
// In removeWhole's words (novox/hq ADR 0118): written over, so given back, not deleted.
|
||||
step.Verb, step.Why = "restore", "no longer declared; the original the mesh wrote over goes back "+
|
||||
"from "+orphan.Kept+", unless the file was changed since the mesh last wrote it"
|
||||
case orphan.Type == string(declaration.TypeAccess) && orphan.User != "" && len(orphan.Groups) > 0:
|
||||
// In takeBackGranted's words (novox/hq ADR 0328 §11: no account is in a group for data but a library's).
|
||||
step.Verb, step.Why = "restore", "no longer declared; the path is left as it is, and "+orphan.User+
|
||||
" leaves "+strings.Join(orphan.Groups, ", ")+", which the mesh put it in, unless another declared "+
|
||||
"resource still asks for it"
|
||||
case orphan.Type == string(declaration.TypeService):
|
||||
// What removal will do, said before it does it (novox/hq ADR 0118), in removeService's
|
||||
// words. "restore" only where it may stop or disable something — the record cannot say
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The replay of novox/hq issue 358, written only with what the node-engine had before its fix, so it can be laid
|
||||
// over the commit before (mesh-lab's replays register): a mesh directory outside every home was given its mode
|
||||
// and owner by path, so a parent swapped for a link between two applies carried root's chmod to the link's target.
|
||||
|
||||
// aMeshDirectory applies a directory declared at <base>/parent/state once, as the mesh's own, and answers it
|
||||
// with the record a second apply would read.
|
||||
func aMeshDirectory(t *testing.T) (base string, r *declaration.Directory, previous store.Applied) {
|
||||
t.Helper()
|
||||
base = t.TempDir()
|
||||
r = &declaration.Directory{ID: "mod.state", Path: filepath.Join(base, "parent", "state"), Mode: "0700"}
|
||||
out, err := applyDirectory(r, store.Applied{}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("the first apply: %v", err)
|
||||
}
|
||||
if out.Action != "created" {
|
||||
t.Fatalf("the first apply said %q, want created", out.Action)
|
||||
}
|
||||
return base, r, store.Applied{ID: r.ID, Type: "directory", Target: r.Path}
|
||||
}
|
||||
|
||||
// swapParentForLink moves <base>/parent away and puts a link to elsewhere (as written) in its place, where
|
||||
// elsewhere holds a directory state of mode 0755: what an account that can write <base> does between two applies.
|
||||
func swapParentForLink(t *testing.T, base, to string) (target string) {
|
||||
t.Helper()
|
||||
elsewhere := filepath.Join(base, "elsewhere")
|
||||
target = filepath.Join(elsewhere, "state")
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(target, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Rename(filepath.Join(base, "parent"), filepath.Join(base, "moved")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(to, filepath.Join(base, "parent")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
func permOf(t *testing.T, path string) os.FileMode {
|
||||
t.Helper()
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.Mode().Perm()
|
||||
}
|
||||
|
||||
// The acceptance of issue 358: a parent swapped for a link after an apply leaves the link's target untouched, and
|
||||
// the refusal names the link and who could have made it.
|
||||
func TestAParentSwappedForALinkLeavesItsTargetUntouched(t *testing.T) {
|
||||
base, r, previous := aMeshDirectory(t)
|
||||
target := swapParentForLink(t, base, filepath.Join(base, "elsewhere"))
|
||||
|
||||
_, err := applyDirectory(r, previous, true)
|
||||
var refusal *PlacementRefusedError
|
||||
if !errors.As(err, &refusal) {
|
||||
t.Fatalf("the directory reached through a link was not refused: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), filepath.Join(base, "parent")) || !strings.Contains(err.Error(), "could replace") {
|
||||
t.Errorf("the refusal does not name the link and who could replace it: %v", err)
|
||||
}
|
||||
if got := permOf(t, target); got != 0o755 {
|
||||
t.Errorf("the link's target was given mode %o (was 755): root's chmod followed the link", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/accounts"
|
||||
)
|
||||
|
||||
// Root's numbers, which no module identity is and whose data is never kept (novox/hq ADR 0328 §15).
|
||||
//
|
||||
// A module identity whose uid or gid is 0, or whose gid is a group whose members act as root (accounts.RootGroups,
|
||||
// the account judge's own closed list), would be a way to root through a container or a shared folder: refused,
|
||||
// naming why. A shared folder owned by root or such a group is the machine's own data, never kept.
|
||||
|
||||
// RootGroupIDs is the number of every group whose members act as root (accounts.RootGroups), as this machine
|
||||
// numbers them: read from /etc/group itself and from the machine's name service (`getent group`), since the engine
|
||||
// is built without cgo and Go's own lookup reads /etc/group alone — a group a name service holds, or one number
|
||||
// carrying several names, would otherwise pass a check made by name. Read at most once a minute, every look and
|
||||
// apply judging against the same answer, and the name service given five seconds in all. A variable so a test can
|
||||
// stand in.
|
||||
var RootGroupIDs = func() map[int]string {
|
||||
rootIDs.Lock()
|
||||
defer rootIDs.Unlock()
|
||||
if rootIDs.ids == nil || time.Since(rootIDs.at) > time.Minute {
|
||||
rootIDs.ids, rootIDs.at = readRootGroupIDs(), time.Now()
|
||||
}
|
||||
return rootIDs.ids
|
||||
}
|
||||
|
||||
var rootIDs struct {
|
||||
sync.Mutex
|
||||
ids map[int]string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func readRootGroupIDs() map[int]string {
|
||||
out := map[int]string{}
|
||||
if raw, err := os.ReadFile("/etc/group"); err == nil {
|
||||
for name, gid := range groupEntries(string(raw)) {
|
||||
if slices.Contains(accounts.RootGroups, name) {
|
||||
out[gid] = name
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
for _, name := range accounts.RootGroups {
|
||||
raw, err := exec.CommandContext(ctx, "getent", "group", name).Output()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for found, gid := range groupEntries(string(raw)) {
|
||||
if found == name {
|
||||
out[gid] = name
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// groupEntries reads group database lines, name:password:gid:members, as name to number.
|
||||
func groupEntries(raw string) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
fields := strings.Split(strings.TrimSpace(line), ":")
|
||||
if len(fields) < 3 || strings.HasPrefix(fields[0], "#") {
|
||||
continue
|
||||
}
|
||||
if gid, err := strconv.Atoi(fields[2]); err == nil {
|
||||
out[fields[0]] = gid
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// GroupName is the name this machine's group database gives a number, empty when none. A variable so a test can
|
||||
// stand in for groups it cannot make.
|
||||
var GroupName = func(gid int) string {
|
||||
if g, err := user.LookupGroupId(strconv.Itoa(gid)); err == nil {
|
||||
return g.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// rootGroup is why gid is a root group here — 0, a number a root group has, or a number named as one — or "".
|
||||
func rootGroup(gid int) string {
|
||||
if gid == 0 {
|
||||
return "group 0"
|
||||
}
|
||||
if name, is := RootGroupIDs()[gid]; is {
|
||||
return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name)
|
||||
}
|
||||
if name := GroupName(gid); slices.Contains(accounts.RootGroups, name) {
|
||||
return fmt.Sprintf("gid %d, which is %s here, whose members act as root", gid, name)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// rootOwned is why a folder owned by uid and gid is root's data (ADR 0325 §12, kept for the legacy reader), or "": owned by root, or by a
|
||||
// root group.
|
||||
func rootOwned(uid, gid uint32) string {
|
||||
if uid == 0 {
|
||||
return "owned by root"
|
||||
}
|
||||
if why := rootGroup(int(gid)); why != "" {
|
||||
return "owned by " + why
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// rootIdentity is why a module identity's numbers are refused (ADR 0328 §15), or nil.
|
||||
func rootIdentity(uid, gid int) error {
|
||||
if uid <= 0 {
|
||||
return fmt.Errorf("its uid is %d, and a module's identity is never root (novox/hq ADR 0328 §15)", uid)
|
||||
}
|
||||
if why := rootGroup(gid); why != "" {
|
||||
return fmt.Errorf("its gid is %s, and a module's identity is never in a root group (novox/hq ADR 0328 §15)", why)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handed is every group a record that went in this apply put an account in, and another declared resource still
|
||||
// asks for: account → group. The resource that still asks records it as the mesh's when it is applied, so the group
|
||||
// is taken back once nobody asks for it any more, rather than staying for ever because the record that put it
|
||||
// there went first (novox/hq ADR 0252 §3).
|
||||
type Handed map[string]map[string]bool
|
||||
|
||||
func (h Handed) hand(account, group string) {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
if h[account] == nil {
|
||||
h[account] = map[string]bool{}
|
||||
}
|
||||
h[account][group] = true
|
||||
}
|
||||
|
||||
func (h Handed) has(account, group string) bool { return h[account][group] }
|
||||
@@ -35,7 +35,7 @@ const FirstLoginUID = 1000
|
||||
// wanted is every group the whole declaration asks an account to be in (novox/hq ADR 0252), so a group
|
||||
// this resource stops asking for is kept while another resource asks for it.
|
||||
func applyUser(ctx context.Context, sys system.System, r *declaration.User, run Runner,
|
||||
previous store.Applied, wanted Wanted) (Outcome, error) {
|
||||
previous store.Applied, wanted Wanted, handed Handed) (Outcome, error) {
|
||||
out := begin(r)
|
||||
out.Action = "unchanged"
|
||||
// What was found is carried from the record for as long as the resource is recorded — for this
|
||||
@@ -100,7 +100,7 @@ func applyUser(ctx context.Context, sys system.System, r *declaration.User, run
|
||||
// Groups before the shell, so that a failure here comes before the shell is changed: a record
|
||||
// is written only for an apply that worked, and a shell changed by a failed one would be read
|
||||
// next time as the account's own, and the one it replaced lost.
|
||||
if err := applyGroups(ctx, sys, r, run, previous, wanted, &out); err != nil {
|
||||
if err := applyGroups(ctx, sys, r, run, previous, wanted, handed, &out); err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
@@ -207,7 +207,7 @@ func onOff(on bool) string {
|
||||
// And every group the mesh put the account in, while no other declared resource asks for it (novox/hq
|
||||
// ADR 0252), on the same rule: never fatal, and never a group the account was in before.
|
||||
func removeUser(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
wanted Wanted) (string, string, error) {
|
||||
wanted Wanted, handed Handed) (string, string, error) {
|
||||
const kept = "the account is kept; the host never deletes a login"
|
||||
login, exists, err := system.LookUpUser(ctx, system.Runner(run), a.Target)
|
||||
if err != nil {
|
||||
@@ -226,7 +226,7 @@ func removeUser(ctx context.Context, sys system.System, a store.Applied, run Run
|
||||
action = "restored"
|
||||
}
|
||||
}
|
||||
if gave, said := giveGroupsBack(ctx, sys, a, run, wanted); said != "" {
|
||||
if gave, said := giveGroupsBack(ctx, sys, a, run, wanted, handed); said != "" {
|
||||
detail += "; " + said
|
||||
if gave {
|
||||
action = "restored"
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A bounded walk under a library's root (novox/hq ADR 0328 §7, issue 567).
|
||||
//
|
||||
// **Walked by descriptor.** The root is opened as a mesh directory is (meshWalk: no link another account could
|
||||
// make, the root itself no link), and everything under it is reached from its parent's descriptor with O_NOFOLLOW:
|
||||
// a link met is passed by, a folder whose device is not the root's is not entered, and an entry opened is checked
|
||||
// to be the one that was looked at. An entry's group and mode are set through its own descriptor, so a name
|
||||
// swapped for a link between the look and the change changes nothing.
|
||||
//
|
||||
// **Bounded, and resumed.** A library is millions of files, and a walk that never ends is a machine that cannot
|
||||
// say what it did. Each walk reads at most so many entries for at most so long; one that meets its bound says how
|
||||
// far it got (a cursor, the names of each level below the root, kept in the engine's state and never said) and the
|
||||
// next goes on from there, in the order of the names, so successive walks cover the library whole.
|
||||
|
||||
// WalkLimits bound one walk under a root: how many entries it reads, and for how long.
|
||||
type WalkLimits struct {
|
||||
Entries int
|
||||
Time time.Duration
|
||||
}
|
||||
|
||||
// ActLimits bound one walk of the take act; one stopped at its bound goes on from its cursor at the next look.
|
||||
var ActLimits = WalkLimits{Entries: 5_000_000, Time: 30 * time.Minute}
|
||||
|
||||
// WalkCounts is what one walk did or found. Counts only: never a path below the root.
|
||||
type WalkCounts struct {
|
||||
// Folders and Files are the entries changed; Kept the ones that already had what they need.
|
||||
Folders, Files, Kept int
|
||||
// Links are links passed by, Other entries neither a file nor a folder, Mounts folders on another filesystem
|
||||
// not entered; Linked files with another name an account owns, given the group; Shared files with another
|
||||
// name root owns, left as they are; Stripped files whose setuid or setgid bit the group change took.
|
||||
Links, Other, Mounts, Linked, Shared, Stripped int
|
||||
// Read is every entry looked at; Failed the entries that could not be changed (an immutable file, say).
|
||||
Read, Failed int
|
||||
}
|
||||
|
||||
// ErrWalkBound is a walk that met its bound before it was through: what it did stays, and the cursor says where
|
||||
// the next one goes on.
|
||||
var ErrWalkBound = errors.New("the walk met its bound")
|
||||
|
||||
// keptPath refuses a root that is not stated exactly: absolute and clean, so `..` and a doubled separator never
|
||||
// name it.
|
||||
func keptPath(path string) error {
|
||||
if !filepath.IsAbs(path) || filepath.Clean(path) != path {
|
||||
return errors.New("the root is not stated as the engine states one (absolute, no `..`, no doubled or " +
|
||||
"trailing separator); nothing was changed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,535 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
// The descriptor half of walk.go and library.go (novox/hq ADR 0328 §6, §7, §9).
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// walkStats is how a walk reads an entry and an open descriptor: the machine's own calls, or a test's stand-in
|
||||
// for a device it cannot mount.
|
||||
type walkStats struct {
|
||||
at func(dirfd int, name string, st *unix.Stat_t) error
|
||||
fd func(fd int, st *unix.Stat_t) error
|
||||
}
|
||||
|
||||
var machineStats = walkStats{
|
||||
at: func(dirfd int, name string, st *unix.Stat_t) error {
|
||||
return unix.Fstatat(dirfd, name, st, unix.AT_SYMLINK_NOFOLLOW)
|
||||
},
|
||||
fd: unix.Fstat,
|
||||
}
|
||||
|
||||
// What a visit did to one entry, as flags: nothing (it had what it needs), or changed, and beside it whether it was a
|
||||
// file with another name (linked, changed; shared, left), and whether its setuid or setgid bit went.
|
||||
const (
|
||||
visitKept = 0
|
||||
visitChanged = 1 << iota
|
||||
visitLinked // a file with another name, owned by an account: given the group
|
||||
visitShared // a file with another name, owned by root: left as it is
|
||||
visitStripped // a file whose setuid or setgid bit the group change took, and which stays taken
|
||||
)
|
||||
|
||||
// visitFunc is what a walk does to one folder or file, open on fd, whose stat is st: what it did, or the error
|
||||
// counted on that entry alone (errStopWalk stops the walk).
|
||||
type visitFunc func(fd int, st unix.Stat_t, folder bool) (int, error)
|
||||
|
||||
// errStopWalk wraps an error that stops the whole walk, never one entry's alone.
|
||||
type errStopWalk struct{ err error }
|
||||
|
||||
func (e errStopWalk) Error() string { return e.err.Error() }
|
||||
func (e errStopWalk) Unwrap() error { return e.err }
|
||||
|
||||
// treeWalk is one walk under a library's root, from the descriptor of the root itself.
|
||||
type treeWalk struct {
|
||||
dev uint64
|
||||
deadline time.Time
|
||||
limit int
|
||||
stats walkStats
|
||||
visit visitFunc
|
||||
counts WalkCounts
|
||||
// at is the names of the levels below the root down to the entry being walked; Cursor, once the walk met its
|
||||
// bound, where the next goes on.
|
||||
at []string
|
||||
Cursor []string
|
||||
// Failed is the first failure on one entry, in words with no path.
|
||||
Failed string
|
||||
}
|
||||
|
||||
// openRoot opens a library's root as a mesh directory is opened (no link another account could make, the root
|
||||
// itself no link) and starts a walk from it on the root's own device.
|
||||
func openRoot(path string, limits WalkLimits, stats walkStats, visit visitFunc) (*treeWalk, int, unix.Stat_t, error) {
|
||||
var st unix.Stat_t
|
||||
if err := keptPath(path); err != nil {
|
||||
return nil, -1, st, err
|
||||
}
|
||||
mw := &meshWalk{want: path}
|
||||
fd, err := mw.open()
|
||||
if errors.Is(err, errNotThereYet) {
|
||||
return nil, -1, st, errors.New("the root is not there; nothing was changed")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, -1, st, errors.New("the root is not walked: " + pathless(err))
|
||||
}
|
||||
if mw.At != path {
|
||||
unix.Close(fd)
|
||||
return nil, -1, st, errors.New("the root is reached through a link, and the engine walks no root through " +
|
||||
"one; nothing was changed")
|
||||
}
|
||||
if err := stats.fd(fd, &st); err != nil {
|
||||
unix.Close(fd)
|
||||
return nil, -1, st, fmt.Errorf("the root cannot be read: %v; nothing was changed", err)
|
||||
}
|
||||
return &treeWalk{dev: uint64(st.Dev), deadline: time.Now().Add(limits.Time), limit: limits.Entries,
|
||||
stats: stats, visit: visit}, fd, st, nil
|
||||
}
|
||||
|
||||
// walkRoot walks path whole, or from resume, with visit: the walk, with what it did and the cursor where it stopped
|
||||
// when it met its bound (with ErrWalkBound), and the walk's own error otherwise.
|
||||
func walkRoot(path string, limits WalkLimits, stats walkStats, resume []string, visit visitFunc) (*treeWalk, error) {
|
||||
w, fd, st, err := openRoot(path, limits, stats, visit)
|
||||
if err != nil {
|
||||
return &treeWalk{}, err
|
||||
}
|
||||
return w, w.dir(fd, st, resume)
|
||||
}
|
||||
|
||||
// read counts one entry, and says the bound when it is met, keeping where the walk stood as the cursor.
|
||||
func (w *treeWalk) read() error {
|
||||
if w.counts.Read >= w.limit || time.Now().After(w.deadline) {
|
||||
w.Cursor = append([]string(nil), w.at...)
|
||||
return ErrWalkBound
|
||||
}
|
||||
w.counts.Read++
|
||||
return nil
|
||||
}
|
||||
|
||||
// dir is one folder, open on fd, whose stat is st: the folder itself (unless the walk resumes below it, which an
|
||||
// earlier walk visited), then every entry in it in the order of their names. It closes fd.
|
||||
func (w *treeWalk) dir(fd int, st unix.Stat_t, resume []string) error {
|
||||
f := os.NewFile(uintptr(fd), "")
|
||||
defer f.Close()
|
||||
if len(resume) == 0 {
|
||||
if err := w.read(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.visited(fd, st, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
names, err := f.Readdirnames(-1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("a folder could not be read: %v", err)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
var below []string
|
||||
if len(resume) > 0 {
|
||||
if name < resume[0] {
|
||||
continue
|
||||
}
|
||||
if name == resume[0] {
|
||||
below = resume[1:]
|
||||
}
|
||||
resume = nil
|
||||
}
|
||||
w.at = append(w.at, name)
|
||||
err := w.entry(fd, name, below)
|
||||
w.at = w.at[:len(w.at)-1]
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// entry is one name in the folder open on dirfd: a link passed by, a folder on this device walked, a file visited
|
||||
// through its own descriptor, anything else left.
|
||||
func (w *treeWalk) entry(dirfd int, name string, resume []string) error {
|
||||
var st unix.Stat_t
|
||||
if err := w.stats.at(dirfd, name, &st); err != nil {
|
||||
if errors.Is(err, unix.ENOENT) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("an entry could not be read: %v", err)
|
||||
}
|
||||
switch st.Mode & unix.S_IFMT {
|
||||
case unix.S_IFLNK:
|
||||
w.counts.Links++
|
||||
return nil
|
||||
case unix.S_IFDIR, unix.S_IFREG:
|
||||
default:
|
||||
w.counts.Other++
|
||||
return nil
|
||||
}
|
||||
if uint64(st.Dev) != w.dev {
|
||||
w.counts.Mounts++
|
||||
return nil
|
||||
}
|
||||
isDir := st.Mode&unix.S_IFMT == unix.S_IFDIR
|
||||
flags := unix.O_RDONLY | unix.O_NOFOLLOW | unix.O_CLOEXEC | unix.O_NOCTTY | unix.O_NONBLOCK
|
||||
if isDir {
|
||||
flags |= unix.O_DIRECTORY
|
||||
}
|
||||
fd, err := unix.Openat(dirfd, name, flags, 0)
|
||||
switch {
|
||||
case errors.Is(err, unix.ENOENT):
|
||||
return nil
|
||||
case errors.Is(err, unix.ELOOP), errors.Is(err, unix.ENOTDIR):
|
||||
// Swapped for a link, or for something else, after it was looked at: passed by.
|
||||
w.counts.Links++
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("an entry could not be opened: %v", err)
|
||||
}
|
||||
var opened unix.Stat_t
|
||||
if err := w.stats.fd(fd, &opened); err != nil {
|
||||
unix.Close(fd)
|
||||
return fmt.Errorf("an entry could not be read: %v", err)
|
||||
}
|
||||
if opened.Dev != st.Dev || opened.Ino != st.Ino || opened.Mode&unix.S_IFMT != st.Mode&unix.S_IFMT {
|
||||
unix.Close(fd)
|
||||
return errors.New("an entry changed while it was walked, and the walk stopped there")
|
||||
}
|
||||
if isDir {
|
||||
return w.dir(fd, opened, resume)
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
if err := w.read(); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.visited(fd, opened, false)
|
||||
}
|
||||
|
||||
// visited visits one entry and counts what it did. An error that stops the walk stops it; any other failure on
|
||||
// one entry (an immutable file, say) is counted failed, its words kept once, and the walk goes on.
|
||||
func (w *treeWalk) visited(fd int, st unix.Stat_t, folder bool) error {
|
||||
did, err := w.visit(fd, st, folder)
|
||||
var stop errStopWalk
|
||||
if errors.As(err, &stop) {
|
||||
return stop.err
|
||||
}
|
||||
if err != nil {
|
||||
w.counts.Failed++
|
||||
if w.Failed == "" {
|
||||
w.Failed = err.Error()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case did&visitShared != 0:
|
||||
w.counts.Shared++
|
||||
case did&visitChanged != 0 && folder:
|
||||
w.counts.Folders++
|
||||
case did&visitChanged != 0:
|
||||
w.counts.Files++
|
||||
default:
|
||||
w.counts.Kept++
|
||||
}
|
||||
if did&visitLinked != 0 {
|
||||
w.counts.Linked++
|
||||
}
|
||||
if did&visitStripped != 0 {
|
||||
w.counts.Stripped++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// groupVisit gives each entry the library's group gid, group read and write, and on a folder group enter and the
|
||||
// setgid bit, so what is made beneath keeps the group (ADR 0328 §6, §7). The owner, the owner's bits and others'
|
||||
// bits are never changed — but for what the group change itself takes:
|
||||
//
|
||||
// - **A file's setuid and setgid bits stay taken.** The kernel drops them when a file's group changes; a setgid
|
||||
// file in the library's group would hand anyone who runs it the group, and a setuid one is no library data. They
|
||||
// are never put back, setgid is taken off a file the group change left it on, and each such file is counted.
|
||||
// - **A file with another name** (the *arr apps hardlink downloads into the library) is given the group when an
|
||||
// account owns it: the kernel's protected_hardlinks already keeps an account from linking a file it does not
|
||||
// own. One root owns is left as it is: its other name may be anywhere, and the group would reach it. Both are
|
||||
// counted.
|
||||
func groupVisit(gid int) visitFunc {
|
||||
return func(fd int, st unix.Stat_t, folder bool) (int, error) {
|
||||
did := visitKept
|
||||
if !folder && st.Nlink > 1 {
|
||||
if st.Uid == 0 {
|
||||
return visitShared, nil
|
||||
}
|
||||
did |= visitLinked
|
||||
}
|
||||
mode := uint32(st.Mode) & 0o7777
|
||||
want := mode | 0o060
|
||||
if folder {
|
||||
want = mode | 0o070 | unix.S_ISGID
|
||||
}
|
||||
regroup := int(st.Gid) != gid
|
||||
if !folder && regroup {
|
||||
want &^= unix.S_ISUID | unix.S_ISGID
|
||||
}
|
||||
if !regroup && want == mode {
|
||||
return did &^ visitLinked, nil
|
||||
}
|
||||
if regroup {
|
||||
if err := unix.Fchown(fd, -1, gid); err != nil {
|
||||
return 0, fmt.Errorf("an entry's group could not be set: %v", err)
|
||||
}
|
||||
}
|
||||
if err := unix.Fchmod(fd, want); err != nil {
|
||||
return 0, fmt.Errorf("an entry's mode could not be set: %v", err)
|
||||
}
|
||||
if !folder && mode&(unix.S_ISUID|unix.S_ISGID) != 0 && want&(unix.S_ISUID|unix.S_ISGID) == 0 {
|
||||
did |= visitStripped
|
||||
}
|
||||
return did | visitChanged, nil
|
||||
}
|
||||
}
|
||||
|
||||
// folderEmpty is whether the folder open on fd holds nothing yet: a root the mesh may keep without asking.
|
||||
func folderEmpty(fd int) (bool, error) {
|
||||
dup, err := unix.Dup(fd)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
f := os.NewFile(uintptr(dup), "")
|
||||
defer f.Close()
|
||||
names, err := f.Readdirnames(1)
|
||||
switch {
|
||||
case len(names) > 0:
|
||||
return false, nil
|
||||
case err == nil, errors.Is(err, io.EOF):
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the folder could not be read: %v", err)
|
||||
}
|
||||
|
||||
// FolderFacts is what the engine reads of a library's root before keeping it: its owner, group, mode and whether
|
||||
// it holds data.
|
||||
type FolderFacts struct {
|
||||
UID, GID uint32
|
||||
Mode uint32
|
||||
Empty bool
|
||||
}
|
||||
|
||||
// readFolder reads a folder's facts through the descriptor a mesh directory is opened by.
|
||||
func readFolder(path string) (FolderFacts, error) {
|
||||
if err := keptPath(path); err != nil {
|
||||
return FolderFacts{}, err
|
||||
}
|
||||
mw := &meshWalk{want: path}
|
||||
fd, err := mw.open()
|
||||
if err != nil {
|
||||
return FolderFacts{}, err
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
if mw.At != path {
|
||||
return FolderFacts{}, errors.New("the folder is reached through a link, and the engine keeps no folder " +
|
||||
"through one")
|
||||
}
|
||||
var st unix.Stat_t
|
||||
if err := unix.Fstat(fd, &st); err != nil {
|
||||
return FolderFacts{}, err
|
||||
}
|
||||
empty, err := folderEmpty(fd)
|
||||
if err != nil {
|
||||
return FolderFacts{}, err
|
||||
}
|
||||
return FolderFacts{UID: st.Uid, GID: st.Gid, Mode: uint32(st.Mode) & 0o7777, Empty: empty}, nil
|
||||
}
|
||||
|
||||
// keepFolders keeps a library's root and each of its requested sub-paths — the folders themselves, never what is
|
||||
// in them — in the group gid with group read, write and enter and the setgid bit (ADR 0328 §6). A sub-path that is
|
||||
// missing is made, each level with the root's owner, the group, and the root's owner's and others' bits beside
|
||||
// group rwx and setgid (§9). No link is followed (the root is reached as a mesh directory is, and every level below
|
||||
// it by O_NOFOLLOW), nothing is made or changed on another filesystem, and the owner and the owner's and others'
|
||||
// bits of a folder that was there are never changed. How many folders it changed and made.
|
||||
func keepFolders(root string, paths []string, gid int) (changed, made int, err error) {
|
||||
_, fd, rst, err := openRoot(root, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
keep := func(fd int, st unix.Stat_t) error {
|
||||
mode := uint32(st.Mode) & 0o7777
|
||||
want := mode | 0o070 | unix.S_ISGID
|
||||
if int(st.Gid) == gid && want == mode {
|
||||
return nil
|
||||
}
|
||||
if int(st.Gid) != gid {
|
||||
if err := unix.Fchown(fd, -1, gid); err != nil {
|
||||
return fmt.Errorf("a folder's group could not be set: %v", err)
|
||||
}
|
||||
}
|
||||
if err := unix.Fchmod(fd, want); err != nil {
|
||||
return fmt.Errorf("a folder's mode could not be set: %v", err)
|
||||
}
|
||||
changed++
|
||||
return nil
|
||||
}
|
||||
if err := keep(fd, rst); err != nil {
|
||||
return changed, made, err
|
||||
}
|
||||
madeMode := uint32(rst.Mode)&0o0707 | 0o070 | unix.S_ISGID
|
||||
for _, p := range paths {
|
||||
n, err := keepPath(fd, rst, p, gid, madeMode, keep)
|
||||
made += n
|
||||
if err != nil {
|
||||
return changed, made, err
|
||||
}
|
||||
}
|
||||
return changed, made, nil
|
||||
}
|
||||
|
||||
// keepPath walks one sub-path down from the root open on rootFd, making each missing level, and keeps the last.
|
||||
func keepPath(rootFd int, rst unix.Stat_t, p string, gid int, madeMode uint32,
|
||||
keep func(int, unix.Stat_t) error) (int, error) {
|
||||
made := 0
|
||||
cur, err := unix.Dup(rootFd)
|
||||
if err != nil {
|
||||
return made, err
|
||||
}
|
||||
defer func() { unix.Close(cur) }()
|
||||
const dirFlags = unix.O_RDONLY | unix.O_DIRECTORY | unix.O_NOFOLLOW | unix.O_CLOEXEC
|
||||
levels := strings.Split(p, "/")
|
||||
for i, name := range levels {
|
||||
next, err := unix.Openat(cur, name, dirFlags, 0)
|
||||
if errors.Is(err, unix.ENOENT) {
|
||||
// Made for its maker alone (the engine, as root), so it is nobody else's until it is the root's.
|
||||
if mkErr := unix.Mkdirat(cur, name, 0o700); mkErr != nil && !errors.Is(mkErr, unix.EEXIST) {
|
||||
return made, fmt.Errorf("a level of the path %s could not be made: %v", p, mkErr)
|
||||
}
|
||||
if next, err = unix.Openat(cur, name, dirFlags, 0); err == nil {
|
||||
if err = unix.Fchown(next, int(rst.Uid), gid); err == nil {
|
||||
err = unix.Fchmod(next, madeMode)
|
||||
}
|
||||
if err != nil {
|
||||
unix.Close(next)
|
||||
return made, fmt.Errorf("a level of the path %s made could not be given the root's owner "+
|
||||
"and the library's group: %v", p, err)
|
||||
}
|
||||
made++
|
||||
}
|
||||
}
|
||||
if errors.Is(err, unix.ELOOP) || errors.Is(err, unix.ENOTDIR) {
|
||||
return made, fmt.Errorf("the path %s is not a folder below the root (a link, or a file), and the "+
|
||||
"engine keeps no path through one", p)
|
||||
}
|
||||
if err != nil {
|
||||
return made, fmt.Errorf("the path %s could not be opened: %v", p, err)
|
||||
}
|
||||
unix.Close(cur)
|
||||
cur = next
|
||||
var st unix.Stat_t
|
||||
if err := unix.Fstat(cur, &st); err != nil {
|
||||
return made, err
|
||||
}
|
||||
if st.Dev != rst.Dev {
|
||||
return made, fmt.Errorf("the path %s lies on another filesystem than the root, and the engine keeps "+
|
||||
"nothing there", p)
|
||||
}
|
||||
if i == len(levels)-1 {
|
||||
if err := keep(cur, st); err != nil {
|
||||
return made, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return made, nil
|
||||
}
|
||||
|
||||
// MountFsid is the kernel's id of the filesystem at point (statfs f_fsid), in hex; empty when it cannot be read.
|
||||
// A variable so a test can stand in for a filesystem it cannot mount.
|
||||
var MountFsid = func(point string) string {
|
||||
var st unix.Statfs_t
|
||||
if err := unix.Statfs(point, &st); err != nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%08x%08x", uint32(st.Fsid.Val[0]), uint32(st.Fsid.Val[1]))
|
||||
}
|
||||
|
||||
// MountUUID is the UUID of the filesystem at point when it is on a block device: the name in /dev/disk/by-uuid of
|
||||
// the device the mount's own device number is. Empty for one on no block device (NFS, FUSE, ZFS, a btrfs
|
||||
// subvolume's own device). A variable so a test can stand in for disks it cannot make.
|
||||
var MountUUID = func(point string) string {
|
||||
var st unix.Stat_t
|
||||
if err := unix.Stat(point, &st); err != nil {
|
||||
return ""
|
||||
}
|
||||
const dir = "/dev/disk/by-uuid"
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, e := range entries {
|
||||
var dev unix.Stat_t
|
||||
if unix.Stat(dir+"/"+e.Name(), &dev) == nil && dev.Mode&unix.S_IFMT == unix.S_IFBLK && dev.Rdev == st.Dev {
|
||||
return e.Name()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SampleLimits bound the sample that judges whether a root's data is already the library's: so many entries a
|
||||
// folder, so many levels down, for so long. A sample that runs out of time is not in shape: it asks.
|
||||
var SampleLimits = struct {
|
||||
PerFolder, Levels int
|
||||
Time time.Duration
|
||||
}{PerFolder: 20, Levels: 3, Time: 2 * time.Second}
|
||||
|
||||
// sampleInShape is whether the first entries of each folder under root, a few levels down, are all in the group gid
|
||||
// with group read and write, and folders with group enter and setgid (ADR 0328 §7). Reads only, by descriptor: no
|
||||
// link followed, no other filesystem entered. A file with more than one name is not judged (the take act leaves it).
|
||||
func sampleInShape(root string, gid int) bool {
|
||||
_, fd, st, err := openRoot(root, WalkLimits{Entries: 1, Time: time.Minute}, machineStats, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
deadline := time.Now().Add(SampleLimits.Time)
|
||||
return sampleFolder(fd, uint64(st.Dev), gid, SampleLimits.Levels, deadline)
|
||||
}
|
||||
|
||||
// sampleFolder judges the folder open on fd's first entries, and the folders among them levels further. It closes fd.
|
||||
func sampleFolder(fd int, dev uint64, gid, levels int, deadline time.Time) bool {
|
||||
f := os.NewFile(uintptr(fd), "")
|
||||
defer f.Close()
|
||||
names, err := f.Readdirnames(SampleLimits.PerFolder)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return false
|
||||
}
|
||||
for _, name := range names {
|
||||
if time.Now().After(deadline) {
|
||||
return false
|
||||
}
|
||||
var st unix.Stat_t
|
||||
if err := unix.Fstatat(fd, name, &st, unix.AT_SYMLINK_NOFOLLOW); err != nil {
|
||||
return false
|
||||
}
|
||||
if uint64(st.Dev) != dev {
|
||||
continue
|
||||
}
|
||||
mode := uint32(st.Mode)
|
||||
switch mode & unix.S_IFMT {
|
||||
case unix.S_IFREG:
|
||||
if st.Nlink == 1 && (int(st.Gid) != gid || mode&0o060 != 0o060) {
|
||||
return false
|
||||
}
|
||||
case unix.S_IFDIR:
|
||||
if int(st.Gid) != gid || mode&0o2070 != 0o2070 {
|
||||
return false
|
||||
}
|
||||
if levels > 1 {
|
||||
next, err := unix.Openat(fd, name, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
|
||||
if err != nil || !sampleFolder(next, dev, gid, levels-1, deadline) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
//go:build !linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// The node-engine runs on Linux: a library is kept only by descriptor, which this system cannot. Everything refuses,
|
||||
// saying so, and changes nothing.
|
||||
|
||||
var errNoKeeping = errors.New("a library is kept only by descriptor, which this system cannot; nothing was changed")
|
||||
|
||||
// FolderFacts is what the engine reads of a library's root before keeping it.
|
||||
type FolderFacts struct {
|
||||
UID, GID uint32
|
||||
Mode uint32
|
||||
Empty bool
|
||||
}
|
||||
|
||||
func readFolder(string) (FolderFacts, error) { return FolderFacts{}, errNoKeeping }
|
||||
func keepFolders(string, []string, int) (int, int, error) { return 0, 0, errNoKeeping }
|
||||
func ReconcileLibraries(context.Context, *declaration.Declaration) []string { return nil }
|
||||
|
||||
func TakeLibrary(context.Context, *declaration.Declaration, string, string, string, string) (<-chan struct{},
|
||||
string, error) {
|
||||
return nil, "", errNoKeeping
|
||||
}
|
||||
|
||||
// MountFsid reads no filesystem id where a library is not kept.
|
||||
var MountFsid = func(string) string { return "" }
|
||||
|
||||
// MountUUID reads no UUID where a library is not kept.
|
||||
var MountUUID = func(string) string { return "" }
|
||||
|
||||
func sampleInShape(string, int) bool { return false }
|
||||
@@ -106,11 +106,11 @@ type AdoptedRewrite struct {
|
||||
}
|
||||
|
||||
// RewriteAdopted makes the produced bundle one for an adopted machine: the foundation's own filter
|
||||
// taken out, and the mesh's guard put in its place, guarding on this node the store's port, the
|
||||
// broker's management port and the broker's plaintext port. The last is published on every
|
||||
// interface and the foundation's filter admits it from the private network only, so a found
|
||||
// firewall that filters only incoming traffic would leave it reachable from anywhere (novox/hq ADR
|
||||
// 0103). Every one is a port of a module genesis takes. The nftables package stays: the guard is loaded with it, and
|
||||
// taken out, and the mesh's guard put in its place, guarding on this node the store's port. It is
|
||||
// published on every interface and the foundation's filter admits it from the private network only,
|
||||
// so a found firewall that filters only incoming traffic would leave it reachable from anywhere
|
||||
// (novox/hq ADR 0103). The bus is reached from anywhere on purpose — a node enrols over it — and its
|
||||
// monitoring port is published on loopback only, so neither is guarded. The nftables package stays: the guard is loaded with it, and
|
||||
// installing a package loads no table. Openings are not the bundle's — the first push declares
|
||||
// them, once there is a controller to derive them.
|
||||
func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) {
|
||||
@@ -128,11 +128,11 @@ func RewriteAdopted(r *Rewritten, p FoundationPorts) (AdoptedRewrite, error) {
|
||||
out.Removed = append(out.Removed, id)
|
||||
}
|
||||
|
||||
out.Guarded = []int{p.Store, p.Management, p.AMQP}
|
||||
out.Guarded = []int{p.Store}
|
||||
var text bytes.Buffer
|
||||
text.WriteString(",\n // The mesh's guard (novox/hq ADR 0100): this machine is adopted, so its own firewall\n" +
|
||||
" // stays in force and the foundation's filter is not loaded. The guard only refuses: the\n" +
|
||||
" // store's port and the broker's management and plaintext ports, except from the machine and\n" +
|
||||
" // store's port, except from the machine and\n" +
|
||||
" // the private network.")
|
||||
for _, res := range guardResources(out.Guarded) {
|
||||
var one bytes.Buffer
|
||||
@@ -185,7 +185,7 @@ func (r Rewritten) declares(id string) bool {
|
||||
// container runtime's configuration whole — and neither is anything the operator installs later.
|
||||
var genesisTakes = map[string]bool{
|
||||
RegistryModule: true, ControlPlaneModule: true, BuilderModule: true,
|
||||
"postgres": true, "lavinmq": true, "mesh-vault": true, "mesh-catalog": true,
|
||||
"postgres": true, BusModule: true, "mesh-vault": true, "mesh-catalog": true,
|
||||
}
|
||||
|
||||
// takeIfAdopted takes one of genesis's own modules on an adopted node, once it is assigned and
|
||||
|
||||
@@ -70,7 +70,7 @@ func TestTheGuardRefusesOnlyWhatIsAddressedToThisMachine(t *testing.T) {
|
||||
|
||||
func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) {
|
||||
r := producedBundle(t)
|
||||
p := FoundationPorts{Store: 5433, Management: 15673, AMQP: 5773}
|
||||
p := FoundationPorts{Store: 5433}
|
||||
if _, err := RewritePorts(&r, p, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -101,9 +101,9 @@ func TestAnAdoptedBundleLoadsNoDroppingTableAndExactlyTheGuard(t *testing.T) {
|
||||
if len(guards) != 1 {
|
||||
t.Fatalf("%d guard table(s)", len(guards))
|
||||
}
|
||||
// The store's, the broker's plaintext and its management port: each one the filter admits
|
||||
// from the private network only (novox/hq ADR 0103).
|
||||
if !strings.Contains(guards[0].Content, "tcp dport { 5433, 5773, 15673 } drop") {
|
||||
// The store's: the one foundation port the filter admits from the private network only
|
||||
// (novox/hq ADR 0103). The bus is reached from anywhere, its monitoring only on loopback.
|
||||
if !strings.Contains(guards[0].Content, "tcp dport 5433 drop") && !strings.Contains(guards[0].Content, "tcp dport { 5433 } drop") {
|
||||
t.Errorf("the guard does not refuse this node's ports: %s", guards[0].Content)
|
||||
}
|
||||
if strings.Count(guards[0].Content, "accept") != 1 || !strings.Contains(guards[0].Content, "policy accept") {
|
||||
|
||||
@@ -500,7 +500,7 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
for _, c := range []struct {
|
||||
what, path string
|
||||
made bool
|
||||
}{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"broker admin", BrokerAdminFile, creds.BrokerMade}} {
|
||||
}{{"store superuser", StoreSuperuserFile, creds.StoreMade}, {"bus controller", BusControllerFile, creds.BusMade}} {
|
||||
switch {
|
||||
case c.made && o.DryRun:
|
||||
say(fmt.Sprintf(" %-17s would be made and kept at %s (0600)", c.what, c.path))
|
||||
@@ -510,9 +510,9 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
say(fmt.Sprintf(" %-17s already at %s — kept", c.what, c.path))
|
||||
}
|
||||
}
|
||||
say(fmt.Sprintf(" credentials the template's bootstrap and guest are gone: %d store and %d broker "+
|
||||
"connection(s) rewritten, the store reads its password from a file, the broker's admin is changed once it answers",
|
||||
root.StoreURLs, root.BrokerURLs))
|
||||
say(fmt.Sprintf(" credentials the template's bootstrap is gone: %d store and %d bus "+
|
||||
"connection(s) rewritten, the store reads its password from a file, the bus is raised with the controller's account",
|
||||
root.StoreURLs, root.BusURLs))
|
||||
if rewritten.Renamed {
|
||||
say(fmt.Sprintf(" control plane %s, renamed from %s",
|
||||
rewritten.TempName, rewritten.WasCalled))
|
||||
@@ -744,8 +744,8 @@ func Run(ctx context.Context, o Options, d Deps, say func(string)) (Result, erro
|
||||
}
|
||||
|
||||
// ---- broker ---------------------------------------------------------------------------
|
||||
say("broker — the foundation's broker, adopted as the lavinmq module: one bus, not two")
|
||||
if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, say); err != nil {
|
||||
say("broker — the foundation's bus, adopted as the nats module: one bus, not two")
|
||||
if err := InstallBroker(ctx, o, permanentControl, rewritten.Declaration, d.Fetch, say); err != nil {
|
||||
return result, failed(StepBroker, err)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BuilderModule is the module that lets a mesh produce anything at all.
|
||||
const BuilderModule = "builder"
|
||||
// BuilderModule is the module that lets a mesh produce anything at all: the build agent, which holds
|
||||
// the node seat `node-build-agent` (novox/hq ADR 0190). The one-holder `builder` module it replaced
|
||||
// left the catalogue on 2026-10-03, and genesis went on asking for it (issue 596).
|
||||
const BuilderModule = "build-agent"
|
||||
|
||||
// BuilderRepository is what its image is called in this mesh's own registry.
|
||||
const BuilderRepository = "mesh-builder"
|
||||
@@ -50,10 +53,10 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
|
||||
manifest, err := readManifest(o.Catalogue, BuilderModule)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%w\n"+
|
||||
"This is the manifest that makes the builder an ordinary module. Without it the mesh "+
|
||||
"This is the manifest that makes the build agent an ordinary module. Without it the mesh "+
|
||||
"has the image and no way to run it, so nothing can be built here", err)
|
||||
}
|
||||
pinned, places, err := pinPlaceholder(manifest, published.Reference, BuilderModule)
|
||||
pinned, places, err := pinTheCarriedBuilder(manifest, published.Reference)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -66,21 +69,72 @@ func InstallBuilder(ctx context.Context, o Options, d Deps, control controlPlane
|
||||
}
|
||||
|
||||
// Before the push, so the account is in the declaration the machine receives rather than in
|
||||
// the one after it. A builder that arrives without its credential starts, finds nothing it may
|
||||
// read, and waits — which looks exactly like a builder with no work.
|
||||
account := o.Node + "-" + BuilderModule
|
||||
if _, err := control.tell(ctx, "builder", "issue", account, "--node", o.Node); err != nil {
|
||||
// Said and carried on. An account that already exists is the ordinary case on a re-run,
|
||||
// and the push below is what makes either state true on the machine.
|
||||
if !strings.Contains(err.Error(), "already") {
|
||||
return out, fmt.Errorf("the builder has no broker account, so it can take no work: %w", err)
|
||||
}
|
||||
say(" broker account " + account + " — already issued")
|
||||
} else {
|
||||
out.Account = true
|
||||
say(" broker account " + account + ", scoped to what it consumes and emits")
|
||||
// the one after it. A build agent that arrives without its credential starts, finds nothing it
|
||||
// may read, and waits — which looks exactly like an agent with no work. The same act as any
|
||||
// module's account (`module issue`), which a re-run repeats harmlessly.
|
||||
if _, err := control.tell(ctx, "module", "issue", BuilderModule, "--node", o.Node); err != nil {
|
||||
return out, fmt.Errorf("the build agent has no account on the bus, so it can take no work: %w", err)
|
||||
}
|
||||
out.Account = true
|
||||
say(" bus account " + BuilderModule + " on " + o.Node + ", scoped to what it consumes and emits")
|
||||
|
||||
out.Installed.Pushed, err = pushNode(ctx, o, control, say)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// pinTheCarriedBuilder names the image this installer carried and published in the build agent's
|
||||
// manifest, in place of the artifact the mesh would build for it.
|
||||
//
|
||||
// **The one module genesis cannot have built** (novox/hq ADR 0073): the build agent is what builds,
|
||||
// and at this step the mesh has no build agent. The catalogue's manifest names its server container by
|
||||
// the artifact it builds (`"artifact": "server"`, an image built from mesh-controller's
|
||||
// cmd/mesh-builder); the image this installer carries is that build, made from the same program. So
|
||||
// every container built from that image artifact runs the published reference instead, and the rest
|
||||
// of the manifest — its build, its seat, its requirements — is the catalogue's as written. The mesh
|
||||
// rebuilds it from the catalogue like any module from then on.
|
||||
//
|
||||
// A manifest still carrying the old placeholder digest is pinned the old way. One naming neither an
|
||||
// image artifact nor a placeholder is refused: there would be nothing to put the carried image in.
|
||||
func pinTheCarriedBuilder(manifest []byte, reference string) ([]byte, int, error) {
|
||||
if bytes.Contains(manifest, []byte(placeholderDigest)) {
|
||||
return pinPlaceholder(manifest, reference, BuilderModule)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
return nil, 0, fmt.Errorf("the %s module's manifest is not readable: %w", BuilderModule, err)
|
||||
}
|
||||
images := map[string]bool{}
|
||||
if build, ok := m["build"].(map[string]any); ok {
|
||||
artifacts, _ := build["artifacts"].([]any)
|
||||
for _, a := range artifacts {
|
||||
if art, ok := a.(map[string]any); ok && art["kind"] == "image" {
|
||||
if name, ok := art["name"].(string); ok {
|
||||
images[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
places := 0
|
||||
resources, _ := m["resources"].([]any)
|
||||
for _, r := range resources {
|
||||
res, ok := r.(map[string]any)
|
||||
if !ok || res["type"] != "container" {
|
||||
continue
|
||||
}
|
||||
if name, ok := res["artifact"].(string); ok && images[name] {
|
||||
delete(res, "artifact")
|
||||
res["image"] = reference
|
||||
places++
|
||||
}
|
||||
}
|
||||
if places == 0 {
|
||||
return nil, 0, fmt.Errorf(
|
||||
"the %s module's manifest runs no container from an image it builds, so there is nowhere to "+
|
||||
"put the builder this installer carried and published (%s)", BuilderModule, reference)
|
||||
}
|
||||
pinned, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return append(pinned, '\n'), places, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -44,15 +45,15 @@ func TestTheBuildersPackageRegistryPortIsASettingAndNotTheManifest(t *testing.T)
|
||||
[]byte(theBuildersBinding), quietly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := c.settings["builder-settings.json"]; got != `{"serves":{"port":3100}}` {
|
||||
if got := c.settings[BuilderModule+"-settings.json"]; got != `{"serves":{"port":3100}}` {
|
||||
t.Errorf("the builder was told %q about the package registry's port", got)
|
||||
}
|
||||
// The manifest reaches the mesh as the catalogue wrote it. A port rewritten into it here is a
|
||||
// port the next registration from the catalogue silently takes back.
|
||||
if got := c.settings["builder-module.json"]; got != theBuildersBinding {
|
||||
if got := c.settings[BuilderModule+"-module.json"]; got != theBuildersBinding {
|
||||
t.Errorf("the installer changed the builder's manifest:\n%s", got)
|
||||
}
|
||||
set, push := c.index("settings set builder"), c.index("push anchor")
|
||||
set, push := c.index("settings set "+BuilderModule), c.index("push anchor")
|
||||
if set < 0 || push < 0 || set > push {
|
||||
t.Fatalf("the port was not set before the push that raises the builder: %v", c.told)
|
||||
}
|
||||
@@ -167,3 +168,73 @@ func TestAForgeManifestTheCatalogueDoesNotHaveIsNamed(t *testing.T) {
|
||||
t.Errorf("a missing forge manifest was not explained: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// theCataloguesModule is the catalogue's own manifest of a module, never a copy: the clone beside a merge
|
||||
// check (MESH_CHECK_BESIDE), where it must be; else a checkout beside this one; else the test skips aloud.
|
||||
func theCataloguesModule(t *testing.T, module string) []byte {
|
||||
t.Helper()
|
||||
inside := filepath.Join("mesh-catalog", "modules", module, "module.json")
|
||||
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
|
||||
raw, err := os.ReadFile(filepath.Join(root, inside))
|
||||
if err != nil {
|
||||
t.Fatalf("the catalogue beside this check (MESH_CHECK_BESIDE=%s) has no %s: %v", root, inside, err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "..", inside))
|
||||
if err != nil {
|
||||
t.Skipf("NOT JUDGED: no catalogue beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it")
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// **Genesis installs the build agent the catalogue holds** (novox/hq issue 596, ADR 0190): the
|
||||
// one-holder `builder` left the catalogue on 2026-10-03, and genesis stopped at step 13 asking for it.
|
||||
// The catalogue's own manifest is read, and the image the installer carried is put where the manifest
|
||||
// names the image it builds.
|
||||
func TestGenesisInstallsTheCataloguesBuildAgent(t *testing.T) {
|
||||
manifest := theCataloguesModule(t, BuilderModule)
|
||||
const reference = "127.0.0.1:5000/mesh-builder@sha256:" + "4444444444444444444444444444444444444444444444444444444444444444"
|
||||
pinned, places, err := pinTheCarriedBuilder(manifest, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"claims"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(pinned, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != BuilderModule || len(m.Claims) == 0 || m.Claims[0].Name != "node-build-agent" {
|
||||
t.Fatalf("genesis installs %q claiming %v; the build agent holds node-build-agent (ADR 0190)", m.Module, m.Claims)
|
||||
}
|
||||
running := 0
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" {
|
||||
continue
|
||||
}
|
||||
if _, still := r["artifact"]; still {
|
||||
t.Errorf("the container %v still names an artifact nothing has built yet", r["name"])
|
||||
}
|
||||
if r["image"] == reference {
|
||||
running++
|
||||
}
|
||||
}
|
||||
if places != 1 || running != 1 {
|
||||
t.Errorf("the carried builder runs in %d container(s) (%d place(s) said); the build agent runs one", running, places)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest that runs nothing from an image it builds has nowhere to put the carried builder, and is
|
||||
// refused rather than registered running nothing.
|
||||
func TestABuildAgentManifestWithNowhereForTheCarriedBuilderIsRefused(t *testing.T) {
|
||||
_, _, err := pinTheCarriedBuilder([]byte(`{"module":"build-agent","resources":[{"id":"server","type":"container","name":"x","artifact":"server"}],"build":{"artifacts":[{"name":"server","kind":"bundle"}]}}`),
|
||||
"127.0.0.1:5000/mesh-builder@sha256:"+strings.Repeat("4", 64))
|
||||
if err == nil || !strings.Contains(err.Error(), "nowhere to put") {
|
||||
t.Fatalf("a manifest with no image artifact was pinned: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -311,6 +311,17 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) {
|
||||
for name, path := range m.OwnSecrets {
|
||||
secretAt[path] = name
|
||||
}
|
||||
// **A file the module itself declares is its own, not a secret** (novox/hq issue 597). The
|
||||
// controller reads its merge window's settings from a file its manifest writes from its settings
|
||||
// (MESH_MERGE_WINDOW_FILE, ADR 0276): the mesh writes it with the module's declaration, and there
|
||||
// is nothing for genesis to carry in. Only a `…_FILE` naming neither an own-secret nor a file of
|
||||
// the module is refused.
|
||||
declared := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if r.Type == "file" && r.Path != "" {
|
||||
declared[r.Path] = true
|
||||
}
|
||||
}
|
||||
|
||||
wanted := map[string]string{}
|
||||
for _, r := range m.Resources {
|
||||
@@ -333,10 +344,13 @@ func secretsByVariableIn(manifest []byte) (map[string]string, error) {
|
||||
on = from
|
||||
}
|
||||
secret, named := secretAt[on]
|
||||
if !named && declared[on] {
|
||||
continue
|
||||
}
|
||||
if !named {
|
||||
return nil, fmt.Errorf(
|
||||
"the %s module's container reads %s from %s, and no own-secret of that "+
|
||||
"module writes that file.\n"+
|
||||
"the %s module's container reads %s from %s, and neither an own-secret nor "+
|
||||
"a file of that module writes that file.\n"+
|
||||
"So the mesh would seal nothing there and the control plane would find "+
|
||||
"an empty file where a connection string has to be. The manifest has to "+
|
||||
"name the two ends the same, directly or through a mount",
|
||||
|
||||
@@ -30,14 +30,13 @@ const theControlPlaneModule = `{
|
||||
"inventory": "/var/lib/mesh/mesh-controller/inventory",
|
||||
"identity": "/var/lib/mesh/mesh-controller/identity",
|
||||
"licences": "/var/lib/mesh/mesh-controller/licences",
|
||||
"broker": "/var/lib/mesh/mesh-controller/broker",
|
||||
"broker-management": "/var/lib/mesh/mesh-controller/broker-management"
|
||||
"bus": "/var/lib/mesh/mesh-controller/bus"
|
||||
},
|
||||
"resources": [
|
||||
{"id": "mesh-state", "type": "directory", "path": "/var/lib/mesh/mesh-controller", "mode": "0700"},
|
||||
{"id": "broker-env", "type": "file", "path": "/var/lib/mesh/mesh-controller/broker.env",
|
||||
"mode": "0600",
|
||||
"content": "MESH_BROKER_AMQP=${secret:broker}\nMESH_BROKER_MANAGEMENT=${secret:broker-management}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
|
||||
"content": "MESH_BUS_NATS=${secret:bus}\nMESH_BROKER_ADDRESS=${machine:at}:5671\n"},
|
||||
{"id": "server", "type": "container", "name": "mesh-controller",
|
||||
"image": "` + builtImage + `",
|
||||
"network": "host", "args": ["serve"],
|
||||
@@ -133,11 +132,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
|
||||
// them at /run/secrets. Matching on the path alone would find nothing and refuse a correct
|
||||
// manifest, which is exactly the ordinary case in the catalogue.
|
||||
for variable, secret := range map[string]string{
|
||||
"MESH_STORE_INVENTORY": "inventory",
|
||||
"MESH_STORE_IDENTITY": "identity",
|
||||
"MESH_STORE_LICENCES": "licences",
|
||||
"MESH_BROKER_AMQP": "broker",
|
||||
"MESH_BROKER_MANAGEMENT": "broker-management",
|
||||
"MESH_STORE_INVENTORY": "inventory",
|
||||
"MESH_STORE_IDENTITY": "identity",
|
||||
"MESH_STORE_LICENCES": "licences",
|
||||
"MESH_BUS_NATS": "bus",
|
||||
} {
|
||||
if wanted[variable] != secret {
|
||||
t.Errorf("%s would be accepted as %q, want %q", variable, wanted[variable], secret)
|
||||
@@ -161,9 +159,10 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Three stores and both halves of the broker: everything the foundation made and nothing else.
|
||||
if len(delivered) != 5 {
|
||||
t.Fatalf("%d values were delivered, and the foundation names five: %v",
|
||||
// Three stores and the controller's account on the bus: everything the foundation made and
|
||||
// nothing else.
|
||||
if len(delivered) != 4 {
|
||||
t.Fatalf("%d values were delivered, and the foundation names four: %v",
|
||||
len(delivered), delivered)
|
||||
}
|
||||
for _, secret := range delivered {
|
||||
@@ -177,12 +176,15 @@ func TestTheStoreConnectionsComeFromTheBundleThatMadeThem(t *testing.T) {
|
||||
// secret in a mesh is one the mesh made; `secret accept` is only for what predates the mesh.
|
||||
func TestASecretTheFoundationNeverMadeIsLeftToTheMesh(t *testing.T) {
|
||||
extra := strings.Replace(theControlPlaneModule,
|
||||
`"broker": "/var/lib/mesh/mesh-controller/broker",`,
|
||||
`"broker": "/var/lib/mesh/mesh-controller/broker",
|
||||
"something-new": "/var/lib/mesh/mesh-controller/something-new",`, 1)
|
||||
`"bus": "/var/lib/mesh/mesh-controller/bus"`,
|
||||
`"bus": "/var/lib/mesh/mesh-controller/bus",
|
||||
"something-new": "/var/lib/mesh/mesh-controller/something-new"`, 1)
|
||||
extra = strings.Replace(extra,
|
||||
`"content": "MESH_BROKER_AMQP=${secret:broker}\n`,
|
||||
`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BROKER_AMQP=${secret:broker}\n`, 1)
|
||||
`"content": "MESH_BUS_NATS=${secret:bus}\n`,
|
||||
`"content": "MESH_SOMETHING_NEW=${secret:something-new}\nMESH_BUS_NATS=${secret:bus}\n`, 1)
|
||||
if extra == theControlPlaneModule {
|
||||
t.Fatal("the fixture no longer says what this test adds to")
|
||||
}
|
||||
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
|
||||
@@ -94,6 +94,13 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
||||
switch mine, err := identity.Load(where); {
|
||||
case err == nil && mine.Node == o.Node:
|
||||
say(" already enrolled " + o.Node + " — " + where)
|
||||
// **Enrolled is not adopted** (novox/hq ADR 0331 §5). Until the bus is the nats module, its
|
||||
// user list is the installer's to place, so it is placed again, before anything dials the
|
||||
// bus as a node: the controller's composition is the one that is right, whatever an earlier
|
||||
// run left there. The bundle seeds the list only where none is (create-once, ADR 0333 §4).
|
||||
if err := placeUntilAdopted(ctx, control, say); err != nil {
|
||||
return out, err
|
||||
}
|
||||
case err == nil:
|
||||
return out, fmt.Errorf(
|
||||
"this machine is already node %q and was asked to become %q.\n"+
|
||||
@@ -159,10 +166,16 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla
|
||||
|
||||
// busAccounts and busContainer are where the installer's bundle raises the bus: the file its
|
||||
// configuration includes, and the container that reads it. The installer raised them, so it is the
|
||||
// one that knows them (examples/foundation-first-node-nats.lock).
|
||||
// one that knows them (examples/foundation-first-node.lock). A bridge until the bus is adopted as the
|
||||
// nats module, which receives the list in its declaration from then on (novox/hq ADR 0331 §5).
|
||||
//
|
||||
// **The module's path and the module's container** (novox/hq ADR 0333 §4): the bundle raises the bus
|
||||
// where the `nats` module keeps it, so the list placed here is the one the module goes on writing, in
|
||||
// the same file, read by the same container. The tests hold both to the bundle and to the module's
|
||||
// manifest (`bus-users` and its server container).
|
||||
const (
|
||||
busAccounts = "/var/lib/mesh-bus-conf/accounts.conf"
|
||||
busContainer = "mesh-broker"
|
||||
busAccounts = "/var/lib/nats-module/conf/accounts.conf"
|
||||
busContainer = "mesh-broker-nats"
|
||||
)
|
||||
|
||||
// placeTheBusUsers writes the mesh's composed user list beside the bus the installer raised, and makes
|
||||
@@ -191,6 +204,27 @@ func placeTheBusUsers(ctx context.Context, control controlPlane, say func(string
|
||||
return nil
|
||||
}
|
||||
|
||||
// placeUntilAdopted places the bus's users unless the nats module has them now (novox/hq ADR 0331
|
||||
// §5): from the bus's adoption on, the module receives the list in its declaration, and the
|
||||
// installer's bridge is not run again.
|
||||
//
|
||||
// **Placed again rather than left alone.** The other way out — writing the bundle's list only
|
||||
// where no list exists yet — needs a file the applier creates and then never reconciles, which no
|
||||
// resource of a declaration is; and it would keep whatever list was there, while the controller's
|
||||
// composition is the one that is right. So a re-run converges the file on what the controller says.
|
||||
// The module is asked of the controller by name (`module list`), which is where adoption registers it.
|
||||
func placeUntilAdopted(ctx context.Context, control controlPlane, say func(string)) error {
|
||||
known, err := control.tell(ctx, "module", "list")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mentions(known, BusModule) {
|
||||
say(" bus users the " + BusModule + " module's — not placed by the installer")
|
||||
return nil
|
||||
}
|
||||
return placeTheBusUsers(ctx, control, say)
|
||||
}
|
||||
|
||||
// runTheHost makes sure something on this machine is listening to the mesh, and proves it.
|
||||
//
|
||||
// **The installer does not install the service, and says so.** A unit file is a packaging decision
|
||||
|
||||
@@ -59,6 +59,9 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) {
|
||||
switch {
|
||||
case strings.Contains(joined, "node list"):
|
||||
return "anchor here 01J0\n", nil
|
||||
// The bus is the nats module already, so its users are the module's.
|
||||
case strings.Contains(joined, "module list"):
|
||||
return "nats 1\npostgres 1\n", nil
|
||||
// An agent is running here too. Both halves are needed: enrolling makes the mesh hear from
|
||||
// a machine once, so the first answer alone also describes a machine with no agent at all.
|
||||
case name == "pgrep":
|
||||
@@ -82,6 +85,11 @@ func TestAMachineThatHasAlreadyEnrolledIsNotEnrolledAgain(t *testing.T) {
|
||||
t.Errorf("a token was issued for a machine that already holds an identity: %v",
|
||||
runtime.commands)
|
||||
}
|
||||
// Nor are the bus's users placed: the bus is adopted, so they are the nats module's, and the
|
||||
// installer's bridge is not run again (novox/hq ADR 0331 §5).
|
||||
if runtime.ran("broker accounts") || runtime.ran("accounts.conf") {
|
||||
t.Errorf("the bus's users were placed on a re-run: %v", runtime.commands)
|
||||
}
|
||||
if out.Agent != "already running" {
|
||||
t.Errorf("the host agent is reported as %q", out.Agent)
|
||||
}
|
||||
@@ -95,6 +103,8 @@ func TestAMeshThatHasHeardFromAMachineWithNoAgentStartsOne(t *testing.T) {
|
||||
runtime := &asked{answer: func(name string, args []string) (string, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(joined, "module list"):
|
||||
return "nats 1\n", nil // the bus is adopted: no users to place
|
||||
case strings.Contains(joined, "node list"):
|
||||
// Exactly what enrolling leaves behind, with nothing running.
|
||||
return "anchor here 01J0\n", nil
|
||||
@@ -159,6 +169,8 @@ func TestAHostThatIsRunningAndUnheardOfIsNotAnInstall(t *testing.T) {
|
||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(joined, "module list"):
|
||||
return "nats 1\n", nil // the bus is adopted: no users to place
|
||||
case strings.Contains(joined, "node list"):
|
||||
// Enrolled, and never spoken.
|
||||
return "anchor never spoken 01J0\n", nil
|
||||
@@ -190,6 +202,8 @@ func TestAMachineWithNoHostServiceIsRefusedRatherThanGivenOne(t *testing.T) {
|
||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(joined, "module list"):
|
||||
return "nats 1\n", nil // the bus is adopted: no users to place
|
||||
case strings.Contains(joined, "node list"):
|
||||
return "anchor never spoken 01J0\n", nil
|
||||
case args[0] == "show":
|
||||
@@ -301,8 +315,8 @@ func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
|
||||
|
||||
placed, enrol := []int{}, -1
|
||||
for i, c := range runtime.commands {
|
||||
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker") &&
|
||||
strings.Contains(c, "/var/lib/mesh-bus-conf/accounts.conf") {
|
||||
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "kill -s HUP mesh-broker-nats") &&
|
||||
strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") {
|
||||
placed = append(placed, i)
|
||||
}
|
||||
if strings.HasPrefix(c, "/usr/local/bin/mesh-host enrol") {
|
||||
@@ -314,3 +328,49 @@ func TestAFirstNodeIsLetOntoTheBusItRaised(t *testing.T) {
|
||||
"(placed at %v, enrolled at %d):\n%s", placed, enrol, strings.Join(runtime.commands, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
// A re-run on a machine that enrolled and whose bus is not yet the nats module places the
|
||||
// controller's list again (novox/hq ADR 0331 §5, review of mesh-host #96). The apply just wrote the
|
||||
// bundle's list — the controller alone — over the one placed at enrolment; left so, the next time the
|
||||
// bus re-reads it every node and the enrolment account are refused.
|
||||
func TestARerunOnAnEnrolledMachineWhoseBusIsNotAdoptedPlacesTheUsersAgain(t *testing.T) {
|
||||
runtime := &asked{answer: func(name string, args []string) (string, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(joined, "node list"):
|
||||
return "anchor here 01J0\n", nil
|
||||
case strings.Contains(joined, "module list"):
|
||||
return "mesh-controller 1\npostgres 1\n", nil
|
||||
case strings.Contains(joined, "broker accounts"):
|
||||
return "accounts {\n MESH { users = [ { user: \"node.anchor\" } ] }\n}\n", nil
|
||||
case name == "pgrep":
|
||||
return "4242\n", nil
|
||||
case name == "sh":
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("unexpected: %s %v", name, args)
|
||||
}}
|
||||
|
||||
out, err := Enrol(context.Background(), Options{
|
||||
Node: "anchor", State: alreadyEnrolled(t, "anchor"), Timeout: time.Second,
|
||||
Host: "/usr/local/bin/mesh-host", HostInBackground: true,
|
||||
}, arch(t), controlPlane{container: "temp-mesh-controller", run: runtime.run, timeout: time.Second},
|
||||
func(string) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Joined || runtime.ran("token issue") {
|
||||
t.Errorf("a re-run enrolled again: %v", runtime.commands)
|
||||
}
|
||||
placed := false
|
||||
for _, c := range runtime.commands {
|
||||
if strings.HasPrefix(c, "sh -c") && strings.Contains(c, "node.anchor") &&
|
||||
strings.Contains(c, "/var/lib/nats-module/conf/accounts.conf") && strings.Contains(c, "kill -s HUP mesh-broker-nats") {
|
||||
placed = true
|
||||
}
|
||||
}
|
||||
if !placed {
|
||||
t.Fatalf("the controller's list was not placed again on a re-run before adoption:\n%s",
|
||||
strings.Join(runtime.commands, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,3 +180,85 @@ func TestAProcessFormNamingNoOneReplacementIsRefused(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// theStepNineSecretsOf is what step 9 delivers for a process-form controller manifest: its genesis
|
||||
// container made, pinned, and its secrets read from that container's environment.
|
||||
func theStepNineSecretsOf(t *testing.T, manifest []byte) (map[string]string, error) {
|
||||
t.Helper()
|
||||
form, err := processFormOf(manifest)
|
||||
if err != nil || !form.Found {
|
||||
t.Fatalf("the manifest has no process form: %v", err)
|
||||
}
|
||||
genesis, err := genesisForm(manifest, form, builtImage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pinned, _, err := pinImage(genesis, builtImage, "registry.internal:5000/mesh-controller@sha256:"+strings.Repeat("e", 64), ControlPlaneModule)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return secretsByVariableIn(pinned)
|
||||
}
|
||||
|
||||
// **A file the controller's manifest writes itself is not a secret** (novox/hq issue 597). The merge
|
||||
// window's settings reach the controller in a file its manifest declares from its settings
|
||||
// (MESH_MERGE_WINDOW_FILE, ADR 0276); genesis refused it as a secret nobody writes, and stopped at
|
||||
// step 11 of 24.
|
||||
func TestAFileTheControllersManifestWritesIsNotASecret(t *testing.T) {
|
||||
manifest := strings.Replace(processFormManifest,
|
||||
`{"id": "controller", "type": "process",`,
|
||||
`{"id": "merge-window", "type": "file", "path": "${dir:mesh-state}/merge-window.json", "mode": "0600",
|
||||
"owner": "mesh-controller", "content": "{\"merge-window\": \"${setting:merge-window}\"}\n"},
|
||||
{"id": "controller", "type": "process",`, 1)
|
||||
manifest = strings.Replace(manifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`,
|
||||
`"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus",
|
||||
"MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1)
|
||||
if !strings.Contains(manifest, "MESH_MERGE_WINDOW_FILE") || !strings.Contains(manifest, `"id": "merge-window"`) {
|
||||
t.Fatal("the fixture was not changed")
|
||||
}
|
||||
wanted, err := theStepNineSecretsOf(t, []byte(manifest))
|
||||
if err != nil {
|
||||
t.Fatalf("step 9 refuses a file the manifest writes itself: %v", err)
|
||||
}
|
||||
if _, asked := wanted["MESH_MERGE_WINDOW"]; asked {
|
||||
t.Errorf("the merge window's settings are delivered as a secret: %v", wanted)
|
||||
}
|
||||
if wanted["MESH_STORE_INVENTORY"] != "inventory" || wanted["MESH_BUS_NATS"] != "bus" {
|
||||
t.Errorf("step 9 delivers %v; the store's and the bus's secrets are still wanted", wanted)
|
||||
}
|
||||
}
|
||||
|
||||
// And a `…_FILE` that neither an own-secret nor a file of the module writes is still refused.
|
||||
func TestAFileNothingOfTheControllerWritesIsStillRefused(t *testing.T) {
|
||||
manifest := strings.Replace(processFormManifest, `"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus"}`,
|
||||
`"MESH_BUS_NATS_FILE": "${dir:mesh-state}/bus",
|
||||
"MESH_MERGE_WINDOW_FILE": "${dir:mesh-state}/merge-window.json"}`, 1)
|
||||
if _, err := theStepNineSecretsOf(t, []byte(manifest)); err == nil || !strings.Contains(err.Error(), "MESH_MERGE_WINDOW_FILE") {
|
||||
t.Fatalf("a file nothing writes was accepted, or refused without its name: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller's own manifest, as the mesh runs it, passes step 9 (issue 597): in a merge check the
|
||||
// node-engine's check has the controller cloned beside it (MESH_CHECK_BESIDE), where it must be; else a
|
||||
// checkout beside this one; else it skips aloud.
|
||||
func TestTheControllersOwnManifestPassesStepNine(t *testing.T) {
|
||||
var manifest []byte
|
||||
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
|
||||
raw, err := os.ReadFile(filepath.Join(root, "mesh-controller", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("the controller is not beside this check in MESH_CHECK_BESIDE=%s: %v", root, err)
|
||||
}
|
||||
manifest = raw
|
||||
} else if raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-controller", "module.json")); err == nil {
|
||||
manifest = raw
|
||||
} else {
|
||||
t.Skip("NOT JUDGED: no controller beside this checkout and no MESH_CHECK_BESIDE; the merge check judges it")
|
||||
}
|
||||
wanted, err := theStepNineSecretsOf(t, manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("genesis's step 9 refuses the controller the mesh runs: %v", err)
|
||||
}
|
||||
if wanted["MESH_STORE_INVENTORY"] != "inventory" {
|
||||
t.Errorf("step 9 delivers %v", wanted)
|
||||
}
|
||||
}
|
||||
|
||||
+243
-19
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
@@ -14,8 +16,8 @@ import (
|
||||
// Genesis raises a store and a broker before any module system exists, because the control plane
|
||||
// cannot ask provisioning for the store it keeps its own records in or the broker it is reached over
|
||||
// (novox/hq ADR 0006). That leaves two servers behind: the foundation's, and a second one the
|
||||
// `postgres`/`lavinmq` modules used to raise for other modules to use. This turns the foundation's
|
||||
// own servers into those modules, so a mesh runs ONE postgres and ONE lavinmq — the control plane's
|
||||
// `postgres` and bus modules used to raise for other modules to use. This turns the foundation's
|
||||
// own servers into those modules, so a mesh runs ONE postgres and ONE bus — the control plane's
|
||||
// contexts and the database of each module that asks for one, in the same server (WBS 3.1/3.2).
|
||||
//
|
||||
// **Adopted in place, not replaced.** The control plane is stateless and is swapped for a fresh
|
||||
@@ -155,37 +157,251 @@ func deliverCredential(ctx context.Context, o Options, control controlPlane, mod
|
||||
return nil
|
||||
}
|
||||
|
||||
// InstallBroker makes the foundation's broker the `lavinmq` module, adopted in place — the same
|
||||
// shape as InstallStore, for the same reasons. The administrator's password is the one genesis
|
||||
// gave the image's default account (rootsecrets.go), carried in so the module's provisioner can
|
||||
// reach the management API as it.
|
||||
func InstallBroker(ctx context.Context, o Options, control controlPlane,
|
||||
foundation *declaration.Declaration, say func(string)) error {
|
||||
// BusModule is the module the foundation's bus is adopted as (novox/hq ADR 0331 §1). The previous
|
||||
// broker's module, lavinmq, left the catalogue with the move to one bus (ADR 0131).
|
||||
const BusModule = "nats"
|
||||
|
||||
const module = "lavinmq"
|
||||
// InstallBroker makes the foundation's bus the `nats` module, adopted in place — the same shape as
|
||||
// InstallStore, for the same reasons: the bus holds the mesh's streams, so it is kept, and from the
|
||||
// module's first push the bus's user list reaches it in the module's declaration like any module's
|
||||
// (ADR 0331 §1, §5). The installer's own placing of that list (enrol.go placeTheBusUsers) was the
|
||||
// bridge to here and is not run again.
|
||||
//
|
||||
// **The bundle raises the bus where the module keeps it** (novox/hq ADR 0333 §1): the same container
|
||||
// name, the same streams directory, configuration directory, user list and certificate. That is checked
|
||||
// here against the module's own manifest before anything is registered, and a difference is refused
|
||||
// naming each one — under another name or on other paths the module would raise a second bus beside the
|
||||
// first, or an empty one in its place.
|
||||
//
|
||||
// **The image is the one thing that changes, once** (ADR 0333 §3). The store's module pins the upstream
|
||||
// postgres the foundation runs; the bus's module builds its server image (it carries the mesh's snapshot
|
||||
// tool), so no pinned reference can be equal to the foundation's. The applier recreates the container on
|
||||
// the module's image, on the same streams, users and certificate. This says so before the push and waits
|
||||
// after it until the container runs the module's image and the bus answers, so nothing after this step
|
||||
// dials a bus that is restarting.
|
||||
func InstallBroker(ctx context.Context, o Options, control controlPlane,
|
||||
foundation *declaration.Declaration, fetch func(context.Context, string) (int, string, error),
|
||||
say func(string)) error {
|
||||
|
||||
const module = BusModule
|
||||
manifest, err := readManifest(o.Catalogue, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
broker, err := brokerIn(foundation)
|
||||
bus, err := brokerIn(foundation)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := serverMatchesFoundation(manifest, broker, module); err != nil {
|
||||
if err := busLayoutIsTheModules(manifest, foundation, module); err != nil {
|
||||
return err
|
||||
}
|
||||
say(" adopting " + broker.Name + " — the broker the foundation raised, unchanged")
|
||||
if err := installProvider(ctx, o, control, module, manifest, "(the provisioner; the server is adopted, not built)",
|
||||
say(" adopting " + bus.Name + " — the bus the foundation raised, its streams kept")
|
||||
if err := installProvider(ctx, o, control, module, manifest, "(the server, adopting the foundation's bus)",
|
||||
func() error {
|
||||
return deliverCredential(ctx, o, control, module, "admin", BrokerAdminFile, "the broker's administrator", say)
|
||||
},
|
||||
say); err != nil {
|
||||
if running := runningImage(ctx, control, bus.Name); running != "" && running != bus.Image {
|
||||
say(" already restarted " + bus.Name + " — it runs the module's image")
|
||||
return nil
|
||||
}
|
||||
say(" restarting " + bus.Name + " — once, onto the " + module + " module's image; its " +
|
||||
"streams, users and certificate stay where they are (novox/hq ADR 0333 §3)")
|
||||
return nil
|
||||
}, say); err != nil {
|
||||
return err
|
||||
}
|
||||
say(" adopted " + broker.Name + " — the foundation's broker is now the " + module + " module")
|
||||
if err := waitForTheAdoptedBus(ctx, o, control, bus, fetch, say); err != nil {
|
||||
return err
|
||||
}
|
||||
say(" adopted " + bus.Name + " — the foundation's bus is now the " + module +
|
||||
" module, and its user list is the module's from here on")
|
||||
return nil
|
||||
}
|
||||
|
||||
// busMonitor is where the bus says it is healthy: its monitoring port, published on the machine's own
|
||||
// loopback by the bundle and by the module alike (both "127.0.0.1:8222:8222"). /healthz answers ok only
|
||||
// once the server is up and JetStream has its streams back.
|
||||
const busMonitor = "http://127.0.0.1:8222/healthz"
|
||||
|
||||
// runningImage is the image the named container was created from, or "" when there is none.
|
||||
func runningImage(ctx context.Context, control controlPlane, name string) string {
|
||||
asking, cancel := context.WithTimeout(ctx, control.timeout)
|
||||
defer cancel()
|
||||
out, err := control.run(asking, "docker", "inspect", "--format", "{{.Config.Image}}", name)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(out)
|
||||
}
|
||||
|
||||
// waitForTheAdoptedBus waits until the bus container runs an image other than the one the bundle raised
|
||||
// it from — the module's, after the applier's one recreate — and the bus answers on its monitor.
|
||||
//
|
||||
// Bounded by --wait, and asked at least once. A bus still on the bundle's image when it runs out is
|
||||
// refused, naming what it runs: the push was sent and the node has not applied it, and every step after
|
||||
// this one reaches the mesh through that bus.
|
||||
func waitForTheAdoptedBus(ctx context.Context, o Options, control controlPlane, bus *declaration.Container,
|
||||
fetch func(context.Context, string) (int, string, error), say func(string)) error {
|
||||
|
||||
deadline := time.Now().Add(o.Wait)
|
||||
var running, said string
|
||||
for {
|
||||
running = runningImage(ctx, control, bus.Name)
|
||||
if running != "" && running != bus.Image {
|
||||
code, body, err := fetch(ctx, busMonitor)
|
||||
if err == nil && code == http.StatusOK {
|
||||
say(" bus answers " + bus.Name + " on " + running)
|
||||
return nil
|
||||
}
|
||||
said = strings.TrimSpace(body)
|
||||
if err != nil {
|
||||
said = err.Error()
|
||||
}
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(answerEvery):
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case running == "":
|
||||
return fmt.Errorf("the bus container %s is not there after the %s module was pushed, so nothing "+
|
||||
"after this step could reach the mesh. The node-engine's journal says why it did not raise it",
|
||||
bus.Name, BusModule)
|
||||
case running == bus.Image:
|
||||
return fmt.Errorf("%s still runs the bundle's image %s %s after the %s module was pushed: the node "+
|
||||
"has not applied it. Its streams are untouched; run this installer again once the node-engine "+
|
||||
"applies (its journal says why it did not)", bus.Name, bus.Image, o.Wait, BusModule)
|
||||
}
|
||||
return fmt.Errorf("%s runs the %s module's image %s and its monitor (%s) did not answer ok within %s: %s",
|
||||
bus.Name, BusModule, running, busMonitor, o.Wait, orUnknown(said))
|
||||
}
|
||||
|
||||
// BusUsersID is the bundle's resource for the bus's first user list.
|
||||
const BusUsersID = "bus-accounts"
|
||||
|
||||
// busLayout is where a bus lives: its container, the machine's directories it mounts as its streams,
|
||||
// its configuration and its certificate, and the file its user list is.
|
||||
type busLayout struct {
|
||||
Container, Streams, Conf, Certificate, Users string
|
||||
}
|
||||
|
||||
// Where the server reads each, inside its container: the same in the bundle and the module, since both
|
||||
// run the same server with the same configuration.
|
||||
const (
|
||||
busStreamsInside = "/data"
|
||||
busConfInside = "/etc/nats"
|
||||
busCertificateInside = "/tls"
|
||||
)
|
||||
|
||||
// foundationBusLayout reads the layout of the bus the bundle raises.
|
||||
func foundationBusLayout(foundation *declaration.Declaration) (busLayout, error) {
|
||||
bus, err := brokerIn(foundation)
|
||||
if err != nil {
|
||||
return busLayout{}, err
|
||||
}
|
||||
at := mountedFrom(bus.Volumes)
|
||||
l := busLayout{Container: bus.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
|
||||
Certificate: at[busCertificateInside]}
|
||||
for _, r := range foundation.Resources {
|
||||
if f, ok := r.(*declaration.File); ok && r.Identity() == BusUsersID {
|
||||
l.Users = f.Path
|
||||
}
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// moduleBusLayout reads the layout of the bus the module declares: its server container, the mounts of
|
||||
// that container with every `${access:<id>}` resolved to the access's path, and its `bus-users`.
|
||||
func moduleBusLayout(manifest []byte, module string) (busLayout, error) {
|
||||
var m struct {
|
||||
BusUsers string `json:"bus-users"`
|
||||
Resources []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Volumes []string `json:"volumes"`
|
||||
} `json:"resources"`
|
||||
Accesses []struct {
|
||||
ID string `json:"id"`
|
||||
Path string `json:"path"`
|
||||
} `json:"accesses"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
return busLayout{}, fmt.Errorf("the %s module's manifest is not readable: %w", module, err)
|
||||
}
|
||||
var containers []int
|
||||
for i, r := range m.Resources {
|
||||
if r.Type == "container" {
|
||||
containers = append(containers, i)
|
||||
}
|
||||
}
|
||||
if len(containers) != 1 {
|
||||
var names []string
|
||||
for _, i := range containers {
|
||||
names = append(names, m.Resources[i].Name)
|
||||
}
|
||||
return busLayout{}, fmt.Errorf("the %s module declares %s; the bus it adopts is one server, so it "+
|
||||
"declares one container", module, describeNames(names))
|
||||
}
|
||||
server := m.Resources[containers[0]]
|
||||
resolved := make([]string, 0, len(server.Volumes))
|
||||
for _, v := range server.Volumes {
|
||||
for _, a := range m.Accesses {
|
||||
v = strings.ReplaceAll(v, "${access:"+a.ID+"}", a.Path)
|
||||
}
|
||||
resolved = append(resolved, v)
|
||||
}
|
||||
at := mountedFrom(resolved)
|
||||
return busLayout{Container: server.Name, Streams: at[busStreamsInside], Conf: at[busConfInside],
|
||||
Certificate: at[busCertificateInside], Users: m.BusUsers}, nil
|
||||
}
|
||||
|
||||
// busLayoutIsTheModules checks that the bus the bundle raised is where the module keeps it, field by
|
||||
// field, so the applier takes it over rather than raising a second bus beside it or an empty one in its
|
||||
// place (novox/hq ADR 0333 §1). Every difference is named, with both sides.
|
||||
func busLayoutIsTheModules(manifest []byte, foundation *declaration.Declaration, module string) error {
|
||||
theirs, err := moduleBusLayout(manifest, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ours, err := foundationBusLayout(foundation)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var differ []string
|
||||
for _, f := range []struct{ what, module, bundle string }{
|
||||
{"the server container", theirs.Container, ours.Container},
|
||||
{"its streams (" + busStreamsInside + ")", theirs.Streams, ours.Streams},
|
||||
{"its configuration (" + busConfInside + ")", theirs.Conf, ours.Conf},
|
||||
{"its certificate (" + busCertificateInside + ")", theirs.Certificate, ours.Certificate},
|
||||
{"its user list", theirs.Users, ours.Users},
|
||||
} {
|
||||
if f.module == "" || f.module != f.bundle {
|
||||
differ = append(differ, fmt.Sprintf("%s: the module's is %q, the bundle's %q", f.what,
|
||||
f.module, f.bundle))
|
||||
}
|
||||
}
|
||||
if len(differ) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"the bus the foundation raised is not where the %s module keeps it:\n - %s\n"+
|
||||
"The module adopts the foundation's bus in place, so the bundle raises it with the module's "+
|
||||
"name and paths; otherwise adopting it would raise a second bus beside the first, or an empty "+
|
||||
"one on other paths (novox/hq ADR 0333 §1). Nothing has been registered",
|
||||
module, strings.Join(differ, "\n - "))
|
||||
}
|
||||
|
||||
func describeNames(names []string) string {
|
||||
if len(names) == 0 {
|
||||
return "no container at all"
|
||||
}
|
||||
return "the containers " + strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// InstallVault installs the vault as a foundation module (novox/hq ADR 0085, amended). Nothing to
|
||||
// adopt and nothing to carry in: it is its own runtime, built from the catalogue like any provider,
|
||||
// and from its first push it keeps the export of every operator-sealed secret on its own disk.
|
||||
@@ -207,9 +423,9 @@ func storeIn(d *declaration.Declaration) (*declaration.Container, error) {
|
||||
return foundationContainer(d, StoreID, "store")
|
||||
}
|
||||
|
||||
// brokerIn finds the broker container in the bundle this installer produced.
|
||||
// brokerIn finds the bus's container in the bundle this installer produced.
|
||||
func brokerIn(d *declaration.Declaration) (*declaration.Container, error) {
|
||||
return foundationContainer(d, BrokerID, "broker")
|
||||
return foundationContainer(d, BrokerID, "bus")
|
||||
}
|
||||
|
||||
func foundationContainer(d *declaration.Declaration, id, what string) (*declaration.Container, error) {
|
||||
@@ -262,3 +478,11 @@ func serverMatchesFoundation(manifest []byte, store *declaration.Container, modu
|
||||
"store with. Its server container has to carry the name the foundation raised",
|
||||
module, store.Name)
|
||||
}
|
||||
|
||||
// orUnknown is s, or a word for nothing: trust.go held it, and this branch takes trust.go out.
|
||||
func orUnknown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "(nothing)"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// aCatalogue is a catalogue checkout holding these modules' manifests.
|
||||
func aCatalogue(t *testing.T, manifests map[string]string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
for module, manifest := range manifests {
|
||||
at := filepath.Join(dir, catalogueDir, module)
|
||||
if err := os.MkdirAll(at, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(at, "module.json"), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
// aBusModule is the nats module's manifest as far as adoption reads it: its server container, the
|
||||
// directories it mounts, its certificate through an access, and its user list — the shape of the
|
||||
// catalogue's modules/nats/module.json.
|
||||
func aBusModule(l busLayout) string {
|
||||
return `{"module": "nats", "version": "1",
|
||||
"claims": [{"name": "mesh-broker", "scope": "mesh"}],
|
||||
"bus-users": "` + l.Users + `",
|
||||
"resources": [
|
||||
{"id": "server", "type": "container", "name": "` + l.Container + `", "ports": ["4222:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["` + l.Streams + `:/data", "` + l.Conf + `:/etc/nats:ro", "${access:tls}:/tls:ro"], "artifact": "server"}],
|
||||
"accesses": [{"id": "tls", "path": "` + l.Certificate + `", "mode": "read"}],
|
||||
"build": {"artifacts": [{"name": "server", "kind": "image", "from": "Dockerfile"}]}}`
|
||||
}
|
||||
|
||||
// theModulesLayout is where the catalogue's nats module keeps its bus (mesh-catalog modules/nats).
|
||||
var theModulesLayout = busLayout{Container: "mesh-broker-nats", Streams: "/var/lib/mesh-broker-nats",
|
||||
Conf: "/var/lib/nats-module/conf", Certificate: "/var/lib/mesh-broker-tls",
|
||||
Users: "/var/lib/nats-module/conf/accounts.conf"}
|
||||
|
||||
// theModulesImage is what the bus runs once the applier recreated it on the module's built image.
|
||||
const theModulesImage = "127.0.0.1:5000/nats@sha256:3333333333333333333333333333333333333333333333333333333333333333"
|
||||
|
||||
// aBusThatAdopts answers the broker step as a node that applied the push: the bus container runs the
|
||||
// module's image, and its monitor says ok.
|
||||
func aBusThatAdopts() (*asked, func(context.Context, string) (int, string, error)) {
|
||||
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
|
||||
"inspect --format {{.Config.Image}} mesh-broker-nats": theModulesImage + "\n",
|
||||
})}
|
||||
return runtime, func(_ context.Context, url string) (int, string, error) {
|
||||
if url != busMonitor {
|
||||
return 0, "", fmt.Errorf("asked %s", url)
|
||||
}
|
||||
return http.StatusOK, `{"status":"ok"}`, nil
|
||||
}
|
||||
}
|
||||
|
||||
func busStepWith(t *testing.T, o Options, runtime *asked,
|
||||
fetch func(context.Context, string) (int, string, error)) ([]string, error) {
|
||||
t.Helper()
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control := controlPlane{container: "mesh-controller", run: runtime.run, timeout: time.Second}
|
||||
var said []string
|
||||
err = InstallBroker(context.Background(), o, control, rewritten.Declaration, fetch,
|
||||
func(line string) { said = append(said, line) })
|
||||
return said, err
|
||||
}
|
||||
|
||||
func busStep(t *testing.T, catalogue string) (*asked, error) {
|
||||
t.Helper()
|
||||
runtime, fetch := aBusThatAdopts()
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
_, err := busStepWith(t, o, runtime, fetch)
|
||||
return runtime, err
|
||||
}
|
||||
|
||||
// The broker step adopts the foundation's bus as the nats module (novox/hq ADR 0331 §1): the module
|
||||
// the mesh goes on managing, registered from the catalogue — never lavinmq, which left it.
|
||||
func TestTheBrokerStepAdoptsTheBusAsTheNatsModule(t *testing.T) {
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
runtime, err := busStep(t, catalogue)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !runtime.ran("module add /nats-module.json") {
|
||||
t.Fatalf("the nats module was not registered: %v", runtime.commands)
|
||||
}
|
||||
if runtime.ran("lavinmq") {
|
||||
t.Errorf("the broker step still speaks of lavinmq: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// A catalogue that holds no nats module fails the step by name, before anything is registered —
|
||||
// which is what today's step did for lavinmq, silently adopting a module that no longer exists.
|
||||
func TestTheBrokerStepFailsForAModuleTheCatalogueDoesNotHold(t *testing.T) {
|
||||
catalogue := aCatalogue(t, map[string]string{"lavinmq": `{"module": "lavinmq", "version": "1"}`})
|
||||
runtime, err := busStep(t, catalogue)
|
||||
if err == nil {
|
||||
t.Fatal("a catalogue without the nats module was adopted from")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "nats module") {
|
||||
t.Errorf("the refusal does not name the module it looked for: %v", err)
|
||||
}
|
||||
if runtime.ran("module add") {
|
||||
t.Errorf("something was registered: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// A bus module that keeps its bus anywhere else than the bundle raised it would raise a second bus
|
||||
// beside the first, or an empty one on other paths (novox/hq ADR 0333 §1). It is refused naming every
|
||||
// difference with both sides, before anything is registered. The layout is the one the bundle had
|
||||
// before ADR 0333, kept as a module's here: the refusal genesis stopped at, at step 18 (issue 590).
|
||||
func TestABusModuleWhoseLayoutIsNotTheFoundationsIsRefused(t *testing.T) {
|
||||
before := busLayout{Container: "mesh-broker", Streams: "mesh-broker-data", Conf: "/var/lib/mesh-bus-conf",
|
||||
Certificate: "mesh-broker-tls", Users: "/var/lib/mesh-bus-conf/accounts.conf"}
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(before)})
|
||||
runtime, err := busStep(t, catalogue)
|
||||
if err == nil {
|
||||
t.Fatal("a bus module kept elsewhere was adopted")
|
||||
}
|
||||
for _, named := range []string{
|
||||
`"mesh-broker"`, `"mesh-broker-nats"`, `"mesh-broker-data"`, `"/var/lib/mesh-broker-nats"`,
|
||||
`"/var/lib/mesh-bus-conf"`, `"/var/lib/nats-module/conf"`, `"mesh-broker-tls"`, `"/var/lib/mesh-broker-tls"`,
|
||||
`"/var/lib/mesh-bus-conf/accounts.conf"`, `"/var/lib/nats-module/conf/accounts.conf"`,
|
||||
} {
|
||||
if !strings.Contains(err.Error(), named) {
|
||||
t.Errorf("the refusal does not name %s: %v", named, err)
|
||||
}
|
||||
}
|
||||
if runtime.ran("module add") {
|
||||
t.Errorf("something was registered: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// One field moved is refused as surely as all of them: a module whose streams are elsewhere would be
|
||||
// raised empty on the bus's own name.
|
||||
func TestABusModuleWithItsStreamsElsewhereIsRefused(t *testing.T) {
|
||||
moved := theModulesLayout
|
||||
moved.Streams = "/srv/nats"
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(moved)})
|
||||
runtime, err := busStep(t, catalogue)
|
||||
if err == nil || !strings.Contains(err.Error(), `"/srv/nats"`) || !strings.Contains(err.Error(), "streams") {
|
||||
t.Fatalf("a module with its streams elsewhere was not refused naming them: %v", err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "certificate") {
|
||||
t.Errorf("the refusal names what agrees: %v", err)
|
||||
}
|
||||
if runtime.ran("module add") {
|
||||
t.Errorf("something was registered: %v", runtime.commands)
|
||||
}
|
||||
}
|
||||
|
||||
// The node's bus port is kept (novox/hq ADR 0333 §2): the module declares 4222, and the broker step
|
||||
// gives it the setting that maps that to the port the bundle published, 5671 unless given — before the
|
||||
// push, so the first declaration already names it.
|
||||
func TestTheBrokerStepGivesTheModuleTheNodesBusPort(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
given int
|
||||
want string
|
||||
}{{0, `{"ports":{"4222":5671}}`}, {5771, `{"ports":{"4222":5771}}`}} {
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
runtime, fetch := aBusThatAdopts()
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
o.Ports = FoundationPorts{Bus: c.given}
|
||||
said, err := busStepWith(t, o, runtime, fetch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set, pushed := -1, -1
|
||||
for i, command := range runtime.commands {
|
||||
if strings.Contains(command, "settings set nats /nats-settings.json --node anchor") && set < 0 {
|
||||
set = i
|
||||
}
|
||||
if strings.Contains(command, "push anchor") && pushed < 0 {
|
||||
pushed = i
|
||||
}
|
||||
}
|
||||
if set < 0 || pushed < 0 || set > pushed {
|
||||
t.Fatalf("bus port %d: the module's port setting was not given before the push: %v", c.given, runtime.commands)
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "settings nats on anchor: "+c.want) {
|
||||
t.Errorf("bus port %d: the setting given is not %s:\n%s", c.given, c.want, strings.Join(said, "\n"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The one restart is said before it happens, and the step goes on only once the bus runs the module's
|
||||
// image and answers (novox/hq ADR 0333 §3).
|
||||
func TestTheBrokerStepSaysTheRestartAndWaitsForTheBusToAnswer(t *testing.T) {
|
||||
previous := answerEvery
|
||||
answerEvery = time.Millisecond
|
||||
defer func() { answerEvery = previous }()
|
||||
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
|
||||
|
||||
// The push is sent; the node takes a few passes to recreate the bus, and the server a few more to
|
||||
// answer.
|
||||
pushed, inspected, monitored := false, 0, 0
|
||||
runtime := &asked{}
|
||||
agrees := aMeshThatAgrees(nil)
|
||||
runtime.answer = func(name string, args []string) (string, error) {
|
||||
joined := strings.Join(args, " ")
|
||||
switch {
|
||||
case strings.Contains(joined, "push anchor"):
|
||||
pushed = true
|
||||
case strings.Contains(joined, "inspect --format {{.Config.Image}} mesh-broker-nats"):
|
||||
if !pushed {
|
||||
return bundleImage + "\n", nil
|
||||
}
|
||||
inspected++
|
||||
if inspected < 3 {
|
||||
return bundleImage + "\n", nil
|
||||
}
|
||||
return theModulesImage + "\n", nil
|
||||
}
|
||||
return agrees(name, args)
|
||||
}
|
||||
fetch := func(context.Context, string) (int, string, error) {
|
||||
monitored++
|
||||
if monitored < 3 {
|
||||
return http.StatusServiceUnavailable, `{"status":"unavailable","error":"JetStream stream catching up"}`, nil
|
||||
}
|
||||
return http.StatusOK, `{"status":"ok"}`, nil
|
||||
}
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
o.Wait = 10 * time.Second
|
||||
said, err := busStepWith(t, o, runtime, fetch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.Join(said, "\n")
|
||||
restart := strings.Index(text, "restarting mesh-broker-nats — once")
|
||||
answers := strings.Index(text, "bus answers mesh-broker-nats on "+theModulesImage)
|
||||
adopted := strings.Index(text, "adopted mesh-broker-nats")
|
||||
if restart < 0 || answers < restart || adopted < answers {
|
||||
t.Fatalf("the step did not say the restart, then the bus answering, then the adoption:\n%s", text)
|
||||
}
|
||||
if inspected < 3 || monitored < 3 {
|
||||
t.Errorf("the step went on before the bus ran the module's image and answered (asked its image %d "+
|
||||
"times, its monitor %d)", inspected, monitored)
|
||||
}
|
||||
}
|
||||
|
||||
// A bus still on the bundle's image when the wait runs out is refused, naming the image: every step after
|
||||
// this one reaches the mesh through that bus, and the push was not applied.
|
||||
func TestABusTheNodeDidNotRecreateIsRefused(t *testing.T) {
|
||||
previous := answerEvery
|
||||
answerEvery = time.Millisecond
|
||||
defer func() { answerEvery = previous }()
|
||||
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundleImage := containerNamed(rewritten.Declaration, "mesh-broker-nats").Image
|
||||
runtime := &asked{answer: aMeshThatAgrees(map[string]string{
|
||||
"inspect --format {{.Config.Image}} mesh-broker-nats": bundleImage + "\n",
|
||||
})}
|
||||
fetch := func(context.Context, string) (int, string, error) { return http.StatusOK, `{"status":"ok"}`, nil }
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
o.Wait = 20 * time.Millisecond
|
||||
_, err = busStepWith(t, o, runtime, fetch)
|
||||
if err == nil {
|
||||
t.Fatal("a bus the node never recreated was taken as adopted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), bundleImage) || !strings.Contains(err.Error(), "not applied") {
|
||||
t.Errorf("the refusal does not say the bus still runs the bundle's image: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A re-run after the adoption finds the bus on the module's image already, and says so rather than
|
||||
// announcing a restart that does not happen.
|
||||
func TestARerunAfterTheAdoptionSaysTheBusIsTheModulesAlready(t *testing.T) {
|
||||
catalogue := aCatalogue(t, map[string]string{BusModule: aBusModule(theModulesLayout)})
|
||||
runtime, fetch := aBusThatAdopts()
|
||||
o := installing(t, catalogue)
|
||||
o.CatalogSource = Source{Repository: "git://forge/mesh-catalog.git"}
|
||||
said, err := busStepWith(t, o, runtime, fetch)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := strings.Join(said, "\n")
|
||||
if strings.Contains(text, "restarting") || !strings.Contains(text, "already restarted mesh-broker-nats") {
|
||||
t.Errorf("a re-run announced a restart, or did not say the bus is the module's already:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// theCataloguesBusModule is the catalogue's own nats manifest, never a copy of it (novox/hq ADR 0333,
|
||||
// "How it is checked"): in a merge check the clone the build seat puts beside this repository
|
||||
// (MESH_CHECK_BESIDE, the catalogue at its main), where it must be; elsewhere a checkout of the catalogue
|
||||
// beside this one. With neither, the test says so and skips — a skip is not a pass, and the merge check
|
||||
// never skips it.
|
||||
func theCataloguesBusModule(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
const inside = "mesh-catalog/modules/nats/module.json"
|
||||
if root := os.Getenv("MESH_CHECK_BESIDE"); root != "" {
|
||||
raw, err := os.ReadFile(filepath.Join(root, inside))
|
||||
if err != nil {
|
||||
t.Fatalf("the catalogue is not beside this check in MESH_CHECK_BESIDE=%s, so the bundle's bus "+
|
||||
"cannot be held to the module's: %v", root, err)
|
||||
}
|
||||
t.Logf("judged against %s as cloned beside this check", filepath.Join(root, inside))
|
||||
return raw
|
||||
}
|
||||
beside := filepath.Join("..", "..", "..", inside)
|
||||
raw, err := os.ReadFile(beside)
|
||||
if err != nil {
|
||||
t.Skipf("NOT JUDGED: no catalogue beside this checkout (%s) and no MESH_CHECK_BESIDE, so the bundle's "+
|
||||
"bus is not held to the nats module's manifest here; the merge check holds it", beside)
|
||||
}
|
||||
t.Logf("judged against the catalogue checked out at %s", beside)
|
||||
return raw
|
||||
}
|
||||
|
||||
// **The genesis bundle takes the nats module's layout** (novox/hq ADR 0333 §1), read from the module's
|
||||
// manifest: the bus's container, its streams, its configuration, its certificate and its user list are
|
||||
// where the module keeps them, so adopting the bus keeps it. Before ADR 0333 the bundle raised
|
||||
// `mesh-broker` on volumes the module never mounts, and genesis stopped at step 18 (issue 590).
|
||||
func TestTheBundleRaisesTheBusWhereTheCataloguesModuleKeepsIt(t *testing.T) {
|
||||
manifest := theCataloguesBusModule(t)
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busLayoutIsTheModules(manifest, rewritten.Declaration, BusModule); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// And the port the installer's setting maps is the one the module declares (ADR 0333 §2).
|
||||
var m struct {
|
||||
Resources []struct {
|
||||
Type string `json:"type"`
|
||||
Ports []string `json:"ports"`
|
||||
} `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(manifest, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := false
|
||||
for _, r := range m.Resources {
|
||||
for _, p := range r.Ports {
|
||||
if p == fmt.Sprintf("%d:4222", busModulePort) {
|
||||
declared = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !declared {
|
||||
t.Errorf("the nats module does not publish its server on %d, the port the installer's setting maps "+
|
||||
"to the node's bus port", busModulePort)
|
||||
}
|
||||
}
|
||||
|
||||
// The bundle as the installer reads it agrees with the module's layout as this repository states it,
|
||||
// and with where the installer's bridge writes (ADR 0333 §4): the bridge places the users in the file the
|
||||
// bus reads, and makes the container that reads it re-read it.
|
||||
func TestTheBridgeWritesWhereTheBundleRaisesTheBus(t *testing.T) {
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ours, err := foundationBusLayout(rewritten.Declaration)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ours != theModulesLayout {
|
||||
t.Errorf("the bundle raises the bus at %+v; the nats module keeps it at %+v", ours, theModulesLayout)
|
||||
}
|
||||
if ours.Users != busAccounts || ours.Container != busContainer {
|
||||
t.Errorf("the bridge writes %s and signals %s; the bundle's bus reads %s and is %s",
|
||||
busAccounts, busContainer, ours.Users, ours.Container)
|
||||
}
|
||||
if filepath.Dir(ours.Users) != ours.Conf {
|
||||
t.Errorf("the user list %s is not in the configuration directory %s the bus reads", ours.Users, ours.Conf)
|
||||
}
|
||||
}
|
||||
|
||||
// The bundle's bus configuration and user list are seeds (create-once): an installer run again after
|
||||
// the adoption applies the bundle again, and must not put the bundle's one user back over the users the
|
||||
// module writes there (ADR 0333 §4).
|
||||
func TestTheBundlesBusFilesAreSeeds(t *testing.T) {
|
||||
rewritten, err := Rewrite(theRealBundle(t), held)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seeds := map[string]bool{}
|
||||
for _, r := range rewritten.Declaration.Resources {
|
||||
if f, ok := r.(*declaration.File); ok && f.CreateOnce {
|
||||
seeds[f.Path] = true
|
||||
}
|
||||
}
|
||||
for _, path := range []string{busAccounts, theModulesLayout.Conf + "/nats.conf"} {
|
||||
if !seeds[path] {
|
||||
t.Errorf("the bundle writes %s over whatever is there; it is a seed, written only where nothing is", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -144,8 +144,8 @@ func RaisePackageRegistry(ctx context.Context, o Options, d Deps, control contro
|
||||
// recordTheForgesPort makes the port given for the package registry a setting of the module that
|
||||
// serves it, on this node (novox/hq 04-ISSUES/085, ADR 0100).
|
||||
//
|
||||
// **The forge is the one foundation port that was not a setting.** The store's, the bus's, the
|
||||
// broker's management port and the registry's each become a `ports` setting of the module that
|
||||
// **The forge is the one foundation port that was not a setting.** The store's, the bus's and the
|
||||
// registry's each become a `ports` setting of the module that
|
||||
// binds them, set where that module is registered and assigned; the forge is raised by hand here,
|
||||
// long before its module can be built, so there was no registration to hang it on and the number
|
||||
// lived in rewritten manifest text instead. So the manifest is registered here — not assigned, and
|
||||
|
||||
+32
-29
@@ -20,24 +20,21 @@ import (
|
||||
// FoundationPorts are the machine's ports the foundation binds (novox/hq ADR 0100).
|
||||
//
|
||||
// **The node's, not the catalogue's.** A machine in use may already hold one — a predecessor's
|
||||
// registry on 5000, its broker's management port — and a port fixed in the bundle and the manifests
|
||||
// registry on 5000, its bus on 5671 — and a port fixed in the bundle and the manifests
|
||||
// surfaces as a container that fails to bind, and one changed at genesis would be changed back when
|
||||
// the foundation is adopted as modules. So each is an input here, checked free, rewritten into the
|
||||
// bundle, and handed to the controller as that node's setting for the module that binds it.
|
||||
type FoundationPorts struct {
|
||||
Store int `json:"store"`
|
||||
Bus int `json:"bus"`
|
||||
AMQP int `json:"amqp"`
|
||||
Management int `json:"management"`
|
||||
Registry int `json:"registry"`
|
||||
Packages int `json:"packages"`
|
||||
Hub int `json:"hub"`
|
||||
Store int `json:"store"`
|
||||
Bus int `json:"bus"`
|
||||
Registry int `json:"registry"`
|
||||
Packages int `json:"packages"`
|
||||
Hub int `json:"hub"`
|
||||
}
|
||||
|
||||
// DefaultPorts are the catalogue's numbers.
|
||||
func DefaultPorts() FoundationPorts {
|
||||
return FoundationPorts{Store: 5432, Bus: 5671, AMQP: 5672, Management: 15672, Registry: 5000,
|
||||
Packages: 3000, Hub: 51820}
|
||||
return FoundationPorts{Store: 5432, Bus: 5671, Registry: 5000, Packages: 3000, Hub: 51820}
|
||||
}
|
||||
|
||||
// DefaultOverlayRange is the controller's default private-network range.
|
||||
@@ -47,7 +44,7 @@ const DefaultOverlayRange = "10.42.0.0/16"
|
||||
func (p FoundationPorts) orDefaults() FoundationPorts {
|
||||
d := DefaultPorts()
|
||||
for _, f := range []struct{ got, def *int }{
|
||||
{&p.Store, &d.Store}, {&p.Bus, &d.Bus}, {&p.AMQP, &d.AMQP}, {&p.Management, &d.Management},
|
||||
{&p.Store, &d.Store}, {&p.Bus, &d.Bus},
|
||||
{&p.Registry, &d.Registry}, {&p.Packages, &d.Packages}, {&p.Hub, &d.Hub},
|
||||
} {
|
||||
if *f.got == 0 {
|
||||
@@ -60,8 +57,7 @@ func (p FoundationPorts) orDefaults() FoundationPorts {
|
||||
// named is each port with what it is and its protocol, in a fixed order.
|
||||
func (p FoundationPorts) named() []namedPort {
|
||||
return []namedPort{
|
||||
{"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the broker's AMQP", "tcp", p.AMQP},
|
||||
{"the broker's management", "tcp", p.Management}, {"the registry", "tcp", p.Registry},
|
||||
{"the store", "tcp", p.Store}, {"the bus", "tcp", p.Bus}, {"the registry", "tcp", p.Registry},
|
||||
{"the package registry", "tcp", p.Packages}, {"the private network's hub", "udp", p.Hub},
|
||||
}
|
||||
}
|
||||
@@ -88,8 +84,11 @@ func (p FoundationPorts) Check() error {
|
||||
}
|
||||
|
||||
// moduleSettings is what each foundation module is told about its ports on this node: the port it
|
||||
// declares, to the machine's port it is given. Only what differs from the catalogue — a converged
|
||||
// genesis on the defaults sets nothing, and so changes nothing it did before.
|
||||
// declares, to the machine's port it is given. Only what differs from the catalogue's manifest is
|
||||
// set. For the store, the registry and the forge the manifest declares the foundation's default, so
|
||||
// a genesis on the defaults sets nothing for them. The bus is the exception: the nats module
|
||||
// declares 4222 and the foundation publishes the bus on 5671, the port every enrolment token
|
||||
// names, so the bus always carries a setting.
|
||||
func (p FoundationPorts) moduleSettings() map[string]map[string]int {
|
||||
d := DefaultPorts()
|
||||
out := map[string]map[string]int{}
|
||||
@@ -103,14 +102,18 @@ func (p FoundationPorts) moduleSettings() map[string]map[string]int {
|
||||
out[module][strconv.Itoa(declared)] = given
|
||||
}
|
||||
add("postgres", d.Store, p.Store)
|
||||
add("lavinmq", d.Bus, p.Bus)
|
||||
add("lavinmq", d.AMQP, p.AMQP)
|
||||
add("lavinmq", d.Management, p.Management)
|
||||
// The bus's module declares the server's own port, and the foundation published the bus on the
|
||||
// node's: adopted in place, it keeps the port every token already names (ADR 0331 §1).
|
||||
add(BusModule, busModulePort, p.Bus)
|
||||
add(RegistryModule, d.Registry, p.Registry)
|
||||
add(ForgeModule, d.Packages, p.Packages)
|
||||
return out
|
||||
}
|
||||
|
||||
// busModulePort is the machine's port the nats module declares for the bus (mesh-catalog
|
||||
// modules/nats, "4222:4222").
|
||||
const busModulePort = 4222
|
||||
|
||||
// PortsSetting is the controller's settings key for a module's given ports.
|
||||
const PortsSetting = "ports"
|
||||
|
||||
@@ -200,10 +203,10 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe
|
||||
if p.Store != d.Store {
|
||||
replace(`"ports": ["5432:5432"]`, fmt.Sprintf(`"ports": ["%d:5432"]`, p.Store), "the store's published port")
|
||||
}
|
||||
if p.Bus != d.Bus || p.AMQP != d.AMQP || p.Management != d.Management {
|
||||
replace(`"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"]`,
|
||||
fmt.Sprintf(`"ports": ["%d:5671", "%d:5672", "127.0.0.1:%d:15672"]`, p.Bus, p.AMQP, p.Management),
|
||||
"the broker's published ports")
|
||||
if p.Bus != d.Bus {
|
||||
replace(`"ports": ["5671:4222", "127.0.0.1:8222:8222"]`,
|
||||
fmt.Sprintf(`"ports": ["%d:4222", "127.0.0.1:8222:8222"]`, p.Bus),
|
||||
"the bus's published ports")
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
@@ -221,9 +224,9 @@ func RewritePorts(r *Rewritten, p FoundationPorts, overlayRange string) (PortsRe
|
||||
value := control.Env[key]
|
||||
now := value
|
||||
now = strings.ReplaceAll(now, "@127.0.0.1:5432/", fmt.Sprintf("@127.0.0.1:%d/", p.Store))
|
||||
now = strings.ReplaceAll(now, "@127.0.0.1:5672/", fmt.Sprintf("@127.0.0.1:%d/", p.AMQP))
|
||||
if strings.HasSuffix(now, "@127.0.0.1:15672") {
|
||||
now = strings.TrimSuffix(now, "15672") + strconv.Itoa(p.Management)
|
||||
// The bus, as the control plane dials it (MESH_BUS_NATS) on the machine's loopback.
|
||||
if strings.HasSuffix(now, "@127.0.0.1:5671") {
|
||||
now = strings.TrimSuffix(now, "5671") + strconv.Itoa(p.Bus)
|
||||
}
|
||||
if key == brokerAddressVar {
|
||||
if host, port, splitErr := net.SplitHostPort(value); splitErr == nil && port == "5671" {
|
||||
@@ -316,8 +319,8 @@ func PortsFree(ctx context.Context, run Runner, p FoundationPorts, ours func(rea
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("the foundation's ports must be free before anything is raised:\n - %s\n"+
|
||||
"Give it another with the matching flag (--store-port, --bus-port, --amqp-port, "+
|
||||
"--management-port, --registry-port, --packages-port, --hub-port); nothing was changed",
|
||||
"Give it another with the matching flag (--store-port, --bus-port, "+
|
||||
"--registry-port, --packages-port, --hub-port); nothing was changed",
|
||||
strings.Join(problems, "\n - "))
|
||||
}
|
||||
return nil
|
||||
@@ -504,8 +507,8 @@ func CheckTheMachine(ctx context.Context, o Options, run Runner, bundle *declara
|
||||
if err := PortsFree(ctx, run, p, ours); err != nil {
|
||||
return err
|
||||
}
|
||||
say(fmt.Sprintf(" ports free store %d, bus %d, amqp %d, management %d, registry %d, packages %d, hub %d/udp",
|
||||
p.Store, p.Bus, p.AMQP, p.Management, p.Registry, p.Packages, p.Hub))
|
||||
say(fmt.Sprintf(" ports free store %d, bus %d, registry %d, packages %d, hub %d/udp",
|
||||
p.Store, p.Bus, p.Registry, p.Packages, p.Hub))
|
||||
if o.Tunnel != "" {
|
||||
// One tunnel and one range: the mesh's range IS the found tunnel's, so the rule that the
|
||||
// two must not overlap applies only where a found tunnel is left running beside the mesh's
|
||||
|
||||
@@ -31,7 +31,7 @@ func producedBundle(t *testing.T) Rewritten {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
|
||||
if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
@@ -60,7 +60,7 @@ func TestTheDefaultPortsLeaveTheBundleAsItWas(t *testing.T) {
|
||||
|
||||
func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
|
||||
r := producedBundle(t)
|
||||
p := FoundationPorts{Store: 5433, Bus: 5771, AMQP: 5772, Management: 15673, Registry: 5100}
|
||||
p := FoundationPorts{Store: 5433, Bus: 5771, Registry: 5100}
|
||||
got, err := RewritePorts(&r, p, "10.77.0.0/16")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -72,9 +72,9 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
|
||||
if len(storeC.Ports) != 1 || storeC.Ports[0] != "5433:5432" {
|
||||
t.Errorf("the store publishes %v", storeC.Ports)
|
||||
}
|
||||
broker := containerNamed(r.Declaration, "mesh-broker")
|
||||
if strings.Join(broker.Ports, " ") != "5771:5671 5772:5672 127.0.0.1:15673:15672" {
|
||||
t.Errorf("the broker publishes %v", broker.Ports)
|
||||
bus := containerNamed(r.Declaration, "mesh-broker-nats")
|
||||
if strings.Join(bus.Ports, " ") != "5771:4222 127.0.0.1:8222:8222" {
|
||||
t.Errorf("the bus publishes %v", bus.Ports)
|
||||
}
|
||||
control, err := controlPlaneIn(r.Declaration)
|
||||
if err != nil {
|
||||
@@ -85,9 +85,8 @@ func TestGivenPortsMoveOnlyTheMachinesSide(t *testing.T) {
|
||||
t.Errorf("%s still dials %s", key, value)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(control.Env["MESH_BROKER_AMQP"], "@127.0.0.1:5772/") ||
|
||||
!strings.HasSuffix(control.Env["MESH_BROKER_MANAGEMENT"], "@127.0.0.1:15673") {
|
||||
t.Errorf("the broker is dialled at %s and %s", control.Env["MESH_BROKER_AMQP"], control.Env["MESH_BROKER_MANAGEMENT"])
|
||||
if !strings.HasSuffix(control.Env[BusVar], "@127.0.0.1:5771") {
|
||||
t.Errorf("the control plane dials the bus at %s", control.Env[BusVar])
|
||||
}
|
||||
if control.Env["MESH_BROKER_ADDRESS"] != "192.0.2.10:5771" || r.BrokerAddress != "192.0.2.10:5771" {
|
||||
t.Errorf("nodes are told to dial %s (%s)", control.Env["MESH_BROKER_ADDRESS"], r.BrokerAddress)
|
||||
@@ -169,20 +168,20 @@ func noneOurs(reachable.Reach) bool { return false }
|
||||
func TestABusyPortIsRefusedNamingItsHolder(t *testing.T) {
|
||||
m := machineRunner{
|
||||
ss: "tcp LISTEN 0 4096 0.0.0.0:5000 0.0.0.0:* users:((\"docker-proxy\",pid=1,fd=7))\n" +
|
||||
"tcp LISTEN 0 4096 127.0.0.1:15672 0.0.0.0:* users:((\"beam.smp\",pid=2,fd=7))\n",
|
||||
"tcp LISTEN 0 4096 0.0.0.0:5671 0.0.0.0:* users:((\"beam.smp\",pid=2,fd=7))\n",
|
||||
ps: "predecessor-registry\t0.0.0.0:5000->5000/tcp\n",
|
||||
}
|
||||
err := PortsFree(context.Background(), m.run, DefaultPorts(), noneOurs)
|
||||
if err == nil {
|
||||
t.Fatal("held ports were not refused")
|
||||
}
|
||||
for _, want := range []string{"predecessor-registry", "beam.smp", "tcp/5000", "tcp/15672", "--registry-port"} {
|
||||
for _, want := range []string{"predecessor-registry", "beam.smp", "tcp/5000", "tcp/5671", "--registry-port"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
p := DefaultPorts()
|
||||
p.Registry, p.Management = 5100, 15673
|
||||
p.Registry, p.Bus = 5100, 5771
|
||||
if err := PortsFree(context.Background(), m.run, p, noneOurs); err != nil {
|
||||
t.Errorf("other ports given and still refused: %v", err)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,28 @@ import (
|
||||
"github.com/novox/mesh-host/internal/profile"
|
||||
)
|
||||
|
||||
// CheckTemplate is what preflight asks of the bundle template before anything is changed: that it
|
||||
// is a declaration, that it raises a control plane, and that it names the bus the way this
|
||||
// installer fills it (novox/hq ADR 0331 §3, §4).
|
||||
//
|
||||
// Parsed here as well as at the rewrite, because a template that is not a declaration should cost
|
||||
// a second rather than an image load and a written file. And a control plane given no bus never
|
||||
// answers: the verify step would wait its whole timeout to say so, and a template like that is a
|
||||
// fault in the bundle, said at once.
|
||||
func CheckTemplate(template []byte) (*declaration.Declaration, error) {
|
||||
parsed, err := declaration.ParseFileTrusted(template)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bundle template is not a declaration: %w", err)
|
||||
}
|
||||
if _, err := controlPlaneIn(parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := BusPlaceholders(template, parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// DefaultRegistry is where an image reference that names no host comes from.
|
||||
const DefaultRegistry = "registry-1.docker.io:443"
|
||||
|
||||
@@ -76,13 +98,8 @@ func Preflight(ctx context.Context, o Options, d Deps, say func(string)) ([]byte
|
||||
"it. Point --bundle at one; mesh-host's examples/foundation-first-node.lock is "+
|
||||
"the shape", err)
|
||||
}
|
||||
// Parsed here as well as at the rewrite, because a template that is not a declaration should
|
||||
// cost a second rather than an image load and a written file.
|
||||
parsed, err := declaration.ParseFileTrusted(template)
|
||||
parsed, err := CheckTemplate(template)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bundle template is not a declaration: %w", err)
|
||||
}
|
||||
if _, err := controlPlaneIn(parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
say(fmt.Sprintf(" bundle template %s (%d resources)", o.Template, len(parsed.Resources)))
|
||||
|
||||
@@ -149,3 +149,21 @@ func TestWhereAnImageWouldBeFetchedFrom(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A template whose temporary control plane is given no bus is refused at preflight, before anything
|
||||
// is changed, rather than raised and waited on at the verify step until its timeout (novox/hq issue
|
||||
// 146, ADR 0331 §4): that control plane says it has no MESH_BUS_NATS every few seconds, for ever.
|
||||
func TestATemplateWhoseControlPlaneHasNoBusIsRefusedBeforeAnythingIsRaised(t *testing.T) {
|
||||
template := string(theRealBundle(t))
|
||||
if _, err := CheckTemplate([]byte(template)); err != nil {
|
||||
t.Fatalf("the shipped bundle is refused at preflight: %v", err)
|
||||
}
|
||||
const bus = `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`
|
||||
if !strings.Contains(template, bus) {
|
||||
t.Fatalf("the shipped bundle does not give its control plane %s", bus)
|
||||
}
|
||||
_, err := CheckTemplate([]byte(strings.Replace(template, bus, "", 1)))
|
||||
if err == nil || !strings.Contains(err.Error(), BusVar) {
|
||||
t.Fatalf("a control plane with no bus was let through preflight: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,7 +99,7 @@ type Rewritten struct {
|
||||
// migration pointing at an image no registry serves, and the apply would fail in the middle —
|
||||
// after the store is up, before the broker. They are one image and they move together.
|
||||
//
|
||||
// **Third-party images are not touched.** postgres and lavinmq keep the `name@sha256:` references
|
||||
// **Third-party images are not touched.** postgres and nats keep the `name@sha256:` references
|
||||
// the template carries and are pulled from wherever those name (novox/hq ADR 0006). This is
|
||||
// checked afterwards rather than merely intended: the produced bundle is re-parsed and every other
|
||||
// container's image is compared against what it was.
|
||||
|
||||
@@ -183,7 +183,7 @@ func TestTheProducedBundleKeepsTheTemplatesComments(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const remembered = "NAMED VOLUME"
|
||||
const remembered = "A filter before anything listens"
|
||||
if !strings.Contains(string(out.Bundle), remembered) {
|
||||
t.Errorf("the produced bundle lost the template's comments; %q is gone", remembered)
|
||||
}
|
||||
|
||||
+185
-102
@@ -4,49 +4,58 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// The mesh's root credentials, made at genesis rather than copied from the template.
|
||||
//
|
||||
// **The template carries `bootstrap` and `guest`, and a mesh raised from it kept them** (novox/hq
|
||||
// issue 071). The store's superuser and the broker's administrator are the two credentials every
|
||||
// other one rests on, and they were the two that were not secret: constants in a file anybody can
|
||||
// read, carried into the mesh by `secret accept` and marked as something the mesh must never
|
||||
// replace — which is correct for a credential that already created the databases, and made the
|
||||
// well-known value permanent.
|
||||
// **The template carries `bootstrap`, and a mesh raised from it kept it** (novox/hq issue 071). The
|
||||
// store's superuser and the controller's account on the bus are the two credentials every other one
|
||||
// rests on, and they were the two that were not secret: constants in a file anybody can read,
|
||||
// carried into the mesh by `secret accept` and marked as something the mesh must never replace —
|
||||
// which is correct for a credential that already created the databases, and made the well-known
|
||||
// value permanent.
|
||||
//
|
||||
// So the installer makes them. Two random values, **made once and kept on this machine** at the
|
||||
// paths the postgres and lavinmq modules declare as their own secrets — so that when phase three
|
||||
// adopts the store and the broker, `secret accept` carries in exactly the value the servers were
|
||||
// raised with, and the host's later write of the sealed secret lands the same bytes in the same
|
||||
// file. A second run finds the files and changes nothing, which is what lets the installer say
|
||||
// "already done" about a store it must not restart.
|
||||
// So the installer makes them. Two random values, **made once and kept on this machine** — the
|
||||
// store's at the path the postgres module declares as its own secret, so that when phase three
|
||||
// adopts the store `secret accept` carries in exactly the value the server was raised with; the
|
||||
// bus's beside the other things only genesis holds. A second run finds the files and changes
|
||||
// nothing, which is what lets the installer say "already done" about a store it must not restart.
|
||||
//
|
||||
// **The store reads its password from a file, not its environment.** `POSTGRES_PASSWORD` in a
|
||||
// container's environment is in `docker inspect` for ever; the module that adopts the store
|
||||
// declares the same file mount, so the two specs are one and the applier reconciles rather than
|
||||
// recreates (phase3.go). The broker has no such file: its image's default administrator is changed
|
||||
// in place by an action once the broker answers, and the produced bundle carries that action.
|
||||
// recreates (phase3.go).
|
||||
//
|
||||
// **The bus has no administrator to change; it has a user list** (novox/hq ADR 0331 §3). The
|
||||
// bundle carries the first list — the controller's own account at `bootstrap` — and the
|
||||
// controller's address names the same account. Both are the bundle's placeholders: the address's
|
||||
// `nats://controller:bootstrap@` and the controller's entry in the carried list. The installer
|
||||
// puts the made password in the first and its bcrypt hash in the second, so the bus is raised with
|
||||
// the account the controller dials with. The controller records a hash of what it actually uses
|
||||
// (mesh-controller internal/broker, CredentialIn), so its own compositions carry the same account
|
||||
// onward, and the permanent controller is given it by `secret accept` (control.go deliverStores).
|
||||
const (
|
||||
// StoreSuperuserFile is where the store's superuser password lives on the machine — the
|
||||
// postgres module's own-secret path, so genesis and adoption write the same file.
|
||||
StoreSuperuserFile = "/var/lib/postgres/superuser.secret"
|
||||
// BrokerAdminFile is the same for the broker's administrator — the lavinmq module's.
|
||||
BrokerAdminFile = "/var/lib/lavinmq-module/admin.secret"
|
||||
// BrokerAdminUser is the broker's administrator. The image's default account, kept by name
|
||||
// and given a password that is not the image's default; a renamed account would have to be
|
||||
// created before anything can authenticate, and the thing that creates accounts is the thing
|
||||
// that has to authenticate first.
|
||||
BrokerAdminUser = "guest"
|
||||
// BusControllerFile is where the controller's password on the bus lives on the machine. Not a
|
||||
// module's path: no module reads it — the permanent controller is handed the address the
|
||||
// temporary one was given — and genesis keeps it only so a second run puts the same account
|
||||
// in the same two places.
|
||||
BusControllerFile = "/var/lib/mesh-genesis/bus-controller.secret"
|
||||
// BusControllerHashFile is the bcrypt hash of it, kept with it. bcrypt salts every hash, so a
|
||||
// hash made afresh on every run would rewrite the bus's user list on every run; kept, the
|
||||
// produced bundle is the same bundle twice.
|
||||
BusControllerHashFile = BusControllerFile + ".bcrypt"
|
||||
|
||||
storeSuperuserMount = "/run/secrets/superuser"
|
||||
|
||||
@@ -55,15 +64,81 @@ const (
|
||||
templateStorePassword = `"POSTGRES_PASSWORD": "bootstrap"`
|
||||
templateStoreVolumes = `"volumes": ["mesh-store-data:/var/lib/postgresql/data"]`
|
||||
templateStoreURL = "postgres:bootstrap@"
|
||||
templateBrokerURL = "guest:guest@"
|
||||
templateBrokerReady = "\"verify\": [\"lavinmqctl\", \"status\"]\n },"
|
||||
brokerAdminMarker = "/var/lib/lavinmq/.mesh-admin"
|
||||
|
||||
// BusURLPlaceholder is the controller's account in its bus address, as the template names it.
|
||||
BusURLPlaceholder = "nats://" + busControllerUser + ":bootstrap@"
|
||||
// BusUserPlaceholder is the controller's entry in the user list the template carries, up to
|
||||
// its password's hash — written as it stands inside the bundle's JSON string, quotes escaped.
|
||||
BusUserPlaceholder = `{ user: \"` + busControllerUser + `\", password: \"`
|
||||
busControllerUser = "controller"
|
||||
)
|
||||
|
||||
// RootCredentials are the two values, and whether this run made them.
|
||||
// BusVar is where the control plane finds the mesh's bus, with its own account (mesh-controller
|
||||
// internal/broker NATSVar), and CertificateVar the certificate it pins the bus to (ADR 0004).
|
||||
const (
|
||||
BusVar = "MESH_BUS_NATS"
|
||||
CertificateVar = "MESH_BROKER_CERTIFICATE"
|
||||
)
|
||||
|
||||
// BusPlaceholders refuses a template whose bus connection this installer does not recognise,
|
||||
// naming the placeholder it looked for (novox/hq ADR 0331 §3, §4).
|
||||
//
|
||||
// Asked at preflight, before anything is changed, and again where the bundle is rewritten. A
|
||||
// template with no `MESH_BUS_NATS` raises a control plane that cannot reach the mesh's bus and
|
||||
// says so every few seconds for ever (issue 146): the verify step would wait out its whole timeout
|
||||
// for an answer that cannot come. That is a fault in the bundle, and it is found here instead.
|
||||
func BusPlaceholders(template []byte, d *declaration.Declaration) error {
|
||||
control, err := controlPlaneIn(d)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address := strings.TrimSpace(control.Env[BusVar])
|
||||
if address == "" {
|
||||
return fmt.Errorf("the template's control plane (%q) is given no %s, so it could not reach the mesh's "+
|
||||
"bus and would never answer. A genesis template names the bus as %s<address>, which this "+
|
||||
"installer fills with the account it makes; mesh-host's examples/foundation-first-node.lock "+
|
||||
"is the shape", control.Name, BusVar, BusURLPlaceholder)
|
||||
}
|
||||
if !strings.HasPrefix(address, BusURLPlaceholder) {
|
||||
return fmt.Errorf("the template's control plane reaches the bus at %s, and this installer looked for "+
|
||||
"%s<address> there: the account it makes goes where that placeholder is, and it will not "+
|
||||
"guess where else a bus connection could be", redactURL(address), BusURLPlaceholder)
|
||||
}
|
||||
if strings.TrimSpace(control.Env[CertificateVar]) == "" {
|
||||
return fmt.Errorf("the template's control plane is given no %s, so it could not pin the bus's "+
|
||||
"certificate (novox/hq ADR 0004) and would refuse to dial it", CertificateVar)
|
||||
}
|
||||
if n := bytes.Count(template, []byte(BusUserPlaceholder)); n != 1 {
|
||||
return fmt.Errorf("the template's user list for the bus names the controller's account %d times, and "+
|
||||
"this installer looked for it once, as %s<hash>: the bus has to be raised with the account the "+
|
||||
"controller dials with, and that is where it goes", n, BusUserPlaceholder)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// redactURL is a bus address with any password in it replaced, for an error that quotes it.
|
||||
func redactURL(address string) string {
|
||||
scheme, rest, has := strings.Cut(address, "://")
|
||||
if !has {
|
||||
rest, scheme = address, ""
|
||||
}
|
||||
at := strings.LastIndex(rest, "@")
|
||||
user, _, hasPassword := strings.Cut(rest[:max(at, 0)], ":")
|
||||
if at < 0 || !hasPassword {
|
||||
return address
|
||||
}
|
||||
if scheme != "" {
|
||||
scheme += "://"
|
||||
}
|
||||
return scheme + user + ":…" + rest[at:]
|
||||
}
|
||||
|
||||
// RootCredentials are the two values, and whether this run made them. BusHash is the bcrypt hash of
|
||||
// Bus, as the bus's user list carries it.
|
||||
type RootCredentials struct {
|
||||
Store, Broker string
|
||||
StoreMade, BrokerMade bool
|
||||
Store, Bus string
|
||||
BusHash string
|
||||
StoreMade, BusMade bool
|
||||
}
|
||||
|
||||
// RootSecrets reads the credentials this machine already holds, or makes them.
|
||||
@@ -71,17 +146,44 @@ type RootCredentials struct {
|
||||
// A dry run makes them in memory and writes nothing — so the bundle it reports is the shape of the
|
||||
// real one, and a machine that was only asked is not left holding half a genesis.
|
||||
func RootSecrets(dryRun bool) (RootCredentials, error) {
|
||||
return rootSecretsAt(StoreSuperuserFile, BusControllerFile, BusControllerHashFile, dryRun)
|
||||
}
|
||||
|
||||
func rootSecretsAt(storeFile, busFile, hashFile string, dryRun bool) (RootCredentials, error) {
|
||||
var out RootCredentials
|
||||
var err error
|
||||
if out.Store, out.StoreMade, err = keptOrMade(StoreSuperuserFile, dryRun); err != nil {
|
||||
if out.Store, out.StoreMade, err = keptOrMade(storeFile, dryRun); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.Broker, out.BrokerMade, err = keptOrMade(BrokerAdminFile, dryRun); err != nil {
|
||||
if out.Bus, out.BusMade, err = keptOrMade(busFile, dryRun); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.BusHash, err = keptHash(hashFile, out.Bus, dryRun); err != nil {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// keptHash is the bcrypt hash of password kept at path, made again only when the kept one is
|
||||
// missing or no longer the hash of this password.
|
||||
func keptHash(path, password string, dryRun bool) (string, error) {
|
||||
if kept, err := readCredentialFile(path); err == nil {
|
||||
if bcrypt.CompareHashAndPassword([]byte(kept), []byte(password)) == nil {
|
||||
return kept, nil
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if dryRun {
|
||||
return string(hash), nil
|
||||
}
|
||||
return string(hash), writeCredentialFile(path, string(hash))
|
||||
}
|
||||
|
||||
func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
|
||||
value, err = readCredentialFile(path)
|
||||
if err == nil {
|
||||
@@ -97,8 +199,13 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
|
||||
if dryRun {
|
||||
return value, true, nil
|
||||
}
|
||||
return value, true, writeCredentialFile(path, value)
|
||||
}
|
||||
|
||||
// writeCredentialFile writes a credential as genesis keeps it.
|
||||
func writeCredentialFile(path, value string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return "", false, err
|
||||
return err
|
||||
}
|
||||
// Written whole and renamed into place, at 0600, owned by whoever runs the installer — root,
|
||||
// which is also who the host runs as when it later writes the sealed copy here.
|
||||
@@ -112,12 +219,9 @@ func keptOrMade(path string, dryRun bool) (value string, made bool, err error) {
|
||||
// tolerates either ending, so a file an earlier genesis wrote still reads.
|
||||
tmp := path + ".genesis"
|
||||
if err := os.WriteFile(tmp, []byte(value), 0o600); err != nil {
|
||||
return "", false, err
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return value, true, nil
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// readCredentialFile is a credential as genesis keeps it: the value alone, its line ending gone.
|
||||
@@ -134,13 +238,6 @@ func readCredentialFile(path string) (string, error) {
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// credentialFingerprint names a credential without being one — what the broker-admin action
|
||||
// leaves on the broker's volume, so its verify holds for this value and not for any value.
|
||||
func credentialFingerprint(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// freshSecret is the same shape the controller mints: 30 random bytes as unpadded base64url, 40
|
||||
// characters, URL-safe — it lands inside connection strings.
|
||||
func freshSecret() (string, error) {
|
||||
@@ -153,55 +250,48 @@ func freshSecret() (string, error) {
|
||||
|
||||
// Masking makes a reporter that never says the credentials this run made.
|
||||
//
|
||||
// The applier reports each action with its command line, and two of them now carry a real
|
||||
// password — the context schemas' connection strings and the broker's change_password. Those
|
||||
// lines go to a terminal and to whatever keeps the transcript, which for the lab is a file. The
|
||||
// exact values are known here, so they are replaced wherever they appear, in every line said.
|
||||
// The applier reports each action with its command line, and the context schemas' connection
|
||||
// strings carry a real password. Those lines go to a terminal and to whatever keeps the
|
||||
// transcript, which for the lab is a file. The exact values are known here, so they are replaced
|
||||
// wherever they appear, in every line said.
|
||||
func Masking(say func(string), c RootCredentials) func(string) {
|
||||
replacer := strings.NewReplacer(c.Store, "…", c.Broker, "…")
|
||||
if c.Store == "" || c.Broker == "" {
|
||||
if c.Store == "" || c.Bus == "" {
|
||||
return say
|
||||
}
|
||||
replacer := strings.NewReplacer(c.Store, "…", c.Bus, "…")
|
||||
return func(line string) { say(replacer.Replace(line)) }
|
||||
}
|
||||
|
||||
// RefuseExistingServers stops a run that would put a made credential in front of a server raised
|
||||
// RefuseExistingServers stops a run that would put a made credential in front of a store raised
|
||||
// by an earlier installer with the template's.
|
||||
//
|
||||
// The store's password is set by initdb, once, on an empty volume; the broker's by the action
|
||||
// above, once. A machine that already holds `mesh-store-data` or `mesh-broker-data` and has no
|
||||
// credential file was raised with `bootstrap` and `guest`, and minting new values here would make a
|
||||
// bundle that dials with passwords the servers do not have — failing three steps later, in the
|
||||
// schemas, with nothing pointing back here. Refused by name instead, with the way forward.
|
||||
// The store's password is set by initdb, once, on an empty volume. A machine that already holds
|
||||
// `mesh-store-data` and has no credential file was raised with `bootstrap`, and minting a new
|
||||
// value here would make a bundle that dials with a password the server does not have — failing
|
||||
// three steps later, in the schemas, with nothing pointing back here. Refused by name instead,
|
||||
// with the way forward.
|
||||
//
|
||||
// **The bus is not asked.** Its accounts live in the user list beside its configuration, which
|
||||
// the bundle writes and the bus re-reads, not in its data volume — so a bus raised earlier takes
|
||||
// the made account as readily as a new one.
|
||||
func RefuseExistingServers(ctx context.Context, run Runner, c RootCredentials) error {
|
||||
for _, check := range []struct {
|
||||
made bool
|
||||
volume string
|
||||
what string
|
||||
file string
|
||||
}{
|
||||
{c.StoreMade, "mesh-store-data", "store", StoreSuperuserFile},
|
||||
{c.BrokerMade, "mesh-broker-data", "broker", BrokerAdminFile},
|
||||
} {
|
||||
if !check.made {
|
||||
continue
|
||||
}
|
||||
if _, err := run(ctx, "docker", "volume", "inspect", check.volume); err != nil {
|
||||
continue // no such volume: a fresh machine, which is the case this installer makes
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"this machine already holds the %s's data (volume %s) and no credential at %s, so it was raised "+
|
||||
"by an earlier installer with the template's password. A new one made here would not open it. "+
|
||||
"Put the password the %s has into %s (0600, the value alone) and run again; then change it "+
|
||||
"on the server and accept the new value — this installer does not rotate a running %s",
|
||||
check.what, check.volume, check.file, check.what, check.file, check.what)
|
||||
if !c.StoreMade {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
if _, err := run(ctx, "docker", "volume", "inspect", "mesh-store-data"); err != nil {
|
||||
return nil // no such volume: a fresh machine, which is the case this installer makes
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"this machine already holds the store's data (volume mesh-store-data) and no credential at %s, so it was raised "+
|
||||
"by an earlier installer with the template's password. A new one made here would not open it. "+
|
||||
"Put the password the store has into %s (0600, the value alone) and run again; then change it "+
|
||||
"on the server and accept the new value — this installer does not rotate a running store",
|
||||
StoreSuperuserFile, StoreSuperuserFile)
|
||||
}
|
||||
|
||||
// RootRewrite says what RewriteRoot did to the bundle.
|
||||
type RootRewrite struct {
|
||||
StoreURLs, BrokerURLs int
|
||||
StoreURLs, BusURLs int
|
||||
}
|
||||
|
||||
// RewriteRoot puts the made credentials into the produced bundle, in place of the template's.
|
||||
@@ -225,37 +315,30 @@ func RewriteRoot(r *Rewritten, c RootCredentials) (RootRewrite, error) {
|
||||
"the store's volumes"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
// Everything that dials the store or the broker with the template's credentials.
|
||||
// Everything that dials the store with the template's credentials.
|
||||
out.StoreURLs = bytes.Count(bundle, []byte(templateStoreURL))
|
||||
if out.StoreURLs == 0 {
|
||||
return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateStoreURL)
|
||||
}
|
||||
bundle = bytes.ReplaceAll(bundle, []byte(templateStoreURL), []byte("postgres:"+c.Store+"@"))
|
||||
out.BrokerURLs = bytes.Count(bundle, []byte(templateBrokerURL))
|
||||
if out.BrokerURLs == 0 {
|
||||
return out, fmt.Errorf("the template names no %q connection, so this installer cannot tell what it would be leaving well-known", templateBrokerURL)
|
||||
}
|
||||
bundle = bytes.ReplaceAll(bundle, []byte(templateBrokerURL), []byte(BrokerAdminUser+":"+c.Broker+"@"))
|
||||
|
||||
// The broker's administrator, changed once the broker answers and before anything dials it.
|
||||
// Verified by a marker on the broker's own data volume holding this password's fingerprint —
|
||||
// the image carries nothing that can try a password from inside, and a marker that merely
|
||||
// existed would let a regenerated password go unapplied for ever. What proves the password
|
||||
// works is the control plane answering over it, a few resources later. The marker ends in a
|
||||
// newline because the verify reads it with the shell's `read`, which fails at end of file
|
||||
// without one — an action that ran and a verify that said no, once, in the lab.
|
||||
fp := credentialFingerprint(c.Broker)
|
||||
action := templateBrokerReady + "\n" +
|
||||
" {\n" +
|
||||
" \"id\": \"broker-admin\",\n" +
|
||||
" \"type\": \"action\",\n" +
|
||||
" \"in\": \"mesh-broker\",\n" +
|
||||
" \"command\": [\"sh\", \"-c\", \"lavinmqctl change_password " + BrokerAdminUser + " '" + c.Broker + "' && echo " + fp + " > " + brokerAdminMarker + "\"],\n" +
|
||||
" \"verify\": [\"sh\", \"-c\", \"read m < " + brokerAdminMarker + " && [ \\\"$m\\\" = " + fp + " ]\"]\n" +
|
||||
" },"
|
||||
if bundle, err = replaceOnce(bundle, templateBrokerReady, action, "the broker's readiness check"); err != nil {
|
||||
// The bus: the controller's account, in its address and in the user list the bus is raised
|
||||
// with, the same account in both (novox/hq ADR 0331 §3).
|
||||
if err := BusPlaceholders(bundle, r.Declaration); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.BusURLs = bytes.Count(bundle, []byte(BusURLPlaceholder))
|
||||
bundle = bytes.ReplaceAll(bundle, []byte(BusURLPlaceholder),
|
||||
[]byte("nats://"+busControllerUser+":"+c.Bus+"@"))
|
||||
at := bytes.Index(bundle, []byte(BusUserPlaceholder)) + len(BusUserPlaceholder)
|
||||
end := bytes.Index(bundle[at:], []byte(`\"`))
|
||||
if end < 0 {
|
||||
return out, fmt.Errorf("the controller's entry in the template's user list for the bus has no end to its password")
|
||||
}
|
||||
if !strings.HasPrefix(c.BusHash, "$2") {
|
||||
return out, fmt.Errorf("the controller's bus password was given no bcrypt hash, so the bus could not be raised with it; this is the installer's fault")
|
||||
}
|
||||
bundle = append(append(append([]byte{}, bundle[:at]...), c.BusHash...), bundle[at+end:]...)
|
||||
|
||||
parsed, err := declaration.ParseFileTrusted(bundle)
|
||||
if err != nil {
|
||||
|
||||
@@ -5,13 +5,24 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
)
|
||||
|
||||
// The produced bundle carries no well-known credential: the store reads its password from the
|
||||
// file genesis made, every connection string names the made values, and the broker's default
|
||||
// administrator is changed by an action before anything dials it (novox/hq issue 071).
|
||||
func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
|
||||
// testCredentials are made values with a real bcrypt hash of the bus password, as RootSecrets makes.
|
||||
func testCredentials(t *testing.T, store, bus string) RootCredentials {
|
||||
t.Helper()
|
||||
hash, err := bcrypt.GenerateFromPassword([]byte(bus), bcrypt.MinCost)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return RootCredentials{Store: store, Bus: bus, BusHash: string(hash)}
|
||||
}
|
||||
|
||||
// theExample is the one genesis bundle, rewritten for an image as the installer does.
|
||||
func theExample(t *testing.T) Rewritten {
|
||||
t.Helper()
|
||||
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
|
||||
if err != nil {
|
||||
t.Skip("no example bundle beside this checkout")
|
||||
@@ -20,81 +31,142 @@ func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
creds := RootCredentials{Store: "STORE-PW-40-characters-of-random-base64u", Broker: "BROKER-PW-40-characters-of-random-base64"}
|
||||
return r
|
||||
}
|
||||
|
||||
// The produced bundle carries no well-known credential: the store reads its password from the
|
||||
// file genesis made, and the controller's account on the bus is the made one in both places the
|
||||
// bundle names it — the address the control plane dials and the user list the bus is raised with
|
||||
// (novox/hq issue 071, ADR 0331 §3).
|
||||
func TestTheProducedBundleCarriesNoWellKnownCredential(t *testing.T) {
|
||||
r := theExample(t)
|
||||
creds := testCredentials(t, "STORE-PW-40-characters-of-random-base64u", "BUS-PW-40-characters-of-random-base64uuu")
|
||||
got, err := RewriteRoot(&r, creds)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(r.Bundle)
|
||||
for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "guest:guest@"} {
|
||||
for _, gone := range []string{`"bootstrap"`, "postgres:bootstrap@", "controller:bootstrap@"} {
|
||||
if strings.Contains(text, gone) {
|
||||
t.Errorf("the produced bundle still says %s", gone)
|
||||
}
|
||||
}
|
||||
if got.StoreURLs < 3 || got.BrokerURLs < 2 {
|
||||
t.Errorf("rewrote %d store and %d broker connections; the template has three and two", got.StoreURLs, got.BrokerURLs)
|
||||
if got.StoreURLs < 3 || got.BusURLs != 1 {
|
||||
t.Errorf("rewrote %d store and %d bus connections; the template has three and one", got.StoreURLs, got.BusURLs)
|
||||
}
|
||||
var store, action bool
|
||||
var store, accounts, control bool
|
||||
for _, res := range r.Declaration.Resources {
|
||||
switch x := res.(type) {
|
||||
case *declaration.Container:
|
||||
if x.Name != "mesh-store" {
|
||||
switch x.Name {
|
||||
case "mesh-store":
|
||||
store = true
|
||||
if _, has := x.Env["POSTGRES_PASSWORD"]; has {
|
||||
t.Error("the store still takes its password from its environment")
|
||||
}
|
||||
if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
|
||||
t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
|
||||
}
|
||||
if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
|
||||
t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
|
||||
}
|
||||
case r.TempName:
|
||||
control = true
|
||||
if want := "nats://controller:" + creds.Bus + "@127.0.0.1:5671"; x.Env[BusVar] != want {
|
||||
t.Errorf("the temporary control plane dials the bus at %q, not with the made account", x.Env[BusVar])
|
||||
}
|
||||
}
|
||||
case *declaration.File:
|
||||
if x.ID != "bus-accounts" {
|
||||
continue
|
||||
}
|
||||
store = true
|
||||
if _, has := x.Env["POSTGRES_PASSWORD"]; has {
|
||||
t.Error("the store still takes its password from its environment")
|
||||
accounts = true
|
||||
hash := hashOf(t, x.Content, "controller")
|
||||
if bcrypt.CompareHashAndPassword([]byte(hash), []byte(creds.Bus)) != nil {
|
||||
t.Errorf("the bus is raised with a controller account whose hash %q is not the made password's", hash)
|
||||
}
|
||||
if x.Env["POSTGRES_PASSWORD_FILE"] != storeSuperuserMount {
|
||||
t.Errorf("the store reads its password from %q", x.Env["POSTGRES_PASSWORD_FILE"])
|
||||
}
|
||||
if !strings.Contains(strings.Join(x.Volumes, " "), StoreSuperuserFile+":"+storeSuperuserMount) {
|
||||
t.Errorf("the store does not mount %s: %v", StoreSuperuserFile, x.Volumes)
|
||||
}
|
||||
case *declaration.Action:
|
||||
if x.ID != "broker-admin" {
|
||||
continue
|
||||
}
|
||||
action = true
|
||||
if x.In != "mesh-broker" || !strings.Contains(strings.Join(x.Command, " "), "change_password "+BrokerAdminUser+" '"+creds.Broker+"'") {
|
||||
t.Errorf("the broker-admin action is %v in %q", x.Command, x.In)
|
||||
if bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")) == nil {
|
||||
t.Error("the bus is still raised with the controller at bootstrap")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !store || !action {
|
||||
t.Fatalf("store=%v action=%v", store, action)
|
||||
}
|
||||
// The order matters: the broker's password changes after it answers and before the control
|
||||
// plane, which dials it with the new one, is raised.
|
||||
var readyAt, adminAt, controlAt int
|
||||
for i, res := range r.Declaration.Resources {
|
||||
switch res.Identity() {
|
||||
case "broker-ready":
|
||||
readyAt = i
|
||||
case "broker-admin":
|
||||
adminAt = i
|
||||
case "control-plane":
|
||||
controlAt = i
|
||||
}
|
||||
}
|
||||
if !(readyAt < adminAt && adminAt < controlAt) {
|
||||
t.Errorf("order ready=%d admin=%d control=%d", readyAt, adminAt, controlAt)
|
||||
if !store || !accounts || !control {
|
||||
t.Fatalf("store=%v accounts=%v control=%v", store, accounts, control)
|
||||
}
|
||||
}
|
||||
|
||||
// A template that no longer says what this expects is refused, not half-rewritten.
|
||||
func TestATemplateWithoutTheKnownCredentialsIsRefused(t *testing.T) {
|
||||
// hashOf is the password hash a NATS user list gives one user.
|
||||
func hashOf(t *testing.T, list, user string) string {
|
||||
t.Helper()
|
||||
marker := `{ user: "` + user + `", password: "`
|
||||
at := strings.Index(list, marker)
|
||||
if at < 0 {
|
||||
t.Fatalf("the user list names no %s:\n%s", user, list)
|
||||
}
|
||||
rest := list[at+len(marker):]
|
||||
return rest[:strings.Index(rest, `"`)]
|
||||
}
|
||||
|
||||
// The bus's placeholders are what the installer looks for, and a template naming no bus connection
|
||||
// it knows is refused at preflight, naming the placeholder (novox/hq ADR 0331 §3, §4).
|
||||
func TestABundleNamingNoBusConnectionItKnowsIsRefusedByItsPlaceholder(t *testing.T) {
|
||||
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
|
||||
if err != nil {
|
||||
t.Skip("no example bundle beside this checkout")
|
||||
}
|
||||
changed := strings.Replace(string(template), `"POSTGRES_PASSWORD": "bootstrap"`, `"POSTGRES_PASSWORD": "other"`, 1)
|
||||
r, err := Rewrite([]byte(changed), "sha256:"+strings.Repeat("ab", 32))
|
||||
parse := func(b string) *declaration.Declaration {
|
||||
d, err := declaration.ParseFileTrusted([]byte(b))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
if err := BusPlaceholders(template, parse(string(template))); err != nil {
|
||||
t.Fatalf("the shipped bundle is refused: %v", err)
|
||||
}
|
||||
for _, c := range []struct{ what, from, to, names string }{
|
||||
{"no bus address", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",`, "", BusVar},
|
||||
{"the previous broker's", `"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671"`,
|
||||
`"MESH_BUS_NATS": "amqp://guest:guest@127.0.0.1:5671"`, BusURLPlaceholder},
|
||||
{"no pinned certificate", `"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"`, `"MESH_OTHER": "x"`, CertificateVar},
|
||||
{"no controller in the user list", `{ user: \"controller\"`, `{ user: \"someone\"`, BusUserPlaceholder},
|
||||
} {
|
||||
if !strings.Contains(string(template), c.from) {
|
||||
t.Fatalf("%s: the bundle does not say %s", c.what, c.from)
|
||||
}
|
||||
changed := strings.Replace(string(template), c.from, c.to, 1)
|
||||
err := BusPlaceholders([]byte(changed), parse(changed))
|
||||
if err == nil {
|
||||
t.Errorf("%s: accepted", c.what)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), c.names) {
|
||||
t.Errorf("%s: refused without naming %s: %v", c.what, c.names, err)
|
||||
}
|
||||
if strings.Contains(err.Error(), "guest:guest") {
|
||||
t.Errorf("%s: the refusal quotes a password: %v", c.what, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The account is made once, with its hash, and a second run puts the same bytes in the bundle — a
|
||||
// hash salted afresh every run would rewrite the bus's user list every run.
|
||||
func TestTheBusAccountIsKeptWithItsHash(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
first, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RewriteRoot(&r, RootCredentials{Store: "x", Broker: "y"}); err == nil {
|
||||
t.Fatal("a template with an unknown store password was rewritten")
|
||||
if !first.BusMade || len(first.Bus) != 40 || bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) != nil {
|
||||
t.Fatalf("first run: made=%v hash ok=%v", first.BusMade, bcrypt.CompareHashAndPassword([]byte(first.BusHash), []byte(first.Bus)) == nil)
|
||||
}
|
||||
second, err := rootSecretsAt(dir+"/store", dir+"/bus", dir+"/bus.bcrypt", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.BusMade || second.Bus != first.Bus || second.BusHash != first.BusHash {
|
||||
t.Fatalf("a second run changed the bus account: made=%v same=%v same hash=%v",
|
||||
second.BusMade, second.Bus == first.Bus, second.BusHash == first.BusHash)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,60 +197,25 @@ func TestRootSecretsAreKeptAcrossRuns(t *testing.T) {
|
||||
// Nothing the installer says after making the credentials contains them.
|
||||
func TestTheTranscriptNeverSaysTheCredentials(t *testing.T) {
|
||||
var said []string
|
||||
say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Broker: "BROKER-PW"})
|
||||
say := Masking(func(l string) { said = append(said, l) }, RootCredentials{Store: "STORE-PW", Bus: "BUS-PW"})
|
||||
say("created context-schemas (docker run -e MESH_STORE_INVENTORY=postgres://postgres:STORE-PW@127.0.0.1:5432/inventory)")
|
||||
say("failed broker-admin (sh -c lavinmqctl change_password guest 'BROKER-PW' && echo x)")
|
||||
say("failed control-plane (MESH_BUS_NATS=nats://controller:BUS-PW@127.0.0.1:5671)")
|
||||
for _, l := range said {
|
||||
if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BROKER-PW") {
|
||||
if strings.Contains(l, "STORE-PW") || strings.Contains(l, "BUS-PW") {
|
||||
t.Errorf("said a credential: %s", l)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "guest '…'") {
|
||||
if !strings.Contains(said[0], "postgres:…@") || !strings.Contains(said[1], "controller:…@") {
|
||||
t.Errorf("the lines were not the same lines with the values masked: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// The broker-admin marker is written with a line ending, because the verify reads it with `read`.
|
||||
func TestTheBrokerAdminMarkerHasALineEnding(t *testing.T) {
|
||||
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
|
||||
if err != nil {
|
||||
t.Skip("no example bundle beside this checkout")
|
||||
}
|
||||
r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, res := range r.Declaration.Resources {
|
||||
a, ok := res.(*declaration.Action)
|
||||
if !ok || a.ID != "broker-admin" {
|
||||
continue
|
||||
}
|
||||
cmd := strings.Join(a.Command, " ")
|
||||
if !strings.Contains(cmd, "&& echo ") || strings.Contains(cmd, "printf %s") {
|
||||
t.Errorf("the marker is written without a line ending: %s", cmd)
|
||||
}
|
||||
if !strings.Contains(strings.Join(a.Verify, " "), "read m <") {
|
||||
t.Errorf("the verify does not read the marker: %v", a.Verify)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The foundation filters before anything listens: nftables is in place before the store, and its
|
||||
// rules refuse the store's and broker's client ports from outside while keeping ssh, the bus a node
|
||||
// enrols over and the registry a node pulls from (novox/hq issue 054).
|
||||
func TestTheFoundationFiltersBeforeAnythingListens(t *testing.T) {
|
||||
template, err := os.ReadFile("../../examples/foundation-first-node.lock")
|
||||
if err != nil {
|
||||
t.Skip("no example bundle beside this checkout")
|
||||
}
|
||||
r, err := Rewrite(template, "sha256:"+strings.Repeat("ab", 32))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := RewriteRoot(&r, RootCredentials{Store: "s", Broker: "b"}); err != nil {
|
||||
r := theExample(t)
|
||||
if _, err := RewriteRoot(&r, testCredentials(t, "s", "b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var filterAt, loadedAt, storeAt, brokerAt = -1, -1, -1, -1
|
||||
|
||||
@@ -89,7 +89,7 @@ func TestAFoundationThatIsUpAndAnsweringIsAccepted(t *testing.T) {
|
||||
// run-once and scheduled shapes are excluded on purpose — a step that has exited is not a
|
||||
// fault. The name is `temp-mesh-controller` because the permanent one is a module and takes the
|
||||
// plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all.
|
||||
want := []string{"mesh-store", "mesh-broker", "temp-mesh-controller"}
|
||||
want := []string{"mesh-store", "mesh-broker-nats", "temp-mesh-controller"}
|
||||
if len(verified.Running) != len(want) {
|
||||
t.Fatalf("confirmed %v running, want %v", verified.Running, want)
|
||||
}
|
||||
@@ -132,7 +132,7 @@ func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) {
|
||||
// than being told only that something is not what it should be.
|
||||
func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
|
||||
runtime := &asked{answer: func(_ string, args []string) (string, error) {
|
||||
if args[0] == "container" && args[len(args)-1] == "mesh-broker" {
|
||||
if args[0] == "container" && args[len(args)-1] == "mesh-broker-nats" {
|
||||
return "false exited\n", nil
|
||||
}
|
||||
if args[0] == "container" {
|
||||
@@ -146,7 +146,7 @@ func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("a container that had exited was reported as part of a running foundation")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") {
|
||||
if !strings.Contains(err.Error(), "mesh-broker-nats") || !strings.Contains(err.Error(), "exited") {
|
||||
t.Errorf("the failure does not say which container is in what state: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"slices"
|
||||
@@ -132,7 +133,13 @@ func (d *Directory) validate(where string, _ bool) []string {
|
||||
if d.Path == "" {
|
||||
problems = append(problems, where+": a directory needs a path")
|
||||
}
|
||||
return append(problems, checkMode(where, d.Mode)...)
|
||||
// A directory's mode may carry the setgid bit (2xxx), so what is made beneath it keeps its group (novox/hq ADR
|
||||
// 0334 §1); never setuid or sticky, which mean nothing the mesh asks of a directory it owns.
|
||||
mode := d.Mode
|
||||
if len(mode) == 4 && mode[0] == '2' {
|
||||
mode = "0" + mode[1:]
|
||||
}
|
||||
return append(problems, checkMode(where, mode)...)
|
||||
}
|
||||
|
||||
// File is a file with literal content. The host renders nothing.
|
||||
@@ -456,14 +463,51 @@ const (
|
||||
// **The distinction 04-ISSUES/036 and 026 turn on.** A `directory` resource is the mesh's own —
|
||||
// it creates it, sets its owner and mode, and removes it when empty ([ADR 0030](novox/hq)). A
|
||||
// media library, a download spool is the operator's: it existed before the mesh, several modules
|
||||
// read and write it at once, and the mesh must not create, chown, reconcile or remove it. The
|
||||
// host confirms it is there and mounts it; nothing else.
|
||||
// read and write it at once, and the mesh must not create, chown, chmod or remove it. The host
|
||||
// confirms it is there and mounts it; nothing else. Library data is a library's (novox/hq ADR 0328),
|
||||
// not an access.
|
||||
type Access struct {
|
||||
ID string `json:"id"`
|
||||
Type Type `json:"type"`
|
||||
Path string `json:"path"`
|
||||
// Mode is how this module reaches the path: read or read-write. Absent narrows to read.
|
||||
Mode string `json:"mode,omitempty"`
|
||||
|
||||
// Users and Taken are what a controller speaking ADR 0325 sends (the identities that use the folder, and
|
||||
// that the operator's warrant was carried out): accepted, so its declaration is not refused whole, and not
|
||||
// acted on since ADR 0328 superseded ADR 0325 — no ACL entry is kept.
|
||||
Users []AccessUser `json:"users,omitempty"`
|
||||
Taken bool `json:"taken,omitempty"`
|
||||
|
||||
// Grant is what a controller older than ADR 0325 still sends (ADR 0312's group grant, superseded): accepted
|
||||
// so its declaration is not refused whole, and never acted on — no account is put in a group for data (§5).
|
||||
Grant json.RawMessage `json:"grant,omitempty"`
|
||||
}
|
||||
|
||||
// The uses an access's user has (ADR 0051 §3), and where it runs (ADR 0325 §4).
|
||||
const (
|
||||
FromContainer = "container"
|
||||
FromToolRunner = "tool-runner"
|
||||
)
|
||||
|
||||
// AccessUser is one identity that uses an accessed folder: its name and uid as the controller allocated them, its
|
||||
// use, and where it runs. A tool runner's account may come with no uid (0 or absent): the engine reads it by name.
|
||||
type AccessUser struct {
|
||||
Identity string `json:"identity"`
|
||||
UID int `json:"uid,omitempty"`
|
||||
Use string `json:"use"`
|
||||
From string `json:"from"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads a user as strictly as a resource is read.
|
||||
func (u *AccessUser) UnmarshalJSON(raw []byte) error {
|
||||
type plain AccessUser
|
||||
var p plain
|
||||
if err := strictDecode(raw, &p); err != nil {
|
||||
return fmt.Errorf("an access's user: %w", err)
|
||||
}
|
||||
*u = AccessUser(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Access) Identity() string { return a.ID }
|
||||
@@ -482,9 +526,43 @@ func (a *Access) validate(where string, _ bool) []string {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: an access is %q or %q, not %q", where, AccessRead, AccessReadWrite, a.Mode))
|
||||
}
|
||||
for _, u := range a.Users {
|
||||
switch {
|
||||
case u.Identity == "" || !accountName.MatchString(u.Identity):
|
||||
problems = append(problems, fmt.Sprintf("%s: a user of an access is an account's name, and %q is not one",
|
||||
where, u.Identity))
|
||||
case u.UID < 0 || (u.UID == 0 && u.From != FromToolRunner):
|
||||
// Never root (novox/hq ADR 0325 §11): an entry for uid 0 would be a right root has anyway, and a
|
||||
// number below it is no account. A tool runner's account alone may come without its number: the
|
||||
// controller cannot know the operator's account's, and the engine reads it by name
|
||||
// (apply.ResolveRunners), refusing one that is root's.
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s has uid %d, and a module's identity is never "+
|
||||
"root or below it (novox/hq ADR 0325 §11)", where, u.Identity, u.UID))
|
||||
}
|
||||
if u.Use != AccessRead && u.Use != AccessReadWrite {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s uses it %q, and a use is %q or %q", where,
|
||||
u.Identity, u.Use, AccessRead, AccessReadWrite))
|
||||
}
|
||||
if u.Use == AccessReadWrite && a.Mode != AccessReadWrite {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s writes an access declared %s", where,
|
||||
u.Identity, orRead(a.Mode)))
|
||||
}
|
||||
if u.From != FromContainer && u.From != FromToolRunner {
|
||||
problems = append(problems, fmt.Sprintf("%s: the user %s runs from %q, and that is %q or %q", where,
|
||||
u.Identity, u.From, FromContainer, FromToolRunner))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// orRead is an access's mode as said: absent narrows to read.
|
||||
func orRead(mode string) string {
|
||||
if mode == "" {
|
||||
return AccessRead
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
func (u *User) Identity() string { return u.ID }
|
||||
func (u *User) Kind() Type { return TypeUser }
|
||||
func (u *User) Target() string { return u.Name }
|
||||
@@ -1160,6 +1238,78 @@ type Container struct {
|
||||
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||
Health *Health `json:"health,omitempty"`
|
||||
|
||||
// RunAs is who the container runs as, resolved by the controller from the module's identity and the
|
||||
// image's declared interface (novox/hq ADR 0325 §2, §3): told by environment (the image reads PUID and
|
||||
// PGID, say) or by the runtime's own user option. Part of the spec, so a change recreates it. Absent: the
|
||||
// container runs as its image says. Its key is `identity`, the manifest's word; the field is not, because
|
||||
// a resource's Identity is its id.
|
||||
RunAs *ContainerIdentity `json:"identity,omitempty"`
|
||||
}
|
||||
|
||||
// The ways an image is told who to run as (novox/hq ADR 0325 §2).
|
||||
const (
|
||||
IdentityByEnv = "env"
|
||||
IdentityByRun = "run"
|
||||
)
|
||||
|
||||
// ContainerIdentity is who a container runs as, and how its image is told.
|
||||
type ContainerIdentity struct {
|
||||
// Mode is IdentityByEnv or IdentityByRun.
|
||||
Mode string `json:"mode"`
|
||||
// UserEnv and GroupEnv are the variables the image reads the user's and the group's number from, for
|
||||
// IdentityByEnv: "PUID", "PGID".
|
||||
UserEnv string `json:"user_env,omitempty"`
|
||||
GroupEnv string `json:"group_env,omitempty"`
|
||||
UID int `json:"uid"`
|
||||
GID int `json:"gid"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads an identity as strictly as a resource is read.
|
||||
func (i *ContainerIdentity) UnmarshalJSON(raw []byte) error {
|
||||
type plain ContainerIdentity
|
||||
var p plain
|
||||
if err := strictDecode(raw, &p); err != nil {
|
||||
return fmt.Errorf("a container's identity: %w", err)
|
||||
}
|
||||
*i = ContainerIdentity(p)
|
||||
return nil
|
||||
}
|
||||
|
||||
// envName is a variable's name as a runtime takes it.
|
||||
var envName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
func (i *ContainerIdentity) problems(where string, env map[string]string) []string {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
switch i.Mode {
|
||||
case IdentityByEnv:
|
||||
if !envName.MatchString(i.UserEnv) || (i.GroupEnv != "" && !envName.MatchString(i.GroupEnv)) {
|
||||
problems = append(problems, fmt.Sprintf("%s: an identity told by environment names the variables "+
|
||||
"it is told in, and %q and %q are not both names", where, i.UserEnv, i.GroupEnv))
|
||||
}
|
||||
for _, name := range []string{i.UserEnv, i.GroupEnv} {
|
||||
if _, set := env[name]; set && name != "" {
|
||||
problems = append(problems, fmt.Sprintf("%s: env sets %s, which the identity sets; one value "+
|
||||
"for one variable", where, name))
|
||||
}
|
||||
}
|
||||
case IdentityByRun:
|
||||
if i.UserEnv != "" || i.GroupEnv != "" {
|
||||
problems = append(problems, where+": an identity told by the runtime's user option names no variable")
|
||||
}
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf("%s: an identity is told by %q or %q, not %q", where,
|
||||
IdentityByEnv, IdentityByRun, i.Mode))
|
||||
}
|
||||
// Never root (novox/hq ADR 0325 §11): refused here, so nothing of the declaration is made or started.
|
||||
if i.UID <= 0 || i.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: the identity is uid %d, gid %d, and a module's identity is "+
|
||||
"never root, nor a number below it (novox/hq ADR 0325 §11)", where, i.UID, i.GID))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func (c *Container) Identity() string { return c.ID }
|
||||
@@ -1242,6 +1392,7 @@ func (c *Container) validate(where string, _ bool) []string {
|
||||
problems = append(problems, where+": networks is for a container that keeps running; a step "+
|
||||
"runs and exits, and joins nothing afterwards")
|
||||
}
|
||||
problems = append(problems, c.RunAs.problems(where+": container "+c.Name, c.Env)...)
|
||||
return append(problems, checkImage(where, c.Image)...)
|
||||
}
|
||||
|
||||
@@ -1392,6 +1543,185 @@ type Declaration struct {
|
||||
// the generation of what it puts back, which may be older than what the machine applied, and is
|
||||
// not refused for it.
|
||||
PutBack bool
|
||||
|
||||
// Identities is the identity of every module assigned to this machine (novox/hq ADR 0328 §4): an account
|
||||
// and a group `mesh-<module>` with the numbers the controller allocated once for the module, the same on
|
||||
// every machine. The engine makes each before any resource is applied, and removes what it made of one no
|
||||
// longer declared. Absent is a controller older than ADR 0325.
|
||||
Identities []Identity
|
||||
|
||||
// Groups is the group of every library this machine is reached by (novox/hq ADR 0328 §3): `mesh-<library>`
|
||||
// with the number the controller allocated once, mesh-wide. The engine makes each, and removes what it made of
|
||||
// one no longer declared. Absent is a controller older than ADR 0328.
|
||||
Groups []Group
|
||||
|
||||
// Libraries is every library this machine is reached by (ADR 0328 §1, §6, §9): its root here, its group, whether
|
||||
// this is its home, whether the operator's warrant to take it was carried out, the sub-paths its modules ask for,
|
||||
// and the tool runner's account to put in its group (§11). Absent is a controller older than ADR 0328.
|
||||
Libraries []Library
|
||||
}
|
||||
|
||||
// Identity is one module's identity, as the controller allocated it (ADR 0328 §4): its user `mesh-<module>` with
|
||||
// uid, its own group of the same name and number, its primary group gid (its own, or its first library's group),
|
||||
// and the groups of its other libraries as added groups.
|
||||
type Identity struct {
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
UID int `json:"uid"`
|
||||
GID int `json:"gid"`
|
||||
Groups []int `json:"groups,omitempty"`
|
||||
// Runner is the tool runner's account, put in the module's own group (novox/hq ADR 0334 §2) when its tools run
|
||||
// on this machine. Absent for a module that declares no tools here, and from a controller older than ADR 0334.
|
||||
Runner string `json:"runner,omitempty"`
|
||||
}
|
||||
|
||||
// Group is one library's group, as the controller allocated it (ADR 0328 §3).
|
||||
type Group struct {
|
||||
Name string `json:"name"`
|
||||
GID int `json:"gid"`
|
||||
}
|
||||
|
||||
// GroupPrefix starts every library group's name, as IdentityPrefix starts every module's.
|
||||
const GroupPrefix = "mesh-"
|
||||
|
||||
// LibraryAccessPrefix starts the access a library's wait and its take act name: `library:<name>`.
|
||||
const LibraryAccessPrefix = "library:"
|
||||
|
||||
// Library is one library as this machine sees it (ADR 0328).
|
||||
type Library struct {
|
||||
Library string `json:"library"`
|
||||
// Root is the library's root on this machine: on its home the data itself, elsewhere the mount point the mounts
|
||||
// module mounts it at.
|
||||
Root string `json:"root"`
|
||||
Group string `json:"group"`
|
||||
GID int `json:"gid"`
|
||||
// Home is that the library's data lives on this machine: only there does the engine keep it.
|
||||
Home bool `json:"home"`
|
||||
// Taken is the controller saying the operator's warrant to take it was carried out (§7).
|
||||
Taken bool `json:"taken,omitempty"`
|
||||
// Paths are the sub-paths its modules ask for, relative to the root.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
// Runner is the tool runner's account, put in the library's group (§11). Absent when no module's tools use it.
|
||||
Runner string `json:"runner,omitempty"`
|
||||
}
|
||||
|
||||
// checkLibraries judges the groups and libraries on arrival: every library's group declared, named for it, never
|
||||
// root's number; paths that stay below the root; a runner that is an account's name; and no two roots that nest
|
||||
// (§13).
|
||||
func checkLibraries(groups []Group, libs []Library) []string {
|
||||
var problems []string
|
||||
byName := map[string]int{}
|
||||
numbers := map[int]string{}
|
||||
for _, g := range groups {
|
||||
where := fmt.Sprintf("group %q", g.Name)
|
||||
if !strings.HasPrefix(g.Name, GroupPrefix) || !accountName.MatchString(g.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s: a library's group is named %s<library>", where, GroupPrefix))
|
||||
}
|
||||
if g.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: gid %d, and a library's group is never root, nor a number "+
|
||||
"below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where, g.GID))
|
||||
}
|
||||
if _, twice := byName[g.Name]; twice {
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
if other, used := numbers[g.GID]; used {
|
||||
problems = append(problems, fmt.Sprintf("%s: gid %d is %s's too", where, g.GID, other))
|
||||
}
|
||||
byName[g.Name], numbers[g.GID] = g.GID, g.Name
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, l := range libs {
|
||||
where := fmt.Sprintf("library %q", l.Library)
|
||||
switch {
|
||||
case l.Library == "" || strings.ContainsAny(l.Library, "/ \t\n"):
|
||||
problems = append(problems, where+": a library is named by one word")
|
||||
case seen[l.Library]:
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
seen[l.Library] = true
|
||||
if !filepath.IsAbs(l.Root) || filepath.Clean(l.Root) != l.Root || l.Root == "/" {
|
||||
problems = append(problems, fmt.Sprintf("%s: its root %q is not an absolute, clean path below /", where,
|
||||
l.Root))
|
||||
}
|
||||
if gid, ok := byName[l.Group]; !ok || gid != l.GID {
|
||||
problems = append(problems, fmt.Sprintf("%s: its group %s (gid %d) is not one this declaration makes",
|
||||
where, l.Group, l.GID))
|
||||
}
|
||||
if l.Group != GroupPrefix+l.Library {
|
||||
problems = append(problems, fmt.Sprintf("%s: its group is %s%s, and %s is not", where, GroupPrefix,
|
||||
l.Library, l.Group))
|
||||
}
|
||||
for _, p := range l.Paths {
|
||||
if p == "" || filepath.IsAbs(p) || filepath.Clean(p) != p || p == ".." || strings.HasPrefix(p, "../") {
|
||||
problems = append(problems, fmt.Sprintf("%s: the path %q is not a clean path below its root", where, p))
|
||||
}
|
||||
}
|
||||
if l.Runner != "" && !accountName.MatchString(l.Runner) {
|
||||
problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, l.Runner))
|
||||
}
|
||||
}
|
||||
for i, a := range libs {
|
||||
for _, b := range libs[i+1:] {
|
||||
if a.Root != "" && b.Root != "" && (a.Root == b.Root || strings.HasPrefix(a.Root, b.Root+"/") ||
|
||||
strings.HasPrefix(b.Root, a.Root+"/")) {
|
||||
problems = append(problems, fmt.Sprintf("library %q: its root and library %q's nest, and libraries "+
|
||||
"never overlap (novox/hq ADR 0328 §13)", a.Library, b.Library))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// IdentityPrefix starts every module identity's name.
|
||||
const IdentityPrefix = "mesh-"
|
||||
|
||||
// checkIdentities judges the identities on arrival: each named for its module, never root, and no name or number
|
||||
// twice — two modules with one account would write as each other.
|
||||
func checkIdentities(ids []Identity, groups []Group) []string {
|
||||
var problems []string
|
||||
names, uids := map[string]bool{}, map[int]string{}
|
||||
libraryGroups := map[int]bool{}
|
||||
for _, g := range groups {
|
||||
libraryGroups[g.GID] = true
|
||||
}
|
||||
for _, id := range ids {
|
||||
where := fmt.Sprintf("identity %q", id.Name)
|
||||
if id.Module == "" || id.Name != IdentityPrefix+id.Module || !accountName.MatchString(id.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s: a module's identity is named %s<module>, and this one is "+
|
||||
"for the module %q", where, IdentityPrefix, id.Module))
|
||||
}
|
||||
if id.UID <= 0 || id.GID <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d, gid %d, and a module's identity is never root, "+
|
||||
"nor a number below it (novox/hq ADR 0328 §15); nothing of this declaration was made", where,
|
||||
id.UID, id.GID))
|
||||
}
|
||||
if names[id.Name] {
|
||||
problems = append(problems, where+": declared twice")
|
||||
}
|
||||
if other, used := uids[id.UID]; used {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d is %s's too", where, id.UID, other))
|
||||
}
|
||||
// Its primary group is its own (gid == uid) or a library's this declaration makes (ADR 0328 §4); so is
|
||||
// every added group. A module's own number is never a library group's.
|
||||
if id.GID != id.UID && !libraryGroups[id.GID] {
|
||||
problems = append(problems, fmt.Sprintf("%s: its primary group gid %d is neither its own (%d) nor a "+
|
||||
"library's this declaration makes", where, id.GID, id.UID))
|
||||
}
|
||||
if libraryGroups[id.UID] {
|
||||
problems = append(problems, fmt.Sprintf("%s: uid %d is a library group's number too", where, id.UID))
|
||||
}
|
||||
for _, g := range id.Groups {
|
||||
if !libraryGroups[g] {
|
||||
problems = append(problems, fmt.Sprintf("%s: its added group gid %d is not a library's this "+
|
||||
"declaration makes", where, g))
|
||||
}
|
||||
}
|
||||
if id.Runner != "" && !accountName.MatchString(id.Runner) {
|
||||
problems = append(problems, fmt.Sprintf("%s: its runner %q is not an account's name", where, id.Runner))
|
||||
}
|
||||
names[id.Name], uids[id.UID] = true, id.Name
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// LeftOutModuleOf says which left-out module a recorded resource belongs to, if any: its id is the
|
||||
@@ -1561,6 +1891,13 @@ type envelope struct {
|
||||
// sends them only to a host whose reports say `reads_generation`.
|
||||
Generation int64 `json:"generation,omitempty"`
|
||||
PutBack bool `json:"put_back,omitempty"`
|
||||
// Identities is optional on the wire too (novox/hq ADR 0325 §3, ADR 0328 §4). **An older host refuses this key**; a
|
||||
// controller sends it only to a host whose health says link.IdentityContract.
|
||||
Identities []Identity `json:"identities,omitempty"`
|
||||
// Groups and Libraries are optional too (novox/hq ADR 0328). **An older host refuses these keys**; a controller
|
||||
// sends them only to a host whose health says link.LibraryContract.
|
||||
Groups []Group `json:"groups,omitempty"`
|
||||
Libraries []Library `json:"libraries,omitempty"`
|
||||
}
|
||||
|
||||
func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
@@ -1578,8 +1915,10 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
|
||||
}
|
||||
|
||||
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
|
||||
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack}
|
||||
var problems []string
|
||||
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack,
|
||||
Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries}
|
||||
problems := checkIdentities(env.Identities, env.Groups)
|
||||
problems = append(problems, checkLibraries(env.Groups, env.Libraries)...)
|
||||
if env.Generation < 0 {
|
||||
// Below zero would read as "none claimed" and pass every refusal of an older generation.
|
||||
problems = append(problems, fmt.Sprintf("an assignment generation below zero (%d) is not one the "+
|
||||
@@ -1732,6 +2071,10 @@ func checkWhileStopped(resources []Resource) []string {
|
||||
return problems
|
||||
}
|
||||
|
||||
// accountName is what an identity's or an access user's name may be: the portable account name the user tools
|
||||
// take, optionally ending in `$`, never starting with a dash — it is handed to them as an argument.
|
||||
var accountName = regexp.MustCompile(`^[A-Za-z0-9_.][A-Za-z0-9_.-]*\$?$`)
|
||||
|
||||
func strictDecode(raw []byte, into any) error {
|
||||
// DisallowUnknownFields is the whole point rather than strictness for its own sake: a
|
||||
// field the host does not know is a thing the control plane believes it asked for.
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package declaration
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const anImage = "docker.io/library/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
// The fields ADR 0325 adds, read as the controller sends them (novox/hq ADR 0325 §2, §3, §5, issue 567).
|
||||
func TestIdentitiesUsersAndARunAsAreReadAsSent(t *testing.T) {
|
||||
d, err := Parse([]byte(`{"declaration":1,
|
||||
"identities":[{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}],
|
||||
"resources":[
|
||||
{"id":"radarr.app","type":"container","name":"radarr","image":"` + anImage + `",
|
||||
"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":61001}},
|
||||
{"id":"plex.app","type":"container","name":"plex","image":"` + anImage + `","identity":{"mode":"run","uid":61002,"gid":61002}},
|
||||
{"id":"radarr.movies","type":"access","path":"/srv/media/movies","mode":"read-write","taken":true,
|
||||
"users":[{"identity":"mesh-radarr","uid":61001,"use":"read-write","from":"container"},
|
||||
{"identity":"nox","uid":61003,"use":"read","from":"tool-runner"}]},
|
||||
{"id":"old.movies","type":"access","path":"/srv/media/movies","mode":"read-write","grant":{"account":"nox"}}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.Identities) != 1 || d.Identities[0].UID != 61001 {
|
||||
t.Fatalf("identities: %+v", d.Identities)
|
||||
}
|
||||
if c := d.Resources[0].(*Container); c.RunAs == nil || c.RunAs.UserEnv != "PUID" {
|
||||
t.Fatalf("a container's identity: %+v", c.RunAs)
|
||||
}
|
||||
if a := d.Resources[2].(*Access); !a.Taken || len(a.Users) != 2 || a.Users[1].From != FromToolRunner {
|
||||
t.Fatalf("an access's users: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// What the engine refuses on arrival: a root number, an identity not named for its module, a field it does not
|
||||
// know, a user that writes a read access, a variable set twice.
|
||||
func TestWhatADeclarationMayNotSayOfAnIdentity(t *testing.T) {
|
||||
for want, raw := range map[string]string{
|
||||
"never root": `"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]`,
|
||||
"named mesh-<module>": `"identities":[{"module":"m","name":"m","uid":61001,"gid":61001}],` +
|
||||
`"resources":[]`,
|
||||
"is mesh-a's too": `"identities":[{"module":"a","name":"mesh-a","uid":61001,"gid":61001},` +
|
||||
`{"module":"b","name":"mesh-b","uid":61001,"gid":61002}],"resources":[]`,
|
||||
"unknown field": `"identities":[{"module":"m","name":"mesh-m","uid":61001,"gid":61001,"home":"/"}],` +
|
||||
`"resources":[]`,
|
||||
"writes an access declared read": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","mode":"read",` +
|
||||
`"users":[{"identity":"mesh-m","uid":61001,"use":"read-write","from":"container"}]}]`,
|
||||
"uid 0": `"resources":[{"id":"m.a","type":"access","path":"/srv/x","users":[{"identity":"root","uid":0,` +
|
||||
`"use":"read","from":"container"}]}]`,
|
||||
"one value": `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` +
|
||||
`"env":{"PUID":"1000"},"identity":{"mode":"env","user_env":"PUID","uid":61001,"gid":61001}}]`,
|
||||
`"env" or "run"`: `"resources":[{"id":"m.c","type":"container","name":"c","image":"` + anImage + `",` +
|
||||
`"identity":{"mode":"sudo","uid":61001,"gid":61001}}]`,
|
||||
} {
|
||||
_, err := Parse([]byte(`{"declaration":1,` + raw + `}`))
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("want a refusal saying %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a
|
||||
// signing context of its own. The controller's side holds the same names and words.
|
||||
func TestTheTakeActsAskKeepsItsFieldNamesAndContext(t *testing.T) {
|
||||
body, _ := json.Marshal(AccessActAsk{Act: ActTake, Access: "radarr.movies", Warrant: "w"})
|
||||
if got := keysOf(t, body); got != "access act warrant" {
|
||||
t.Fatalf("the act's fields are %q", got)
|
||||
}
|
||||
if AskAccessActSubject("ace") != "mesh.node.ace.ask.access-act" {
|
||||
t.Fatalf("the subject is %q", AskAccessActSubject("ace"))
|
||||
}
|
||||
if AccessActContext != "novox-mesh access-act v2\n" {
|
||||
t.Fatalf("the signing context is %q", AccessActContext)
|
||||
}
|
||||
}
|
||||
|
||||
// signedAct is an act's ask as the controller makes it: the hand-over's fields beside the act's, signed over the
|
||||
// act's context and its bytes.
|
||||
func signedAct(t *testing.T, key ed25519.PrivateKey, ask HandOverAsk, act AccessActAsk) []byte {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(struct {
|
||||
HandOverAsk
|
||||
AccessActAsk
|
||||
}{ask, act})
|
||||
body, _ := json.Marshal(SignedHandOver{Ask: raw, Signature: ed25519.Sign(key, append([]byte(AccessActContext), raw...))})
|
||||
return body
|
||||
}
|
||||
|
||||
// **Nothing is done without a warrant**, nor for any act but take, nor signed as a hand-over (ADR 0325 §8).
|
||||
func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
|
||||
m, key := aMember(t)
|
||||
now := time.Now()
|
||||
var given []AccessAct
|
||||
handle := func(a AccessAct) (string, error) { given = append(given, a); return "started", nil }
|
||||
check := HandOverCheck{Signer: m.Signer, Node: m.Node, Nonces: &Nonces{}}
|
||||
ask := func(nonce string) HandOverAsk {
|
||||
return HandOverAsk{Node: m.Node, Path: "/srv/media/movies", By: "the operator", Expires: now.Add(HandOverGood),
|
||||
Nonce: nonce}
|
||||
}
|
||||
for why, body := range map[string][]byte{
|
||||
"no warrant": signedAct(t, key, ask(aNonce+"01"), AccessActAsk{Act: ActTake, Access: "radarr.movies"}),
|
||||
"another act": signedAct(t, key, ask(aNonce+"02"), AccessActAsk{Act: "share", Access: "radarr.movies", Warrant: "w"}),
|
||||
"no access": signedAct(t, key, ask(aNonce+"03"), AccessActAsk{Act: ActTake, Warrant: "w"}),
|
||||
"a hand-over's": signedAsk(t, key, ask(aNonce+"04")),
|
||||
"a slash in the warrant": signedAct(t, key, ask(aNonce+"05"), AccessActAsk{Act: ActTake, Access: "radarr.movies",
|
||||
Warrant: "w/1"}),
|
||||
"two lines as access": signedAct(t, key, ask(aNonce+"07"), AccessActAsk{Act: ActTake, Access: "radarr.\nmovies",
|
||||
Warrant: "w"}),
|
||||
} {
|
||||
if a := AnswerAccessAct(check, handle, body); a.Refused == "" || !strings.HasSuffix(a.Refused, "nothing was changed") {
|
||||
t.Errorf("%s: not refused, or refused without saying nothing was changed: %+v", why, a)
|
||||
}
|
||||
}
|
||||
if len(given) != 0 {
|
||||
t.Fatalf("a refused ask reached the act: %+v", given)
|
||||
}
|
||||
a := AnswerAccessAct(check, handle, signedAct(t, key, ask(aNonce+"06"), AccessActAsk{Act: ActTake,
|
||||
Access: "radarr.movies", Warrant: "w-1"}))
|
||||
if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" {
|
||||
t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given)
|
||||
}
|
||||
}
|
||||
|
||||
// The controller names an access by its declared resource id exactly, which may hold a `/`: it reaches the act,
|
||||
// which matches it against what was declared.
|
||||
func TestATakeActNamesItsAccessByItsDeclaredId(t *testing.T) {
|
||||
m, key := aMember(t)
|
||||
var given []AccessAct
|
||||
handle := func(a AccessAct) (string, error) { given = append(given, a); return "started", nil }
|
||||
check := HandOverCheck{Signer: m.Signer, Node: m.Node, Nonces: &Nonces{}}
|
||||
ask := HandOverAsk{Node: m.Node, Path: "/services/media/movies", By: "the operator",
|
||||
Expires: time.Now().Add(HandOverGood), Nonce: aNonce + "08"}
|
||||
id := "radarr.access-services/media/movies"
|
||||
a := AnswerAccessAct(check, handle, signedAct(t, key, ask, AccessActAsk{Act: ActTake, Access: id, Warrant: "w-1"}))
|
||||
if a.Refused != "" || len(given) != 1 || given[0].Access != id {
|
||||
t.Fatalf("an access id with a slash did not reach the act as declared: %+v %+v", a, given)
|
||||
}
|
||||
}
|
||||
+179
-14
@@ -136,20 +136,27 @@ func AnswerHandOver(check HandOverCheck, handle HandOverFunc, body []byte) HandO
|
||||
// one short line naming who asked, and a path when withPath and none otherwise. The nonce is taken last, after
|
||||
// every other check passed. It answers the ask with who asked trimmed, or why it is refused.
|
||||
func believed(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, string) {
|
||||
ask, _, refused := believedBytes(check, context, withPath, body)
|
||||
return ask, refused
|
||||
}
|
||||
|
||||
// believedBytes is believed with the ask's bytes as signed, for an ask that carries more than a hand-over's
|
||||
// fields: read out of them only once they are believed.
|
||||
func believedBytes(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, []byte, string) {
|
||||
if len(body) > MaxHandOverAsk {
|
||||
return HandOverAsk{}, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body))
|
||||
return HandOverAsk{}, nil, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body))
|
||||
}
|
||||
var signed SignedHandOver
|
||||
if err := json.Unmarshal(body, &signed); err != nil {
|
||||
return HandOverAsk{}, "the ask is not a signed ask: " + err.Error()
|
||||
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
|
||||
}
|
||||
if len(check.Signer) != ed25519.PublicKeySize ||
|
||||
!ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) {
|
||||
return HandOverAsk{}, "the ask is not signed by the mesh"
|
||||
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
|
||||
}
|
||||
var ask HandOverAsk
|
||||
if err := json.Unmarshal(signed.Ask, &ask); err != nil {
|
||||
return HandOverAsk{}, "the signed ask is not an ask: " + err.Error()
|
||||
return HandOverAsk{}, nil, "the signed ask is not an ask: " + err.Error()
|
||||
}
|
||||
now := time.Now()
|
||||
if check.Now != nil {
|
||||
@@ -163,25 +170,25 @@ func believed(check HandOverCheck, context string, withPath bool, body []byte) (
|
||||
nonce, nonceErr := hex.DecodeString(ask.Nonce)
|
||||
switch {
|
||||
case check.Node == "" || ask.Node != check.Node:
|
||||
return HandOverAsk{}, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node)
|
||||
return HandOverAsk{}, nil, fmt.Sprintf("the ask is for %q, and this is %q", ask.Node, check.Node)
|
||||
case ask.Expires.IsZero() || !now.Before(ask.Expires):
|
||||
return HandOverAsk{}, "the ask has expired"
|
||||
return HandOverAsk{}, nil, "the ask has expired"
|
||||
case ask.Expires.After(now.Add(HandOverGood + HandOverSkew)):
|
||||
return HandOverAsk{}, "the ask is good for longer than an ask may be"
|
||||
return HandOverAsk{}, nil, "the ask is good for longer than an ask may be"
|
||||
case nonceErr != nil || len(nonce) < 16:
|
||||
return HandOverAsk{}, "the ask carries no nonce of its own"
|
||||
return HandOverAsk{}, nil, "the ask carries no nonce of its own"
|
||||
case ask.By == "":
|
||||
return HandOverAsk{}, "the ask names nobody as asking"
|
||||
return HandOverAsk{}, nil, "the ask names nobody as asking"
|
||||
case len(ask.By) > MaxHandOverBy || strings.ContainsAny(ask.By, "\r\n\x00"):
|
||||
return HandOverAsk{}, "who asked is not one short line"
|
||||
return HandOverAsk{}, nil, "who asked is not one short line"
|
||||
case withPath && ask.Path == "":
|
||||
return HandOverAsk{}, "the ask names no directory"
|
||||
return HandOverAsk{}, nil, "the ask names no directory"
|
||||
case !withPath && ask.Path != "":
|
||||
return HandOverAsk{}, "the ask names a path, and this ask takes none"
|
||||
return HandOverAsk{}, nil, "the ask names a path, and this ask takes none"
|
||||
case !nonces.take(ask.Nonce, ask.Expires, now):
|
||||
return HandOverAsk{}, "the ask was taken already"
|
||||
return HandOverAsk{}, nil, "the ask was taken already"
|
||||
}
|
||||
return ask, ""
|
||||
return ask, signed.Ask, ""
|
||||
}
|
||||
|
||||
// HandOverReply is what a hand-over request is answered with, and whether to answer: a request with no reply
|
||||
@@ -262,3 +269,161 @@ func SetuidSearchReply(check HandOverCheck, handle SetuidSearchFunc, reply strin
|
||||
type SetuidSearches interface {
|
||||
HearSetuidSearches(SetuidSearchFunc)
|
||||
}
|
||||
|
||||
// A judgement of this node's directories asked over the bus (novox/hq issue 358).
|
||||
//
|
||||
// Before the node-engine gives a mesh directory its mode and owner only through a walk that follows no link
|
||||
// another account could make, the operator is told which directories that walk would refuse on each node. The
|
||||
// controller asks this node's engine on AskJudgeDirectoriesSubject, signed as a hand-over is and held to the
|
||||
// same checks (believed), under a signing context of its own. The engine judges every directory of the
|
||||
// declaration it keeps, read-only — nothing is made, chmodded or chowned — and answers one line per directory.
|
||||
|
||||
// AskJudgeDirectoriesSubject is where the controller asks this node's engine to judge its directories.
|
||||
func AskJudgeDirectoriesSubject(node string) string {
|
||||
return "mesh.node." + node + ".ask.judge-directories"
|
||||
}
|
||||
|
||||
// JudgeDirectoriesContext is prefixed to a judgement ask's bytes before signing: never another ask's, nor a
|
||||
// declaration's.
|
||||
const JudgeDirectoriesContext = "novox-mesh judge-directories v1\n"
|
||||
|
||||
// JudgeDirectoriesFunc judges this node's declared directories, read-only, asked by by: the words to answer with.
|
||||
type JudgeDirectoriesFunc func(by string) (string, error)
|
||||
|
||||
func refusedJudgement(why string) HandOverAnswer {
|
||||
return HandOverAnswer{Refused: why + "; nothing was judged"}
|
||||
}
|
||||
|
||||
// AnswerJudgeDirectories is the engine's judgement of one ask's bytes: believed as a hand-over is, a path refused,
|
||||
// and only then the handler.
|
||||
func AnswerJudgeDirectories(check HandOverCheck, handle JudgeDirectoriesFunc, body []byte) HandOverAnswer {
|
||||
if handle == nil {
|
||||
return refusedJudgement("this engine judges no directory on an ask")
|
||||
}
|
||||
ask, refused := believed(check, JudgeDirectoriesContext, false, body)
|
||||
if refused != "" {
|
||||
return refusedJudgement(refused)
|
||||
}
|
||||
said, err := handle(ask.By)
|
||||
if err != nil {
|
||||
return refusedJudgement(err.Error())
|
||||
}
|
||||
return HandOverAnswer{Said: said}
|
||||
}
|
||||
|
||||
// JudgeDirectoriesReply is what a judgement request is answered with, and whether to answer, as HandOverReply.
|
||||
func JudgeDirectoriesReply(check HandOverCheck, handle JudgeDirectoriesFunc, reply string, body []byte) ([]byte, bool) {
|
||||
if reply == "" {
|
||||
return nil, false
|
||||
}
|
||||
answer, err := json.Marshal(AnswerJudgeDirectories(check, handle, body))
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return answer, true
|
||||
}
|
||||
|
||||
// DirectoryJudgements is a link that hears the controller asking this node to judge its directories. Optional.
|
||||
type DirectoryJudgements interface {
|
||||
HearDirectoryJudgements(JudgeDirectoriesFunc)
|
||||
}
|
||||
|
||||
// The take act on a library, asked over the bus (novox/hq ADR 0328 §7, issue 567).
|
||||
//
|
||||
// A library whose root already holds data is kept only on the operator's word: it waits (§8), the controller asks
|
||||
// the operator once for that library (ADR 0259), and on the approved warrant asks the library's home engine on
|
||||
// AskAccessActSubject, signed as a hand-over is and held to the same checks (believed), under a signing context of
|
||||
// its own, naming the act, the access `library:<name>`, its root as the engine states it, and the warrant. The engine
|
||||
// judges every value against what it was last declared and what is on disk, answers at once that the act started or
|
||||
// why it is refused, sets the library's group, group write and setgid on everything under the root (following no
|
||||
// link, entering no other filesystem), and states the act on the library's verdict (Act).
|
||||
|
||||
// AskAccessActSubject is where the controller asks this node's engine to run an act on one access's folder.
|
||||
func AskAccessActSubject(node string) string { return "mesh.node." + node + ".ask.access-act" }
|
||||
|
||||
// AccessActContext is prefixed to an act ask's bytes before signing: never another ask's, nor a declaration's.
|
||||
const AccessActContext = "novox-mesh access-act v2\n"
|
||||
|
||||
// ActTake is the one act asked: the library's group, group write and setgid set on everything under its root, on
|
||||
// a warrant (ADR 0328 §7).
|
||||
const ActTake = "take"
|
||||
|
||||
// AccessActAsk is an act's own fields, beside a hand-over's (node, path, by, expires, nonce): which act, the
|
||||
// access by its declared resource id exactly as the declaration names it (`/` allowed), and the warrant the
|
||||
// operator approved.
|
||||
type AccessActAsk struct {
|
||||
Act string `json:"act"`
|
||||
Access string `json:"access"`
|
||||
Warrant string `json:"warrant"`
|
||||
}
|
||||
|
||||
// AccessAct is one act asked and believed, as the handler is given it.
|
||||
type AccessAct struct {
|
||||
Act, Access, Path, Warrant, By string
|
||||
}
|
||||
|
||||
// MaxActField bounds an access id and a warrant id.
|
||||
const MaxActField = 200
|
||||
|
||||
// AccessActFunc judges and runs one act: the words to answer with, or the refusal.
|
||||
type AccessActFunc func(AccessAct) (string, error)
|
||||
|
||||
func refusedAct(why string) HandOverAnswer {
|
||||
return HandOverAnswer{Refused: why + "; nothing was changed"}
|
||||
}
|
||||
|
||||
// AnswerAccessAct is the engine's judgement of one act ask's bytes: believed as a hand-over is, then the act, the
|
||||
// access and the warrant — **nothing is done without a warrant** — and only then the handler.
|
||||
func AnswerAccessAct(check HandOverCheck, handle AccessActFunc, body []byte) HandOverAnswer {
|
||||
if handle == nil {
|
||||
return refusedAct("this engine runs no act on an accessed folder")
|
||||
}
|
||||
ask, signed, refused := believedBytes(check, AccessActContext, true, body)
|
||||
if refused != "" {
|
||||
return refusedAct(refused)
|
||||
}
|
||||
var act AccessActAsk
|
||||
if err := json.Unmarshal(signed, &act); err != nil {
|
||||
return refusedAct("the signed ask is not an act's ask: " + err.Error())
|
||||
}
|
||||
act.Access, act.Warrant = strings.TrimSpace(act.Access), strings.TrimSpace(act.Warrant)
|
||||
switch {
|
||||
case act.Warrant == "":
|
||||
return refusedAct("the ask names no warrant, and an act on an accessed folder runs only on one")
|
||||
case act.Access == "":
|
||||
return refusedAct("the ask names no access")
|
||||
case act.Act != ActTake:
|
||||
return refusedAct(fmt.Sprintf("the ask names the act %q, and the one act asked is %q", act.Act, ActTake))
|
||||
}
|
||||
// The access is the declared resource's id exactly as the declaration names it — which may hold a `/`
|
||||
// (radarr.access-services/media/movies) — and the engine matches it against what it was declared, refusing
|
||||
// anything else; the warrant is one short name.
|
||||
if len(act.Access) > MaxActField || strings.ContainsAny(act.Access, "\r\n\x00") {
|
||||
return refusedAct("the ask's access is not one short line")
|
||||
}
|
||||
if len(act.Warrant) > MaxActField || strings.ContainsAny(act.Warrant, "\r\n\x00/") {
|
||||
return refusedAct("the ask's warrant is not one short name")
|
||||
}
|
||||
said, err := handle(AccessAct{Act: act.Act, Access: act.Access, Path: ask.Path, Warrant: act.Warrant, By: ask.By})
|
||||
if err != nil {
|
||||
return refusedAct(err.Error())
|
||||
}
|
||||
return HandOverAnswer{Said: said}
|
||||
}
|
||||
|
||||
// AccessActReply is what an act request is answered with, and whether to answer, as HandOverReply.
|
||||
func AccessActReply(check HandOverCheck, handle AccessActFunc, reply string, body []byte) ([]byte, bool) {
|
||||
if reply == "" {
|
||||
return nil, false
|
||||
}
|
||||
answer, err := json.Marshal(AnswerAccessAct(check, handle, body))
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return answer, true
|
||||
}
|
||||
|
||||
// AccessActs is a link that hears the controller asking this node to run an act on an accessed folder. Optional.
|
||||
type AccessActs interface {
|
||||
HearAccessActs(AccessActFunc)
|
||||
}
|
||||
|
||||
@@ -64,7 +64,11 @@ type natsLink struct {
|
||||
handing *nats.Subscription
|
||||
// searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361).
|
||||
searching *nats.Subscription
|
||||
node string
|
||||
// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
|
||||
judging *nats.Subscription
|
||||
// acting hears the controller asking for the take act on a library (novox/hq ADR 0328 §7).
|
||||
acting *nats.Subscription
|
||||
node string
|
||||
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
|
||||
signer ed25519.PublicKey
|
||||
arrived chan Declaration
|
||||
@@ -238,6 +242,43 @@ func (l *natsLink) HearSetuidSearches(handle SetuidSearchFunc) {
|
||||
}
|
||||
}
|
||||
|
||||
// HearDirectoryJudgements listens for the controller asking this node to judge its declared directories,
|
||||
// read-only (novox/hq issue 358), on this machine's own subject, verified as a hand-over is
|
||||
// (AnswerJudgeDirectories). A bus whose user list is older than the grant refuses the subscription, said by the
|
||||
// client; the operator's line then says that nothing on this node answers.
|
||||
func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) {
|
||||
if handle == nil {
|
||||
return
|
||||
}
|
||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
||||
judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) {
|
||||
if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer {
|
||||
_ = msg.Respond(body)
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
l.judging = judging
|
||||
}
|
||||
}
|
||||
|
||||
// HearAccessActs listens for the controller asking this node to run the take act on an access's folder (novox/hq ADR
|
||||
// 0328 §7), on this machine's own subject, verified as a hand-over is (AnswerAccessAct). A bus whose user list is older
|
||||
// than the grant refuses the subscription, said by the client; the controller then hears no answer.
|
||||
func (l *natsLink) HearAccessActs(handle AccessActFunc) {
|
||||
if handle == nil {
|
||||
return
|
||||
}
|
||||
check := HandOverCheck{Signer: l.signer, Node: l.node}
|
||||
acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) {
|
||||
if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer {
|
||||
_ = msg.Respond(body)
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
l.acting = acting
|
||||
}
|
||||
}
|
||||
|
||||
func (l *natsLink) Declarations() <-chan Declaration { return l.arrived }
|
||||
func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
|
||||
func (l *natsLink) Lost() <-chan error { return l.lost }
|
||||
@@ -255,6 +296,12 @@ func (l *natsLink) Close() {
|
||||
if l.searching != nil {
|
||||
_ = l.searching.Unsubscribe()
|
||||
}
|
||||
if l.acting != nil {
|
||||
_ = l.acting.Unsubscribe()
|
||||
}
|
||||
if l.judging != nil {
|
||||
_ = l.judging.Unsubscribe()
|
||||
}
|
||||
if l.conn != nil {
|
||||
l.conn.Close()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The subject and the signing context the controller holds the same words for (mesh-controller internal/link).
|
||||
func TestTheDirectoryJudgementAskKeepsItsSubjectAndContext(t *testing.T) {
|
||||
if AskJudgeDirectoriesSubject("laptop") != "mesh.node.laptop.ask.judge-directories" {
|
||||
t.Fatalf("the subject is %q", AskJudgeDirectoriesSubject("laptop"))
|
||||
}
|
||||
if JudgeDirectoriesContext != "novox-mesh judge-directories v1\n" || JudgeDirectoriesContext == HandOverContext ||
|
||||
JudgeDirectoriesContext == SetuidSearchContext {
|
||||
t.Fatalf("the signing context is %q", JudgeDirectoriesContext)
|
||||
}
|
||||
}
|
||||
|
||||
// A judgement ask is believed as a hand-over is: signed by the mesh under its own context, for this node,
|
||||
// unexpired, once, and naming no path.
|
||||
func TestADirectoryJudgementAskIsBelievedOnlySignedForThisNodeAndOnce(t *testing.T) {
|
||||
m, key := aMember(t)
|
||||
now := time.Date(2026, 10, 12, 14, 0, 0, 0, time.UTC)
|
||||
var asked []string
|
||||
handle := func(by string) (string, error) { asked = append(asked, by); return "3 directories declared", nil }
|
||||
check := HandOverCheck{Signer: m.Signer, Node: "n1", Now: func() time.Time { return now }, Nonces: &Nonces{}}
|
||||
good := HandOverAsk{Node: "n1", By: "jo", Expires: now.Add(HandOverGood), Nonce: aNonce}
|
||||
for name, body := range map[string][]byte{
|
||||
"a setuid search's signature": signedSearch(t, key, SetuidSearchContext, good),
|
||||
"another node": signedSearch(t, key, JudgeDirectoriesContext, HandOverAsk{Node: "n2", By: "jo", Expires: good.Expires, Nonce: aNonce}),
|
||||
"a path": signedSearch(t, key, JudgeDirectoriesContext, HandOverAsk{Node: "n1", Path: "/x", By: "jo", Expires: good.Expires, Nonce: aNonce}),
|
||||
"expired": signedSearch(t, key, JudgeDirectoriesContext, HandOverAsk{Node: "n1", By: "jo", Expires: now, Nonce: aNonce}),
|
||||
} {
|
||||
if a := AnswerJudgeDirectories(check, handle, body); !strings.HasSuffix(a.Refused, "nothing was judged") || a.Said != "" {
|
||||
t.Errorf("%s: %+v", name, a)
|
||||
}
|
||||
}
|
||||
if len(asked) != 0 {
|
||||
t.Fatalf("a refused ask reached the handler: %v", asked)
|
||||
}
|
||||
body := signedSearch(t, key, JudgeDirectoriesContext, good)
|
||||
if a := AnswerJudgeDirectories(check, handle, body); a.Said != "3 directories declared" || len(asked) != 1 {
|
||||
t.Fatalf("the good ask: %+v, %v", a, asked)
|
||||
}
|
||||
if a := AnswerJudgeDirectories(check, handle, body); !strings.Contains(a.Refused, "taken already") {
|
||||
t.Fatalf("the same ask again: %+v", a)
|
||||
}
|
||||
if _, answer := JudgeDirectoriesReply(check, handle, "", body); answer {
|
||||
t.Fatal("a request with no reply subject was answered")
|
||||
}
|
||||
}
|
||||
@@ -229,6 +229,11 @@ type Report struct {
|
||||
// as "not known", never as healthy and never as a reason to raise anything. Fresh on every report
|
||||
// this engine makes, and said between reports on HealthSubject when it changes.
|
||||
Health *Health `json:"health,omitempty"`
|
||||
|
||||
// Unapplied is which declared resources this apply did not apply, by id — failed or not attempted. Not
|
||||
// said on the wire (Failed says why, in words): it is what makes a reconcile whose result changed news to
|
||||
// the node-engine, a failure that moved to another resource included (novox/hq issue 233).
|
||||
Unapplied []string `json:"-"`
|
||||
}
|
||||
|
||||
// LivenessContract is the version of the health statement this host keeps (ADR 0240 Phase A: liveness,
|
||||
@@ -246,6 +251,35 @@ const ReadinessContract = 2
|
||||
// refuses the whole declaration for a field it does not know.
|
||||
const RootContract = 3
|
||||
|
||||
// GrantContract is the statement of a host that also reads an access's `grant` (novox/hq ADR 0312): it grants a
|
||||
// read-write access's path group to the containers and the account the grant names, takes it back, and states the
|
||||
// access unhealthy while its group cannot write. Like the fields before it, its presence is what tells the
|
||||
// controller this host may be sent `grant`; an older host refuses the whole declaration for a field it does not
|
||||
// know.
|
||||
const GrantContract = 4
|
||||
|
||||
// IdentityContract is the statement of a host that reads a declaration's `identities`, a container's `identity` and
|
||||
// an access's `users` and `taken` (novox/hq ADR 0325): it makes each module's account, runs a container as its
|
||||
// module's identity, keeps each shared folder usable through ACL entries, states an access it may not keep yet as
|
||||
// waiting with a wait of kind access, and hears the take act. It no longer grants an access's group (ADR 0312 is
|
||||
// superseded): a `grant` still sent is accepted and not acted on. Like the fields before it, its presence is what
|
||||
// tells the controller this host may be sent those fields; an older host refuses the whole declaration for them.
|
||||
const IdentityContract = 5
|
||||
|
||||
// LibraryContract is the statement of a host that reads a declaration's `groups` and `libraries` and an identity's
|
||||
// `groups` (novox/hq ADR 0328, which supersedes ADR 0325): it makes each library's group and each module's user in
|
||||
// its libraries' groups, keeps each library it is home to under its group, holds the containers of a library whose
|
||||
// disk is not there, states a library it may not keep yet as waiting, and hears the take act for `library:<name>`.
|
||||
// It keeps no ACL entries: an access's `users` and `taken` are accepted and not acted on. Like the fields before
|
||||
// it, its presence is what tells the controller this host may be sent those fields.
|
||||
const LibraryContract = 6
|
||||
|
||||
// OwnGroupContract is the statement of a host that also reads an identity's `runner` (novox/hq ADR 0334 §2): it puts
|
||||
// the tool runner's account in the module's own group, takes it back when no identity names it, and says the change
|
||||
// as `identity:<module>`, which the tool runner's unit names under restart-on to be restarted once. Like the fields
|
||||
// before it, its presence is what tells the controller this host may be sent `runner` on an identity.
|
||||
const OwnGroupContract = 7
|
||||
|
||||
// Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in
|
||||
// every report, as an event on each change, and again every minute while anything is not healthy — so a
|
||||
// lost statement is not a lost fault.
|
||||
@@ -284,6 +318,20 @@ const KindAccount = "account"
|
||||
// starting ReasonUsedAsFound, until the operator hands it to the mesh (`nox node hand-over <node> <path>` on the control-node, novox/hq issue 356).
|
||||
const KindDirectory = "directory"
|
||||
|
||||
// KindAccess is an access (novox/hq ADR 0051): no engine since ADR 0328 states one; kept for the controller's reading
|
||||
// of an older engine's statements.
|
||||
const KindAccess = "access"
|
||||
|
||||
// KindLibrary is a library this machine is reached by (novox/hq ADR 0328), said under resource `library:<name>`
|
||||
// with no module: healthy while it is kept (or, not home, while its mount is there), waiting with an access wait of
|
||||
// reason keep or disk, unknown while its filesystem could not be named, unhealthy otherwise.
|
||||
const KindLibrary = "library"
|
||||
|
||||
// KindRunner is a tool runner's account the engine could not put in a group (novox/hq ADR 0334 §2, ADR 0328 §11):
|
||||
// missing, or root's. Said unhealthy as `runner:<module>` (with its module) or `runner:library:<name>`, words with no
|
||||
// path, never failing an apply; gone once the account is right.
|
||||
const KindRunner = "runner"
|
||||
|
||||
// ReasonUsedAsFound starts the reason of a directory used as found.
|
||||
const ReasonUsedAsFound = "used as found:"
|
||||
|
||||
@@ -396,6 +444,78 @@ type ResourceHealth struct {
|
||||
// Waits is what a resource stated waiting waits for (novox/hq ADR 0283): each one own secret or one
|
||||
// setting of the module by name, the part, and what the operator gives, in words. Absent otherwise.
|
||||
Waits []health.Wait `json:"waits,omitempty"`
|
||||
// AccessWaits is what a library stated waiting waits for (novox/hq ADR 0328 §8, §9). Beside Waits rather than in
|
||||
// it because mesh-sdk's health.Wait names a secret or a setting and nothing else; it moves there once that type
|
||||
// has a kind. Absent otherwise.
|
||||
AccessWaits []AccessWait `json:"access-waits,omitempty"`
|
||||
// Act is the last walk under a library's root (ADR 0328 §7): the take on a warrant. Absent on every other
|
||||
// verdict, and on a library never walked.
|
||||
Act *AccessActRecord `json:"act,omitempty"`
|
||||
}
|
||||
|
||||
// WaitKindAccess is the kind of a wait on a library (novox/hq ADR 0328 §8, §9; first ADR 0325 §7).
|
||||
const WaitKindAccess = "access"
|
||||
|
||||
// AccessWait is one library waiting for the operator (ADR 0328 §8, §9): the access `library:<name>`, the part (the
|
||||
// library), and what the operator gives, in words with no path (ADR 0234 §6): "let the mesh keep the library media",
|
||||
// or "mount the disk that holds the library media".
|
||||
type AccessWait struct {
|
||||
// Kind is WaitKindAccess.
|
||||
Kind string `json:"kind"`
|
||||
Access string `json:"access"`
|
||||
Part string `json:"part"`
|
||||
What string `json:"what"`
|
||||
// Placed names, with no path, the filesystem the folder is on now, on a wait for the operator's word: a warrant
|
||||
// given for another filesystem (the folder moved) does not answer it, and the controller asks again.
|
||||
Placed string `json:"placed,omitempty"`
|
||||
// Reason is what the wait is for, for the controller to read without parsing words: WaitKeep (the operator's
|
||||
// word to keep it) or WaitDisk (its disk mounted).
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// The reasons of an access wait.
|
||||
const (
|
||||
WaitKeep = "keep"
|
||||
WaitDisk = "disk"
|
||||
)
|
||||
|
||||
// The states of a walk under an accessed folder.
|
||||
const (
|
||||
ActRunning = "running"
|
||||
ActDone = "done"
|
||||
// ActStopped is a walk that met its bound in entries or time: what it changed stays, and what it did not
|
||||
// reach is not changed. Asked again, or on the next apply, it goes on.
|
||||
ActStopped = "stopped"
|
||||
ActFailed = "failed"
|
||||
)
|
||||
|
||||
// AccessActRecord is one walk under a library's root, as the engine records it (novox/hq ADR 0328 §7). Counts only:
|
||||
// never a path.
|
||||
type AccessActRecord struct {
|
||||
// Act is ActTake.
|
||||
Act string `json:"act"`
|
||||
Access string `json:"access"`
|
||||
Warrant string `json:"warrant,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
State string `json:"state"`
|
||||
Started time.Time `json:"started"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
// Folders and Files are how many it changed; Kept how many already had their entries; Links the links it
|
||||
// passed by unfollowed; Other the entries it left that are neither a file nor a folder; Mounts the folders on
|
||||
// another filesystem it did not enter.
|
||||
Folders int `json:"folders,omitempty"`
|
||||
Files int `json:"files,omitempty"`
|
||||
Kept int `json:"kept,omitempty"`
|
||||
Links int `json:"links,omitempty"`
|
||||
Other int `json:"other,omitempty"`
|
||||
Mounts int `json:"mounts,omitempty"`
|
||||
// Shared is the files with another name root owns, left as they are; Linked those an account owns, given the
|
||||
// group; Stripped the files whose setuid or setgid bit the group change took.
|
||||
Shared int `json:"shared,omitempty"`
|
||||
Linked int `json:"linked,omitempty"`
|
||||
Stripped int `json:"stripped,omitempty"`
|
||||
// Why is what stopped or failed it, in words with no path.
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// SetuidSearch is how far a running search for setuid programs got, as of its last checkpoint. Counts, times
|
||||
|
||||
@@ -59,8 +59,14 @@ type Queue struct {
|
||||
// SetuidSearch throws away the last search for setuid programs and starts a full one, asked by the
|
||||
// controller's terminal over the link (novox/hq issue 361). Nil answers every such ask with a refusal.
|
||||
SetuidSearch SetuidSearchFunc
|
||||
Say Announce
|
||||
Timeout time.Duration
|
||||
// JudgeDirectories judges this node's declared directories read-only, asked by the controller over the
|
||||
// link (novox/hq issue 358). Nil answers every such ask with a refusal.
|
||||
JudgeDirectories JudgeDirectoriesFunc
|
||||
// AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved
|
||||
// warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal.
|
||||
AccessAct AccessActFunc
|
||||
Say Announce
|
||||
Timeout time.Duration
|
||||
|
||||
once sync.Once
|
||||
wake chan struct{}
|
||||
|
||||
@@ -248,6 +248,14 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou
|
||||
if s, ok := link.(SetuidSearches); ok {
|
||||
s.HearSetuidSearches(queue.SetuidSearch)
|
||||
}
|
||||
// And asking for a read-only judgement of its directories (novox/hq issue 358).
|
||||
if j, ok := link.(DirectoryJudgements); ok {
|
||||
j.HearDirectoryJudgements(queue.JudgeDirectories)
|
||||
}
|
||||
// And asking for the take act on a library, on a warrant (novox/hq ADR 0328 §7).
|
||||
if s, ok := link.(AccessActs); ok {
|
||||
s.HearAccessActs(queue.AccessAct)
|
||||
}
|
||||
|
||||
for {
|
||||
// Asked to stop — by standing aside, say — nothing further is taken in.
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Machine is what this machine is made of, as it says itself (novox/hq issue 160): what somebody
|
||||
// deciding where a module should go wants to know, beside what it can do.
|
||||
//
|
||||
// **A fact it could not determine is said, never zeroed.** Every fact below is either present or
|
||||
// named in Unknown with why; a machine reporting 0 bytes of memory would be read as a machine with
|
||||
// none. So a field is omitted only when Unknown says why.
|
||||
type Machine struct {
|
||||
// ObservedAt is when this was read: the age every reader shows beside it.
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
|
||||
// KernelRelease is the running kernel's release, as /proc/sys/kernel/osrelease says it. (The
|
||||
// profile's own `kernel` is the kernel's family, `linux`.)
|
||||
KernelRelease string `json:"kernel_release,omitempty"`
|
||||
// Distribution and its Version, from /etc/os-release: VERSION_ID, or BUILD_ID for a rolling
|
||||
// distribution that has no version (Arch Linux says `rolling`).
|
||||
Distribution string `json:"distribution,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
|
||||
// Processor is its model name; Cores the physical cores and Threads the logical processors.
|
||||
Processor string `json:"processor,omitempty"`
|
||||
Cores int `json:"cores,omitempty"`
|
||||
Threads int `json:"threads,omitempty"`
|
||||
|
||||
MemoryBytes int64 `json:"memory_bytes,omitempty"`
|
||||
|
||||
// Virtual is `physical`, or the kind of virtual machine or container it is (`kvm`, `lxc`), as
|
||||
// systemd-detect-virt names it.
|
||||
Virtual string `json:"virtual,omitempty"`
|
||||
|
||||
// BootedAt is when it booted: its uptime is the time since, so the fact does not change between
|
||||
// two reports of the same boot.
|
||||
BootedAt *time.Time `json:"booted_at,omitempty"`
|
||||
|
||||
Timezone string `json:"timezone,omitempty"`
|
||||
|
||||
// Filesystems is each local filesystem mounted, by where, with its size and what is free.
|
||||
Filesystems []Filesystem `json:"filesystems,omitempty"`
|
||||
|
||||
// Unknown names each fact that could not be determined, and why.
|
||||
Unknown map[string]string `json:"unknown,omitempty"`
|
||||
}
|
||||
|
||||
// Filesystem is one mounted local filesystem.
|
||||
type Filesystem struct {
|
||||
Mount string `json:"mount"`
|
||||
Device string `json:"device"`
|
||||
Type string `json:"type"`
|
||||
// SizeBytes and FreeBytes are what statfs says: free is what an unprivileged writer may still
|
||||
// use (f_bavail), the number a person deciding where data goes wants.
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
FreeBytes int64 `json:"free_bytes"`
|
||||
}
|
||||
|
||||
// The facts, by the names Unknown says them under.
|
||||
const (
|
||||
FactKernelRelease = "kernel release"
|
||||
FactDistribution = "distribution"
|
||||
FactVersion = "version"
|
||||
FactProcessor = "processor"
|
||||
FactCores = "cores"
|
||||
FactThreads = "threads"
|
||||
FactMemory = "memory"
|
||||
FactVirtual = "virtual or physical"
|
||||
FactUptime = "uptime"
|
||||
FactTimezone = "timezone"
|
||||
FactFilesystems = "filesystems"
|
||||
FactPublicAddress = "public address"
|
||||
)
|
||||
|
||||
// MachineFacts is every fact Unknown may name, in the order a reader shows them.
|
||||
var MachineFacts = []string{FactKernelRelease, FactDistribution, FactVersion, FactProcessor, FactCores,
|
||||
FactThreads, FactMemory, FactVirtual, FactUptime, FactTimezone, FactFilesystems, FactPublicAddress}
|
||||
|
||||
// publicAddressUnknown is why the public uplink address is not said. A machine behind a NAT cannot see
|
||||
// its public address from itself, and asking a service outside the mesh is a dependency the mesh does not
|
||||
// take on unasked; how a machine learns it is not decided yet, so it is said as unknown rather than
|
||||
// guessed from a private address.
|
||||
const publicAddressUnknown = "a machine behind a NAT cannot see its public address from itself, and how it " +
|
||||
"learns it is not decided yet"
|
||||
|
||||
// Sources is where the facts are read from. Replaced in tests only for the parsing layer; the real
|
||||
// sources are exercised against this machine as well (novox/hq ADR 0017).
|
||||
type Sources struct {
|
||||
Read func(path string) ([]byte, error)
|
||||
Readlink func(path string) (string, error)
|
||||
Run Runner
|
||||
// Statfs is (size, free) in bytes of the filesystem mounted at a path.
|
||||
Statfs func(path string) (size, free int64, err error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// RealSources reads this machine.
|
||||
func RealSources() Sources {
|
||||
return Sources{Read: os.ReadFile, Readlink: os.Readlink, Run: ExecRunner, Statfs: statfs, Now: time.Now}
|
||||
}
|
||||
|
||||
func statfs(path string) (int64, int64, error) {
|
||||
var s syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &s); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return int64(s.Blocks) * int64(s.Bsize), int64(s.Bavail) * int64(s.Bsize), nil
|
||||
}
|
||||
|
||||
// DescribeMachine reads what this machine is made of. It never fails: a fact that cannot be read is
|
||||
// named in Unknown with why, and the rest are still said.
|
||||
func DescribeMachine(ctx context.Context, src Sources) Machine {
|
||||
if src.Now == nil {
|
||||
src.Now = time.Now
|
||||
}
|
||||
m := Machine{ObservedAt: src.Now().UTC(), Unknown: map[string]string{}}
|
||||
unknown := func(fact string, why string) { m.Unknown[fact] = why }
|
||||
|
||||
if b, err := src.Read("/proc/sys/kernel/osrelease"); err != nil {
|
||||
unknown(FactKernelRelease, err.Error())
|
||||
} else if r := strings.TrimSpace(string(b)); r == "" {
|
||||
unknown(FactKernelRelease, "/proc/sys/kernel/osrelease is empty")
|
||||
} else {
|
||||
m.KernelRelease = r
|
||||
}
|
||||
|
||||
if b, err := src.Read("/etc/os-release"); err != nil {
|
||||
unknown(FactDistribution, err.Error())
|
||||
unknown(FactVersion, err.Error())
|
||||
} else {
|
||||
name, version, whyName, whyVersion := distributionOf(string(b))
|
||||
m.Distribution, m.Version = name, version
|
||||
if whyName != "" {
|
||||
unknown(FactDistribution, whyName)
|
||||
}
|
||||
if whyVersion != "" {
|
||||
unknown(FactVersion, whyVersion)
|
||||
}
|
||||
}
|
||||
|
||||
if b, err := src.Read("/proc/cpuinfo"); err != nil {
|
||||
unknown(FactProcessor, err.Error())
|
||||
unknown(FactCores, err.Error())
|
||||
unknown(FactThreads, err.Error())
|
||||
} else {
|
||||
c := processorOf(string(b))
|
||||
m.Processor, m.Cores, m.Threads = c.model, c.cores, c.threads
|
||||
for fact, why := range c.unknown {
|
||||
unknown(fact, why)
|
||||
}
|
||||
}
|
||||
|
||||
if b, err := src.Read("/proc/meminfo"); err != nil {
|
||||
unknown(FactMemory, err.Error())
|
||||
} else if kb, ok := memTotalKB(string(b)); !ok {
|
||||
unknown(FactMemory, "MemTotal not found in /proc/meminfo")
|
||||
} else {
|
||||
m.MemoryBytes = kb * 1024
|
||||
}
|
||||
|
||||
m.Virtual, _ = virtualOf(ctx, src.Run, unknown)
|
||||
|
||||
if b, err := src.Read("/proc/stat"); err != nil {
|
||||
unknown(FactUptime, err.Error())
|
||||
} else if booted, ok := bootedAt(string(b)); !ok {
|
||||
unknown(FactUptime, "btime not found in /proc/stat")
|
||||
} else {
|
||||
m.BootedAt = &booted
|
||||
}
|
||||
|
||||
if tz, why := timezoneOf(src); why != "" {
|
||||
unknown(FactTimezone, why)
|
||||
} else {
|
||||
m.Timezone = tz
|
||||
}
|
||||
|
||||
if b, err := src.Read("/proc/self/mounts"); err != nil {
|
||||
unknown(FactFilesystems, err.Error())
|
||||
} else {
|
||||
fs, why := filesystemsOf(ctx, string(b), src.Statfs)
|
||||
m.Filesystems = fs
|
||||
if why != "" {
|
||||
unknown(FactFilesystems, why)
|
||||
}
|
||||
}
|
||||
|
||||
unknown(FactPublicAddress, publicAddressUnknown)
|
||||
return m
|
||||
}
|
||||
|
||||
// distributionOf is the distribution's name and version from an os-release file, with why either is
|
||||
// not said. NAME first, then ID; VERSION_ID first, then BUILD_ID.
|
||||
func distributionOf(osRelease string) (name, version, whyName, whyVersion string) {
|
||||
fields := map[string]string{}
|
||||
for _, line := range strings.Split(osRelease, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, found := strings.Cut(line, "=")
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
fields[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`)
|
||||
}
|
||||
name = firstOf(fields["NAME"], fields["ID"])
|
||||
version = firstOf(fields["VERSION_ID"], fields["BUILD_ID"])
|
||||
if name == "" {
|
||||
whyName = "/etc/os-release names neither NAME nor ID"
|
||||
}
|
||||
if version == "" {
|
||||
whyVersion = "/etc/os-release names neither VERSION_ID nor BUILD_ID"
|
||||
}
|
||||
return name, version, whyName, whyVersion
|
||||
}
|
||||
|
||||
func firstOf(values ...string) string {
|
||||
for _, v := range values {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type processor struct {
|
||||
model string
|
||||
cores, threads int
|
||||
unknown map[string]string
|
||||
}
|
||||
|
||||
// processorOf reads /proc/cpuinfo: the model from `model name` (or `Model`, `Hardware`, `cpu model`,
|
||||
// which other architectures use), threads from its `processor` entries, and cores from the distinct
|
||||
// (physical id, core id) pairs — which an architecture that does not say them leaves unknown rather than
|
||||
// guessed equal to the threads.
|
||||
func processorOf(cpuinfo string) processor {
|
||||
p := processor{unknown: map[string]string{}}
|
||||
pairs := map[string]bool{}
|
||||
physical := "0"
|
||||
var models = map[string]string{}
|
||||
for _, line := range strings.Split(cpuinfo, "\n") {
|
||||
key, value, found := strings.Cut(line, ":")
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
key, value = strings.TrimSpace(key), strings.TrimSpace(value)
|
||||
switch key {
|
||||
case "processor":
|
||||
p.threads++
|
||||
case "physical id":
|
||||
physical = value
|
||||
case "core id":
|
||||
pairs[physical+"/"+value] = true
|
||||
case "model name", "Model", "Hardware", "cpu model":
|
||||
if _, had := models[key]; !had {
|
||||
models[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
p.model = firstOf(models["model name"], models["cpu model"], models["Model"], models["Hardware"])
|
||||
p.cores = len(pairs)
|
||||
if p.model == "" {
|
||||
p.unknown[FactProcessor] = "/proc/cpuinfo names no model"
|
||||
}
|
||||
if p.threads == 0 {
|
||||
p.unknown[FactThreads] = "/proc/cpuinfo lists no processor"
|
||||
}
|
||||
if p.cores == 0 {
|
||||
p.unknown[FactCores] = "/proc/cpuinfo says no core id"
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func memTotalKB(meminfo string) (int64, bool) {
|
||||
for _, line := range strings.Split(meminfo, "\n") {
|
||||
if !strings.HasPrefix(line, "MemTotal:") {
|
||||
continue
|
||||
}
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) < 2 {
|
||||
return 0, false
|
||||
}
|
||||
kb, err := strconv.ParseInt(parts[1], 10, 64)
|
||||
if err != nil || kb <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return kb, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// virtualOf asks systemd-detect-virt. It says `none` on a physical machine, and exits 1 doing so.
|
||||
func virtualOf(ctx context.Context, run Runner, unknown func(string, string)) (string, bool) {
|
||||
if run == nil {
|
||||
unknown(FactVirtual, "nothing here runs commands")
|
||||
return "", false
|
||||
}
|
||||
out, err := run(ctx, "systemd-detect-virt")
|
||||
switch said := strings.TrimSpace(out); {
|
||||
case said == "none":
|
||||
return "physical", true
|
||||
case said != "" && err == nil:
|
||||
return said, true
|
||||
case err != nil:
|
||||
unknown(FactVirtual, "systemd-detect-virt: "+err.Error())
|
||||
default:
|
||||
unknown(FactVirtual, "systemd-detect-virt said nothing")
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func bootedAt(stat string) (time.Time, bool) {
|
||||
for _, line := range strings.Split(stat, "\n") {
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) == 2 && parts[0] == "btime" {
|
||||
secs, err := strconv.ParseInt(parts[1], 10, 64)
|
||||
if err != nil || secs <= 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.Unix(secs, 0).UTC(), true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// timezoneOf is the zone /etc/localtime links into the zoneinfo tree, or /etc/timezone's.
|
||||
func timezoneOf(src Sources) (string, string) {
|
||||
var whys []string
|
||||
if src.Readlink != nil {
|
||||
target, err := src.Readlink("/etc/localtime")
|
||||
if err == nil {
|
||||
if _, zone, found := strings.Cut(target, "zoneinfo/"); found && zone != "" {
|
||||
return zone, ""
|
||||
}
|
||||
whys = append(whys, "/etc/localtime links to "+target+", outside a zoneinfo tree")
|
||||
} else {
|
||||
whys = append(whys, "/etc/localtime: "+err.Error())
|
||||
}
|
||||
}
|
||||
if b, err := src.Read("/etc/timezone"); err == nil {
|
||||
if zone := strings.TrimSpace(string(b)); zone != "" {
|
||||
return zone, ""
|
||||
}
|
||||
whys = append(whys, "/etc/timezone is empty")
|
||||
} else {
|
||||
whys = append(whys, err.Error())
|
||||
}
|
||||
return "", strings.Join(whys, "; ")
|
||||
}
|
||||
|
||||
// localTypes are the filesystem types said: those on a local device. A network filesystem is left out on
|
||||
// purpose — statfs on a server that stopped answering blocks, and this runs on every apply.
|
||||
var localTypes = map[string]bool{"ext2": true, "ext3": true, "ext4": true, "xfs": true, "btrfs": true,
|
||||
"zfs": true, "f2fs": true, "vfat": true, "exfat": true, "ntfs": true, "ntfs3": true, "fuseblk": true,
|
||||
"jfs": true, "reiserfs": true, "bcachefs": true}
|
||||
|
||||
// sizing is held while a check of the filesystems' sizes runs.
|
||||
var sizing sync.Mutex
|
||||
|
||||
// filesystemsOf lists each local filesystem once — a device mounted at several places (a btrfs
|
||||
// subvolume, a bind mount) under its shortest mount — with its size and what is free. Bounded by the
|
||||
// context, so a filesystem whose statfs hangs cannot hang the apply that reports it.
|
||||
func filesystemsOf(ctx context.Context, mounts string, stat func(string) (int64, int64, error)) ([]Filesystem, string) {
|
||||
if stat == nil {
|
||||
return nil, "nothing here can ask a filesystem its size"
|
||||
}
|
||||
byDevice := map[string]Filesystem{}
|
||||
for _, line := range strings.Split(mounts, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 3 || !localTypes[f[2]] {
|
||||
continue
|
||||
}
|
||||
mount := unescapeMount(f[1])
|
||||
if had, ok := byDevice[f[0]]; ok && len(had.Mount) <= len(mount) {
|
||||
continue
|
||||
}
|
||||
byDevice[f[0]] = Filesystem{Mount: mount, Device: f[0], Type: f[2]}
|
||||
}
|
||||
if len(byDevice) == 0 {
|
||||
return nil, "/proc/self/mounts lists no local filesystem"
|
||||
}
|
||||
|
||||
// One check at a time, across every report: a filesystem whose statfs hangs keeps the one check that
|
||||
// asked it, and every report after it says the sizes unknown until it finishes, rather than leaving
|
||||
// another goroutine and thread blocked behind it on every apply.
|
||||
if !sizing.TryLock() {
|
||||
return nil, "the last check of the filesystems' sizes has not finished yet"
|
||||
}
|
||||
|
||||
type answer struct {
|
||||
fs []Filesystem
|
||||
why []string
|
||||
}
|
||||
done := make(chan answer, 1)
|
||||
go func() {
|
||||
defer sizing.Unlock()
|
||||
var a answer
|
||||
for _, fs := range byDevice {
|
||||
size, free, err := stat(fs.Mount)
|
||||
if err != nil {
|
||||
a.why = append(a.why, fmt.Sprintf("%s: %v", fs.Mount, err))
|
||||
continue
|
||||
}
|
||||
fs.SizeBytes, fs.FreeBytes = size, free
|
||||
a.fs = append(a.fs, fs)
|
||||
}
|
||||
done <- a
|
||||
}()
|
||||
select {
|
||||
case a := <-done:
|
||||
sort.Slice(a.fs, func(i, j int) bool { return a.fs[i].Mount < a.fs[j].Mount })
|
||||
sort.Strings(a.why)
|
||||
return a.fs, strings.Join(a.why, "; ")
|
||||
case <-ctx.Done():
|
||||
return nil, "asking the filesystems their size did not finish: " + ctx.Err().Error()
|
||||
}
|
||||
}
|
||||
|
||||
// unescapeMount undoes the octal escapes /proc/self/mounts writes a space or tab in a path as.
|
||||
func unescapeMount(path string) string {
|
||||
if !strings.Contains(path, `\`) {
|
||||
return path
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(path); i++ {
|
||||
if path[i] == '\\' && i+3 < len(path) {
|
||||
if n, err := strconv.ParseUint(path[i+1:i+4], 8, 8); err == nil {
|
||||
b.WriteByte(byte(n))
|
||||
i += 3
|
||||
continue
|
||||
}
|
||||
}
|
||||
b.WriteByte(path[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package profile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The parsing layer, with the machine's files given (novox/hq ADR 0017 allows a fake here and only here);
|
||||
// the real machine is asked below.
|
||||
|
||||
var files = map[string]string{
|
||||
"/proc/sys/kernel/osrelease": "7.2.8-arch1-2\n",
|
||||
"/etc/os-release": "NAME=\"Arch Linux\"\nPRETTY_NAME=\"Arch Linux\"\nID=arch\nBUILD_ID=rolling\n",
|
||||
"/proc/cpuinfo": "processor\t: 0\nmodel name\t: AMD Ryzen 9 5900HS\nphysical id\t: 0\ncore id\t\t: 0\n\n" +
|
||||
"processor\t: 1\nmodel name\t: AMD Ryzen 9 5900HS\nphysical id\t: 0\ncore id\t\t: 0\n\n" +
|
||||
"processor\t: 2\nmodel name\t: AMD Ryzen 9 5900HS\nphysical id\t: 0\ncore id\t\t: 1\n",
|
||||
"/proc/meminfo": "MemTotal: 32000000 kB\nMemFree: 1 kB\n",
|
||||
"/proc/stat": "cpu 1 2 3\nbtime 1791800000\n",
|
||||
"/proc/self/mounts": "/dev/nvme0n1p2 / btrfs rw 0 0\n/dev/nvme0n1p2 /home btrfs rw 0 0\n" +
|
||||
"proc /proc proc rw 0 0\ntank/data /srv/my\\040data zfs rw 0 0\n" +
|
||||
"server:/x /mnt/x nfs4 rw 0 0\n/dev/loop3 /snap/core/1 squashfs ro 0 0\n",
|
||||
}
|
||||
|
||||
func given(read map[string]string) Sources {
|
||||
return Sources{
|
||||
Read: func(p string) ([]byte, error) {
|
||||
if s, ok := read[p]; ok {
|
||||
return []byte(s), nil
|
||||
}
|
||||
return nil, &os.PathError{Op: "open", Path: p, Err: os.ErrNotExist}
|
||||
},
|
||||
Readlink: func(string) (string, error) { return "/usr/share/zoneinfo/Europe/Brussels", nil },
|
||||
Run: func(context.Context, string, ...string) (string, error) { return "none\n", errors.New("exited 1") },
|
||||
Statfs: func(p string) (int64, int64, error) { return 1000, 400, nil },
|
||||
Now: func() time.Time { return time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC) },
|
||||
}
|
||||
}
|
||||
|
||||
func TestMachine_saysEveryFactItCanRead(t *testing.T) {
|
||||
m := DescribeMachine(context.Background(), given(files))
|
||||
|
||||
if m.KernelRelease != "7.2.8-arch1-2" || m.Distribution != "Arch Linux" || m.Version != "rolling" {
|
||||
t.Errorf("kernel %q, distribution %q %q", m.KernelRelease, m.Distribution, m.Version)
|
||||
}
|
||||
if m.Processor != "AMD Ryzen 9 5900HS" || m.Cores != 2 || m.Threads != 3 {
|
||||
t.Errorf("processor %q, %d cores, %d threads", m.Processor, m.Cores, m.Threads)
|
||||
}
|
||||
if m.MemoryBytes != 32000000*1024 {
|
||||
t.Errorf("memory %d", m.MemoryBytes)
|
||||
}
|
||||
if m.Virtual != "physical" {
|
||||
t.Errorf("systemd-detect-virt said none and exited 1; read as %q", m.Virtual)
|
||||
}
|
||||
if m.BootedAt == nil || m.BootedAt.Unix() != 1791800000 {
|
||||
t.Errorf("booted at %v", m.BootedAt)
|
||||
}
|
||||
if m.Timezone != "Europe/Brussels" {
|
||||
t.Errorf("timezone %q", m.Timezone)
|
||||
}
|
||||
// One btrfs device under its shortest mount, the zfs dataset with its escaped space, and no network or
|
||||
// read-only image filesystem.
|
||||
var mounts []string
|
||||
for _, f := range m.Filesystems {
|
||||
mounts = append(mounts, f.Mount)
|
||||
if f.SizeBytes != 1000 || f.FreeBytes != 400 {
|
||||
t.Errorf("%s: size %d free %d", f.Mount, f.SizeBytes, f.FreeBytes)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(mounts, ","); got != "/,/srv/my data" {
|
||||
t.Errorf("filesystems %q", got)
|
||||
}
|
||||
// Everything read is not unknown; the public address is, with why.
|
||||
for fact := range m.Unknown {
|
||||
if fact != FactPublicAddress {
|
||||
t.Errorf("%s said unknown though it was readable: %s", fact, m.Unknown[fact])
|
||||
}
|
||||
}
|
||||
if m.Unknown[FactPublicAddress] == "" {
|
||||
t.Error("the public address is not said, and not said unknown either")
|
||||
}
|
||||
}
|
||||
|
||||
// The acceptance of issue 160: a machine that cannot determine a fact says so rather than reporting a zero.
|
||||
func TestMachine_aFactItCannotReadIsSaidUnknownNeverZero(t *testing.T) {
|
||||
src := given(map[string]string{})
|
||||
src.Readlink = func(string) (string, error) { return "", os.ErrNotExist }
|
||||
src.Run = func(context.Context, string, ...string) (string, error) { return "", errors.New("not found") }
|
||||
m := DescribeMachine(context.Background(), src)
|
||||
|
||||
raw, _ := json.Marshal(m)
|
||||
var said map[string]any
|
||||
_ = json.Unmarshal(raw, &said)
|
||||
for key, v := range said {
|
||||
if key == "observed_at" || key == "unknown" {
|
||||
continue
|
||||
}
|
||||
t.Errorf("%s is said as %v from a machine where nothing could be read", key, v)
|
||||
}
|
||||
for _, fact := range MachineFacts {
|
||||
if strings.TrimSpace(m.Unknown[fact]) == "" {
|
||||
t.Errorf("%s is neither said nor said unknown with why", fact)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMachine_aHungFilesystemCannotHangTheReport(t *testing.T) {
|
||||
src := given(files)
|
||||
block := make(chan struct{})
|
||||
var asked atomic.Int32
|
||||
src.Statfs = func(string) (int64, int64, error) { asked.Add(1); <-block; return 0, 0, nil }
|
||||
defer func() {
|
||||
close(block)
|
||||
// The check that hung ends once its filesystem answers, and lets the next one run.
|
||||
for !sizing.TryLock() {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
sizing.Unlock()
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
m := DescribeMachine(ctx, src)
|
||||
if len(m.Filesystems) != 0 || !strings.Contains(m.Unknown[FactFilesystems], "did not finish") {
|
||||
t.Errorf("filesystems %v, unknown %q", m.Filesystems, m.Unknown[FactFilesystems])
|
||||
}
|
||||
stuck := asked.Load()
|
||||
|
||||
// Every report while it hangs says so at once, and asks no filesystem again: no goroutine is left
|
||||
// blocked behind the hung one per apply.
|
||||
for i := 0; i < 3; i++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
m = DescribeMachine(ctx, src)
|
||||
cancel()
|
||||
if !strings.Contains(m.Unknown[FactFilesystems], "has not finished yet") {
|
||||
t.Errorf("a report while the check hangs said %q", m.Unknown[FactFilesystems])
|
||||
}
|
||||
}
|
||||
if asked.Load() != stuck {
|
||||
t.Errorf("filesystems asked %d times while one check hung, was %d", asked.Load(), stuck)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMachine_coresUnsaidAreUnknownNotGuessed(t *testing.T) {
|
||||
p := processorOf("processor\t: 0\nModel\t: Raspberry Pi 4\nprocessor\t: 1\n")
|
||||
if p.model != "Raspberry Pi 4" || p.threads != 2 || p.cores != 0 || p.unknown[FactCores] == "" {
|
||||
t.Errorf("%+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// Against the real machine (ADR 0017): what it says agrees with what this process sees.
|
||||
func TestAgainstThisMachine_machineSaysWhatItIs(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("the facts are read from Linux's own files")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
m := DescribeMachine(ctx, RealSources())
|
||||
if m.MemoryBytes <= 0 && m.Unknown[FactMemory] == "" {
|
||||
t.Error("memory neither said nor unknown")
|
||||
}
|
||||
if m.Threads != 0 && m.Threads < runtime.NumCPU() {
|
||||
t.Errorf("%d threads said, and this process alone may use %d", m.Threads, runtime.NumCPU())
|
||||
}
|
||||
if m.BootedAt != nil && m.BootedAt.After(time.Now()) {
|
||||
t.Errorf("booted in the future: %v", m.BootedAt)
|
||||
}
|
||||
for _, f := range m.Filesystems {
|
||||
if f.SizeBytes <= 0 || f.FreeBytes > f.SizeBytes {
|
||||
t.Errorf("%s: size %d free %d", f.Mount, f.SizeBytes, f.FreeBytes)
|
||||
}
|
||||
}
|
||||
for _, fact := range MachineFacts {
|
||||
if why, unknown := m.Unknown[fact]; unknown && strings.TrimSpace(why) == "" {
|
||||
t.Errorf("%s unknown with no why", fact)
|
||||
}
|
||||
}
|
||||
raw, _ := json.MarshalIndent(m, "", " ")
|
||||
t.Logf("%s", raw)
|
||||
}
|
||||
@@ -68,6 +68,9 @@ type Profile struct {
|
||||
Architecture string `json:"architecture"`
|
||||
Kernel string `json:"kernel"`
|
||||
Capabilities []Verdict `json:"capabilities"`
|
||||
// Machine is what it is made of (novox/hq issue 160), said with every profile: at enrolment, after
|
||||
// every apply, and by the reconcile once a day when nothing else made it speak.
|
||||
Machine *Machine `json:"machine,omitempty"`
|
||||
}
|
||||
|
||||
// Has reports whether a named capability is present. Unknown names are absent, not an error:
|
||||
@@ -114,5 +117,9 @@ func Detect(ctx context.Context, detectors []Detector, timeout time.Duration) Pr
|
||||
sort.Slice(p.Capabilities, func(i, j int) bool {
|
||||
return p.Capabilities[i].Name < p.Capabilities[j].Name
|
||||
})
|
||||
describeCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
m := DescribeMachine(describeCtx, RealSources())
|
||||
cancel()
|
||||
p.Machine = &m
|
||||
return p
|
||||
}
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LibrariesName is the file, beside the node's state, of every library this machine is home to (novox/hq ADR 0328
|
||||
// §6, §7, §9): which filesystem its root was kept on, whether and how it is kept, and the last walk under it. Its own
|
||||
// file for the reason the hand-overs have theirs: the take act runs beside an apply, whose save of the state would
|
||||
// lose a record written into it meanwhile.
|
||||
const LibrariesName = "libraries.json"
|
||||
|
||||
// LibrariesPath is that file beside the state at statePath.
|
||||
func LibrariesPath(statePath string) string {
|
||||
return filepath.Join(filepath.Dir(statePath), LibrariesName)
|
||||
}
|
||||
|
||||
// How a library came to be kept.
|
||||
const (
|
||||
// KeptByWarrant is a root that held data, taken on the operator's word (ADR 0328 §7).
|
||||
KeptByWarrant = "warrant"
|
||||
// KeptEmpty is a root that held nothing yet when it was first kept.
|
||||
KeptEmpty = "empty"
|
||||
// KeptInShape is a root found already in the library's group with group write and setgid.
|
||||
KeptInShape = "in-shape"
|
||||
)
|
||||
|
||||
// LibraryRecord is one library as the engine keeps it on its home. Counts, names and mount points only beside its
|
||||
// root: never a path under it.
|
||||
type LibraryRecord struct {
|
||||
Library string `json:"library"`
|
||||
Root string `json:"root"`
|
||||
// GID is the library's group as it was last kept: what a walk stopped at its bound goes on with.
|
||||
GID int `json:"gid,omitempty"`
|
||||
// Mount is the mount point that held the root when the engine first saw it; Placed and Fs name that filesystem
|
||||
// (the engine's PlacedID, and what it was made from: UUID, filesystem id or source).
|
||||
Mount string `json:"mount,omitempty"`
|
||||
Placed string `json:"placed,omitempty"`
|
||||
Fs string `json:"fs,omitempty"`
|
||||
// Kept is that the mesh keeps it, on the filesystem KeptOn names (a Placed): another filesystem at the root is
|
||||
// judged afresh. How is why; Warrant and By the warrant and who asked, for one taken on a warrant.
|
||||
Kept bool `json:"kept,omitempty"`
|
||||
KeptOn string `json:"kept_on,omitempty"`
|
||||
How string `json:"how,omitempty"`
|
||||
Warrant string `json:"warrant,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at,omitzero"`
|
||||
// Act is the last walk under the root; Cursor where a walk that met its bound stopped (the names of each level
|
||||
// below the root, never said), for the next to go on from.
|
||||
Act *KeptAct `json:"act,omitempty"`
|
||||
// Paths are the sub-paths kept so far on the filesystem KeptOn names: one asked for later that holds data not
|
||||
// in the group waits for the operator's word as a root would, unless the library was taken whole.
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
Cursor []string `json:"cursor,omitempty"`
|
||||
}
|
||||
|
||||
// KeptAct is one walk under a library's root, as the engine records it. Counts only: never a path.
|
||||
type KeptAct struct {
|
||||
Act string `json:"act"`
|
||||
Warrant string `json:"warrant,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
State string `json:"state"`
|
||||
Started time.Time `json:"started"`
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
// On names the filesystem it walked (a Placed).
|
||||
On string `json:"on,omitempty"`
|
||||
// Folders and Files are the entries changed; Kept the ones that already had what they need; Links the links
|
||||
// passed by unfollowed; Other the entries neither a file nor a folder; Mounts the folders on another filesystem
|
||||
// not entered; Shared the files with another name root owns, left as they are.
|
||||
Folders int `json:"folders,omitempty"`
|
||||
Files int `json:"files,omitempty"`
|
||||
Kept int `json:"kept,omitempty"`
|
||||
Links int `json:"links,omitempty"`
|
||||
Other int `json:"other,omitempty"`
|
||||
Mounts int `json:"mounts,omitempty"`
|
||||
Shared int `json:"shared,omitempty"`
|
||||
// Linked is the files with another name an account owns, given the group; Stripped the files whose setuid or
|
||||
// setgid bit the group change took, and which stays taken.
|
||||
Linked int `json:"linked,omitempty"`
|
||||
Stripped int `json:"stripped,omitempty"`
|
||||
// Why is what stopped or failed it, in words with no path.
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// LibraryFile is the libraries' file, one writer at a time in this process.
|
||||
type LibraryFile struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewLibraryFile is the libraries' file beside the state at statePath.
|
||||
func NewLibraryFile(statePath string) *LibraryFile {
|
||||
return &LibraryFile{path: LibrariesPath(statePath)}
|
||||
}
|
||||
|
||||
// All is every library's record, by name. None is an empty map; a file that cannot be read is an error.
|
||||
func (k *LibraryFile) All() (map[string]LibraryRecord, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.load()
|
||||
}
|
||||
|
||||
// Update changes one library's record under the file's lock: change is given the record (empty when there is none)
|
||||
// and whether there was one, and says whether to keep it; a record not kept is removed.
|
||||
func (k *LibraryFile) Update(name string, change func(r *LibraryRecord, had bool) bool) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
all, err := k.load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r, had := all[name]
|
||||
r.Library = name
|
||||
if change(&r, had) {
|
||||
all[name] = r
|
||||
} else {
|
||||
delete(all, name)
|
||||
}
|
||||
return k.save(all)
|
||||
}
|
||||
|
||||
func (k *LibraryFile) load() (map[string]LibraryRecord, error) {
|
||||
raw, err := os.ReadFile(k.path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return map[string]LibraryRecord{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the libraries: %w", err)
|
||||
}
|
||||
out := map[string]LibraryRecord{}
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("the libraries (%s) are unreadable: %w", k.path, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *LibraryFile) save(all map[string]LibraryRecord) error {
|
||||
raw, err := json.MarshalIndent(all, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(k.path), ".libraries-*.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if _, err := tmp.Write(append(raw, '\n')); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp.Name(), k.path)
|
||||
}
|
||||
@@ -241,6 +241,47 @@ type State struct {
|
||||
// network stays declared: undeclaring brings nothing back (ADR 0118), and a private network
|
||||
// assigned again finds the tunnel's configuration retired rather than missing.
|
||||
Retired []Retired `json:"retired,omitempty"`
|
||||
|
||||
// Identities is every module identity the engine made here (novox/hq ADR 0328 §4): its account and group,
|
||||
// by name and number, and which of the two it made. An identity no longer declared is removed, and only
|
||||
// what the engine made: an account or group that was there before is never the mesh's to remove.
|
||||
Identities []MadeIdentity `json:"identities,omitempty"`
|
||||
|
||||
// Groups is every library group the engine made here (novox/hq ADR 0328 §3), removed when no longer declared;
|
||||
// a group that was there before is never the mesh's to remove.
|
||||
Groups []MadeGroup `json:"groups,omitempty"`
|
||||
|
||||
// Members is every library group the engine put a tool runner's account in that it was not in before (ADR 0328
|
||||
// §11, as ADR 0252 records an account's groups): taken back when no longer declared.
|
||||
Members []Membership `json:"members,omitempty"`
|
||||
}
|
||||
|
||||
// MadeGroup is one library group the engine made.
|
||||
type MadeGroup struct {
|
||||
Name string `json:"name"`
|
||||
GID int `json:"gid"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Membership is one account the engine put in one library's group.
|
||||
type Membership struct {
|
||||
Account string `json:"account"`
|
||||
Group string `json:"group"`
|
||||
// Library is the library whose group it is; Module, for a module's own group (ADR 0334 §2), the module.
|
||||
Library string `json:"library,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
}
|
||||
|
||||
// MadeIdentity is one module identity as the engine made it.
|
||||
type MadeIdentity struct {
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
UID int `json:"uid"`
|
||||
GID int `json:"gid"`
|
||||
// MadeUser and MadeGroup say which the engine made: only those are removed.
|
||||
MadeUser bool `json:"made_user,omitempty"`
|
||||
MadeGroup bool `json:"made_group,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Retired is a found configuration the mesh removed once what replaced it was proven (novox/hq
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package system
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A module's identity as a system account (novox/hq ADR 0328 §3, §4): an account and a group with exactly the numbers
|
||||
// the controller allocated, no login shell, no home and no password. Optional beside System: a system that has
|
||||
// none (android) makes no module identity, and the engine says so.
|
||||
|
||||
// CreateSystemGroup makes a system group with gid.
|
||||
func (arch) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error {
|
||||
if _, err := run(ctx, "groupadd", "--system", "--gid", strconv.Itoa(gid), name); err != nil {
|
||||
return fmt.Errorf("cannot create the group %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NoHome is the home field of every module account: a path that is not there, so nothing that follows an account's
|
||||
// home — usermod regrouping every file under it, for one — ever walks a real tree. Never `/`: `usermod --gid` on
|
||||
// an account whose home is `/` regroups every file it owns on the machine.
|
||||
const NoHome = "/nonexistent"
|
||||
|
||||
// CreateSystemAccount makes a system account with uid and the group gid as its own: no home made, NoHome as the
|
||||
// home field, nologin as its shell, and the password useradd leaves locked.
|
||||
func (arch) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int, groups []string) error {
|
||||
args := []string{"--system", "--uid", strconv.Itoa(uid), "--gid", strconv.Itoa(gid)}
|
||||
if len(groups) > 0 {
|
||||
args = append(args, "--groups", strings.Join(groups, ","))
|
||||
}
|
||||
args = append(args, "--no-create-home", "--home-dir", NoHome, "--shell", "/usr/bin/nologin", name)
|
||||
if _, err := run(ctx, "useradd", args...); err != nil {
|
||||
return fmt.Errorf("cannot create the account %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModifyAccount gives an account the mesh made its primary group gid and exactly groups as its added groups.
|
||||
//
|
||||
// **Its home is moved to NoHome first, never its files.** `usermod --gid` regroups every file under the account's
|
||||
// home that the account owns; an account made with `/` as its home (an earlier engine's) would have the whole
|
||||
// machine walked as root. So an account whose home is not NoHome is given NoHome first, without -m: no file moves.
|
||||
func (arch) ModifyAccount(ctx context.Context, run Runner, name string, gid int, groups []string) error {
|
||||
passwd, err := run(ctx, "getent", "passwd", name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read the account %q: %w", name, err)
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(passwd), ":")
|
||||
if len(fields) < 6 {
|
||||
return fmt.Errorf("the user database gave %q for %q, which is not a passwd entry", strings.TrimSpace(passwd),
|
||||
name)
|
||||
}
|
||||
if fields[5] != NoHome {
|
||||
if _, err := run(ctx, "usermod", "--home", NoHome, name); err != nil {
|
||||
return fmt.Errorf("cannot give the account %q no home before its groups: %w", name, err)
|
||||
}
|
||||
}
|
||||
if _, err := run(ctx, "usermod", "--gid", strconv.Itoa(gid), "--groups", strings.Join(groups, ","),
|
||||
name); err != nil {
|
||||
return fmt.Errorf("cannot give the account %q its groups: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveAccount removes an account, leaving every file it owns where it is.
|
||||
func (arch) RemoveAccount(ctx context.Context, run Runner, name string) error {
|
||||
if _, err := run(ctx, "userdel", name); err != nil {
|
||||
return fmt.Errorf("cannot remove the account %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveGroup removes a group.
|
||||
func (arch) RemoveGroup(ctx context.Context, run Runner, name string) error {
|
||||
if _, err := run(ctx, "groupdel", name); err != nil {
|
||||
return fmt.Errorf("cannot remove the group %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateSystemGroup makes a system group with gid, with busybox's addgroup.
|
||||
func (alpine) CreateSystemGroup(ctx context.Context, run Runner, name string, gid int) error {
|
||||
if _, err := run(ctx, "addgroup", "-S", "-g", strconv.Itoa(gid), name); err != nil {
|
||||
return fmt.Errorf("cannot create the group %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateSystemAccount makes a system account with busybox's adduser: -S system, -D no password, -H no home made.
|
||||
func (alpine) CreateSystemAccount(ctx context.Context, run Runner, name string, uid, gid int, groups []string) error {
|
||||
group, err := run(ctx, "getent", "group", strconv.Itoa(gid))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create the account %q: its group gid %d is not here: %w", name, gid, err)
|
||||
}
|
||||
groupName := name
|
||||
if n, _, ok := strings.Cut(group, ":"); ok {
|
||||
groupName = n
|
||||
}
|
||||
if _, err := run(ctx, "adduser", "-S", "-D", "-H", "-h", NoHome, "-s", "/sbin/nologin", "-u", strconv.Itoa(uid),
|
||||
"-G", groupName, name); err != nil {
|
||||
return fmt.Errorf("cannot create the account %q: %w", name, err)
|
||||
}
|
||||
for _, g := range groups {
|
||||
if _, err := run(ctx, "addgroup", name, g); err != nil {
|
||||
return fmt.Errorf("cannot put the account %q in %q: %w", name, g, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ModifyAccount gives an account the mesh made exactly groups as its added groups, with busybox's addgroup and
|
||||
// delgroup. Its primary group busybox cannot change (it has no usermod, and the mesh edits no passwd by hand):
|
||||
// an account whose primary group must change is refused, saying so, and left as it is.
|
||||
func (alpine) ModifyAccount(ctx context.Context, run Runner, name string, gid int, groups []string) error {
|
||||
passwd, err := run(ctx, "getent", "passwd", name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read the account %q: %w", name, err)
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(passwd), ":")
|
||||
if len(fields) < 4 || fields[3] != strconv.Itoa(gid) {
|
||||
return fmt.Errorf("this system cannot change the account %q's primary group to %d: busybox has no usermod, "+
|
||||
"and the mesh edits no passwd by hand: %w", name, gid, ErrUnsupported)
|
||||
}
|
||||
primary := ""
|
||||
if g, err := run(ctx, "getent", "group", fields[3]); err == nil {
|
||||
primary, _, _ = strings.Cut(strings.TrimSpace(g), ":")
|
||||
}
|
||||
out, err := run(ctx, "id", "-nG", name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read the account %q's groups: %w", name, err)
|
||||
}
|
||||
have, want := map[string]bool{}, map[string]bool{}
|
||||
for _, g := range strings.Fields(out) {
|
||||
if g != primary && g != name {
|
||||
have[g] = true
|
||||
}
|
||||
}
|
||||
for _, g := range groups {
|
||||
want[g] = true
|
||||
if !have[g] {
|
||||
if _, err := run(ctx, "addgroup", name, g); err != nil {
|
||||
return fmt.Errorf("cannot put the account %q in %q: %w", name, g, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for g := range have {
|
||||
if !want[g] {
|
||||
if _, err := run(ctx, "delgroup", name, g); err != nil {
|
||||
return fmt.Errorf("cannot take the account %q out of %q: %w", name, g, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveAccount removes an account with busybox's deluser.
|
||||
func (alpine) RemoveAccount(ctx context.Context, run Runner, name string) error {
|
||||
if _, err := run(ctx, "deluser", name); err != nil {
|
||||
return fmt.Errorf("cannot remove the account %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveGroup removes a group with busybox's delgroup.
|
||||
func (alpine) RemoveGroup(ctx context.Context, run Runner, name string) error {
|
||||
if _, err := run(ctx, "delgroup", name); err != nil {
|
||||
return fmt.Errorf("cannot remove the group %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package system
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// busybox has no usermod (novox/hq ADR 0328 §4): an account the mesh made is given its added groups with addgroup
|
||||
// and delgroup, and a change of its primary group is refused, saying why.
|
||||
func TestBusyboxGivesAnAccountItsAddedGroupsAndRefusesAnotherPrimary(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
switch {
|
||||
case name == "getent" && args[0] == "passwd":
|
||||
return "mesh-radarr:x:61001:62000::/:/sbin/nologin\n", nil
|
||||
case name == "getent" && args[0] == "group":
|
||||
return "mesh-media:x:62000:\n", nil
|
||||
case name == "id":
|
||||
return "mesh-media mesh-old\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if err := (alpine{}).ModifyAccount(context.Background(), run, "mesh-radarr", 62000, []string{"mesh-books"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(ran, "\n")
|
||||
if !strings.Contains(got, "addgroup mesh-radarr mesh-books") || !strings.Contains(got, "delgroup mesh-radarr mesh-old") ||
|
||||
strings.Contains(got, "delgroup mesh-radarr mesh-media") {
|
||||
t.Fatalf("not given exactly its added groups:\n%s", got)
|
||||
}
|
||||
err := (alpine{}).ModifyAccount(context.Background(), run, "mesh-radarr", 62005, nil)
|
||||
if !errors.Is(err, ErrUnsupported) || !strings.Contains(err.Error(), "primary group") {
|
||||
t.Fatalf("a change of primary group was not refused saying why: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An account is made with no home, and a change of its groups moves an old home of `/` away first: `usermod --gid`
|
||||
// regroups every file under the home the account owns, and under `/` that is the whole machine.
|
||||
func TestAnAccountsGroupsChangeOnlyWithNoHome(t *testing.T) {
|
||||
var ran []string
|
||||
home := "/"
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
ran = append(ran, line)
|
||||
if name == "getent" {
|
||||
return "mesh-radarr:x:61001:61001::" + home + ":/usr/bin/nologin\n", nil
|
||||
}
|
||||
if name == "usermod" && args[0] == "--home" {
|
||||
home = args[1]
|
||||
}
|
||||
if name == "usermod" && args[0] == "--gid" && home != NoHome {
|
||||
t.Fatalf("usermod --gid ran with the home %s: every file there would be regrouped", home)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
if err := (arch{}).CreateSystemAccount(context.Background(), run, "mesh-radarr", 61001, 61001, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(ran[0], "--no-create-home --home-dir "+NoHome+" ") {
|
||||
t.Fatalf("an account is made with a home: %s", ran[0])
|
||||
}
|
||||
if err := (arch{}).ModifyAccount(context.Background(), run, "mesh-radarr", 62000, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(ran, "\n")
|
||||
if !strings.Contains(got, "usermod --home "+NoHome+" mesh-radarr") || strings.Contains(got, " -m") ||
|
||||
strings.Contains(got, "--move-home") {
|
||||
t.Fatalf("the home was not moved away first, without moving files:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// busybox removes what the mesh made of an identity with deluser and delgroup.
|
||||
func TestBusyboxRemovesAnAccountAndAGroup(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
return "", nil
|
||||
}
|
||||
if err := (alpine{}).RemoveAccount(context.Background(), run, "mesh-radarr"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := (alpine{}).RemoveGroup(context.Background(), run, "mesh-media"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(ran, "\n") != "deluser mesh-radarr\ndelgroup mesh-media" {
|
||||
t.Fatalf("removed with: %v", ran)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user