Compare commits
240
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0184e2a5b8 | ||
|
|
50c229f905 | ||
|
|
69b845c0f7 | ||
|
|
caafd9306e | ||
|
|
0d22776b20 | ||
|
|
1803d25c3a | ||
|
|
d56c576e9a | ||
|
|
fbc6d6330f | ||
|
|
fb0cdabe25 | ||
|
|
0474a2a3cb | ||
|
|
180a7e7b80 | ||
|
|
be94d2689e | ||
|
|
b225c7bb4c | ||
|
|
1d44288514 | ||
|
|
af2d2a2c14 | ||
|
|
59d77dd457 | ||
|
|
23d66187a8 | ||
|
|
4476e58080 | ||
|
|
58de6aa10a | ||
|
|
cdd3f3fd4f | ||
|
|
21d6583b5e | ||
|
|
bb46a6ad0f | ||
|
|
f8afaefba0 | ||
|
|
1e38c42646 | ||
|
|
1241ee7041 | ||
|
|
2dc384d641 | ||
|
|
e4dbb49d2f | ||
|
|
02d9d7b053 | ||
|
|
7fc0ea5478 | ||
|
|
8be1c2ab0d | ||
|
|
ff9386dccc | ||
|
|
bba9081ca1 | ||
|
|
65cd7b4787 | ||
|
|
a78239b2aa | ||
|
|
2ac0c4686a | ||
|
|
5233f7584d | ||
|
|
8858dba25e | ||
|
|
9254180cab | ||
|
|
94f9aa298c | ||
|
|
ee9fcaba20 | ||
|
|
43f2d6020e | ||
|
|
6250b1274b | ||
|
|
7f04d12047 | ||
|
|
e88ddab241 | ||
|
|
06f70e11ee | ||
|
|
61cec66ba8 | ||
|
|
5e2fadce15 | ||
|
|
882e75c954 | ||
|
|
1a190090ab | ||
|
|
fd381567f4 | ||
|
|
1195cf4597 | ||
|
|
99285cb080 | ||
|
|
d75c66dc7e | ||
|
|
e196058f6f | ||
|
|
5e71ab6ae6 | ||
|
|
3f02bd7ff6 | ||
|
|
338797a527 | ||
|
|
2c93f79429 | ||
|
|
5f16cf507a | ||
|
|
b32a27c115 | ||
|
|
c16e625a08 | ||
|
|
1cdb4d1a29 | ||
|
|
0dfbe1a179 | ||
|
|
1bdbf47ffa | ||
|
|
16f84b53da | ||
|
|
57b553ac5a | ||
|
|
11a635796d | ||
|
|
8343c4fb36 | ||
|
|
3e3e43de52 | ||
|
|
175fb45c19 | ||
|
|
219169dbd9 | ||
|
|
b99d0b45bf | ||
|
|
f06058390c | ||
|
|
958ccbde3e | ||
|
|
43d1c2456f | ||
|
|
c46937da04 | ||
|
|
cf455bc4fe | ||
|
|
ecf142e451 | ||
|
|
fa9e4483c8 | ||
|
|
e62c32c8c3 | ||
|
|
2bfde7d659 | ||
|
|
db06feed2e | ||
|
|
4e6b756bd2 | ||
|
|
b833d09d28 | ||
|
|
81758b24a8 | ||
|
|
25ad5ce42c | ||
|
|
285395d523 | ||
|
|
b47df31834 | ||
|
|
fd05c8235d | ||
|
|
98d1f165b7 | ||
|
|
332d8f161b | ||
|
|
5088755e5e | ||
|
|
e72c682450 | ||
|
|
be5f31ce66 | ||
|
|
67db003813 | ||
|
|
e5a06cc73d | ||
|
|
20d0998cdd | ||
|
|
83f16cff8d | ||
|
|
96d0322498 | ||
|
|
b2003e7a38 | ||
|
|
3303797bfb | ||
|
|
d5dd7761c6 | ||
|
|
5f5f37b0b2 | ||
|
|
c30b86ae22 | ||
|
|
1598156752 | ||
|
|
e2b6f37168 | ||
|
|
13bdcb694b | ||
|
|
89ebd0eefe | ||
|
|
b4f6537f80 | ||
|
|
7618f661ea | ||
|
|
3c3628c0ce | ||
|
|
4b4bb09a05 | ||
|
|
11b0a1fb29 | ||
|
|
6ea77844d6 | ||
|
|
2633573012 | ||
|
|
976c9998d6 | ||
|
|
48c3e3834c | ||
|
|
0d12fb11a4 | ||
|
|
03b9c47af4 | ||
|
|
cdbff1777e | ||
|
|
59e6a6fa2a | ||
|
|
8c0bb9d453 | ||
|
|
de809d7e0a | ||
|
|
4065b45ae8 | ||
|
|
09c77d57b8 | ||
|
|
c231d1031f | ||
|
|
e597f9791c | ||
|
|
e3a595d8fe | ||
|
|
7e83137aa8 | ||
|
|
f9e1cd183a | ||
|
|
d9f8bbec0c | ||
|
|
d5eeb860e3 | ||
|
|
ef30c12291 | ||
|
|
fd3259a3f5 | ||
|
|
fdb59bcf4e | ||
|
|
d0298a99da | ||
|
|
4cfa0469fd | ||
|
|
45f437eea6 | ||
|
|
a457b38968 | ||
|
|
c5c4caa2a4 | ||
|
|
5a2bc03910 | ||
|
|
60371aadad | ||
|
|
ce1b97ad97 | ||
|
|
52600ec7bd | ||
|
|
278a94d59b | ||
|
|
9d76759e04 | ||
|
|
a8eb24b49c | ||
|
|
da26b7f130 | ||
|
|
cdcba9ca73 | ||
|
|
454be9b269 | ||
|
|
619e14aa8e | ||
|
|
d6f8b00bf8 | ||
|
|
a9789b5125 | ||
|
|
150fadfe8b | ||
|
|
6a2fb63bfc | ||
|
|
64dfa90c3a | ||
|
|
2474bf2833 | ||
|
|
671f284004 | ||
|
|
89025ad004 | ||
|
|
3798673821 | ||
|
|
8119c470d3 | ||
|
|
aab65b81f6 | ||
|
|
9d9e3a46c4 | ||
|
|
0b391f49a2 | ||
|
|
6469e82384 | ||
|
|
dae79c7643 | ||
|
|
d237dee4c9 | ||
|
|
cd13f7726e | ||
|
|
bff1ac14a1 | ||
|
|
49c9002c04 | ||
|
|
c3b2fce0a2 | ||
|
|
198fd34ec4 | ||
|
|
62af89f85d | ||
|
|
1a26778823 | ||
|
|
b9a293f8c8 | ||
|
|
695e022991 | ||
|
|
0e92ef3d1b | ||
|
|
12bc6a56a3 | ||
|
|
cdcfcd9380 | ||
|
|
130c5d98a6 | ||
|
|
cb7b71dcc5 | ||
|
|
8108dd5c23 | ||
|
|
f0d39b4a4a | ||
|
|
89fd0c06e1 | ||
|
|
17038b2a07 | ||
|
|
4dde21f0ab | ||
|
|
7d1d6bffe3 | ||
|
|
7283209d1a | ||
|
|
64f0ab4051 | ||
|
|
5f147fe6b9 | ||
|
|
0d11b4fc93 | ||
|
|
d34b4f32e1 | ||
|
|
d941d798e9 | ||
|
|
b7f70bc4c0 | ||
|
|
8431ac9656 | ||
|
|
bc2a448512 | ||
|
|
bb60e7b584 | ||
|
|
7a41e12063 | ||
|
|
c7b534a08c | ||
|
|
ccc86d3594 | ||
|
|
6fe82c61bd | ||
|
|
8e95c1d9ed | ||
|
|
cb55a46400 | ||
|
|
37947ca4ee | ||
|
|
d8e785bb4b | ||
|
|
281837847e | ||
|
|
c7551162d6 | ||
|
|
2cffa33bcc | ||
|
|
2a2be687f3 | ||
|
|
303d4037e8 | ||
|
|
5c8e768ef5 | ||
|
|
aad33dfe5c | ||
|
|
ef26dba275 | ||
|
|
d12bba0109 | ||
|
|
c85c2804a0 | ||
|
|
2ff40109ac | ||
|
|
46c16e18fe | ||
|
|
f00f9c2a2f | ||
|
|
21cd036232 | ||
|
|
f0adb177f0 | ||
|
|
d2d9fbbd7d | ||
|
|
2026ef36f3 | ||
|
|
51e97a8c14 | ||
|
|
c7a4bf5c11 | ||
|
|
ec983bfc3d | ||
|
|
dd9ab7927d | ||
|
|
0818e4c29a | ||
|
|
41e640d668 | ||
|
|
a1f9856b39 | ||
|
|
74ed7cbe0a | ||
|
|
36704b5471 | ||
|
|
6e44a27a63 | ||
|
|
dce0eef1ca | ||
|
|
e3c3bb37d0 | ||
|
|
b92fb68116 | ||
|
|
3f7f797168 | ||
|
|
c433e76de2 | ||
|
|
f4efc3cad0 | ||
|
|
a76baa4b47 | ||
|
|
90ce9e9d9f |
@@ -102,6 +102,19 @@ what it did not understand would apply most of a declaration and report success.
|
||||
]}
|
||||
```
|
||||
|
||||
**A directory or archive says how it treats files it did not write** (issue 1024,
|
||||
[`novox/hq` ADR 0422](https://git.novox.be/novox/hq)). `unknown` is `move-out`, `module` or `program`;
|
||||
absent, nothing about unknown paths is done. `knows` lists paths inside it, relative to its `path`, that
|
||||
are not declared but belong there; a trailing `/` names a folder and everything in it. `knows` without
|
||||
`unknown` is refused. For `move-out` the node-engine, after every apply and so on every reconcile, moves
|
||||
each path inside the directory that nothing declared, no archive unpacked and `knows` does not name,
|
||||
whole, into `<home>/.local/state/mesh-moved/<module>/<UTC time>/` beside a `<UTC time>.json` list. The
|
||||
list names each move before it is made. Nothing is deleted. A path younger than a minute waits for the
|
||||
next look; a socket, pipe or device is left and said. It is only for a directory under an account's
|
||||
home on one file system with the kept folder, and otherwise the resource fails and nothing moves.
|
||||
`module` and `program` are read by the module and the program that owns the path, and the node-engine
|
||||
does nothing for them.
|
||||
|
||||
**It converges rather than executes.** Applying twice changes nothing the second time; applying
|
||||
to a drifted machine returns it. A mode is *maintained*, not merely set — a permission applied
|
||||
at creation is not a permission held.
|
||||
@@ -244,7 +257,7 @@ without its neighbour checked out is a repository nobody can build.
|
||||
|
||||
```
|
||||
mesh-controller: ./build/mesh-controller plan <node> --json > /tmp/d.json
|
||||
mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v
|
||||
mesh-sdk: MESH_EMITTED=/tmp/d.json go test ./declaration/ -v (run in mesh-sdk's go/ directory)
|
||||
```
|
||||
|
||||
**What this node says when it joins, read by the mesh:**
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Release 57: config and provisioning
|
||||
|
||||
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
|
||||
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
|
||||
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
|
||||
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
|
||||
@@ -116,6 +116,8 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
|
||||
|
||||
--tools-source the repository the shared base is built from
|
||||
--tools-ref what of it to build (default main)
|
||||
--tools-go-source the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)
|
||||
--tools-go-ref what of it to build (default main)
|
||||
--catalog-source the catalogue REPOSITORY, for building its modules;
|
||||
--catalog is the checkout that says what they are
|
||||
--catalog-ref what of it to build (default main)
|
||||
@@ -130,7 +132,7 @@ const usage = `mesh-bootstrap — make a bare machine into a mesh
|
||||
|
||||
The foundation's ports are this machine's, each checked free before anything is
|
||||
raised and kept as the node's setting for the module that binds it:
|
||||
--store-port 5432 --bus-port 5671 --amqp-port 5672 --management-port 15672
|
||||
--store-port 5432 --bus-port 5671
|
||||
--registry-port 5000 (follows --registry, and must agree with it)
|
||||
--packages-port 3000 --hub-port 51820/udp
|
||||
--overlay-range the private network's range (default 10.42.0.0/16); refused
|
||||
@@ -308,6 +310,10 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
|
||||
"the repository the shared base is built from")
|
||||
set.StringVar(&opts.ToolsSource.Ref, "tools-ref", opts.ToolsSource.Ref,
|
||||
"what of it to build (default main)")
|
||||
set.StringVar(&opts.ToolsGoSource.Repository, "tools-go-source", opts.ToolsGoSource.Repository,
|
||||
"the repository the Go toolchain is built from (default: mesh-tools-go beside --tools-source)")
|
||||
set.StringVar(&opts.ToolsGoSource.Ref, "tools-go-ref", opts.ToolsGoSource.Ref,
|
||||
"what of it to build (default main)")
|
||||
set.StringVar(&opts.CatalogSource.Repository, "catalog-source", opts.CatalogSource.Repository,
|
||||
"the catalogue REPOSITORY, for building its modules — --catalog is the checkout that says what they are")
|
||||
set.StringVar(&opts.CatalogSource.Ref, "catalog-ref", opts.CatalogSource.Ref,
|
||||
@@ -325,9 +331,7 @@ func newFlagSet(opts *bootstrap.Options, jsonOut *bool) *flag.FlagSet {
|
||||
into *int
|
||||
}{
|
||||
{"store-port", "the store", &opts.Ports.Store},
|
||||
{"bus-port", "the bus (amqps)", &opts.Ports.Bus},
|
||||
{"amqp-port", "the broker's AMQP", &opts.Ports.AMQP},
|
||||
{"management-port", "the broker's management, on loopback", &opts.Ports.Management},
|
||||
{"bus-port", "the bus", &opts.Ports.Bus},
|
||||
{"registry-port", "the registry", &opts.Ports.Registry},
|
||||
{"packages-port", "the package registry", &opts.Ports.Packages},
|
||||
{"hub-port", "the private network's hub (udp)", &opts.Ports.Hub},
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// An apply runs in a context the bus's loss cannot end: the link's context ends when the bus goes, and the bus may go
|
||||
// because of the apply itself (it replaces the bus's own container), which then must finish (issue 1945, live on
|
||||
// 2026-10-25 at 11:23: novox's node-engine removed the bus container, its link's context ended, and the docker run that
|
||||
// would have made the new one was never made).
|
||||
func TestAnApplyOutlivesTheLinksContext(t *testing.T) {
|
||||
type key struct{}
|
||||
link, cancel := context.WithCancel(context.WithValue(context.Background(), key{}, "v"))
|
||||
ctx, done := applyContext(link)
|
||||
defer done()
|
||||
cancel()
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("the apply's context ended with the link's: %v", ctx.Err())
|
||||
}
|
||||
if ctx.Value(key{}) != "v" {
|
||||
t.Fatal("the apply's context lost the link's values")
|
||||
}
|
||||
}
|
||||
|
||||
// An apply is bounded all the same: a step that hangs (a docker call that never returns) cannot hold the node agent for
|
||||
// ever (the review of issue 1945).
|
||||
func TestAnApplyIsBounded(t *testing.T) {
|
||||
ctx, done := applyContext(context.Background())
|
||||
defer done()
|
||||
at, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
t.Fatal("an apply has no bound")
|
||||
}
|
||||
if left := time.Until(at); left > applyBound || left < applyBound-time.Minute {
|
||||
t.Fatalf("the bound is %s away, want about %s", left, applyBound)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// applyAndKeep applies a whole declaration the way the carried bundle is applied, and says what came of it as the facts
|
||||
// the node says: what the tests of the apply's steps read. The node-engine applies no whole declaration on its own now
|
||||
// (novox/hq ADR 0539 §9); this is the test's way to put a machine in a state.
|
||||
func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.Declared, sched *apply.Scheduler,
|
||||
say link.Announce) link.Report {
|
||||
declared, err := declaration.Parse(raw)
|
||||
if err != nil {
|
||||
return link.Report{Refused: err.Error()}
|
||||
}
|
||||
got := applyModuleHeld(ctx, opts, declared, nil, sched, say)
|
||||
if got.Refused != "" {
|
||||
return link.Report{Refused: got.Refused}
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, h := range got.Updated.Held {
|
||||
held[h.ID] = true
|
||||
}
|
||||
report := factsOf(ctx, opts, declared, got.Updated, got.Outcome, held)
|
||||
if signed != nil {
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), *signed); err != nil {
|
||||
report.Failed = map[string]string{"keeping the declaration": err.Error()}
|
||||
}
|
||||
}
|
||||
if got.Err != nil {
|
||||
report.Failed = map[string]string{"apply": got.Err.Error()}
|
||||
}
|
||||
report.Failed = withUnanswered(report.Failed, got.Probe)
|
||||
report.Unapplied = unapplied(declared, got.Outcome, got.Err)
|
||||
return report
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// authorityPath is where this engine keeps the mesh's own certificate as it last applied it (public): beside the
|
||||
// certificate issued for the machine's name, read into its record (novox/hq ADR 0555 §3).
|
||||
func authorityPath(state string) string { return filepath.Join(filepath.Dir(state), "authority.crt") }
|
||||
|
||||
// keepAuthority keeps the mesh's own certificate a composition carries (the resource a module's certificate.authority
|
||||
// asks for, catalogue.AuthorityID), so the record says the authority this machine holds. A composition carrying none
|
||||
// changes nothing.
|
||||
func keepAuthority(state string, c delivery.Composed) error {
|
||||
for _, raw := range c.Resources {
|
||||
var r struct {
|
||||
ID string `json:"id"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &r) != nil || r.Content == "" || !strings.HasSuffix(r.ID, "."+catalogue.AuthorityID()) {
|
||||
continue
|
||||
}
|
||||
path := authorityPath(state)
|
||||
if was, err := os.ReadFile(path); err == nil && string(was) == r.Content {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, []byte(r.Content), 0o644)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
||||
// record's keys read it; a composition carrying none changes nothing.
|
||||
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||
res := func(id, content string) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"id": id, "type": "file", "path": "/etc/x", "content": content})
|
||||
return raw
|
||||
}
|
||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("web.conf", "x")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(authorityPath(state)); !os.IsNotExist(err) {
|
||||
t.Fatalf("a composition with no authority kept one: %v", err)
|
||||
}
|
||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recordKeys(state)(); got.Authority != "CA-1" {
|
||||
t.Errorf("the record's keys read the authority %q", got.Authority)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
||||
raw, _ := json.Marshal(map[string]any{"node": node, "module": module, "ok": ok, "at": at, "error": why})
|
||||
return raw
|
||||
}
|
||||
|
||||
// A backup is done when the holder says so for this node and module, held on its words when it says it failed, and held
|
||||
// when it says nothing within the bound; another node's or module's event, or one from before the ask, is not the answer
|
||||
// (R6-A A5, novox/hq ADR 0429 §14).
|
||||
func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
|
||||
since := time.Now()
|
||||
run := func(within time.Duration, events ...[]byte) (done bool, words string) {
|
||||
ch := make(chan []byte, len(events))
|
||||
for _, e := range events {
|
||||
ch <- e
|
||||
}
|
||||
got := awaitBackup(context.Background(), ch, "n1", "db", since, within)
|
||||
return got.Done, got.Words
|
||||
}
|
||||
if done, _ := run(time.Second,
|
||||
backupEvent("n2", "db", true, since, ""), backupEvent("n1", "web", true, since, ""),
|
||||
backupEvent("n1", "db", false, since.Add(-time.Hour), "an old night"),
|
||||
backupEvent("n1", "db", true, since.Add(time.Second), "")); !done {
|
||||
t.Error("the event of this node and module was not the answer")
|
||||
}
|
||||
if done, words := run(time.Second, backupEvent("n1", "db", false, since, "the repository is locked")); done ||
|
||||
!strings.Contains(words, "repository is locked") {
|
||||
t.Errorf("a failed backup: done=%v words=%q", done, words)
|
||||
}
|
||||
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
||||
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module that declares nothing to back up on this machine is not backed up before it moves, and so never held for a
|
||||
// backup that cannot run (issue 1945, live on 2026-10-25: redis and searxng on ace were held: "redis declares nothing to
|
||||
// back up on this machine"). What node-backup.backed-up answers decides it.
|
||||
func TestAModuleWithNothingToBackUpIsNotBackedUp(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
said string
|
||||
nothing bool
|
||||
}{
|
||||
{`[{"module":"postgres","runs":1,"paths":["/var/lib/postgres"]}]`, true},
|
||||
{`[{"module":"redis","runs":0,"paths":null,"data":[{"item":"cache"}]}]`, true},
|
||||
{`[]`, true},
|
||||
{`[{"module":"redis","runs":1,"paths":["/var/lib/redis"]}]`, false},
|
||||
} {
|
||||
nothing, ok := declaresNothingToBackUp(json.RawMessage(c.said), "redis")
|
||||
if !ok || nothing != c.nothing {
|
||||
t.Errorf("%s: nothing %v (read %v), want %v", c.said, nothing, ok, c.nothing)
|
||||
}
|
||||
}
|
||||
if _, ok := declaresNothingToBackUp(json.RawMessage(`{"error":"x"}`), "redis"); ok {
|
||||
t.Error("an answer that is not the list was read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// While a container waits, a pass is asked for every so often, not at some long interval; with nothing waiting, none
|
||||
// (novox/hq ADR 0335 §2, ADR 0360 §11).
|
||||
func TestAWaitIsAskedAgainSoon(t *testing.T) {
|
||||
if BusyAskEvery > time.Minute {
|
||||
t.Fatalf("a waiting container is asked every %s", BusyAskEvery)
|
||||
}
|
||||
var asked atomic.Int32
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
busyWaiting.Store(false)
|
||||
t.Cleanup(func() { busyWaiting.Store(false) })
|
||||
go busyPasses(ctx, func() { asked.Add(1) }, 10*time.Millisecond)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if n := asked.Load(); n != 0 {
|
||||
t.Fatalf("%d passes were asked with nothing waiting", n)
|
||||
}
|
||||
busyWaiting.Store(true)
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for asked.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if asked.Load() == 0 {
|
||||
t.Fatal("no pass was asked while a container waits")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A composition that names a file the provisioner fills without saying it is filled would have this node-engine write
|
||||
// that file empty: its apply is refused, loudly, before anything is written (issue 1945, live on 2026-10-25 at 01:22,
|
||||
// when ace, g14 and shanks wrote claude-code's, node-env's and power's files empty).
|
||||
func TestAFileItCannotProduceIsNeverWrittenEmpty(t *testing.T) {
|
||||
unit := json.RawMessage(`{"id":"power.after-boot-unit","type":"file","path":"/etc/systemd/system/mesh-power-after-boot.service","mode":"0644","owner-at-runtime":"owned"}`)
|
||||
c := delivery.Composed{Node: "ace", Resources: []json.RawMessage{unit}}
|
||||
d, err := declarationOf(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := refuseEmptyWrites(d); err == nil || !strings.Contains(err.Error(), "power.after-boot-unit") {
|
||||
t.Fatalf("a file the provisioner fills, not filled here: %v", err)
|
||||
}
|
||||
c.Filled = []string{"/etc/systemd/system/mesh-power-after-boot.service"}
|
||||
if d, _ = declarationOf(c); refuseEmptyWrites(d) != nil {
|
||||
t.Fatalf("a filled file was refused: %v", refuseEmptyWrites(d))
|
||||
}
|
||||
plain := delivery.Composed{Node: "ace", Resources: []json.RawMessage{json.RawMessage(
|
||||
`{"id":"x.f","type":"file","path":"/etc/x","mode":"0644","content":"x"}`)}}
|
||||
if d, _ = declarationOf(plain); refuseEmptyWrites(d) != nil {
|
||||
t.Fatalf("a file with its content was refused: %v", refuseEmptyWrites(d))
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// **A declaration from an older lease epoch is refused before anything is applied** (novox/hq to-be
|
||||
// 45 §6, ADR 0227 rule 2). The controller that sent it lost its lease and goes on sending; this node
|
||||
// applied the holder's. Refused on the host's own apply path — the one the queue's worker runs — before
|
||||
// the machine is read or touched, with a report naming what was refused and what is held, and the
|
||||
// kept declaration left as it was.
|
||||
func TestADeclarationFromAnOlderEpochIsRefusedBeforeAnythingIsApplied(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
_, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := filepath.Join(dir, "a.conf")
|
||||
declared := func(epoch, sequence int64) store.Declared {
|
||||
body, err := json.Marshal(map[string]any{"declaration": 1, "epoch": epoch, "sequence": sequence,
|
||||
"resources": []any{map[string]any{"id": "a", "type": "file", "path": target, "content": "x\n"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
|
||||
}
|
||||
held := declared(57, 3)
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stale := declared(41, 12)
|
||||
report := applyAndKeep(context.Background(), opts, stale.Declaration, &stale, nil, nil)
|
||||
if report.OlderThan == nil || *report.OlderThan != (link.Order{Epoch: 57, Sequence: 3}) ||
|
||||
report.Order != (link.Order{Epoch: 41, Sequence: 12}) || report.Declared != digestOf(stale.Declaration) ||
|
||||
report.Refused == "" {
|
||||
t.Fatalf("the refusal does not name what was refused and what is held: %+v", report)
|
||||
}
|
||||
if _, err := os.Stat(target); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatal("the refused declaration touched the machine")
|
||||
}
|
||||
kept, err := store.ReadDeclared(store.DeclaredPath(opts.state))
|
||||
if err != nil || string(kept.Declaration) != string(held.Declaration) {
|
||||
t.Fatal("the refused declaration replaced what this node kept")
|
||||
}
|
||||
}
|
||||
|
||||
// The queue's counts are kept beside the state and read back by the next host; unreadable is an
|
||||
// error, never zero.
|
||||
func TestTheNumbersSurviveTheHost(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 0 || refused != 0 {
|
||||
t.Fatalf("a node that never reported read %d, %d, %v", sequence, refused, err)
|
||||
}
|
||||
if err := numbersBeside(state).Save(41, 2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sequence, refused, err := numbersBeside(state).Read(); err != nil || sequence != 41 || refused != 2 {
|
||||
t.Fatalf("read back %d, %d, %v", sequence, refused, err)
|
||||
}
|
||||
if err := os.WriteFile(store.NumbersPath(state), []byte(`{"report_sequence":`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := numbersBeside(state).Read(); err == nil {
|
||||
t.Fatal("unreadable numbers were read as zero")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// aFilesLink is a link that serves objects by name, or fails every read with err.
|
||||
type aFilesLink struct {
|
||||
objects map[string][]byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (a aFilesLink) FileObject(_ context.Context, name string, _ int64) ([]byte, error) {
|
||||
if a.err != nil {
|
||||
return nil, a.err
|
||||
}
|
||||
if b, ok := a.objects[name]; ok {
|
||||
return b, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func referenceDeclaration(path string, body []byte) []byte {
|
||||
return []byte(`{"declaration":1,"resources":[{"id":"m.big","type":"file","path":"` + path + `","digest":"` +
|
||||
declaration.DigestOf(body) + `","size":` + strconv.Itoa(len(body)) + `}]}`)
|
||||
}
|
||||
|
||||
// 1695/AC1 and AC2, through the whole apply: a reference is fetched, verified and written; a differing object
|
||||
// refuses the declaration whole and writes nothing.
|
||||
func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
body := []byte(strings.Repeat("a line of a big file\n", 4000))
|
||||
name, _ := declaration.ObjectName(declaration.DigestOf(body))
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "big")
|
||||
opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
|
||||
raw := referenceDeclaration(target, body)
|
||||
|
||||
bad := append([]byte(nil), body...)
|
||||
bad[10] ^= 1
|
||||
ctx := link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: bad}})
|
||||
report := applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||
if report.Refused == "" || !strings.Contains(report.Refused, "m.big") {
|
||||
t.Errorf("1695/AC2: a differing object did not refuse the declaration: %+v", report)
|
||||
}
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
t.Error("1695/AC2: a file was written from an object that differs")
|
||||
}
|
||||
if failedWith(report.Refused).Transient {
|
||||
t.Errorf("an object that differs is a real failure, and was taken as the bus not answering: %s", report.Refused)
|
||||
}
|
||||
|
||||
// The bus answers no JetStream request while it reloads its accounts (2026-10-25 23:04:24, mesh-vault on the bus
|
||||
// node): the failure is transient, so the agent tries it again, and the detail kept is its cause, the tail.
|
||||
noResponders := fmt.Errorf("fetching the object 2bd98ebd of this node's own object bucket: reading the chunks of "+
|
||||
"object %s: %w", strings.Repeat("x", 300), nats.ErrNoResponders)
|
||||
ctx = link.WithFiles(context.Background(), aFilesLink{err: noResponders})
|
||||
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||
failed := failedWith("a file of this composition could not be had by its digest: file mesh-vault.kept: " + report.Refused)
|
||||
if !failed.Transient || !strings.HasSuffix(failed.Detail, nats.ErrNoResponders.Error()) || len(failed.Detail) > 310 ||
|
||||
!strings.HasPrefix(failed.Detail, "a file of this composition") {
|
||||
t.Errorf("a fetch that met no responders: transient %v, detail %q (%d bytes); want transient, its head and its cause",
|
||||
failed.Transient, failed.Detail, len(failed.Detail))
|
||||
}
|
||||
|
||||
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{name: body}})
|
||||
report = applyAndKeep(ctx, opts, raw, nil, nil, nil)
|
||||
if report.Refused != "" {
|
||||
t.Fatalf("1695/AC1: refused: %s", report.Refused)
|
||||
}
|
||||
got, err := os.ReadFile(target)
|
||||
if err != nil || string(got) != string(body) {
|
||||
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
|
||||
}
|
||||
}
|
||||
|
||||
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
|
||||
// as it is, any other refuses and writes nothing — never an empty file.
|
||||
func TestTheHandApplyNeverWritesAReferenceEmpty(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
body := []byte(strings.Repeat("a line of a big file\n", 4000))
|
||||
dir := t.TempDir()
|
||||
target := filepath.Join(dir, "big")
|
||||
opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
|
||||
raw := referenceDeclaration(target, body)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := runApply(context.Background(), opts, d, raw, fromFile); err == nil || !strings.Contains(err.Error(), "digest") {
|
||||
t.Errorf("1695/AC8: a reference with no file and no bus was not refused: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
t.Error("1695/AC8: a file was written for a reference that could not be had")
|
||||
}
|
||||
if err := os.WriteFile(target, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, _ = declaration.ParseFileTrusted(raw)
|
||||
if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil && !strings.Contains(err.Error(), "applied only in part") {
|
||||
t.Fatalf("1695/AC8: %v", err)
|
||||
}
|
||||
if got, _ := os.ReadFile(target); string(got) != string(body) {
|
||||
t.Errorf("1695/AC8: the file holds %d bytes, want %d", len(got), len(body))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/filling"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
|
||||
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
|
||||
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
|
||||
e := fillingEngine(opts, signer, say)
|
||||
fillingNow.Store(e)
|
||||
defer fillingNow.CompareAndSwap(e, nil)
|
||||
if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
|
||||
say("filled files are not written on request on this machine: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// fillingNow is the filling this engine serves, for knockFilling; nil while none is served.
|
||||
var fillingNow atomic.Pointer[filling.Engine]
|
||||
|
||||
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
|
||||
// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled).
|
||||
// What it told, or why it could not, is said.
|
||||
func knockFilling(module string, say func(string)) {
|
||||
e := fillingNow.Load()
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
told, err := e.Knock(filling.Socket)
|
||||
switch {
|
||||
case err != nil:
|
||||
say(fmt.Sprintf("after %s settled: %v", module, err))
|
||||
case told != "":
|
||||
say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told))
|
||||
}
|
||||
}
|
||||
|
||||
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
|
||||
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
|
||||
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
|
||||
func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
|
||||
return &filling.Engine{
|
||||
Declared: func() (*declaration.Declaration, error) {
|
||||
return declaredNow(opts.state, signer)
|
||||
},
|
||||
UIDOf: filling.UIDOf,
|
||||
Lock: func() (func(), error) {
|
||||
applying.Lock()
|
||||
unlock, err := store.Lock(opts.state, nil)
|
||||
if err != nil {
|
||||
applying.Unlock()
|
||||
return nil, err
|
||||
}
|
||||
return func() { unlock(); applying.Unlock() }, nil
|
||||
},
|
||||
Place: func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (string, string, error) {
|
||||
built, err := system.For(builtFor)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
|
||||
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
|
||||
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
||||
if err := store.Save(opts.state, updated); err != nil {
|
||||
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
|
||||
err.Error()))
|
||||
}
|
||||
action, digest := "", ""
|
||||
for _, o := range report.Outcomes {
|
||||
if o.Target == path && o.Type == string(declaration.TypeFile) {
|
||||
action = o.Action
|
||||
}
|
||||
}
|
||||
if rec, ok := updated.At(string(declaration.TypeFile), path); ok {
|
||||
digest = rec.Wrote
|
||||
}
|
||||
return action, digest, writeErr
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/filling"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0349 §4 (issue 668): a write of a filled file and an apply are one at a time — the write
|
||||
// takes the lock the apply holds, and writes only once the apply let it go — and the write is recorded in the
|
||||
// node's state as an apply's outcome is.
|
||||
func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
me, err := user.Current()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if me.Uid == "0" {
|
||||
t.Skip("run as root, whose account is never the provisioner's")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(dir, "m.conf")
|
||||
raw, err := json.Marshal(map[string]any{"declaration": 1, "provisioner_account": me.Username,
|
||||
"filled_files": []string{path}, "resources": []any{map[string]any{"id": "m.conf", "type": "file",
|
||||
"path": path, "content": "from the declaration\n", "owner-at-runtime": "owned", "owner": me.Username}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state),
|
||||
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := fillingEngine(opts, public, func(string) {})
|
||||
uid := uint32(os.Getuid())
|
||||
if err := e.Accepts(uid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// An apply holds the lock.
|
||||
applying.Lock()
|
||||
done := make(chan filling.Answer)
|
||||
go func() {
|
||||
done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("filled\n")})
|
||||
}()
|
||||
select {
|
||||
case a := <-done:
|
||||
applying.Unlock()
|
||||
t.Fatalf("written while an apply held the lock: %+v", a)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
applying.Unlock()
|
||||
t.Fatal("the file was written while an apply held the lock")
|
||||
}
|
||||
applying.Unlock()
|
||||
a := <-done
|
||||
if a.Refused != "" || a.Failed != "" || a.Action != "created" {
|
||||
t.Fatalf("the write after the apply: %+v", a)
|
||||
}
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil || string(got) != "filled\n" {
|
||||
t.Fatalf("the file holds %q (%v)", got, err)
|
||||
}
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec, ok := known.Find("m.conf"); !ok || rec.Wrote != a.Digest || rec.Target != path {
|
||||
t.Errorf("the write is not recorded in the node's state: %+v, answered %q", rec, a.Digest)
|
||||
}
|
||||
|
||||
// And the cross-process lock beside the state too: a write waits for a `reconcile` run by hand.
|
||||
unlock, err := store.Lock(opts.state, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go func() {
|
||||
done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("again\n")})
|
||||
}()
|
||||
select {
|
||||
case a := <-done:
|
||||
unlock()
|
||||
t.Fatalf("written while another process held the apply's lock: %+v", a)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
unlock()
|
||||
if a := <-done; a.Action != "updated" {
|
||||
t.Fatalf("the second write: %+v", a)
|
||||
}
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// **The generation applied is kept, and a put-back never lowers it**: after a put-back of generation 5
|
||||
// over 7, a send from generation 6 is still older than what this machine applied.
|
||||
func TestThePutBackDoesNotLowerTheGenerationApplied(t *testing.T) {
|
||||
built, err := system.For("arch")
|
||||
if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
|
||||
t.Skip("this machine is not one these tests can apply on")
|
||||
}
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
f := machineThatApplied(t, 7)
|
||||
|
||||
newer := f.declared(t, map[string]any{"sequence": 9, "generation": 8}, "docker.conf", f.stays)
|
||||
if r := applyAndKeep(context.Background(), f.opts, newer.Declaration, &newer, nil, nil); r.Refused != "" {
|
||||
t.Fatalf("a newer generation was refused: %s", r.Refused)
|
||||
}
|
||||
back := f.declared(t, map[string]any{"sequence": 10, "generation": 5, "put_back": true}, "docker.conf", f.stays)
|
||||
if r := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil); r.Refused != "" {
|
||||
t.Fatalf("a put-back was refused: %s", r.Refused)
|
||||
}
|
||||
state, err := store.Load(f.opts.state)
|
||||
if err != nil || state.Generation != 8 {
|
||||
t.Fatalf("the generation applied is %d after a put-back of 5 over 8 (%v)", state.Generation, err)
|
||||
}
|
||||
stale := f.declared(t, map[string]any{"sequence": 11, "generation": 7}, "docker.conf", f.stays)
|
||||
if r := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil); r.OlderGeneration == nil {
|
||||
t.Fatalf("a send from generation 7 after 8 was applied because a put-back came between: %+v", r)
|
||||
}
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The assignment generation a declaration was composed from (novox/hq issue 234).
|
||||
//
|
||||
// On 2026-10-04 a declaration newer in sequence than every other, and composed from a view of the
|
||||
// assignments older than the one this machine had applied, named four fewer modules — and the host,
|
||||
// ordering by sequence alone, undeclared all four. The sequence orders arrival; it cannot tell a later
|
||||
// send carrying earlier content. The generation can: the node-engine refuses a declaration composed from
|
||||
// an older generation than the one it applied, before anything is read from the machine.
|
||||
//
|
||||
// These read what the report says through its JSON, as the controller does, so they run against a
|
||||
// node-engine that has never heard of a generation and fail there for that reason.
|
||||
|
||||
// generationFixture is a machine that applied generation 7: a file the mesh declared, kept in its state,
|
||||
// and the declaration it kept — signed — that named it.
|
||||
type generationFixture struct {
|
||||
opts options
|
||||
private ed25519.PrivateKey
|
||||
kept store.Declared
|
||||
// stays is the file the older generation's declaration does not name.
|
||||
stays string
|
||||
}
|
||||
|
||||
func machineThatApplied(t *testing.T, generation int64) generationFixture {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
_, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stays := filepath.Join(dir, "docker.conf")
|
||||
if err := os.WriteFile(stays, []byte("kept\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The state as a node-engine that applied generation 7 keeps it: the file it owns, and the highest
|
||||
// generation it applied. Written as JSON, so the fixture is the same bytes for any build under test.
|
||||
state := map[string]any{"resources": []any{map[string]any{"id": "docker.conf", "type": "file",
|
||||
"origin": store.OriginDeclared, "target": stays}}, "generation": generation}
|
||||
raw, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(opts.state, raw, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := generationFixture{opts: opts, private: private, stays: stays}
|
||||
f.kept = f.declared(t, map[string]any{"sequence": 3, "generation": generation}, "docker.conf", stays)
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), f.kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// declared is a signed declaration with the envelope's keys given and one file.
|
||||
func (f generationFixture) declared(t *testing.T, envelope map[string]any, id, path string) store.Declared {
|
||||
t.Helper()
|
||||
body := map[string]any{"declaration": 1,
|
||||
"resources": []any{map[string]any{"id": id, "type": "file", "path": path, "content": "x\n"}}}
|
||||
for k, v := range envelope {
|
||||
body[k] = v
|
||||
}
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store.Declared{Declaration: raw, Signature: ed25519.Sign(f.private, raw)}
|
||||
}
|
||||
|
||||
// reportJSON is a report as the controller reads it.
|
||||
func reportJSON(t *testing.T, r link.Report) map[string]any {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// **A newer sequence from an older generation is refused, and nothing is undeclared** (issue 234).
|
||||
func TestANewerSequenceFromAnOlderGenerationIsRefusedAndNothingIsUndeclared(t *testing.T) {
|
||||
f := machineThatApplied(t, 7)
|
||||
other := filepath.Join(filepath.Dir(f.opts.state), "sudo.conf")
|
||||
stale := f.declared(t, map[string]any{"sequence": 9, "generation": 6}, "sudo.conf", other)
|
||||
|
||||
report := applyAndKeep(context.Background(), f.opts, stale.Declaration, &stale, nil, nil)
|
||||
said := reportJSON(t, report)
|
||||
older, _ := said["older_generation"].(map[string]any)
|
||||
if older == nil || older["generation"] != float64(6) || older["applied"] != float64(7) {
|
||||
t.Fatalf("the refusal does not name the generation refused and the one applied: %v", said)
|
||||
}
|
||||
if report.Refused == "" || !strings.Contains(report.Refused, "generation 6") ||
|
||||
!strings.Contains(report.Refused, "generation 7") {
|
||||
t.Errorf("the refusal is not said in words naming both generations: %q", report.Refused)
|
||||
}
|
||||
if report.Declared != digestOf(stale.Declaration) || report.Sequence != 9 {
|
||||
t.Errorf("the refusal does not name the declaration refused, so the mesh cannot name who sent it: %+v", report)
|
||||
}
|
||||
if _, err := os.Stat(f.stays); err != nil {
|
||||
t.Fatalf("a refused declaration undeclared what the machine applied: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(other); err == nil {
|
||||
t.Error("the refused declaration touched the machine")
|
||||
}
|
||||
state, err := store.Load(f.opts.state)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, owned := state.Find("docker.conf"); !owned {
|
||||
t.Error("the refused declaration took a resource out of what this machine owns")
|
||||
}
|
||||
kept, err := store.ReadDeclared(store.DeclaredPath(f.opts.state))
|
||||
if err != nil || string(kept.Declaration) != string(f.kept.Declaration) {
|
||||
t.Error("the refused declaration replaced what this node kept, so the next reconcile would apply it")
|
||||
}
|
||||
}
|
||||
|
||||
// refusedForItsGeneration says a report refused its declaration for the generation it came from.
|
||||
func refusedForItsGeneration(t *testing.T, r link.Report) bool {
|
||||
t.Helper()
|
||||
return reportJSON(t, r)["older_generation"] != nil || strings.Contains(r.Refused, "generation")
|
||||
}
|
||||
|
||||
// refusedAsUnread says a report refused its declaration as one this node-engine cannot read.
|
||||
func refusedAsUnread(r link.Report) bool { return strings.Contains(r.Refused, "not a declaration") }
|
||||
|
||||
// **A gate's put-back is applied, whatever generation it carries**: it carries the generation of what it
|
||||
// puts back, and says it is a put-back (issue 234).
|
||||
func TestAPutBackFromAnOlderGenerationIsNotRefused(t *testing.T) {
|
||||
f := machineThatApplied(t, 7)
|
||||
back := f.declared(t, map[string]any{"sequence": 9, "generation": 5, "put_back": true}, "docker.conf", f.stays)
|
||||
report := applyAndKeep(context.Background(), f.opts, back.Declaration, &back, nil, nil)
|
||||
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
|
||||
t.Fatalf("a put-back was refused: %s", report.Refused)
|
||||
}
|
||||
}
|
||||
|
||||
// **A declaration that carries no generation is applied as it always was**: an older controller sends
|
||||
// none, and a node-engine that refused it would strand every machine the moment the controller is older.
|
||||
func TestADeclarationWithoutAGenerationIsNotRefusedForOne(t *testing.T) {
|
||||
f := machineThatApplied(t, 7)
|
||||
older := f.declared(t, map[string]any{"sequence": 9}, "docker.conf", f.stays)
|
||||
report := applyAndKeep(context.Background(), f.opts, older.Declaration, &older, nil, nil)
|
||||
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
|
||||
t.Fatalf("a declaration without a generation was refused: %s", report.Refused)
|
||||
}
|
||||
// And the same generation again is no older.
|
||||
same := f.declared(t, map[string]any{"sequence": 10, "generation": 7}, "docker.conf", f.stays)
|
||||
report = applyAndKeep(context.Background(), f.opts, same.Declaration, &same, nil, nil)
|
||||
if refusedForItsGeneration(t, report) || refusedAsUnread(report) {
|
||||
t.Fatalf("a declaration of the generation applied was refused: %s", report.Refused)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
)
|
||||
|
||||
// owedBus is a link that takes reports and statements and keeps a have bucket in memory.
|
||||
type owedBus struct {
|
||||
mu sync.Mutex
|
||||
kv map[string][]byte
|
||||
rv map[string]uint64
|
||||
}
|
||||
|
||||
func (b *owedBus) Report(context.Context, string, []byte) error { return nil }
|
||||
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
|
||||
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
|
||||
|
||||
func (b *owedBus) Get(_ context.Context, key string) ([]byte, uint64, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
v, ok := b.kv[key]
|
||||
if !ok {
|
||||
return nil, 0, link.ErrHaveAbsent
|
||||
}
|
||||
return v, b.rv[key], nil
|
||||
}
|
||||
|
||||
func (b *owedBus) Set(_ context.Context, key string, value []byte, revision uint64) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.rv[key] != revision {
|
||||
return errors.New("wrong last sequence")
|
||||
}
|
||||
b.kv[key], b.rv[key] = value, revision+1
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *owedBus) have(key string) (delivery.Have, bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
h, err := delivery.DecodeHave(b.kv[key])
|
||||
return h, err == nil
|
||||
}
|
||||
|
||||
// A module with nothing long-running (xorg) is ok within one look of its apply, not at the five-minute statement: the
|
||||
// facts look inside the apply runs while it is applying and judges nothing, and the running write then owes a look,
|
||||
// which the health loop makes at once (review of fix 15, B1).
|
||||
func TestAModuleWithNothingLongRunningIsOkWithinALookOfItsApply(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
bus := &owedBus{kv: map[string][]byte{}, rv: map[string]uint64{}}
|
||||
queue := &link.Session{Membership: link.Membership{Node: "n1"}, Timeout: time.Second}
|
||||
queue.Attach(ctx, bus)
|
||||
j, err := liveness.Open(filepath.Join(t.TempDir(), "liveness.json"), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fact := link.ModuleFact{Module: "xorg", FP: "src2:" + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("b", 64)}
|
||||
applying := fact
|
||||
applying.Phase = string(delivery.PhaseApplying)
|
||||
queue.NoteModule(ctx, applying)
|
||||
go judgeWhatRuns(ctx, j, queue, func(string) {}, nil)
|
||||
// The facts look inside the apply, while the module is still applying.
|
||||
queue.SayFacts(ctx, link.Report{Health: &link.Health{Contract: link.LivenessContract, At: time.Now().UTC()}})
|
||||
if h, _ := bus.have("n1.xorg"); h.Health != delivery.HealthUnknown {
|
||||
t.Fatalf("judged while applying: %+v", h)
|
||||
}
|
||||
queue.NoteModule(ctx, fact)
|
||||
deadline := time.Now().Add(2 * time.Second) // well within one look (liveness.LookEvery, 15 s)
|
||||
for {
|
||||
h, _ := bus.have("n1.xorg")
|
||||
if h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("not ok within a look of its apply: %+v", h)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// judgeDirectories judges every directory of the declaration this node keeps, read-only (novox/hq issue 358): which
|
||||
// the walk that follows no link another account could make would refuse. Nothing is made, chmodded or chowned.
|
||||
func judgeDirectories(statePath string) (string, error) {
|
||||
d, err := declaredNow(statePath, nil)
|
||||
if errors.Is(err, store.ErrNothingDeclared) {
|
||||
return "", errors.New("this node keeps no declaration, so it is declared no directory")
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("what runs on this node does not read: %w", err)
|
||||
}
|
||||
return apply.JudgeDirectories(d), nil
|
||||
}
|
||||
|
||||
// judgeDirectoriesFrom is judgeDirectories as the controller asks it over the link.
|
||||
func judgeDirectoriesFrom(statePath string) link.JudgeDirectoriesFunc {
|
||||
return func(string) (string, error) { return judgeDirectories(statePath) }
|
||||
}
|
||||
|
||||
// judgeDirectoriesCommand is `nox-mesh-host judge-directories` at this machine, as root: the same judgement, for a
|
||||
// machine the controller cannot ask.
|
||||
func judgeDirectoriesCommand(opts options) error {
|
||||
said, err := judgeDirectories(opts.state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(opts.out, said)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A library is judged on every look (novox/hq ADR 0328 §8, §9).
|
||||
//
|
||||
// Each library this machine is reached by is said as a resource `library:<name>` of kind library, with no module:
|
||||
// healthy while it is kept (or, off its home, while its mount is there), waiting with an access wait of reason keep
|
||||
// or disk, unknown while its filesystem could not be named, unhealthy otherwise. The root is the target, never in a
|
||||
// reason nor in a wait's words: the controller says them on the operator's channel, which withholds a path (ADR
|
||||
// 0234 §6). An access is judged by nothing, as before ADR 0325.
|
||||
|
||||
// libraryJudge reads every library on every look. Safe for the apply and the looking loop at once.
|
||||
type libraryJudge struct {
|
||||
mu sync.Mutex
|
||||
libraries []declaration.Library
|
||||
declared *declaration.Declaration
|
||||
// was is each library's last verdict, with since when and how many looks in a row found it unhealthy.
|
||||
was map[string]link.ResourceHealth
|
||||
|
||||
// reading guards inFlight: every library whose look has not returned yet.
|
||||
reading sync.Mutex
|
||||
inFlight map[string]bool
|
||||
}
|
||||
|
||||
// accesses is the one library judge on this machine (its name from when it judged accesses).
|
||||
var accesses = newLibraryJudge()
|
||||
|
||||
func newLibraryJudge() *libraryJudge {
|
||||
return &libraryJudge{was: map[string]link.ResourceHealth{}, inFlight: map[string]bool{}}
|
||||
}
|
||||
|
||||
// set is what the declaration just applied declares: every library. One no longer declared is forgotten.
|
||||
func (j *libraryJudge) set(d *declaration.Declaration, _ map[string]bool) {
|
||||
var ls []declaration.Library
|
||||
if d != nil {
|
||||
ls = append(ls, d.Libraries...)
|
||||
}
|
||||
sort.Slice(ls, func(a, b int) bool { return ls[a].Library < ls[b].Library })
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
j.libraries, j.declared = ls, d
|
||||
declared := map[string]bool{}
|
||||
for _, l := range ls {
|
||||
declared[apply.LibraryAccess(l.Library)] = true
|
||||
}
|
||||
for id := range j.was {
|
||||
if !declared[id] {
|
||||
delete(j.was, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// declaration is the declaration last set.
|
||||
func (j *libraryJudge) declaration() *declaration.Declaration {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return j.declared
|
||||
}
|
||||
|
||||
// lookBound is how long a look waits for one library to answer. A hard network mount whose server is down blocks
|
||||
// a stat for ever, and one library must never stop every health statement this machine makes.
|
||||
var lookBound = 5 * time.Second
|
||||
|
||||
// verdict is one library's judgement in a look.
|
||||
type verdict struct {
|
||||
state, reason string
|
||||
waits []link.AccessWait
|
||||
act *link.AccessActRecord
|
||||
}
|
||||
|
||||
// judgeBounded judges one library within lookBound, outside any lock. A look that has not answered is said so;
|
||||
// while it still hangs, later looks start no other on the same library.
|
||||
func (j *libraryJudge) judgeBounded(l declaration.Library) (verdict, bool) {
|
||||
j.reading.Lock()
|
||||
if j.inFlight[l.Library] {
|
||||
j.reading.Unlock()
|
||||
return verdict{}, false
|
||||
}
|
||||
j.inFlight[l.Library] = true
|
||||
j.reading.Unlock()
|
||||
done := make(chan verdict, 1)
|
||||
go func() {
|
||||
v := judgeLibrary(l)
|
||||
j.reading.Lock()
|
||||
delete(j.inFlight, l.Library)
|
||||
j.reading.Unlock()
|
||||
done <- v
|
||||
}()
|
||||
select {
|
||||
case v := <-done:
|
||||
return v, true
|
||||
case <-time.After(lookBound):
|
||||
return verdict{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// judgeLibrary is one library's judgement now (ADR 0328 §7, §8, §9).
|
||||
func judgeLibrary(l declaration.Library) verdict {
|
||||
v := apply.JudgeLibrary(context.Background(), l)
|
||||
access := apply.LibraryAccess(l.Library)
|
||||
var out verdict
|
||||
if a := v.Act; a != nil {
|
||||
out.act = &link.AccessActRecord{Act: a.Act, Access: access, Warrant: a.Warrant, By: a.By, State: a.State,
|
||||
Started: a.Started, Ended: a.Ended, Folders: a.Folders, Files: a.Files, Kept: a.Kept, Links: a.Links,
|
||||
Other: a.Other, Mounts: a.Mounts, Shared: a.Shared, Linked: a.Linked, Stripped: a.Stripped, Why: a.Why}
|
||||
}
|
||||
switch v.Kind {
|
||||
case apply.VerdictWaitKeep:
|
||||
out.state, out.reason = link.StateWaiting, "its root holds data the mesh has not taken on the filesystem "+
|
||||
"it is on now, and waits for the operator's word (novox/hq ADR 0328 §7, §8)"
|
||||
out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
|
||||
What: apply.KeepWords(l.Library), Placed: v.Placed, Reason: link.WaitKeep}}
|
||||
case apply.VerdictWaitDisk:
|
||||
out.state, out.reason = link.StateWaiting, v.Why+" (novox/hq ADR 0328 §9)"
|
||||
out.waits = []link.AccessWait{{Kind: link.WaitKindAccess, Access: access, Part: l.Library,
|
||||
What: apply.DiskWords(l.Library), Reason: link.WaitDisk}}
|
||||
case apply.VerdictUnknown:
|
||||
// Not a fault and not the operator's: said as not known, and judged again on the next look.
|
||||
out.state, out.reason = link.StateUnknown, v.Why+"; judged again on the next look"
|
||||
case apply.VerdictBroken:
|
||||
out.state, out.reason = link.StateUnhealthy, v.Why
|
||||
default:
|
||||
out.state = link.StateHealthy
|
||||
switch {
|
||||
case v.Act != nil && v.Act.State == link.ActRunning:
|
||||
out.state, out.reason = link.StateStarting, "the take act is setting its group under its root"
|
||||
case v.Act != nil && (v.Act.State == link.ActStopped || v.Act.State == link.ActFailed):
|
||||
out.state, out.reason = link.StateStarting, "the take act goes on from where it stopped: "+v.Act.Why
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// look reads every library now. Whether any verdict changed since the last look, for the loop to say it.
|
||||
func (j *libraryJudge) look(now time.Time) ([]link.ResourceHealth, bool) {
|
||||
j.mu.Lock()
|
||||
ls := append([]declaration.Library(nil), j.libraries...)
|
||||
j.mu.Unlock()
|
||||
|
||||
read := make([]verdict, len(ls))
|
||||
for i, l := range ls {
|
||||
v, answered := j.judgeBounded(l)
|
||||
if !answered {
|
||||
v = verdict{state: link.StateUnknown, reason: "its root does not answer: judging it did not return " +
|
||||
"within " + lookBound.String()}
|
||||
}
|
||||
read[i] = v
|
||||
}
|
||||
|
||||
// And every tool runner's account the last apply could not put in a group, as a resource of its own (ADR 0334
|
||||
// §2): unhealthy, never failing the apply.
|
||||
warnings := apply.RunnerWarnings()
|
||||
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
changed := false
|
||||
out := make([]link.ResourceHealth, 0, len(ls)+len(warnings))
|
||||
for _, w := range warnings {
|
||||
v := link.ResourceHealth{Module: w.Module, Resource: w.Resource, Kind: link.KindRunner, State: link.StateUnhealthy,
|
||||
Reason: "the tool runner's account joins no group: " + w.Why, Since: now.UTC(), Streak: 1}
|
||||
if was, had := j.was[w.Resource]; had && was.Reason == v.Reason {
|
||||
v.Since, v.Streak = was.Since, was.Streak+1
|
||||
} else {
|
||||
changed = true
|
||||
}
|
||||
j.was[w.Resource] = v
|
||||
out = append(out, v)
|
||||
}
|
||||
for i, l := range ls {
|
||||
r := read[i]
|
||||
id := apply.LibraryAccess(l.Library)
|
||||
v := link.ResourceHealth{Resource: id, Kind: link.KindLibrary, Target: l.Root, State: r.state,
|
||||
Reason: r.reason, Since: now.UTC(), AccessWaits: r.waits, Act: r.act}
|
||||
if was, had := j.was[id]; had && was.State == r.state && was.Reason == r.reason {
|
||||
v.Since = was.Since
|
||||
v.Streak = was.Streak
|
||||
} else {
|
||||
changed = true
|
||||
}
|
||||
if r.state == link.StateUnhealthy {
|
||||
v.Streak++
|
||||
}
|
||||
j.was[id] = v
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, changed
|
||||
}
|
||||
|
||||
// withAccesses adds to a statement every library, as a resource of kind library.
|
||||
func withAccesses(h *link.Health, vs []link.ResourceHealth) *link.Health {
|
||||
h.Resources = append(h.Resources, vs...)
|
||||
return h
|
||||
}
|
||||
|
||||
// accessesHealthy says no library in vs is anything but healthy.
|
||||
func accessesHealthy(vs []link.ResourceHealth) bool {
|
||||
for _, v := range vs {
|
||||
if v.State != link.StateHealthy {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// keeper goes on with take acts stopped at their bound, beside the applies, one at a time (ADR 0328 §7).
|
||||
type keeper struct {
|
||||
running atomic.Bool
|
||||
say func(string)
|
||||
}
|
||||
|
||||
// keeping is the one keeper on this machine.
|
||||
var keeping = &keeper{say: func(string) {}}
|
||||
|
||||
// poke goes on with every stopped take act of the declaration last applied, unless that is running already.
|
||||
func (k *keeper) poke(ctx context.Context) {
|
||||
d := accesses.declaration()
|
||||
if d == nil || len(d.Libraries) == 0 || !k.running.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer k.running.Store(false)
|
||||
for _, line := range apply.ReconcileLibraries(ctx, d) {
|
||||
k.say(line)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// startKeeping has the keeper say what it does.
|
||||
func startKeeping(ctx context.Context, say func(string)) {
|
||||
keeping.say = say
|
||||
keeping.poke(ctx)
|
||||
}
|
||||
|
||||
// accessActFrom is the take act as the controller asks it over the link (ADR 0328 §7): judged against the
|
||||
// declaration this node keeps, and run beside the applies.
|
||||
func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc {
|
||||
return func(act link.AccessAct) (string, error) {
|
||||
d, err := declaredNow(statePath, nil)
|
||||
if errors.Is(err, store.ErrNothingDeclared) {
|
||||
return "", errors.New("this node keeps no declaration, so it is declared no library")
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("what runs on this node does not read: %w", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
_, said, err := apply.TakeLibrary(ctx, d, act.Access, filepath.Clean(act.Path), act.Warrant, act.By)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-host: the take act on %s was refused: %v\n", act.Access, err)
|
||||
return "", err
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
}
|
||||
|
||||
// withWaiting is held with every id of waiting beside it: what liveness does not judge.
|
||||
func withWaiting(held, waiting map[string]bool) map[string]bool {
|
||||
if len(waiting) == 0 {
|
||||
return held
|
||||
}
|
||||
out := map[string]bool{}
|
||||
for id := range held {
|
||||
out[id] = true
|
||||
}
|
||||
for id := range waiting {
|
||||
out[id] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// A library whose root holds data never taken is said waiting with one access wait: the access `library:<name>`, the
|
||||
// part its name, words with no path, the filesystem's name, and reason keep (novox/hq ADR 0328 §8); one whose root
|
||||
// is absent waits with reason disk (§9). Neither carries a module.
|
||||
func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
|
||||
was := apply.Libraries
|
||||
apply.Libraries = apply.NewLibraryMemory()
|
||||
defer func() { apply.Libraries = was }()
|
||||
dir := t.TempDir()
|
||||
root := filepath.Join(dir, "media")
|
||||
if err := os.MkdirAll(filepath.Join(root, "a film"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"groups":[{"name":"mesh-media","gid":62000},` +
|
||||
`{"name":"mesh-books","gid":62001}],"libraries":[` +
|
||||
`{"library":"media","root":"` + root + `","group":"mesh-media","gid":62000,"home":true},` +
|
||||
`{"library":"books","root":"` + filepath.Join(dir, "books") + `","group":"mesh-books","gid":62001,"home":true}],` +
|
||||
`"owns_nothing":true,"resources":[]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := newLibraryJudge()
|
||||
j.set(d, nil)
|
||||
vs, _ := j.look(time.Now())
|
||||
if len(vs) != 2 {
|
||||
t.Fatalf("the libraries are not said: %+v", vs)
|
||||
}
|
||||
books, media := vs[0], vs[1]
|
||||
if media.Kind != link.KindLibrary || media.Resource != "library:media" || media.Module != "" ||
|
||||
media.State != link.StateWaiting || len(media.AccessWaits) != 1 {
|
||||
t.Fatalf("a library holding data is not said waiting: %+v", media)
|
||||
}
|
||||
raw, _ := json.Marshal(media.AccessWaits[0])
|
||||
want := `{"kind":"access","access":"library:media","part":"media","what":"let the mesh keep the library media",` +
|
||||
`"placed":"` + media.AccessWaits[0].Placed + `","reason":"keep"}`
|
||||
if string(raw) != want || !strings.HasPrefix(media.AccessWaits[0].Placed, "fs-") {
|
||||
t.Fatalf("the wait reads %s", raw)
|
||||
}
|
||||
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
|
||||
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
|
||||
}
|
||||
}
|
||||
+392
-579
File diff suppressed because it is too large
Load Diff
+5
-296
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
@@ -12,10 +11,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/bundle"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
@@ -98,26 +95,6 @@ func TestNoCommandIsNotAnError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyNeedsExactlyOneDeclaration(t *testing.T) {
|
||||
// `apply` takes a file where every other command takes nothing, so the leftover-argument
|
||||
// rule has an exception — and an exception is where a parser stops refusing things it
|
||||
// should. Both directions are checked.
|
||||
if _, _, err := parseArgs([]string{"apply"}); err == nil {
|
||||
t.Error("apply with no file was accepted")
|
||||
}
|
||||
if _, _, err := parseArgs([]string{"apply", "a.json", "b.json"}); err == nil {
|
||||
t.Error("apply with two files was accepted")
|
||||
}
|
||||
|
||||
command, opts, err := parseArgs([]string{"apply", "decl.json", "--dry-run"})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if command != "apply" || opts.file != "decl.json" || !opts.dryRun {
|
||||
t.Errorf("parsed as command=%q file=%q dry-run=%v", command, opts.file, opts.dryRun)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheStateHasADocumentedDefault(t *testing.T) {
|
||||
// A host that wrote its state somewhere unexpected would forget what it owns on the next
|
||||
// run, and then leave everything it had applied behind forever.
|
||||
@@ -135,93 +112,22 @@ func TestAFlagAfterAPositionalIsRead(t *testing.T) {
|
||||
// off the front fixed the flag after the COMMAND and not the flag after its ARGUMENT: the
|
||||
// standard library stops at the first non-flag argument wherever that argument is.
|
||||
for _, args := range [][]string{
|
||||
{"apply", "decl.json", "--dry-run", "--json"},
|
||||
{"apply", "--dry-run", "decl.json", "--json"},
|
||||
{"apply", "--dry-run", "--json", "decl.json"},
|
||||
{"hand-over", "decl.json", "--dry-run", "--json"},
|
||||
{"hand-over", "--dry-run", "decl.json", "--json"},
|
||||
{"hand-over", "--dry-run", "--json", "decl.json"},
|
||||
} {
|
||||
command, opts, err := parseArgs(args)
|
||||
if err != nil {
|
||||
t.Errorf("%v: unexpected error: %v", args, err)
|
||||
continue
|
||||
}
|
||||
if command != "apply" || opts.file != "decl.json" || !opts.dryRun || !opts.json {
|
||||
if command != "hand-over" || opts.file != "decl.json" || !opts.dryRun || !opts.json {
|
||||
t.Errorf("%v parsed as file=%q dry-run=%v json=%v",
|
||||
args, opts.file, opts.dryRun, opts.json)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds
|
||||
// or its firewall changed — which is how a predecessor still writing is caught, without a report
|
||||
// every five minutes saying nothing new.
|
||||
func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) {
|
||||
w := &adoptionWatch{}
|
||||
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}}
|
||||
if !w.changed(held) {
|
||||
t.Fatal("the first report of a hold was not said")
|
||||
}
|
||||
again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}}
|
||||
if w.changed(again) {
|
||||
t.Error("the same holds in another order were said again")
|
||||
}
|
||||
rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}}
|
||||
if !w.changed(rewritten) {
|
||||
t.Error("a held file rewritten by something else was not said")
|
||||
}
|
||||
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
|
||||
t.Error("a changed firewall was not said")
|
||||
}
|
||||
// What filters the machine is part of it (novox/hq ADR 0168): a predecessor's chain removed by
|
||||
// hand, or the found firewall enabled again, is said without being asked.
|
||||
filtered := link.Report{Firewall: "none", Held: rewritten.Held,
|
||||
Filters: []link.Filter{{Where: "chain HAL-MESH-ONLY (iptables-legacy)", Owner: "other", Refuses: "-j DROP"}}}
|
||||
if !w.changed(filtered) {
|
||||
t.Error("a filter appearing was not said")
|
||||
}
|
||||
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
|
||||
t.Error("a filter removed was not said")
|
||||
}
|
||||
if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held, FoundFirewall: &link.FoundFirewall{Kind: "ufw", Active: true}}) {
|
||||
t.Error("the found firewall coming back was not said")
|
||||
}
|
||||
if !worthSaying(link.Report{Filters: filtered.Filters}) {
|
||||
t.Error("a report carrying only what filters the machine is not worth saying")
|
||||
}
|
||||
}
|
||||
|
||||
// What the mesh heard — a delivery's account as much as a reconcile's — counts as said: the queue
|
||||
// tells the watch every report the broker took (link.Queue.Heard).
|
||||
func TestWhatTheMeshHeardCountsAsSaid(t *testing.T) {
|
||||
w := &adoptionWatch{}
|
||||
report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}}
|
||||
heard := w.said
|
||||
heard(report)
|
||||
if w.changed(report) {
|
||||
t.Error("a reconcile repeated what the mesh had just heard")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) {
|
||||
// The controller previews a flip from what the node last said is reachable; a port that opened
|
||||
// since must reach it without waiting for the next delivery (novox/hq ADR 0100).
|
||||
w := &adoptionWatch{}
|
||||
before := link.Report{Firewall: "ufw", Reachable: []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
|
||||
if !w.changed(before) {
|
||||
t.Fatal("the first report was not said")
|
||||
}
|
||||
reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
|
||||
if w.changed(reordered) {
|
||||
t.Error("the same reachable set was said again")
|
||||
}
|
||||
opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable,
|
||||
link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})}
|
||||
if !w.changed(opened) {
|
||||
t.Error("a newly published port was not said")
|
||||
}
|
||||
}
|
||||
|
||||
// Defends the node's own record: the link and the reconcile loop both apply, and each reads the
|
||||
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
|
||||
// what the other recorded.
|
||||
@@ -270,24 +176,6 @@ func TestOnlyOneApplyRunsAtATime(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued
|
||||
// and lost — the link down when the reconcile spoke — it must be said again.
|
||||
func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) {
|
||||
w := &adoptionWatch{}
|
||||
held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}}
|
||||
if !w.differs(held) {
|
||||
t.Fatal("the first report of a change was not new")
|
||||
}
|
||||
// The link was down: nothing published it, so nothing says it was said.
|
||||
if !w.differs(held) {
|
||||
t.Error("a change that never reached the mesh was counted as said")
|
||||
}
|
||||
w.said(held)
|
||||
if w.differs(held) {
|
||||
t.Error("a change the mesh was told was said again")
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
|
||||
// the point of application, whichever command delivered it, naming both — an adopted control-node
|
||||
// once applied its converged genesis bundle and closed itself for forty-five minutes.
|
||||
@@ -468,125 +356,6 @@ func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to
|
||||
// that — never to the bundle the host carries, which genesis consumed.
|
||||
func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) {
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
opts := stateWithMode(t, store.ModeAdopted)
|
||||
|
||||
controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said,
|
||||
Signature: ed25519.Sign(controllerPrivate, said)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Told, and unable to prove by whom: refused, and the bundle is not applied in its place.
|
||||
_, _, _, err = reconcileSource(opts)
|
||||
if err == nil || !strings.Contains(err.Error(), "not applied in its place") {
|
||||
t.Errorf("a node that cannot prove what it was told fell back to something: %v", err)
|
||||
}
|
||||
|
||||
mine, err := identity.Generate("workstation")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0",
|
||||
Signer: controllerPublic, Password: "issued"}
|
||||
if err := identity.Save(identity.Path(opts.state), mine); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, raw, from, err := reconcileSource(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil {
|
||||
t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw))
|
||||
}
|
||||
|
||||
// And before the mesh has said anything: the bundle, as it always was. A test binary carries
|
||||
// only the placeholder, and asking for it is what proves the path.
|
||||
if err := os.Remove(store.DeclaredPath(opts.state)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, _, err = reconcileSource(opts)
|
||||
if !errors.Is(err, bundle.ErrEmpty) {
|
||||
t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine says which links face outside without being asked.**
|
||||
//
|
||||
// The mesh composes no filter for a machine that has not said (novox/hq ADR 0140), and a machine only
|
||||
// speaks unasked when this fingerprint changes. Left out of it, a machine that has just learnt to say
|
||||
// could speak only when a declaration arrived — and a declaration cannot be composed until it has
|
||||
// spoken. A machine waiting for a push that is waiting for the machine.
|
||||
func TestANewOutwardLinkIsSaidUnasked(t *testing.T) {
|
||||
w := &adoptionWatch{}
|
||||
first := link.Report{Firewall: "none"}
|
||||
if !w.differs(first) {
|
||||
t.Fatal("the first report should differ from nothing")
|
||||
}
|
||||
w.said(first)
|
||||
|
||||
// Only the links changed, and nothing about adoption.
|
||||
learnt := link.Report{Firewall: "none", Outward: []string{"eth0"}}
|
||||
if !w.differs(learnt) {
|
||||
t.Fatal("a machine that has just learnt which links face outside would never say so, " +
|
||||
"and could then never be sent a filter")
|
||||
}
|
||||
w.said(learnt)
|
||||
if w.differs(link.Report{Firewall: "none", Outward: []string{"eth0"}}) {
|
||||
t.Fatal("the same links are reported as a change, so the machine would speak on every reconcile")
|
||||
}
|
||||
|
||||
// And a link that changes — a laptop moving from a cable to a radio — is said too, because the
|
||||
// filter is written around the old one until it is.
|
||||
if !w.differs(link.Report{Firewall: "none", Outward: []string{"wlan0"}}) {
|
||||
t.Fatal("a changed outward link is not said, so the filter stays written around the old one")
|
||||
}
|
||||
}
|
||||
|
||||
// **A converged machine can say which links face outside, unasked.**
|
||||
//
|
||||
// A reconcile is otherwise silent, and the condition deciding when it speaks asked only what an
|
||||
// adopted node reports — what it holds, and the firewall it found. A converged node has neither, so
|
||||
// it could speak only in reply to a declaration, and the mesh composes no declaration for a node
|
||||
// that has not said which links face outside (novox/hq ADR 0140). Measured: three converged machines
|
||||
// sat silent while the control plane refused to send them a filter.
|
||||
func TestAConvergedMachineSaysItsOutwardLinksUnasked(t *testing.T) {
|
||||
// A converged node's reconcile: nothing held, no found firewall, and the links it can see.
|
||||
converged := link.Report{Outward: []string{"eth0"}}
|
||||
if !worthSaying(converged) {
|
||||
t.Fatal("a converged machine cannot say which links face outside, so it can never be " +
|
||||
"sent a filter — a machine waiting for a push that is waiting for the machine")
|
||||
}
|
||||
|
||||
// An adopted node's reasons still hold, because that is how a predecessor still writing is caught.
|
||||
if !worthSaying(link.Report{Firewall: "ufw"}) {
|
||||
t.Fatal("an adopted machine no longer says which firewall it found")
|
||||
}
|
||||
if !worthSaying(link.Report{Held: []link.Held{{ID: "a-file"}}}) {
|
||||
t.Fatal("an adopted machine no longer says what it holds")
|
||||
}
|
||||
|
||||
// And a reconcile with nothing to say stays silent, or every machine speaks every five minutes
|
||||
// about nothing.
|
||||
if worthSaying(link.Report{}) {
|
||||
t.Fatal("a reconcile with nothing to say speaks anyway")
|
||||
}
|
||||
|
||||
// A refused report says nothing about the machine; the refusal is for the console.
|
||||
if worthSaying(link.Report{Outward: []string{"eth0"}, Refused: "not for this node"}) {
|
||||
t.Fatal("a refused report is offered as news about the machine")
|
||||
}
|
||||
}
|
||||
|
||||
// **A host running as a service says what its apply did.**
|
||||
//
|
||||
// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
|
||||
@@ -611,63 +380,3 @@ func TestTheApplysLogIsNeverNil(t *testing.T) {
|
||||
t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends novox/hq issues 257 and 261: a reconcile due while the link applies a newer declaration
|
||||
// applies that newer one once its turn comes — never the one kept when its timer fired. Read before
|
||||
// waiting, it gave back a module assigned a second earlier, and reported a declaration the mesh no
|
||||
// longer recorded as sent.
|
||||
func TestAReconcileAppliesWhatWasKeptWhenItsTurnComes(t *testing.T) {
|
||||
built, err := system.For("arch")
|
||||
if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
|
||||
t.Skip("this machine is not one these tests can apply on")
|
||||
}
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := filepath.Join(dir, "a.conf")
|
||||
kept := func(content string) store.Declared {
|
||||
body := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` + target +
|
||||
`","content":"` + content + `\n"}]}`)
|
||||
return store.Declared{Declaration: body, Signature: ed25519.Sign(private, body)}
|
||||
}
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("older")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The link is applying: the reconcile's timer fires and it waits its turn.
|
||||
applying.Lock()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := reapplyKept(context.Background(), opts, public, nil, nil)
|
||||
done <- err
|
||||
}()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
// The link's apply ends by keeping the newer declaration, and lets go.
|
||||
if err := store.SaveDeclared(store.DeclaredPath(opts.state), kept("newer")); err != nil {
|
||||
applying.Unlock()
|
||||
t.Fatal(err)
|
||||
}
|
||||
applying.Unlock()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the reconcile never ran once the node was free")
|
||||
}
|
||||
got, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "newer\n" {
|
||||
t.Errorf("the reconcile applied %q, what was kept before the newer declaration", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,14 +11,14 @@ import (
|
||||
// serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account
|
||||
// from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served
|
||||
// is said once, and this engine goes on applying; mesh-cli then says the engine does not answer.
|
||||
func serveMeshCLI(ctx context.Context, queue *link.Queue, say func(string)) {
|
||||
func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) {
|
||||
if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil {
|
||||
say("mesh-cli is not answered on this machine: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// forwardOver asks the controller over the queue's link, and follows the line to what came of it.
|
||||
func forwardOver(queue *link.Queue) meshcli.Forward {
|
||||
func forwardOver(queue *link.Session) meshcli.Forward {
|
||||
return func(ctx context.Context, asked meshcli.Asked) cli.Answer {
|
||||
return meshcli.AskTheController(ctx, queue.AskController, asked)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
)
|
||||
|
||||
// 1132/AC1, AC3: the report carries one fact per module the declaration names, with the build the controller sent for
|
||||
// it; a module not yet taken on an adopted node is not one.
|
||||
func TestTheReportCarriesOneFactPerModuleWithItsBuild(t *testing.T) {
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"builds":{"a":"b-1","b":"b-2"},"resources":[
|
||||
{"id":"a.dir","type":"directory","path":"/srv/a","module":"a"},
|
||||
{"id":"b.dir","type":"directory","path":"/srv/b","module":"b"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failure := &apply.Error{Resource: "b.dir", Err: errors.New("no space"), Failures: []apply.Failure{{Resource: "b.dir", Err: errors.New("no space")}}}
|
||||
facts := moduleFacts(d, apply.Report{}, failure)
|
||||
if len(facts) != 2 || facts[0].Module != "a" || facts[0].Build != "b-1" || facts[0].Failed ||
|
||||
facts[1].Module != "b" || facts[1].Build != "b-2" || !facts[1].Failed || facts[1].Detail == "" {
|
||||
t.Fatalf("facts %+v", facts)
|
||||
}
|
||||
adopted, err := declaration.Parse([]byte(`{"declaration":1,"adoption":{"taken":["a"],"untaken":{"b":["b.dir"]}},"resources":[
|
||||
{"id":"a.dir","type":"directory","path":"/srv/a","module":"a"},
|
||||
{"id":"b.dir","type":"directory","path":"/srv/b","module":"b"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if facts := moduleFacts(adopted, apply.Report{}, nil); len(facts) != 1 || facts[0].Module != "a" {
|
||||
t.Fatalf("an untaken module is a fact: %+v", facts)
|
||||
}
|
||||
if moduleFacts(nil, apply.Report{}, nil) != nil {
|
||||
t.Fatal("facts without a declaration")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
osuser "os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
)
|
||||
|
||||
// TestAMoveOutDirectoryKeepsOnlyWhatItKnows drives the node-engine whole through its command (issue 1024, novox/hq
|
||||
// ADR 0422): a directory under an account's home that says move-out keeps the file the mesh declared and moves a
|
||||
// file nobody declared into the kept folder, named in a list, and the run says it moved it. It needs no bus: the
|
||||
// apply is the one a delivery and the five-minute reconcile both run.
|
||||
func TestAMoveOutDirectoryKeepsOnlyWhatItKnows(t *testing.T) {
|
||||
me, err := osuser.Current()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The user database lists an account as a person's only at uid 1000 or more; root, as the forge's merge check
|
||||
// runs, is listed under a number of its own (it can give files to it), by the same name.
|
||||
listedUID := me.Uid
|
||||
if uid, _ := strconv.Atoi(me.Uid); uid < 1000 || uid == 65534 {
|
||||
listedUID = "4242"
|
||||
}
|
||||
root := t.TempDir()
|
||||
home := filepath.Join(root, "home", "agent")
|
||||
dir := filepath.Join(home, ".ssh")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
passwd := filepath.Join(root, "passwd")
|
||||
line := fmt.Sprintf("%s:x:%s:%s::%s:/bin/sh\n", me.Username, listedUID, me.Gid, home)
|
||||
if err := os.WriteFile(passwd, []byte(line), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(apply.UseAccountsFileForTest(passwd))
|
||||
// Independent of the machine: an arch host with a pacman on PATH that answers and holds nothing, as
|
||||
// TestReplay233 puts one there (it answers with an error); nothing here declares a package.
|
||||
bin := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
wasBuilt := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = wasBuilt })
|
||||
|
||||
stray := filepath.Join(dir, "id_stray")
|
||||
if err := os.WriteFile(stray, []byte("somebody else's key"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := time.Now().Add(-time.Hour)
|
||||
if err := os.Chtimes(stray, old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pipe := filepath.Join(dir, "agent.sock")
|
||||
if err := syscall.Mkfifo(pipe, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(pipe, old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts := options{state: filepath.Join(root, "state.json")}
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"ssh.dir","type":"directory","path":"` + dir + `","owner":"` + me.Username + `","unknown":"move-out"},` +
|
||||
`{"id":"ssh.config","type":"file","path":"` + dir + `/config","content":"Host *\n","owner":"` + me.Username + `"}]}`)
|
||||
|
||||
var said []string
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(s string) { said = append(said, s) })
|
||||
|
||||
if report.Refused != "" || len(report.Failed) > 0 {
|
||||
t.Fatalf("refused or failed: %q %v", report.Refused, report.Failed)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "config")); err != nil {
|
||||
t.Errorf("the declared file is not there: %v", err)
|
||||
}
|
||||
if _, err := os.Lstat(stray); err == nil {
|
||||
t.Error("the file nobody declared stayed in the directory")
|
||||
}
|
||||
lists, _ := filepath.Glob(filepath.Join(home, ".local", "state", "mesh-moved", "ssh", "*.json"))
|
||||
if len(lists) != 1 {
|
||||
t.Fatalf("one list expected: %v", lists)
|
||||
}
|
||||
rawList, err := os.ReadFile(lists[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var entries []struct{ From, To string }
|
||||
if err := json.Unmarshal(rawList, &entries); err != nil || len(entries) != 1 {
|
||||
t.Fatalf("the list: %s %v", rawList, err)
|
||||
}
|
||||
if got, err := os.ReadFile(entries[0].To); err != nil || string(got) != "somebody else's key" {
|
||||
t.Errorf("the moved file is not whole in the kept folder: %q %v", got, err)
|
||||
}
|
||||
if len(report.MovedOut) != 1 || report.MovedOut[0].From != stray || report.MovedOut[0].To != entries[0].To {
|
||||
t.Errorf("the report does not carry the move: %+v", report.MovedOut)
|
||||
}
|
||||
if len(report.Left) != 1 || report.Left[0].Path != pipe {
|
||||
t.Errorf("the report does not list the pipe left in place: %+v", report.Left)
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "moved "+stray) {
|
||||
t.Errorf("the run does not say it moved the file: %v", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/nacl/box"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// The move of a value sealed to this machine into the vault (design 57's V3, issue 676; internal/link
|
||||
// movesecret.go). The controller cannot open what it holds; this machine can open what was sealed to it, and so
|
||||
// hands it to the vault, sealed to a key the vault made for the call. The value is in this process for that one
|
||||
// call and written nowhere.
|
||||
|
||||
// vaultAsk asks one of the vault's verbs in this engine's own name and answers its result.
|
||||
type vaultAsk func(verb string, args any) (json.RawMessage, error)
|
||||
|
||||
// moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over
|
||||
// the link open now.
|
||||
func moveSecretFrom(statePath string, queue *link.Session) link.MoveSecretFunc {
|
||||
return func(m link.MoveSecret) (link.MovedSecret, error) {
|
||||
key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath))
|
||||
if err != nil {
|
||||
return link.MovedSecret{}, err
|
||||
}
|
||||
ask := func(verb string, args any) (json.RawMessage, error) {
|
||||
return queue.AskVault(context.Background(), verb, args)
|
||||
}
|
||||
moved, err := moveIntoTheVault(ask, key, m)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
|
||||
}
|
||||
return moved, err
|
||||
}
|
||||
}
|
||||
|
||||
// moveIntoTheVault opens the row's value with this machine's sealing key, hands it to the vault's move-in in its two
|
||||
// steps, and answers what the vault kept once its read-back — sealed to a key made here for this one call — opens
|
||||
// to the same value.
|
||||
func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret) (link.MovedSecret, error) {
|
||||
if m.SealedTo != "" && m.SealedTo != key.Public {
|
||||
return link.MovedSecret{}, errors.New("the controller's store says this value was sealed to a sealing key this " +
|
||||
"machine no longer has, so it cannot be opened here")
|
||||
}
|
||||
value, err := key.Unseal(m.Sealed)
|
||||
if err != nil {
|
||||
return link.MovedSecret{}, fmt.Errorf("the value does not open with this machine's sealing key: %w", err)
|
||||
}
|
||||
defer wipe(value)
|
||||
|
||||
// This machine's sealing key, stated in its own name: the key the vault seals the core's credentials to for the
|
||||
// controller to carry here (novox/hq ADR 0349 §16, issue 763), never a key the controller names.
|
||||
first := map[string]any{"origin": m.Origin, "generation": m.Generation, "sealing_key": key.Public}
|
||||
if !m.MadeAt.IsZero() {
|
||||
first["made_at"] = m.MadeAt.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
if m.Own() {
|
||||
first["node"], first["module"], first["name"] = m.Node, m.Module, m.Name
|
||||
} else {
|
||||
first["provider"], first["provision"] = m.Provider, m.Provision
|
||||
first["consumer"], first["consumer_module"] = m.Consumer, m.ConsumerModule
|
||||
if m.Local != "" {
|
||||
first["local"] = m.Local
|
||||
}
|
||||
}
|
||||
raw, err := ask("move-in", first)
|
||||
if err != nil {
|
||||
return link.MovedSecret{}, fmt.Errorf("the vault did not open the move: %w", err)
|
||||
}
|
||||
var opened struct {
|
||||
Call string `json:"call"`
|
||||
SealTo string `json:"seal_to"`
|
||||
Secret string `json:"secret"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
|
||||
return link.MovedSecret{}, errors.New("the vault answered no call and no key to seal to")
|
||||
}
|
||||
sealed, err := identity.Seal(opened.SealTo, value)
|
||||
if err != nil {
|
||||
return link.MovedSecret{}, fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
|
||||
}
|
||||
backPub, backPriv, err := box.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
return link.MovedSecret{}, err
|
||||
}
|
||||
defer wipe(backPriv[:])
|
||||
raw, err = ask("move-in", map[string]any{"call": opened.Call, "sealed": sealed,
|
||||
"read_back_to": base64.StdEncoding.EncodeToString(backPub[:])})
|
||||
if err != nil {
|
||||
return link.MovedSecret{}, fmt.Errorf("the vault did not take %s: %w", opened.Secret, err)
|
||||
}
|
||||
var kept struct {
|
||||
Name string `json:"name"`
|
||||
Generation int `json:"generation"`
|
||||
Unchanged bool `json:"unchanged"`
|
||||
ReadBack string `json:"read_back"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &kept); err != nil || kept.Name == "" || kept.ReadBack == "" {
|
||||
return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
|
||||
}
|
||||
blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
|
||||
if err != nil {
|
||||
return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s is not sealed", kept.Name)
|
||||
}
|
||||
back, ok := box.OpenAnonymous(nil, blob, backPub, backPriv)
|
||||
if !ok {
|
||||
return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s does not open with the key made for it", kept.Name)
|
||||
}
|
||||
defer wipe(back)
|
||||
if subtle.ConstantTimeCompare(back, value) != 1 {
|
||||
return link.MovedSecret{}, fmt.Errorf("the vault keeps %s, but it read back another value than the one handed "+
|
||||
"in: the move of it did not hold", kept.Name)
|
||||
}
|
||||
return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
|
||||
}
|
||||
|
||||
func wipe(b []byte) {
|
||||
for i := range b {
|
||||
b[i] = 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/nacl/box"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// standInVault answers move-in's two steps as the vault does: a key made for the call, the value opened with it,
|
||||
// and the value kept read back sealed to the key the second step sent. lie makes its read-back another value.
|
||||
type standInVault struct {
|
||||
t *testing.T
|
||||
pub *[32]byte
|
||||
priv *[32]byte
|
||||
opened map[string]any
|
||||
kept string
|
||||
lie string
|
||||
refuse string
|
||||
carried []string
|
||||
}
|
||||
|
||||
func (v *standInVault) ask(verb string, args any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(args)
|
||||
v.carried = append(v.carried, string(raw))
|
||||
if verb != "move-in" {
|
||||
v.t.Fatalf("asked %s", verb)
|
||||
}
|
||||
if v.refuse != "" {
|
||||
return nil, errors.New(v.refuse)
|
||||
}
|
||||
var a map[string]any
|
||||
_ = json.Unmarshal(raw, &a)
|
||||
if a["call"] == nil {
|
||||
v.opened = a
|
||||
pub, priv, _ := box.GenerateKey(rand.Reader)
|
||||
v.pub, v.priv = pub, priv
|
||||
return json.Marshal(map[string]string{"call": "c1", "seal_to": base64.StdEncoding.EncodeToString(pub[:]),
|
||||
"secret": "own/anchor/postgres/superuser-password"})
|
||||
}
|
||||
blob, _ := base64.StdEncoding.DecodeString(a["sealed"].(string))
|
||||
value, ok := box.OpenAnonymous(nil, blob, v.pub, v.priv)
|
||||
if !ok {
|
||||
v.t.Fatal("what the engine sealed does not open with the call's key")
|
||||
}
|
||||
v.kept = string(value)
|
||||
back := v.kept
|
||||
if v.lie != "" {
|
||||
back = v.lie
|
||||
}
|
||||
key, _ := base64.StdEncoding.DecodeString(a["read_back_to"].(string))
|
||||
var to [32]byte
|
||||
copy(to[:], key)
|
||||
boxed, _ := box.SealAnonymous(nil, []byte(back), &to, rand.Reader)
|
||||
return json.Marshal(map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 1,
|
||||
"read_back": base64.StdEncoding.EncodeToString(boxed)})
|
||||
}
|
||||
|
||||
const theValue = "the-superusers-password"
|
||||
|
||||
func aMoveOf(t *testing.T, key identity.SealingKey) link.MoveSecret {
|
||||
t.Helper()
|
||||
sealed, err := identity.Seal(key.Public, []byte(theValue))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return link.MoveSecret{Node: "anchor", By: "the move", MoveSecretAsk: link.MoveSecretAsk{Module: "postgres",
|
||||
Name: "superuser-password", Origin: "made", Generation: "g-1", Sealed: sealed, SealedTo: key.Public}}
|
||||
}
|
||||
|
||||
// The engine opens what was sealed to its machine, hands it to the vault sealed to the call's key — never in the
|
||||
// clear — and answers what the vault kept only once the read-back opens to the same value.
|
||||
func TestTheEngineMovesAValueSealedToItAndChecksTheReadBack(t *testing.T) {
|
||||
key, err := identity.GenerateSealingKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vault := &standInVault{t: t}
|
||||
moved, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if moved.Secret != "own/anchor/postgres/superuser-password" || moved.Generation != 1 || vault.kept != theValue {
|
||||
t.Fatalf("moved %+v, the vault kept %q", moved, vault.kept)
|
||||
}
|
||||
if vault.opened["module"] != "postgres" || vault.opened["name"] != "superuser-password" || vault.opened["node"] != "anchor" ||
|
||||
vault.opened["origin"] != "made" || vault.opened["generation"] != "g-1" ||
|
||||
vault.opened["sealing_key"] != key.Public {
|
||||
t.Errorf("the first step named %v", vault.opened)
|
||||
}
|
||||
if strings.Contains(strings.Join(vault.carried, " "), theValue) {
|
||||
t.Fatal("the value crossed the bus in the clear")
|
||||
}
|
||||
}
|
||||
|
||||
// A read-back that is not the value handed in is a failed move, said; so is a value this machine cannot open, and
|
||||
// the vault's refusal.
|
||||
func TestTheEngineSaysAMoveThatDidNotHold(t *testing.T) {
|
||||
key, _ := identity.GenerateSealingKey()
|
||||
vault := &standInVault{t: t, lie: "something-else"}
|
||||
if _, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key)); err == nil ||
|
||||
!strings.Contains(err.Error(), "read back") {
|
||||
t.Fatalf("a read-back of another value: %v", err)
|
||||
}
|
||||
other, _ := identity.GenerateSealingKey()
|
||||
if _, err := moveIntoTheVault((&standInVault{t: t}).ask, key, aMoveOf(t, other)); err == nil ||
|
||||
!strings.Contains(err.Error(), "sealing key") {
|
||||
t.Fatalf("a value sealed to another key: %v", err)
|
||||
}
|
||||
refusing := &standInVault{t: t, refuse: "the move never replaces a value the vault holds"}
|
||||
if _, err := moveIntoTheVault(refusing.ask, key, aMoveOf(t, key)); err == nil ||
|
||||
!strings.Contains(err.Error(), "never replaces") {
|
||||
t.Fatalf("the vault's refusal: %v", err)
|
||||
}
|
||||
// A pair credential names both its ends.
|
||||
pair := aMoveOf(t, key)
|
||||
pair.Module, pair.Name = "", ""
|
||||
pair.Provider, pair.Provision, pair.Consumer, pair.ConsumerModule = "anchor", "database", "shanks", "gitea"
|
||||
ends := &standInVault{t: t}
|
||||
if _, err := moveIntoTheVault(ends.ask, key, pair); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ends.opened["provider"] != "anchor" || ends.opened["consumer"] != "shanks" || ends.opened["consumer_module"] != "gitea" ||
|
||||
ends.opened["node"] != nil {
|
||||
t.Errorf("the pair's first step named %v", ends.opened)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/network"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The checker runs gofmt, go vet and both test levels by Go's convention, and the build module starts the bus the
|
||||
// integration level needs (novox/hq ADR 0529): no script of the repository's own runs beside them.
|
||||
func TestTheRepositoryHoldsNoCheckScript(t *testing.T) {
|
||||
if _, err := os.Stat(filepath.Join("..", "..", "merge-check.sh")); err == nil {
|
||||
t.Fatal("merge-check.sh is back: the checker's levels run the tests")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,910 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup
|
||||
// it asks and the apply of one module's composition.
|
||||
|
||||
// bucketWatch is a bucket of the bus as the agent watches it, through the session's link.
|
||||
type bucketWatch struct {
|
||||
session *link.Session
|
||||
bucket string
|
||||
}
|
||||
|
||||
func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Update, error) {
|
||||
in, err := w.session.KVWatch(ctx, w.bucket, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(chan agent.Update, 64)
|
||||
go func() {
|
||||
defer close(out)
|
||||
for u := range in {
|
||||
select {
|
||||
case out <- agent.Update{Key: u.Key, Value: u.Value, Revision: u.Revision, Deleted: u.Deleted, Caught: u.Caught}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sessionHaves writes the haves the agent says, through the session.
|
||||
type sessionHaves struct{ session *link.Session }
|
||||
|
||||
func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) {
|
||||
if module == agent.NodeModule {
|
||||
// The node's own key: the digest of its own composition it runs, and what that carries sealed.
|
||||
got, ok := h.session.ReadNodeHave(ctx)
|
||||
if !ok || got.Composed == "" {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.Sealed}, true
|
||||
}
|
||||
got, ok := h.session.ReadHave(ctx, module)
|
||||
if !ok {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.Sealed}, true
|
||||
}
|
||||
|
||||
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
if x.Module == agent.NodeModule {
|
||||
// The node's own key holds its facts and the digest of the composition it runs.
|
||||
if x.Phase == "" || x.Phase == string(delivery.PhaseRunning) {
|
||||
h.session.NoteNode(ctx, x.Composed, x.Sealed)
|
||||
}
|
||||
return
|
||||
}
|
||||
phase := x.Phase
|
||||
if phase == string(delivery.PhaseRunning) {
|
||||
phase = ""
|
||||
}
|
||||
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
|
||||
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
|
||||
Sealed: x.Sealed})
|
||||
}
|
||||
|
||||
func (h sessionHaves) Removed(ctx context.Context, module string) {
|
||||
h.session.NoteRemoved(ctx, module, "")
|
||||
}
|
||||
|
||||
// shorten keeps a have's detail within what a have value holds: its head, which says what failed, and its tail, which is
|
||||
// the cause (an error wrapped five times names the bus's "no responders" only at its end: 2026-10-25 23:04, mesh-vault on
|
||||
// the bus node said only which object it read).
|
||||
func shorten(s string) string {
|
||||
const most, head = 300, 100
|
||||
if len(s) <= most {
|
||||
return s
|
||||
}
|
||||
const gap = " … "
|
||||
tail := s[len(s)-(most-head-len(gap)):]
|
||||
return strings.ToValidUTF8(s[:head], "") + gap + strings.ToValidUTF8(tail, "")
|
||||
}
|
||||
|
||||
// failedWith is a failed apply that says why, transient when the words are the bus not answering for a moment
|
||||
// (link.TransientWords): the agent then tries it again without a new revision.
|
||||
func failedWith(why string) agent.Result {
|
||||
return agent.Result{Failed: true, Transient: link.TransientWords(why), Detail: shorten(why)}
|
||||
}
|
||||
|
||||
// backupWithin is how long a backup is given from its ask to say it is done (ADR 0429 §14).
|
||||
var backupWithin = 10 * time.Minute
|
||||
|
||||
// backedUpEvent is where the holder of node-backup says a backup taken by hand ended, also one that could not start.
|
||||
const backedUpEvent = "mesh.mod.restic.event.backed-up"
|
||||
|
||||
// backupNode asks the holder of node-backup on this node to back a module up now, and waits for its event to say how it
|
||||
// went: the holder answers `now` when it has started, and says `backed-up` when the backup ends.
|
||||
func backupNode(session *link.Session) func(ctx context.Context, module string) agent.Backup {
|
||||
return func(ctx context.Context, module string) agent.Backup {
|
||||
// Heard before the ask, so an event that comes at once is not lost.
|
||||
events, stop, err := session.Hear(backedUpEvent)
|
||||
if err != nil {
|
||||
return agent.Backup{Words: "the end of the backup cannot be heard: " + err.Error()}
|
||||
}
|
||||
defer stop()
|
||||
// A module this machine backs nothing of up moves without a backup: never held for one that cannot run (issue
|
||||
// 1945). An answer that cannot be had or read asks for the backup as before.
|
||||
if said, err := session.AskSeat(ctx, "node-backup", "backed-up", map[string]any{"module": module}, 30*time.Second); err == nil {
|
||||
if nothing, read := declaresNothingToBackUp(said, module); read && nothing {
|
||||
return agent.Backup{Done: true, Words: module + " declares nothing to back up on this machine"}
|
||||
}
|
||||
}
|
||||
asked := time.Now()
|
||||
if _, err := session.AskSeat(ctx, "node-backup", "now", map[string]any{"module": module}, 30*time.Second); err != nil {
|
||||
return agent.Backup{Words: err.Error()}
|
||||
}
|
||||
return awaitBackup(ctx, events, session.Membership.Node, module, asked, backupWithin)
|
||||
}
|
||||
}
|
||||
|
||||
// awaitBackup waits for the event that says the backup of a module on this node ended, within the bound: done when it
|
||||
// says ok, held on its words when it does not, and held when it says nothing within the bound. An event of another node
|
||||
// or module, or from before the ask, is not the answer.
|
||||
func awaitBackup(ctx context.Context, events <-chan []byte, node, module string, since time.Time, within time.Duration) agent.Backup {
|
||||
bound := time.NewTimer(within)
|
||||
defer bound.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return agent.Backup{Words: "the node agent was stopped while the backup ran"}
|
||||
case <-bound.C:
|
||||
return agent.Backup{Words: fmt.Sprintf("the backup did not say it was done within %s", within)}
|
||||
case raw := <-events:
|
||||
var e struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
OK bool `json:"ok"`
|
||||
At time.Time `json:"at"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(raw, &e) != nil || e.Node != node || e.Module != module || e.At.Before(since.Add(-time.Second)) {
|
||||
continue
|
||||
}
|
||||
if e.OK {
|
||||
return agent.Backup{Done: true}
|
||||
}
|
||||
words := "the backup failed"
|
||||
if e.Error != "" {
|
||||
words += ": " + e.Error
|
||||
}
|
||||
return agent.Backup{Words: words}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// declarationOf is the declaration the compositions make together: their resources, and the node's own parts when the
|
||||
// node's composition is among them. It is what the node judges and says its health by, and what one module's apply is
|
||||
// made of (with a single composition).
|
||||
//
|
||||
// **A resource the node's own composition shares with a module's is the module's** (issue 1945): from 2026-10-25 12:43
|
||||
// the controller put modules it knew no fingerprint of the new form for into the node's own composition while their
|
||||
// wants named their own, and the declaration joined of both declared each such file twice, which the filling refuses
|
||||
// ("2 resources declare X") on every node. The node's own copy is left out, and which were left out is said once.
|
||||
func declarationOf(cs ...delivery.Composed) (*declaration.Declaration, error) {
|
||||
d := &declaration.Declaration{Version: declaration.Version}
|
||||
var modules []delivery.Composed
|
||||
for _, c := range cs {
|
||||
if c.Module != "" {
|
||||
modules = append(modules, c)
|
||||
}
|
||||
}
|
||||
if len(modules) > 0 {
|
||||
joined := make([]delivery.Composed, 0, len(cs))
|
||||
for _, c := range cs {
|
||||
if c.Module == "" {
|
||||
var dropped []string
|
||||
c, dropped = ownWithout(c, modules)
|
||||
sayDroppedOnce(dropped)
|
||||
}
|
||||
joined = append(joined, c)
|
||||
}
|
||||
cs = joined
|
||||
}
|
||||
for _, c := range cs {
|
||||
for _, raw := range c.Resources {
|
||||
r, err := declaration.ParseResource(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("a resource of %q: %w", c.Module, err)
|
||||
}
|
||||
d.Resources = append(d.Resources, r)
|
||||
}
|
||||
d.Filled = append(d.Filled, c.Filled...)
|
||||
if c.Module == "" {
|
||||
d.Adoption = c.Own.Adoption
|
||||
d.Identities = append(d.Identities, c.Own.Identities...)
|
||||
d.Groups = append(d.Groups, c.Own.Groups...)
|
||||
d.Libraries = append(d.Libraries, c.Own.Libraries...)
|
||||
d.Provisioner = c.Own.Provisioner
|
||||
}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// ownWithout is the node's own composition without the resources a module's composition among modules also declares:
|
||||
// the same kind and id. A resource that shares only its target (a unit or container of the same name, another id) is
|
||||
// not the module's, and stays: dropped, its record would be an orphan the apply removes, stopping the module's unit. It
|
||||
// answers what it left out, as "<id> (<module>)".
|
||||
func ownWithout(own delivery.Composed, modules []delivery.Composed) (delivery.Composed, []string) {
|
||||
byID := map[string]string{}
|
||||
for _, c := range modules {
|
||||
if c.Module == "" {
|
||||
continue
|
||||
}
|
||||
for _, raw := range c.Resources {
|
||||
r, err := declaration.ParseResource(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
byID[string(r.Kind())+"\x00"+r.Identity()] = c.Module
|
||||
}
|
||||
}
|
||||
if len(byID) == 0 {
|
||||
return own, nil
|
||||
}
|
||||
var dropped []string
|
||||
kept := make([]json.RawMessage, 0, len(own.Resources))
|
||||
for _, raw := range own.Resources {
|
||||
r, err := declaration.ParseResource(raw)
|
||||
if err != nil {
|
||||
kept = append(kept, raw) // refused where it is parsed, with its words
|
||||
continue
|
||||
}
|
||||
module, dup := byID[string(r.Kind())+"\x00"+r.Identity()]
|
||||
if dup {
|
||||
dropped = append(dropped, fmt.Sprintf("%s (%s)", r.Identity(), module))
|
||||
continue
|
||||
}
|
||||
kept = append(kept, raw)
|
||||
}
|
||||
if len(dropped) == 0 {
|
||||
return own, nil
|
||||
}
|
||||
own.Resources = kept
|
||||
return own, dropped
|
||||
}
|
||||
|
||||
// droppedSaid is every set of the node's own resources left out that was said, so each is said once.
|
||||
var droppedSaid sync.Map
|
||||
|
||||
// sayDroppedOnce says once which of the node's own resources were left out because a module's composition declares them.
|
||||
func sayDroppedOnce(dropped []string) {
|
||||
if len(dropped) == 0 {
|
||||
return
|
||||
}
|
||||
sort.Strings(dropped)
|
||||
key := strings.Join(dropped, ", ")
|
||||
if _, said := droppedSaid.LoadOrStore(key, true); said {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-host: the node's own composition also declares %d resource(s) a module's composition "+
|
||||
"declares, and they are left out of it: %s (issue 1945)\n", len(dropped), key)
|
||||
}
|
||||
|
||||
func scopeName(c delivery.Composed) string {
|
||||
if c.Module == "" {
|
||||
return agent.NodeModule
|
||||
}
|
||||
return c.Module
|
||||
}
|
||||
|
||||
func resourcesOf(cs []delivery.Composed) []declaration.Resource {
|
||||
var out []declaration.Resource
|
||||
for _, c := range cs {
|
||||
for _, raw := range c.Resources {
|
||||
if r, err := declaration.ParseResource(raw); err == nil {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// operatorKeysFile is the file beside the node's state that holds the operator's public keys, one ed25519 key in base64 a
|
||||
// line (a line that starts with # is a note). The switch's last declaration writes it as a file resource, beside the
|
||||
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
|
||||
const operatorKeysFile = "operator-keys"
|
||||
|
||||
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the
|
||||
// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at
|
||||
// the next composition. A file that cannot be read trusts the controller's alone, and says so.
|
||||
func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey {
|
||||
return func() []ed25519.PublicKey {
|
||||
keys := []ed25519.PublicKey{}
|
||||
if len(controller) == ed25519.PublicKeySize {
|
||||
keys = append(keys, controller)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
|
||||
if err != nil {
|
||||
return keys
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, err := base64.StdEncoding.DecodeString(line)
|
||||
if err != nil || len(key) != ed25519.PublicKeySize {
|
||||
fmt.Fprintf(os.Stderr, "mesh-host: a line of %s is not an ed25519 public key in base64, and is not trusted\n", operatorKeysFile)
|
||||
continue
|
||||
}
|
||||
keys = append(keys, ed25519.PublicKey(key))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
}
|
||||
|
||||
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
||||
// none, ADR 0539 §9) and its filled paths.
|
||||
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
||||
var once sync.Once
|
||||
var by map[string][]json.RawMessage
|
||||
var filled map[string][]string
|
||||
return func() (map[string][]json.RawMessage, map[string][]string) {
|
||||
once.Do(func() {
|
||||
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
by, filled = splitKept(raw)
|
||||
})
|
||||
return by, filled
|
||||
}
|
||||
}
|
||||
|
||||
// splitKept is a kept declaration's resources by the module they say (the node's own under agent.NodeModule) and its
|
||||
// filled paths by the module whose file is at each path; nil for a declaration that does not read.
|
||||
func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) {
|
||||
var kept struct {
|
||||
Resources []json.RawMessage `json:"resources"`
|
||||
Filled []string `json:"filled_files"`
|
||||
}
|
||||
if json.Unmarshal(raw, &kept) != nil {
|
||||
return nil, nil
|
||||
}
|
||||
isFilled := make(map[string]bool, len(kept.Filled))
|
||||
for _, p := range kept.Filled {
|
||||
isFilled[p] = true
|
||||
}
|
||||
by, filled := map[string][]json.RawMessage{}, map[string][]string{}
|
||||
named := map[string]bool{}
|
||||
for _, r := range kept.Resources {
|
||||
var head struct {
|
||||
Module string `json:"module"`
|
||||
Type string `json:"type"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
_ = json.Unmarshal(r, &head)
|
||||
name := head.Module
|
||||
if name == "" {
|
||||
name = agent.NodeModule
|
||||
}
|
||||
by[name] = append(by[name], r)
|
||||
if head.Type == string(declaration.TypeFile) && isFilled[head.Path] && !named[name+"\x00"+head.Path] {
|
||||
named[name+"\x00"+head.Path] = true
|
||||
filled[name] = append(filled[name], head.Path)
|
||||
}
|
||||
}
|
||||
return by, filled
|
||||
}
|
||||
|
||||
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
|
||||
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
|
||||
// account may not read) is said, and tried again with a growing wait, as the link is.
|
||||
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler,
|
||||
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
|
||||
wait := 2 * time.Second
|
||||
for ctx.Err() == nil {
|
||||
linked, cancel, err := session.WhenLinked(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
began := time.Now()
|
||||
node := session.Membership.Node
|
||||
composedBucket, err := delivery.ComposedBucketFor(node)
|
||||
if err != nil {
|
||||
cancel()
|
||||
say("the node agent cannot name this node's composition bucket: " + err.Error())
|
||||
return
|
||||
}
|
||||
var ag *agent.Agent
|
||||
kept := keptOnce(opts, signer)
|
||||
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
|
||||
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
|
||||
if byModule, _ := kept(); byModule != nil {
|
||||
for module, resources := range byModule {
|
||||
if err := keepAuthority(opts.state, delivery.Composed{Module: module, Resources: resources}); err != nil {
|
||||
say("the mesh's certificate the kept declaration carries could not be kept for this machine's record: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
|
||||
// node, and a node whose compositions carry no list writes nothing.
|
||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
|
||||
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
||||
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
||||
time.AfterFunc(reloadSettles, func() {
|
||||
if now := current.get(); now != nil {
|
||||
now.RetryTransient()
|
||||
}
|
||||
})
|
||||
})
|
||||
// A vault key heard (or heard again changed) looks at the list again, so its hashes are handed (ADR 0547 §9).
|
||||
vaultKey.following(users.Again)
|
||||
go func() {
|
||||
if err := users.Run(linked); err != nil && linked.Err() == nil {
|
||||
say("the bus's user list is not followed on this link: " + err.Error())
|
||||
}
|
||||
}()
|
||||
env := agent.Env{
|
||||
Node: node,
|
||||
Wants: bucketWatch{session, delivery.AssignerWantsBucket},
|
||||
Composed: bucketWatch{session, composedBucket},
|
||||
Haves: sessionHaves{session},
|
||||
Kept: func() map[string][]json.RawMessage { by, _ := kept(); return by },
|
||||
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
|
||||
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
|
||||
Recorded: recordedIn(opts.state, say),
|
||||
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
|
||||
got, err := session.KVHistory(ctx, composedBucket, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]agent.Update, 0, len(got))
|
||||
for _, u := range got {
|
||||
out = append(out, agent.Update{Key: u.Key, Value: u.Value, Revision: u.Revision, Deleted: u.Deleted})
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Trusted: trustedKeys(opts, signer),
|
||||
Backup: backupNode(session),
|
||||
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
|
||||
// again (ace, 2026-10-25 21:17).
|
||||
Settled: func(module string) { knockFilling(module, say) },
|
||||
Say: say,
|
||||
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
||||
Backoff: link.Backoff,
|
||||
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
|
||||
actx, done := applyContext(session.WithLink(ctx))
|
||||
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
||||
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
||||
held, herr := users.Hold(c)
|
||||
var res agent.Result
|
||||
if herr != nil {
|
||||
res = failedWith(herr.Error())
|
||||
} else {
|
||||
res = applyComposition(actx, opts, held, others, sched, say)
|
||||
}
|
||||
done()
|
||||
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
|
||||
if !res.Failed {
|
||||
if err := keepAuthority(opts.state, c); err != nil {
|
||||
say("the mesh's certificate in " + c.Module + "'s composition could not be kept for this machine's record: " + err.Error())
|
||||
}
|
||||
}
|
||||
// An apply of the bus module may write an older list than the newest composition holds: looked at again.
|
||||
if module, _ := users.State(); module != "" && module == c.Module {
|
||||
users.Again()
|
||||
}
|
||||
afterApply()
|
||||
sayFacts(ctx, ag)
|
||||
return res
|
||||
},
|
||||
Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result {
|
||||
actx, done := applyContext(session.WithLink(ctx))
|
||||
res := removeComposition(actx, opts, module, others, sched, say)
|
||||
done()
|
||||
afterApply()
|
||||
sayFacts(ctx, ag)
|
||||
return res
|
||||
},
|
||||
}
|
||||
// The node composes its own modules once the catalogue places it (novox/hq ADR 0547 §2, ADR 0549, issue 1126): its
|
||||
// placement key is read at the agent's start; until then it applies the controller's compositions as before.
|
||||
run, stopRun := context.WithCancel(linked)
|
||||
// What the node runs and how it opens a sealed value, for the composer to keep the boxes it runs; and the agent
|
||||
// told when the node cannot compose, so each wanted module's have says it.
|
||||
hooks := composerHooks{
|
||||
running: func() []delivery.Composed {
|
||||
if now := current.get(); now != nil {
|
||||
return now.Everything()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
unseal: sealOpener(opts.state),
|
||||
// And what each module's have says it runs sealed: after a restart the agent has applied nothing yet.
|
||||
held: func(module string) []string {
|
||||
if module == agent.NodeModule {
|
||||
got, _ := session.ReadNodeHave(linked)
|
||||
return got.Sealed
|
||||
}
|
||||
got, ok := session.ReadHave(linked, module)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return got.Sealed
|
||||
},
|
||||
refused: func(why string) {
|
||||
if now := current.get(); now != nil {
|
||||
now.NotComposed(linked, why)
|
||||
}
|
||||
},
|
||||
}
|
||||
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
||||
// its own key, `<node>.node`, under candidates.
|
||||
shadowHooks := hooks
|
||||
shadowHooks.refused = nil
|
||||
shadow := selfcompose.NewShadow(composerEnv(session, node, say, shadowHooks),
|
||||
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
session.NoteCandidates(ctx, answers)
|
||||
})
|
||||
go shadow.Keep(linked, link.Backoff)
|
||||
if composer := selfComposer(linked, session, node, say, hooks); composer != nil {
|
||||
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
||||
} else {
|
||||
// And once the catalogue places it, the agent begins again composing its own: the placement key appearing ends
|
||||
// this run, so a node does not wait for a restart to follow the seed of its placement (ADR 0549 §4).
|
||||
go endWhenPlaced(run, session, node, stopRun, say)
|
||||
}
|
||||
ag = agent.New(env)
|
||||
current.set(ag)
|
||||
err = ag.Run(run)
|
||||
stopRun()
|
||||
cancel()
|
||||
current.set(nil)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if time.Since(began) > 30*time.Second {
|
||||
wait = 2 * time.Second
|
||||
}
|
||||
if err != nil {
|
||||
say(fmt.Sprintf("the node agent stopped: %v — beginning again in %s", err, wait))
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(wait):
|
||||
}
|
||||
if wait *= 2; wait > 2*time.Minute {
|
||||
wait = 2 * time.Minute
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reloadSettles is how long after this engine wrote the bus's user list the bus is taken to have reloaded its accounts:
|
||||
// the bus module's entrypoint looks at the list every 5 s and then signals the server, whose reload of the accounts
|
||||
// answers no JetStream request for about three seconds (on the bus node, 2026-10-25).
|
||||
var reloadSettles = 10 * time.Second
|
||||
|
||||
// agentNow is the agent running now, which the node's facts are built from.
|
||||
type agentNow struct {
|
||||
mu sync.Mutex
|
||||
ag *agent.Agent
|
||||
}
|
||||
|
||||
func (c *agentNow) set(a *agent.Agent) {
|
||||
c.mu.Lock()
|
||||
c.ag = a
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
func (c *agentNow) get() *agent.Agent {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.ag
|
||||
}
|
||||
|
||||
// applyComposition applies one composition: a module's, or the node's own.
|
||||
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed,
|
||||
sched *apply.Scheduler, say link.Announce) agent.Result {
|
||||
name := scopeName(c)
|
||||
if c.Module == "" {
|
||||
// What another module's composition declares is that module's, never the node's own apply's (issue 1945).
|
||||
var dropped []string
|
||||
c, dropped = ownWithout(c, others)
|
||||
sayDroppedOnce(dropped)
|
||||
}
|
||||
d, err := declarationOf(c)
|
||||
if err != nil {
|
||||
return failedWith(err.Error())
|
||||
}
|
||||
if err := refuseEmptyWrites(d); err != nil {
|
||||
return failedWith(err.Error())
|
||||
}
|
||||
scope := apply.Scope{Module: name, Others: resourcesOf(others)}
|
||||
got := applyModuleHeld(ctx, opts, d, &scope, sched, say)
|
||||
if got.Refused != "" {
|
||||
return failedWith(got.Refused)
|
||||
}
|
||||
results := apply.ModuleResults(d, got.Outcome, got.Err)
|
||||
if r, ok := results[name]; ok && r.Failed {
|
||||
return failedWith(r.Detail)
|
||||
}
|
||||
if got.Err != nil {
|
||||
return failedWith(got.Err.Error())
|
||||
}
|
||||
if got.Probe != nil {
|
||||
return failedWith(got.Probe.partial().Error())
|
||||
}
|
||||
return agent.Result{}
|
||||
}
|
||||
|
||||
// removeComposition removes one module's own resources and nothing of another's.
|
||||
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed,
|
||||
sched *apply.Scheduler, say link.Announce) agent.Result {
|
||||
scope := apply.Scope{Module: module, Others: resourcesOf(others)}
|
||||
got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
|
||||
switch {
|
||||
case got.Refused != "":
|
||||
return agent.Result{Failed: true, Detail: shorten(got.Refused)}
|
||||
case got.Err != nil:
|
||||
return agent.Result{Failed: true, Detail: shorten(got.Err.Error())}
|
||||
}
|
||||
return agent.Result{}
|
||||
}
|
||||
|
||||
// busyModules are the modules whose last apply left a container running because it said busy.
|
||||
func busyModules() []string {
|
||||
busyMu.Lock()
|
||||
defer busyMu.Unlock()
|
||||
var out []string
|
||||
for m, w := range busyBy {
|
||||
if w && m != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given.
|
||||
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||
var raw []byte
|
||||
if len(signer) > 0 {
|
||||
var err error
|
||||
if raw, err = store.LoadDeclared(store.DeclaredPath(statePath), signer); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
kept, err := store.ReadDeclared(store.DeclaredPath(statePath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw = kept.Declaration
|
||||
}
|
||||
return declaration.Parse(raw)
|
||||
}
|
||||
|
||||
// whatRuns is what runs on this node as the declaration of it (novox/hq ADR 0539 §9): the compositions the agent runs
|
||||
// and, for every module the agent does not hold yet, what the kept declaration has for it, with the node's own parts
|
||||
// (adoption, identities, groups, libraries) from the node's composition once the agent holds it and from the kept
|
||||
// declaration until then. What the node judges its health by, and what the filling, the take act and the judgement of
|
||||
// directories read.
|
||||
func whatRuns(everything []delivery.Composed, holdsNode bool, kept func() (*declaration.Declaration, error)) (*declaration.Declaration, error) {
|
||||
d, err := declarationOf(everything...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if holdsNode {
|
||||
return d, nil
|
||||
}
|
||||
old, err := kept()
|
||||
if err != nil {
|
||||
if len(everything) > 0 {
|
||||
return d, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
d.Adoption, d.Identities, d.Groups, d.Libraries = old.Adoption, old.Identities, old.Groups, old.Libraries
|
||||
d.Provisioner, d.Builds = old.Provisioner, old.Builds
|
||||
d.Filled = append(d.Filled, old.Filled...)
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// runsHere says what runs on this node; the running serve sets it to the agent's view, and a command run by hand reads
|
||||
// the kept declaration.
|
||||
var runsHere struct {
|
||||
mu sync.Mutex
|
||||
fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error)
|
||||
}
|
||||
|
||||
func setRunsHere(fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error)) {
|
||||
runsHere.mu.Lock()
|
||||
runsHere.fn = fn
|
||||
runsHere.mu.Unlock()
|
||||
}
|
||||
|
||||
// declaredNow is the declaration of what runs on this node now.
|
||||
func declaredNow(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||
runsHere.mu.Lock()
|
||||
fn := runsHere.fn
|
||||
runsHere.mu.Unlock()
|
||||
if fn != nil {
|
||||
return fn(statePath, signer)
|
||||
}
|
||||
return keptDeclaration(statePath, signer)
|
||||
}
|
||||
|
||||
// runsFromAgent is what runs now as the agent in hand sees it.
|
||||
func runsFromAgent(current *agentNow) func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||
return func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||
kept := func() (*declaration.Declaration, error) { return keptDeclaration(statePath, signer) }
|
||||
ag := current.get()
|
||||
if ag == nil {
|
||||
return kept()
|
||||
}
|
||||
return whatRuns(ag.Everything(), ag.HoldsNode(), kept)
|
||||
}
|
||||
}
|
||||
|
||||
// refuseEmptyWrites refuses a declaration that names a file the provisioner fills (owned at runtime, no content, no
|
||||
// bytes, no digest, no sealed value) without saying it is filled: applied, the file would be written empty over what the
|
||||
// provisioner wrote (issue 1945, live on 2026-10-25 at 01:22 on ace, g14 and shanks). The apply fails, naming the file,
|
||||
// and nothing is written.
|
||||
func refuseEmptyWrites(d *declaration.Declaration) error {
|
||||
filled := make(map[string]bool, len(d.Filled))
|
||||
for _, p := range d.Filled {
|
||||
filled[p] = true
|
||||
}
|
||||
var refused []string
|
||||
for _, r := range d.Resources {
|
||||
f, ok := r.(*declaration.File)
|
||||
if !ok || f.OwnerAtRuntime != declaration.RuntimeOwned || filled[f.Path] {
|
||||
continue
|
||||
}
|
||||
if f.Content == "" && f.Bytes == "" && f.Digest == "" && f.Sealed == "" && f.VaultSealed == nil {
|
||||
refused = append(refused, fmt.Sprintf("%s (%s)", f.ID, f.Path))
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
return fmt.Errorf("the composition names %s with no content and does not say the provisioner fills it, so it "+
|
||||
"would be written empty; nothing was applied", strings.Join(refused, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordedIn says the engine's applied state holds a record of every resource of a composition: what the agent believes
|
||||
// of a have or the kept declaration only when it does, so a resource removed since is made again (issue 1945). A state
|
||||
// that cannot be read says yes: the agent then believes what it was told, as before.
|
||||
func recordedIn(statePath string, say func(string)) func(ctx context.Context, c delivery.Composed) bool {
|
||||
return func(_ context.Context, c delivery.Composed) bool {
|
||||
known, err := store.Load(statePath)
|
||||
if err != nil {
|
||||
if say != nil {
|
||||
say(fmt.Sprintf("the applied state cannot be read, so what runs is believed as told: %v", err))
|
||||
}
|
||||
return true
|
||||
}
|
||||
filled := make(map[string]bool, len(c.Filled))
|
||||
for _, p := range c.Filled {
|
||||
filled[p] = true
|
||||
}
|
||||
for _, raw := range c.Resources {
|
||||
r, err := declaration.ParseResource(raw)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
// A file the provisioner client fills is left to it, and has no record until it writes it.
|
||||
if f, ok := r.(*declaration.File); ok && filled[f.Path] {
|
||||
continue
|
||||
}
|
||||
if _, ok := known.Find(r.Identity()); !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// declaresNothingToBackUp reads node-backup.backed-up's answer for a module: true when this machine backs nothing of it
|
||||
// up (no runs, no paths), and whether the answer could be read at all.
|
||||
func declaresNothingToBackUp(said json.RawMessage, module string) (nothing bool, read bool) {
|
||||
var list []struct {
|
||||
Module string `json:"module"`
|
||||
Runs int `json:"runs"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
if err := json.Unmarshal(said, &list); err != nil {
|
||||
return false, false
|
||||
}
|
||||
for _, m := range list {
|
||||
if m.Module == module && (m.Runs > 0 || len(m.Paths) > 0) {
|
||||
return false, true
|
||||
}
|
||||
}
|
||||
return true, true
|
||||
}
|
||||
|
||||
// applyBound is how long one apply or removal may take: long enough for the slowest image pull and backup step, short
|
||||
// enough that a step that hangs cannot hold the node agent for ever.
|
||||
const applyBound = 30 * time.Minute
|
||||
|
||||
// applyContext is the context an apply or a removal runs in: the link's values, never its end, and a bound of its own.
|
||||
// The link's context ends when the bus goes, and the bus may go because of the apply itself (it replaces the bus's own
|
||||
// container), which must then finish: a container removed and never made again took the mesh down (issue 1945, live on
|
||||
// 2026-10-25 at 11:23). applyBound keeps a hung step from blocking the agent for ever.
|
||||
func applyContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), applyBound)
|
||||
}
|
||||
|
||||
// selfComposer is the node's own composer when the catalogue places the node (its key `placement.<node>` in the
|
||||
// indexer's state), and nil, said once, while it does not or the key cannot be read.
|
||||
// composerHooks are what the composer reads of the agent and the machine (selfcompose.Env Running, Unseal, Refused).
|
||||
type composerHooks struct {
|
||||
running func() []delivery.Composed
|
||||
unseal func(string) ([]byte, error)
|
||||
refused func(string)
|
||||
held func(module string) []string
|
||||
}
|
||||
|
||||
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) *selfcompose.Composer {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
say("this node applies the controller's compositions: " + err.Error())
|
||||
return nil
|
||||
}
|
||||
got, err := session.KVHistory(ctx, delivery.PlacementBucket, key)
|
||||
if err != nil || len(got) == 0 || got[len(got)-1].Deleted {
|
||||
why := "the catalogue does not place it yet"
|
||||
if err != nil {
|
||||
why = "its placement cannot be read: " + err.Error()
|
||||
}
|
||||
say("this node applies the controller's compositions: " + why)
|
||||
return nil
|
||||
}
|
||||
say("this node composes its own modules: the catalogue places it (" + key + ")")
|
||||
return selfcompose.New(composerEnv(session, node, say, hooks))
|
||||
}
|
||||
|
||||
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
|
||||
// to placement candidates (selfcompose.Shadow) alike.
|
||||
func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env {
|
||||
return selfcompose.Env{Node: node,
|
||||
Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
return bucketWatch{session, bucket}.Watch(ctx, pattern)
|
||||
},
|
||||
Get: func(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||
h, err := session.KVHistory(ctx, bucket, key)
|
||||
if err != nil || len(h) == 0 || h[len(h)-1].Deleted {
|
||||
return nil, err
|
||||
}
|
||||
return h[len(h)-1].Value, nil
|
||||
},
|
||||
Mesh: frombus.Mesh{InternalSuffix: overlay.Suffix()},
|
||||
Say: say,
|
||||
// Its modules' own secrets, sealed by the vault to this machine's key (ADR 0550 §5).
|
||||
Own: func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
return session.AskVault(ctx, "own", args)
|
||||
},
|
||||
// And the vault's word that a secret changed, on which it composes again (ADR 0547 §2).
|
||||
Hear: session.Hear,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
|
||||
}
|
||||
|
||||
// endWhenPlaced calls stop once this node's placement key appears in the indexer's placement (a value after the watch
|
||||
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
|
||||
// the node then switches at its next start.
|
||||
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, key)
|
||||
if err != nil {
|
||||
say("whether the catalogue places this node is not followed (" + err.Error() + "): it composes its own from its next start")
|
||||
return
|
||||
}
|
||||
for u := range updates {
|
||||
if !u.Caught && !u.Deleted && len(u.Value) > 0 {
|
||||
say("the catalogue places this node now (" + key + "): the node agent begins again, composing its own modules")
|
||||
stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
)
|
||||
|
||||
// The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
|
||||
// bucket and the have bucket, a session on the bus, and the real apply of a module's composition into a directory of the
|
||||
// test. R6-A AC1, AC5 and AC9 (novox/hq ADR 0539).
|
||||
|
||||
const itNode = "it1"
|
||||
|
||||
type itBed struct {
|
||||
t *testing.T
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
// nodeConn and nodeJS are the node's own, under its real grant (buscompose.PermissionsFor, KindNode): the agent runs
|
||||
// on them, so a watch or a read the grant does not allow fails here as it fails on the bus (issue 1126, 2026-10-27
|
||||
// 09:21: a watch of every have key stopped every node agent). conn is the test's, unrestricted, for setting up.
|
||||
nodeConn *nats.Conn
|
||||
nodeJS nats.JetStreamContext
|
||||
wants nats.KeyValue
|
||||
composed nats.KeyValue
|
||||
haves nats.KeyValue
|
||||
dir string
|
||||
opts options
|
||||
session *link.Session
|
||||
controll ed25519.PrivateKey
|
||||
operator ed25519.PrivateKey
|
||||
said []string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what
|
||||
// buscompose.PermissionsFor gives a node (KindNode); it answers a connection of each.
|
||||
func aServer(t *testing.T) (admin, node *nats.Conn) {
|
||||
t.Helper()
|
||||
perms, err := buscompose.PermissionsFor(buscompose.Principal{Kind: buscompose.KindNode, Node: itNode})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
granted := &server.Permissions{Publish: &server.SubjectPermission{Allow: perms.Publish, Deny: perms.PublishDeny},
|
||||
Subscribe: &server.SubjectPermission{Allow: perms.Subscribe}}
|
||||
if perms.AllowResponses {
|
||||
granted.Response = &server.ResponsePermission{MaxMsgs: 1, Expires: buscompose.ResponseTTL}
|
||||
}
|
||||
nodeUser := buscompose.Principal{Kind: buscompose.KindNode, Node: itNode}.Username()
|
||||
srv, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Users: []*server.User{{Username: "test", Password: "test"}, {Username: nodeUser, Password: "node", Permissions: granted}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the embedded server did not start")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
if admin, err = nats.Connect(srv.ClientURL(), nats.UserInfo("test", "test")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(admin.Close)
|
||||
if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(node.Close)
|
||||
return admin, node
|
||||
}
|
||||
|
||||
func newItBed(t *testing.T) *itBed {
|
||||
t.Helper()
|
||||
// Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node.
|
||||
bin := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
|
||||
b := &itBed{t: t, dir: t.TempDir()}
|
||||
b.conn, b.nodeConn = aServer(t)
|
||||
var err error
|
||||
if b.js, err = b.conn.JetStream(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.nodeJS, err = b.nodeConn.JetStream(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composedBucket, _ := delivery.ComposedBucketFor(itNode)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
into *nats.KeyValue
|
||||
history uint8
|
||||
}{{delivery.AssignerWantsBucket, &b.wants, 8}, {composedBucket, &b.composed, delivery.ComposedHistory},
|
||||
{delivery.HaveBucket, &b.haves, delivery.HaveHistory}} {
|
||||
kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: c.name, History: c.history})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*c.into = kv
|
||||
}
|
||||
b.opts = options{state: filepath.Join(b.dir, "state", "state.json"), timeout: 2 * time.Second}
|
||||
if err := os.MkdirAll(filepath.Dir(b.opts.state), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, b.controll, _ = ed25519.GenerateKey(nil)
|
||||
_, b.operator, _ = ed25519.GenerateKey(nil)
|
||||
return b
|
||||
}
|
||||
|
||||
// start runs the node agent under a session on the bus, trusting the controller's key, and the operator's when asked.
|
||||
func (b *itBed) start(trustOperator bool) (stop func()) {
|
||||
b.t.Helper()
|
||||
if trustOperator {
|
||||
pub := b.operator.Public().(ed25519.PublicKey)
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(b.opts.state), operatorKeysFile),
|
||||
[]byte("# the operator\n"+base64.StdEncoding.EncodeToString(pub)+"\n"), 0o600); err != nil {
|
||||
b.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
||||
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
||||
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {},
|
||||
&agentNow{}, func(context.Context, *agent.Agent) {})
|
||||
}()
|
||||
return func() { cancel(); <-done }
|
||||
}
|
||||
|
||||
func (b *itBed) say(line string) {
|
||||
b.mu.Lock()
|
||||
b.said = append(b.said, line)
|
||||
b.mu.Unlock()
|
||||
b.t.Log(line)
|
||||
}
|
||||
|
||||
func fpIt(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
|
||||
// compose writes a module's signed composition of one file into the node's bucket and returns its digest.
|
||||
func (b *itBed) compose(module, fp, content string, data bool, signer ed25519.PrivateKey) string {
|
||||
b.t.Helper()
|
||||
path := filepath.Join(b.dir, module+".conf")
|
||||
res, _ := json.Marshal(map[string]any{"id": module + ".conf", "type": "file", "path": path, "content": content, "module": module})
|
||||
c := delivery.Composed{Node: itNode, Module: module, FP: fp, Data: data, Resources: []json.RawMessage{res}}
|
||||
signed, err := delivery.SignComposed(c, signer)
|
||||
if err != nil {
|
||||
b.t.Fatal(err)
|
||||
}
|
||||
raw, err := delivery.EncodeComposed(signed)
|
||||
if err != nil {
|
||||
b.t.Fatal(err)
|
||||
}
|
||||
dig, _ := delivery.ComposedDigestOf(signed)
|
||||
key, _ := delivery.ComposedKey(itNode, module, fp)
|
||||
if _, err := b.composed.Put(key, raw); err != nil {
|
||||
b.t.Fatal(err)
|
||||
}
|
||||
return dig
|
||||
}
|
||||
|
||||
func (b *itBed) want(module, fp, digest, backup string) {
|
||||
b.t.Helper()
|
||||
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest, Backup: backup})
|
||||
if err != nil {
|
||||
b.t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.WantKey(itNode, module)
|
||||
if _, err := b.wants.Put(key, raw); err != nil {
|
||||
b.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *itBed) file(module string) string {
|
||||
got, _ := os.ReadFile(filepath.Join(b.dir, module+".conf"))
|
||||
return string(got)
|
||||
}
|
||||
|
||||
func (b *itBed) have(module string) delivery.Have {
|
||||
key, _ := delivery.HaveKey(itNode, module)
|
||||
e, err := b.haves.Get(key)
|
||||
if err != nil {
|
||||
return delivery.Have{}
|
||||
}
|
||||
h, _ := delivery.DecodeHave(e.Value())
|
||||
return h
|
||||
}
|
||||
|
||||
func within(t *testing.T, d time.Duration, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(d)
|
||||
for !ok() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal(what)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// R6-A AC1: given a running agent on a bus, when a want naming a digest and that composition are written, the module is
|
||||
// applied within one second and its have names the fingerprint and the digest; a composition whose digest no want names
|
||||
// is not applied.
|
||||
func TestNatsAWantAndItsCompositionAreAppliedWithinASecond(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
stop := b.start(false)
|
||||
defer stop()
|
||||
time.Sleep(300 * time.Millisecond) // the watches are up
|
||||
|
||||
fp := fpIt("a")
|
||||
dig := b.compose("web", fp, "one\n", false, b.controll)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if b.file("web") != "" {
|
||||
t.Fatal("a composition no want names was applied")
|
||||
}
|
||||
began := time.Now()
|
||||
b.want("web", fp, dig, "")
|
||||
within(t, time.Second, "the module was not applied within one second of its want", func() bool { return b.file("web") == "one\n" })
|
||||
t.Logf("applied %s after the want", time.Since(began))
|
||||
within(t, 2*time.Second, "the have does not name the fingerprint and the digest", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
|
||||
// A want naming a digest the composition at its fingerprint does not have applies nothing.
|
||||
fp2 := fpIt("b")
|
||||
b.compose("web", fp2, "two\n", false, b.controll)
|
||||
b.want("web", fp2, "sha256:"+strings.Repeat("0", 64), "")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if b.file("web") != "one\n" {
|
||||
t.Fatalf("a want naming another digest changed the module: %q", b.file("web"))
|
||||
}
|
||||
}
|
||||
|
||||
// R6-A AC5: a data module that ran before asks its backup before any resource changes; a backup that fails holds the
|
||||
// module (nothing applied, the have says held with the backup's words); when its want then says skip, it is applied.
|
||||
func TestNatsADataModuleAsksItsBackupFirstAndIsHeldWhenItFails(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
var mu sync.Mutex
|
||||
var asked []string
|
||||
var onDiskWhenAsked string
|
||||
if _, err := b.conn.Subscribe("mesh.seat.node-backup.tool.now."+itNode, func(m *nats.Msg) {
|
||||
mu.Lock()
|
||||
asked = append(asked, "now")
|
||||
onDiskWhenAsked = b.file("db")
|
||||
mu.Unlock()
|
||||
_ = m.Respond([]byte(`{"result":{"started":"db"}}`))
|
||||
// The holder says the backup ended on its event.
|
||||
ev, _ := json.Marshal(map[string]any{"node": itNode, "module": "db", "ok": false, "at": time.Now().UTC(),
|
||||
"error": "restic: the repository is locked"})
|
||||
_ = b.conn.Publish(backedUpEvent, ev)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop := b.start(false)
|
||||
defer stop()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
fp1, fp2 := fpIt("a"), fpIt("b")
|
||||
b.want("db", fp1, b.compose("db", fp1, "v1\n", true, b.controll), "")
|
||||
within(t, 3*time.Second, "the first composition was not applied", func() bool { return b.file("db") == "v1\n" })
|
||||
mu.Lock()
|
||||
if len(asked) != 0 {
|
||||
t.Fatalf("a module that never ran was backed up: %v", asked)
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
dig2 := b.compose("db", fp2, "v2\n", true, b.controll)
|
||||
b.want("db", fp2, dig2, "")
|
||||
within(t, 3*time.Second, "the module was not held behind the failed backup", func() bool {
|
||||
h := b.have("db")
|
||||
return h.Phase == delivery.PhaseHeld && strings.Contains(h.Detail, "repository is locked")
|
||||
})
|
||||
mu.Lock()
|
||||
if len(asked) != 1 || onDiskWhenAsked != "v1\n" {
|
||||
t.Fatalf("the backup was asked %v, with %q on disk: it is asked before any resource changes", asked, onDiskWhenAsked)
|
||||
}
|
||||
mu.Unlock()
|
||||
if b.file("db") != "v1\n" {
|
||||
t.Fatalf("something was applied behind a failed backup: %q", b.file("db"))
|
||||
}
|
||||
if h := b.have("db"); h.FP != fp1 {
|
||||
t.Fatalf("a held module should still name what it runs: %+v", h)
|
||||
}
|
||||
|
||||
b.want("db", fp2, dig2, delivery.BackupSkip)
|
||||
within(t, 3*time.Second, "the module was not applied once its want said skip", func() bool { return b.file("db") == "v2\n" })
|
||||
within(t, 2*time.Second, "the have does not say running at the new fingerprint", func() bool {
|
||||
h := b.have("db")
|
||||
return h.Phase == delivery.PhaseRunning && h.FP == fp2 && h.Composed == dig2
|
||||
})
|
||||
}
|
||||
|
||||
// R6-A AC9, the hand path end to end: with no controller running, a composition signed by the operator's key and a want
|
||||
// naming it are applied and the have written; the same signed by a key nobody trusts is not.
|
||||
func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
stop := b.start(true)
|
||||
defer stop()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
_, stranger, _ := ed25519.GenerateKey(nil)
|
||||
fpS := fpIt("c")
|
||||
b.want("evil", fpS, b.compose("evil", fpS, "no\n", false, stranger), "")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if b.file("evil") != "" {
|
||||
t.Fatal("a composition signed by a key nobody trusts was applied")
|
||||
}
|
||||
|
||||
fp := fpIt("d")
|
||||
dig := b.compose("rescue", fp, "back\n", false, b.operator)
|
||||
b.want("rescue", fp, dig, "")
|
||||
within(t, 2*time.Second, "the operator's composition was not applied", func() bool { return b.file("rescue") == "back\n" })
|
||||
within(t, 2*time.Second, "the have was not written", func() bool {
|
||||
h := b.have("rescue")
|
||||
return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2, the milestone's H8): the node agent started whole against a real bus, once the catalogue
|
||||
// places its node (the placement written after it started), composes a module from its want, the indexer's manifest, its record and its mesh-config value, and applies it,
|
||||
// with no key in the controller's compositions; a changed config value is composed again on its event and applied as a
|
||||
// revision of what runs. And a catalogue pull request's placement candidate equal to its placement is answered "same"
|
||||
// in the node's own key, and nothing is applied for it (ADR 0559 §4, subtask 10 of the milestone).
|
||||
func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
kvOf := func(bucket string) nats.KeyValue {
|
||||
kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
placement, modules, manifests := kvOf(delivery.PlacementBucket), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket)
|
||||
resolved := kvOf(selfcompose.ResolvedBucket)
|
||||
kvOf(delivery.BuilderBuildsBucket)
|
||||
candidates := kvOf(delivery.PlacementCandidatesBucket)
|
||||
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
|
||||
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(b.dir, "web.conf")
|
||||
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
|
||||
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}]}`
|
||||
digest := delivery.ManifestDigest([]byte(man))
|
||||
mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man})
|
||||
mkey, _ := delivery.ManifestsKey(digest)
|
||||
fp := fpIt("a")
|
||||
modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)})
|
||||
place, _ := delivery.EncodePlacement(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}})
|
||||
pkey, _ := delivery.PlacementKey(itNode)
|
||||
hkey, _ := delivery.HaveNodeKey(itNode)
|
||||
for _, w := range []struct {
|
||||
kv nats.KeyValue
|
||||
key string
|
||||
value []byte
|
||||
}{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec},
|
||||
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour"}`)}} {
|
||||
if _, err := w.kv.Put(w.key, w.value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true})
|
||||
wkey, _ := delivery.WantKey(itNode, "web")
|
||||
if _, err := b.wants.Put(wkey, want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stop := b.start(false)
|
||||
defer stop()
|
||||
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
|
||||
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if b.file("web") != "" {
|
||||
t.Fatal("a node the catalogue does not place composed its own")
|
||||
}
|
||||
if _, err := placement.Put(pkey, place); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
|
||||
within(t, 2*time.Second, "the have does not name the build", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed != "" && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
if keys, _ := b.composed.Keys(); len(keys) != 0 {
|
||||
t.Errorf("the controller's compositions hold %v; the node composed its own", keys)
|
||||
}
|
||||
first := b.have("web").Composed
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 3*time.Second, "a changed config value was not composed and applied again", func() bool { return b.file("web") == "greeting=salut\n" })
|
||||
within(t, 2*time.Second, "the have does not name the revision", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed != first && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
|
||||
// The node says its facts (its record, as the test wrote it), then a candidate equal to its placement is written: the
|
||||
// node answers "same" at the candidate's head and revision in its own key, and applies nothing.
|
||||
b.session.SayFacts(context.Background(), link.Report{Profile: map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "service-manager", "present": true}}}})
|
||||
second := b.have("web").Composed
|
||||
same, _ := delivery.DecodePlacement(place)
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: "feat/x",
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h1"}, Placement: &same})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("feat/x", itNode)
|
||||
crev, err := candidates.Put(ckey, craw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var answer delivery.CandidateAnswer
|
||||
within(t, 10*time.Second, "the node did not answer its candidate", func() bool {
|
||||
e, err := b.haves.Get(hkey)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(e.Value())
|
||||
answer = n.Candidates["feat/x"]
|
||||
return err == nil && answer.Head == "h1" && n.Record != nil
|
||||
})
|
||||
if answer.Verdict != delivery.CandidateSame || answer.Revision != crev || len(answer.Same) != 1 || answer.Same[0] != "web" {
|
||||
t.Errorf("the answer to a candidate equal to the placement: %+v", answer)
|
||||
}
|
||||
if h := b.have("web"); h.Composed != second || b.file("web") != "greeting=salut\n" {
|
||||
t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R6-A AC8: nothing in the node-engine receives, orders or reconciles a whole declaration (novox/hq ADR 0539 §9, ADR
|
||||
// 0429 §8): no package subscribes to a node's declare subject or names the deleted symbols.
|
||||
func TestNoPackageHearsOrOrdersAWholeDeclaration(t *testing.T) {
|
||||
deleted := []string{
|
||||
"DeclareSubject", "mesh.node.\" + node + \".declare", "ReconcileDue", "ReconcileEvery", "olderThanKept",
|
||||
"refuseOlder", "olderGeneration", "appliedGeneration", "reconcileKept", "reapplyKept", "adoptionWatch",
|
||||
"unsaidFile", "numbersFile", "func gather(", "func handleBody(", "Supersedes(", "ReportSequence",
|
||||
"RefusedOlder", "ReadsGeneration", "ReadsModules", "GenerationRefusal", "LeftOutModuleOf",
|
||||
}
|
||||
root := filepath.Join("..", "..")
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "nodeclare_test.go") {
|
||||
return nil
|
||||
}
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, word := range deleted {
|
||||
if strings.Contains(string(body), word) {
|
||||
t.Errorf("%s still names %q", path, word)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
)
|
||||
|
||||
// This machine's sealing key, stated to the vault in the engine's own name (novox/hq ADR 0372, issue 786). The vault
|
||||
// records it from the machine's own account alone and, when it is a first or a changed key, says the machine's core
|
||||
// secrets changed so the controller asks for them again sealed to it. Without this a machine that never moved a
|
||||
// secret in (adopted after feature 57) has no key at the vault. Nothing but the public half crosses.
|
||||
|
||||
const (
|
||||
// nodeKeyRetryFrom and nodeKeyRetryUpTo bound the wait between tries while the vault does not answer: it starts
|
||||
// after the bus and may not serve the verb yet.
|
||||
nodeKeyRetryFrom = 5 * time.Second
|
||||
nodeKeyRetryUpTo = 5 * time.Minute
|
||||
// nodeKeyLook is how often the key file is read to see whether it changed.
|
||||
nodeKeyLook = 30 * time.Second
|
||||
)
|
||||
|
||||
// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
|
||||
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of
|
||||
// the user list on the link open now, told when the key is first heard or changes.
|
||||
type heardKey struct {
|
||||
mu sync.Mutex
|
||||
pem string
|
||||
kick func()
|
||||
}
|
||||
|
||||
// vaultKey is the vault's key this engine heard.
|
||||
var vaultKey = &heardKey{}
|
||||
|
||||
func (h *heardKey) get() string {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.pem
|
||||
}
|
||||
|
||||
// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer
|
||||
// with no key (a vault before this one) changes nothing.
|
||||
func (h *heardKey) heard(answer json.RawMessage) {
|
||||
var a struct {
|
||||
VaultKey string `json:"vault_key"`
|
||||
}
|
||||
if json.Unmarshal(answer, &a) != nil || !strings.Contains(a.VaultKey, "PUBLIC KEY") {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
changed := a.VaultKey != h.pem
|
||||
h.pem = a.VaultKey
|
||||
kick := h.kick
|
||||
h.mu.Unlock()
|
||||
if changed && kick != nil {
|
||||
kick()
|
||||
}
|
||||
}
|
||||
|
||||
// following names the follower told when the key changes, and tells it at once when a key is known.
|
||||
func (h *heardKey) following(kick func()) {
|
||||
h.mu.Lock()
|
||||
h.kick = kick
|
||||
known := h.pem != ""
|
||||
h.mu.Unlock()
|
||||
if known && kick != nil {
|
||||
kick()
|
||||
}
|
||||
}
|
||||
|
||||
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
|
||||
// vault takes the same key again and changes nothing (ADR 0547 clause 9).
|
||||
const nodeKeyReask = 10 * time.Minute
|
||||
|
||||
// keyHearing is what the keeper does with the vault's own key in node-key's answers (ADR 0547 clause 9): heard takes
|
||||
// it from an answer; held says whether one is held, and while none is the key is stated again at every look, so a vault
|
||||
// delivered after this engine is heard within a look; every is how often it is stated again all the same, so a changed
|
||||
// vault key is heard.
|
||||
type keyHearing struct {
|
||||
heard func(json.RawMessage)
|
||||
held func() bool
|
||||
// look is how often it is stated again while no vault key is held; every how often once one is.
|
||||
look, every time.Duration
|
||||
}
|
||||
|
||||
// keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from
|
||||
// retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line
|
||||
// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. With
|
||||
// hearing, each answer goes to it, and the key is stated again while no vault key is held and at its interval.
|
||||
func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string),
|
||||
retryFrom, retryUpTo time.Duration, hearing *keyHearing) {
|
||||
stated, lastFailure := "", ""
|
||||
var askedAt time.Time
|
||||
wait := retryFrom
|
||||
fail := func(why string) {
|
||||
if why != lastFailure {
|
||||
say("mesh-host: the sealing key was not stated to the vault, trying again: " + why)
|
||||
lastFailure = why
|
||||
}
|
||||
wait = min(wait*2, retryUpTo)
|
||||
}
|
||||
look := min(nodeKeyLook, retryFrom)
|
||||
for {
|
||||
delay := look
|
||||
key, err := load()
|
||||
again := false
|
||||
if hearing != nil && stated != "" {
|
||||
since := time.Since(askedAt)
|
||||
held := hearing.held == nil || hearing.held()
|
||||
again = (!held && since >= hearing.look) || (held && hearing.every > 0 && since >= hearing.every)
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
fail(err.Error())
|
||||
delay = wait
|
||||
case key.Public != stated || again:
|
||||
if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
|
||||
fail(err.Error())
|
||||
delay = wait
|
||||
} else {
|
||||
askedAt = time.Now()
|
||||
if hearing != nil && hearing.heard != nil {
|
||||
hearing.heard(answer)
|
||||
}
|
||||
if key.Public != stated {
|
||||
say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
|
||||
}
|
||||
stated, lastFailure, wait = key.Public, "", retryFrom
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
)
|
||||
|
||||
// Issue 786 (novox/hq ADR 0372): the engine states its own sealing key to the vault's node-key at start, retries
|
||||
// while the vault does not answer, and states it again when the key changes; the same key is not stated twice.
|
||||
type keyVault struct {
|
||||
mu sync.Mutex
|
||||
stated []string
|
||||
fail int
|
||||
verbs []string
|
||||
// keyless is how many answers carry no vault key (a vault before ADR 0547 clause 9's, or not yet started); key the
|
||||
// vault's key the others carry.
|
||||
keyless int
|
||||
key string
|
||||
}
|
||||
|
||||
func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
v.verbs = append(v.verbs, verb)
|
||||
if v.fail > 0 {
|
||||
v.fail--
|
||||
return nil, errors.New("the vault does not answer")
|
||||
}
|
||||
raw, _ := json.Marshal(args)
|
||||
var a struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &a)
|
||||
v.stated = append(v.stated, a.Key)
|
||||
if v.key == "" || v.keyless > 0 {
|
||||
v.keyless--
|
||||
return json.RawMessage(`{"recorded":true}`), nil
|
||||
}
|
||||
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": v.key})
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func (v *keyVault) said() []string {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
return append([]string(nil), v.stated...)
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, what string, ok func() bool) {
|
||||
t.Helper()
|
||||
for i := 0; i < 400; i++ {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out: %s", what)
|
||||
}
|
||||
|
||||
func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(t *testing.T) {
|
||||
first, _ := identity.GenerateSealingKey()
|
||||
second, _ := identity.GenerateSealingKey()
|
||||
var mu sync.Mutex
|
||||
current := first
|
||||
load := func() (identity.SealingKey, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return current, nil
|
||||
}
|
||||
vault := &keyVault{fail: 2}
|
||||
var lines []string
|
||||
var lmu sync.Mutex
|
||||
say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() }
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var hmu sync.Mutex
|
||||
var answers []string
|
||||
heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() }
|
||||
go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, &keyHearing{heard: heard})
|
||||
|
||||
waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 })
|
||||
// Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing.
|
||||
waitFor(t, "the answer heard", func() bool { hmu.Lock(); defer hmu.Unlock(); return len(answers) == 1 })
|
||||
if vault.said()[0] != first.Public {
|
||||
t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public)
|
||||
}
|
||||
vault.mu.Lock()
|
||||
verbs := append([]string(nil), vault.verbs...)
|
||||
vault.mu.Unlock()
|
||||
for _, v := range verbs {
|
||||
if v != "node-key" {
|
||||
t.Errorf("asked %s", v)
|
||||
}
|
||||
}
|
||||
// The same key again is not stated again.
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if n := len(vault.said()); n != 1 {
|
||||
t.Errorf("the same key was stated %d times", n)
|
||||
}
|
||||
mu.Lock()
|
||||
current = second
|
||||
mu.Unlock()
|
||||
waitFor(t, "the changed key stated", func() bool { return len(vault.said()) == 2 })
|
||||
if vault.said()[1] != second.Public {
|
||||
t.Errorf("stated %q after the change, want %q", vault.said()[1], second.Public)
|
||||
}
|
||||
lmu.Lock()
|
||||
defer lmu.Unlock()
|
||||
if len(lines) == 0 {
|
||||
t.Error("the engine said nothing")
|
||||
}
|
||||
asksAgainUntilItHearsTheVaultsKey(t)
|
||||
}
|
||||
|
||||
// ADR 0547 clause 9: the vault's key is heard though the engine's own key never changes. While none is held the key
|
||||
// is stated again at each look, so a vault delivered after the engine (or answering no key yet) is heard; once one is
|
||||
// held, again only at the slow interval, so a changed vault key is heard; never more.
|
||||
func asksAgainUntilItHearsTheVaultsKey(t *testing.T) {
|
||||
sealing, _ := identity.GenerateSealingKey()
|
||||
load := func() (identity.SealingKey, error) { return sealing, nil }
|
||||
pem := func(c string) string { return "-----BEGIN PUBLIC KEY-----\n" + c + "\n-----END PUBLIC KEY-----\n" }
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
keyless int
|
||||
every time.Duration
|
||||
change bool
|
||||
wantAtLeast int
|
||||
wantAtMost int
|
||||
}{
|
||||
{name: "a vault with no key yet is asked again until it answers one, then not", keyless: 3, wantAtLeast: 4, wantAtMost: 4},
|
||||
{name: "a held key is asked again at the slow interval, and a changed one heard", every: 20 * time.Millisecond,
|
||||
change: true, wantAtLeast: 3, wantAtMost: 1 << 20},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
vault := &keyVault{keyless: c.keyless, key: pem("AAAA")}
|
||||
h := &heardKey{}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go keepNodeKeyStated(ctx, load, vault.ask, func(string) {}, time.Millisecond, 5*time.Millisecond,
|
||||
&keyHearing{heard: h.heard, held: func() bool { return h.get() != "" }, look: time.Millisecond, every: c.every})
|
||||
waitFor(t, "the vault's key heard", func() bool { return h.get() == pem("AAAA") })
|
||||
if c.change {
|
||||
vault.mu.Lock()
|
||||
vault.key = pem("BBBB")
|
||||
vault.mu.Unlock()
|
||||
waitFor(t, "the changed vault key heard", func() bool { return h.get() == pem("BBBB") })
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
n := len(vault.said())
|
||||
if n < c.wantAtLeast || n > c.wantAtMost {
|
||||
t.Errorf("node-key asked %d times, want %d to %d", n, c.wantAtLeast, c.wantAtMost)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
||||
// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one
|
||||
// (an older vault) or with no public key changes nothing (ADR 0547 clause 9).
|
||||
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
|
||||
if link.NodeKeyContract != 13 {
|
||||
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
|
||||
}
|
||||
h := &heardKey{}
|
||||
kicks := 0
|
||||
h.following(func() { kicks++ })
|
||||
h.heard(json.RawMessage(`{"recorded":true}`))
|
||||
h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
|
||||
if h.get() != "" || kicks != 0 {
|
||||
t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks)
|
||||
}
|
||||
pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
|
||||
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
|
||||
h.heard(answer)
|
||||
h.heard(answer)
|
||||
if h.get() != pem || kicks != 1 {
|
||||
t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks)
|
||||
}
|
||||
h.following(func() { kicks += 10 })
|
||||
if kicks != 11 {
|
||||
t.Fatalf("a follower named after the key was heard was not told: %d", kicks)
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A declaration carries no order, so a host cannot tell an older one from a newer (novox/hq
|
||||
// 04-ISSUES/107). Its only identity was the digest of its bytes: "not the last" could be said,
|
||||
// "older" could not.
|
||||
|
||||
func keptWith(t *testing.T, sequence int64) store.Declared {
|
||||
t.Helper()
|
||||
return keptAt(t, link.Order{Sequence: sequence})
|
||||
}
|
||||
|
||||
func keptAt(t *testing.T, order link.Order) store.Declared {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"declaration": 1, "resources": []any{}, "owns_nothing": true, "sequence": order.Sequence,
|
||||
"epoch": order.Epoch,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store.Declared{Declaration: body, Signature: []byte("x")}
|
||||
}
|
||||
|
||||
func TestAnOlderDeclarationFromTheMeshIsRefused(t *testing.T) {
|
||||
err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 5})
|
||||
if err == nil {
|
||||
t.Fatal("sequence 5 was accepted over a kept 7")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "older") || !strings.Contains(err.Error(), "nothing was applied") {
|
||||
t.Fatalf("the refusal does not say what it is: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANewerOrEqualDeclarationIsNot(t *testing.T) {
|
||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 8}); err != nil {
|
||||
t.Fatalf("sequence 8 was refused over a kept 7: %v", err)
|
||||
}
|
||||
// Equal is the same declaration again, which reconciling is for.
|
||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 7}); err != nil {
|
||||
t.Fatalf("the same sequence was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoOrderClaimedMeansNoOrderCompared(t *testing.T) {
|
||||
// An older controller sends none; a host that received before it understood them kept none.
|
||||
// Refusing on a guess would strand a node the moment the controller is older than the host.
|
||||
if err := refuseOlder(keptWith(t, 7), nil, link.Order{Sequence: 0}); err != nil {
|
||||
t.Fatalf("a declaration claiming no order was refused: %v", err)
|
||||
}
|
||||
if err := refuseOlder(keptWith(t, 0), nil, link.Order{Sequence: 3}); err != nil {
|
||||
t.Fatalf("a declaration was refused against a kept one that claimed no order: %v", err)
|
||||
}
|
||||
if err := refuseOlder(store.Declared{}, store.ErrNothingDeclared, link.Order{Sequence: 3}); err != nil {
|
||||
t.Fatalf("a first declaration was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The lease epoch decides first** (novox/hq to-be 45 §6): a declaration from a controller that lost
|
||||
// its lease is refused whatever its sequence, and a new lease holder's is taken whatever its sequence.
|
||||
func TestAnOlderEpochIsRefusedAndANewerTaken(t *testing.T) {
|
||||
kept := keptAt(t, link.Order{Epoch: 57, Sequence: 3})
|
||||
err := refuseOlder(kept, nil, link.Order{Epoch: 41, Sequence: 12})
|
||||
if err == nil || !strings.Contains(err.Error(), "epoch 41") || !strings.Contains(err.Error(), "epoch 57") {
|
||||
t.Fatalf("an older epoch was not refused by name: %v", err)
|
||||
}
|
||||
if err := refuseOlder(kept, nil, link.Order{Epoch: 58, Sequence: 1}); err != nil {
|
||||
t.Fatalf("a new lease holder's first declaration was refused for its sequence: %v", err)
|
||||
}
|
||||
if err := refuseOlder(kept, nil, link.Order{Epoch: 57, Sequence: 2}); err == nil {
|
||||
t.Fatal("a lower sequence under the same epoch was accepted")
|
||||
}
|
||||
// A controller with no lease — older, or rolled back to a build from before it — is today's
|
||||
// behaviour: no epoch compared.
|
||||
if err := refuseOlder(kept, nil, link.Order{Sequence: 4}); err != nil {
|
||||
t.Fatalf("a declaration with no epoch was refused for one: %v", err)
|
||||
}
|
||||
if held, older := olderThanKept(kept, nil, link.Order{Epoch: 41, Sequence: 12}); !older ||
|
||||
held != (link.Order{Epoch: 57, Sequence: 3}) {
|
||||
t.Fatalf("olderThanKept said %v, holding %+v", older, held)
|
||||
}
|
||||
// What is kept cannot be read: no order to compare, so nothing is refused on a guess.
|
||||
if _, older := olderThanKept(store.Declared{Declaration: []byte("garbled")}, nil,
|
||||
link.Order{Epoch: 41, Sequence: 1}); older {
|
||||
t.Fatal("refused against a kept declaration nobody can read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os/exec"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// packageManagers are the tools a system's probe asks, for a system whose probe is its package manager.
|
||||
var packageManagers = map[string]string{"arch": "pacman", "alpine": "apk"}
|
||||
|
||||
// unanswered is a probe of the package manager that failed before an apply: which tool, and its words.
|
||||
type unanswered struct {
|
||||
tool string
|
||||
words string
|
||||
}
|
||||
|
||||
// failedKey is how a report names the probe among what failed: one key, the same on every report, so the
|
||||
// mesh counts a package manager that keeps not answering as one failure that goes on.
|
||||
func (u *unanswered) failedKey() string { return "the package manager (" + u.tool + ")" }
|
||||
|
||||
// said is the line the apply says before it begins.
|
||||
func (u *unanswered) said() string {
|
||||
return u.tool + " does not answer here, so every package in this declaration fails and the rest of it is " +
|
||||
"applied (novox/hq issue 233): " + u.words
|
||||
}
|
||||
|
||||
// partial is the failure of an apply that applied everything else, when no package was declared to fail.
|
||||
func (u *unanswered) partial() error {
|
||||
return errors.New("applied only in part: " + u.words + "\n\nevery resource that does not need " + u.tool +
|
||||
" was attempted; none here needed it, and " + u.tool + " must answer before this machine applies whole " +
|
||||
"(novox/hq issue 233)")
|
||||
}
|
||||
|
||||
// probePackageManager proves the machine is the system the host was built for, before anything is applied.
|
||||
//
|
||||
// **A package manager that does not answer fails only what needs it** (novox/hq issue 233). The probe was a
|
||||
// refusal of the whole declaration — right for a host installed on a machine it was not built for, and a
|
||||
// wedge when what the probe reads is the mesh's own: a declaration that left the package manager's module
|
||||
// out removed its configuration file, and from then on every declaration was refused, the one that would
|
||||
// write the file back and the node-engine's own upgrade included. A person on the machine was the only way
|
||||
// back. Now the rest applies — files, units, accounts, the node-engine itself — and each package fails with
|
||||
// the probe's words (the system's PackageInstalled asks the same probe first), so a later declaration can
|
||||
// undo what an earlier one did. Before the rest is applied, a declaration composed from an older view of the
|
||||
// assignments than this machine applied has already been refused (issue 234's generation).
|
||||
//
|
||||
// **A package manager that is not there at all still refuses whole.** That is the host installed on a
|
||||
// machine it was not built for — the arch host on Alpine — which is what the probe exists to catch, and no
|
||||
// declaration the mesh sends removes a package manager's program. Only one that is there and errors (a
|
||||
// configuration file gone, a database locked) is the mesh's own fault and applies in part.
|
||||
//
|
||||
// A system with no package manager (android) still refuses whole: what its probe asks is the system itself,
|
||||
// which no resource the mesh declares can remove, and nothing else stands between it and the wrong machine.
|
||||
func probePackageManager(ctx context.Context, sys system.System) (*unanswered, error) {
|
||||
err := sys.Confirm(ctx, apply.ExecRunner)
|
||||
if err == nil {
|
||||
return nil, nil
|
||||
}
|
||||
tool, ok := packageManagers[sys.Name()]
|
||||
if !ok || errors.Is(err, exec.ErrNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
return &unanswered{tool: tool, words: err.Error()}, nil
|
||||
}
|
||||
|
||||
// applyPass is one apply of the declaration over what the node's state says.
|
||||
type applyPass func(known store.State) (apply.Report, store.State, error)
|
||||
|
||||
// applyMindingProbe applies, and **once more when the package manager did not answer and this apply wrote
|
||||
// something** (novox/hq issue 233). What it wrote may be what the package manager needed — the declaration
|
||||
// that names its module again writes its configuration file back — and then the packages go in now rather
|
||||
// than at the next reconcile. Asked again only then: an apply that wrote nothing changed nothing the package
|
||||
// manager reads. The second pass is the same declaration over the state the first left, so what the first
|
||||
// applied reads unchanged and only what needs the package manager moves.
|
||||
//
|
||||
// It returns the last pass's account, the probe as it stands after it (nil once the package manager
|
||||
// answers), and the first pass's outcomes when a second ran, for whoever counts what an apply moved.
|
||||
func applyMindingProbe(ctx context.Context, sys system.System, probe *unanswered, known store.State,
|
||||
pass applyPass, say func(string)) (apply.Report, store.State, error, *unanswered, []apply.Outcome) {
|
||||
report, updated, err := pass(known)
|
||||
if probe == nil || !wrote(report, err) {
|
||||
return report, updated, err, probe, nil
|
||||
}
|
||||
again, refused := probePackageManager(ctx, sys)
|
||||
if refused != nil || again != nil {
|
||||
return report, updated, err, probe, nil
|
||||
}
|
||||
say(probe.tool + " answers now that this apply wrote what it reads; applying again, so its packages are " +
|
||||
"installed in this apply (novox/hq issue 233)")
|
||||
first := append([]apply.Outcome(nil), report.Outcomes...)
|
||||
var partial *apply.Error
|
||||
if errors.As(err, &partial) {
|
||||
first = append(first, partial.Done.Outcomes...)
|
||||
}
|
||||
report, updated, err = pass(updated)
|
||||
return report, updated, err, nil, first
|
||||
}
|
||||
|
||||
// wrote is whether an apply moved anything on the machine, failed or not.
|
||||
func wrote(report apply.Report, err error) bool {
|
||||
if report.Changed() {
|
||||
return true
|
||||
}
|
||||
var partial *apply.Error
|
||||
return errors.As(err, &partial) && partial.Done.Changed()
|
||||
}
|
||||
|
||||
// unapplied is which declared resources an apply did not apply, failed or not attempted, sorted; nil for a
|
||||
// clean one.
|
||||
func unapplied(d *declaration.Declaration, report apply.Report, err error) []string {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
done := map[string]bool{}
|
||||
for _, o := range report.Outcomes {
|
||||
done[o.ID] = o.Action != "skipped"
|
||||
}
|
||||
var partial *apply.Error
|
||||
if errors.As(err, &partial) {
|
||||
for _, o := range partial.Done.Outcomes {
|
||||
done[o.ID] = done[o.ID] || o.Action != "skipped"
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, r := range d.Resources {
|
||||
if !done[r.Identity()] {
|
||||
out = append(out, r.Identity())
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
// Failed, and every resource has an account: what failed is no resource (the firewall found, a removal).
|
||||
out = []string{"(apply)"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withUnanswered adds the probe to what a report says failed.
|
||||
func withUnanswered(failed map[string]string, u *unanswered) map[string]string {
|
||||
if u == nil {
|
||||
return failed
|
||||
}
|
||||
if failed == nil {
|
||||
failed = map[string]string{}
|
||||
}
|
||||
failed[u.failedKey()] = u.said()
|
||||
return failed
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// Defends novox/hq issue 233: a package manager that does not answer is said even when nothing declared
|
||||
// needs it — the apply is partial, never "applied" — and the report names it under one key, so the mesh
|
||||
// counts it as one failure that goes on.
|
||||
func TestAPackageManagerThatDoesNotAnswerIsSaidWithNoPackageDeclared(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json"), out: &strings.Builder{}}
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[{"id":"pacman.conf","type":"file","path":"` + conf +
|
||||
`","content":"[options]\n"}]}`)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = runApply(context.Background(), opts, d, raw, fromFile)
|
||||
if err == nil || !strings.Contains(err.Error(), "applied only in part") || !strings.Contains(err.Error(), "could not be read") {
|
||||
t.Fatalf("an apply whose package manager did not answer was not said to be partial: %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(conf); statErr != nil {
|
||||
t.Fatalf("the file was not written: %v", statErr)
|
||||
}
|
||||
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
if report.Refused != "" {
|
||||
t.Fatalf("refused whole: %s", report.Refused)
|
||||
}
|
||||
why, ok := report.Failed["the package manager (pacman)"]
|
||||
if !ok || !strings.Contains(why, "could not be read") {
|
||||
t.Errorf("the report does not name the package manager with the probe's words: %v", report.Failed)
|
||||
}
|
||||
}
|
||||
|
||||
// A system whose probe asks the system itself, not a tool a resource needs, still refuses whole: nothing
|
||||
// the mesh declares can remove it, and nothing else stands between that host and the wrong machine.
|
||||
func TestASystemWithNoPackageManagerStillRefusesWhole(t *testing.T) {
|
||||
t.Setenv("PATH", t.TempDir())
|
||||
sys, err := system.For("android")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe, err := probePackageManager(context.Background(), sys)
|
||||
if err == nil || probe != nil {
|
||||
t.Fatalf("the android host's failed probe did not refuse: %v, %+v", err, probe)
|
||||
}
|
||||
}
|
||||
|
||||
// pacmanAnsweringOnce is a pacman that does not answer until conf exists, as the workstation's did until its
|
||||
// configuration file was written back, and installs into a directory of its own once it does.
|
||||
func pacmanAnsweringOnce(t *testing.T, conf string) {
|
||||
t.Helper()
|
||||
bin, installed := t.TempDir(), t.TempDir()
|
||||
script := "#!/bin/sh\n" +
|
||||
"[ -f '" + conf + "' ] || { echo \"error: config file " + conf + " could not be read: No such file or directory\" >&2; exit 1; }\n" +
|
||||
"for a; do last=$a; done\n" +
|
||||
"case \"$1\" in\n" +
|
||||
"-Q) [ \"$2\" = pacman ] && exit 0; [ -f '" + installed + "'/\"$2\" ] && exit 0; exit 1;;\n" +
|
||||
"-S) touch '" + installed + "'/\"$last\"; exit 0;;\n" +
|
||||
"esac\nexit 1\n"
|
||||
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
}
|
||||
|
||||
// Defends the review of novox/hq issue 233: **the apply that writes the package manager's configuration back
|
||||
// installs its packages too**. The probe is asked again once the partial apply wrote something, and when it
|
||||
// answers the declaration is applied again over what the first pass left — the packages in this apply, not five
|
||||
// minutes later, and the report clean.
|
||||
func TestThePackagesGoInWhenTheApplyThatRestoresThePackageManagerWroteIt(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
pacmanAnsweringOnce(t, conf)
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
// The package first: within one pass it is asked about before the file is written, which is the order that
|
||||
// leaves it to a second pass.
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"pacman.base","type":"package","package":"base"},` +
|
||||
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"}]}`)
|
||||
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
|
||||
if report.Refused != "" || len(report.Failed) != 0 {
|
||||
t.Fatalf("the apply that made the package manager answer did not install its packages: refused %q, failed %v",
|
||||
report.Refused, report.Failed)
|
||||
}
|
||||
if !slices.Contains(report.Applied, "pacman.base") || !slices.Contains(report.Applied, "pacman.conf") {
|
||||
t.Errorf("the report does not account for both: %v", report.Applied)
|
||||
}
|
||||
}
|
||||
|
||||
// Defends decision (1) of the review of novox/hq issue 233: **a package manager that is not there at all still
|
||||
// refuses whole**. That is the host installed on the wrong machine, which the probe exists to catch.
|
||||
func TestAPackageManagerThatIsNotThereRefusesWhole(t *testing.T) {
|
||||
t.Setenv("PATH", t.TempDir())
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[{"id":"pacman.conf","type":"file","path":"` + conf +
|
||||
`","content":"[options]\n"}]}`)
|
||||
|
||||
report := applyAndKeep(context.Background(), options{state: filepath.Join(dir, "state.json")}, raw, nil, nil, nil)
|
||||
|
||||
if !strings.Contains(report.Refused, "pacman does not answer") {
|
||||
t.Fatalf("a machine with no pacman at all was not refused whole: %+v", report)
|
||||
}
|
||||
if _, err := os.Stat(conf); err == nil {
|
||||
t.Error("the refused declaration touched the machine")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
These tests read files outside this directory (the whole catalogue, the repository, or its root files): mesh-build-go never keeps this package's verdict (issue 1935).
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The record check reads the engine's applied state: a composition one of whose resources has no record there is not
|
||||
// recorded (issue 1945).
|
||||
func TestACompositionIsRecordedOnlyWhenEveryResourceIs(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
var s store.State
|
||||
s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
|
||||
if err := store.Save(path, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := delivery.Composed{Node: "ace", Module: "ca-trust", Resources: []json.RawMessage{
|
||||
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`)}}
|
||||
recorded := recordedIn(path, func(string) {})
|
||||
if !recorded(context.Background(), c) {
|
||||
t.Fatal("a composition whose every resource has a record was not recorded")
|
||||
}
|
||||
c.Resources = append(c.Resources, json.RawMessage(
|
||||
`{"id":"ca-trust.more","type":"file","module":"ca-trust","path":"/etc/ca2","mode":"0644","content":"y"}`))
|
||||
if recorded(context.Background(), c) {
|
||||
t.Fatal("a composition with a resource the state holds no record of was recorded")
|
||||
}
|
||||
}
|
||||
|
||||
// The modules applied again are those with a missing resource, each once; one down but there is not among them.
|
||||
func TestTheModulesAppliedAgainAreThoseWithAMissingResource(t *testing.T) {
|
||||
st := liveness.Statement{Resources: []liveness.State{
|
||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.homeserver"}, State: liveness.Unhealthy, Missing: true},
|
||||
{Resource: liveness.Resource{Module: "matrix", ID: "matrix.db"}, State: liveness.Unhealthy, Missing: true},
|
||||
{Resource: liveness.Resource{Module: "ca-trust", ID: "ca-trust.trust"}, State: liveness.Unhealthy, Missing: true},
|
||||
{Resource: liveness.Resource{Module: "web", ID: "web.server"}, State: liveness.Unhealthy},
|
||||
}}
|
||||
if got := missingModules(st); len(got) != 2 || got[0] != "ca-trust" || got[1] != "matrix" {
|
||||
t.Fatalf("applied again: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module is applied again only for a resource newly missing: one still missing at the next change of the statement is
|
||||
// not acted on again, so a resource that cannot be made is not applied (and backed up) at every unrelated change. One
|
||||
// that comes back and goes again is newly missing again (the review of issue 1945).
|
||||
func TestOnlyANewlyMissingResourceAppliesItsModuleAgain(t *testing.T) {
|
||||
missing := func(ids ...string) liveness.Statement {
|
||||
var st liveness.Statement
|
||||
for _, id := range ids {
|
||||
st.Resources = append(st.Resources, liveness.State{Resource: liveness.Resource{Module: "matrix", ID: id},
|
||||
State: liveness.Unhealthy, Missing: true})
|
||||
}
|
||||
return st
|
||||
}
|
||||
var last map[string]bool
|
||||
var got []string
|
||||
got, last = newlyMissing(last, missing("matrix.homeserver"))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("first missing: %v", got)
|
||||
}
|
||||
got, last = newlyMissing(last, missing("matrix.homeserver"))
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("still missing was acted on again: %v", got)
|
||||
}
|
||||
got, last = newlyMissing(last, missing())
|
||||
got, last = newlyMissing(last, missing("matrix.homeserver"))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("missing again after it came back: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A file the provisioner client fills is left to it by the apply and has no record until the client writes it: it does
|
||||
// not make its composition unrecorded (the review of issue 1945).
|
||||
func TestAFilledFileDoesNotMakeItsCompositionUnrecorded(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
var s store.State
|
||||
s.Record(store.Applied{ID: "ca-trust.trust", Type: "file", Target: "/etc/ca"})
|
||||
if err := store.Save(path, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := delivery.Composed{Node: "ace", Module: "ca-trust", Filled: []string{"/etc/ca-filled"}, Resources: []json.RawMessage{
|
||||
json.RawMessage(`{"id":"ca-trust.trust","type":"file","module":"ca-trust","path":"/etc/ca","mode":"0644","content":"x"}`),
|
||||
json.RawMessage(`{"id":"ca-trust.filled","type":"file","module":"ca-trust","path":"/etc/ca-filled","mode":"0644","owner-at-runtime":"owned"}`)}}
|
||||
if !recordedIn(path, func(string) {})(context.Background(), c) {
|
||||
t.Fatal("a filled file the client has not written made its composition unrecorded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// unansweringPacman puts a pacman on PATH that answers as one whose configuration file is gone, as the
|
||||
// workstation's did on 2026-10-04 (novox/hq issue 233), and makes this an arch host.
|
||||
func unansweringPacman(t *testing.T) {
|
||||
t.Helper()
|
||||
bin := t.TempDir()
|
||||
script := "#!/bin/sh\necho 'error: config file /etc/pacman.conf could not be read: No such file or directory' >&2\nexit 1\n"
|
||||
if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
}
|
||||
|
||||
// TestReplay233 replays novox/hq issue 233. The package manager's module wrote pacman's configuration
|
||||
// file; a declaration that left the module out removed it; from then on pacman did not answer, and the
|
||||
// node-engine refused every declaration whole — the one that named the module again and would have
|
||||
// written the file back included. A person restored the file by hand an hour and a half later.
|
||||
//
|
||||
// Held to: a declaration that restores the file is applied while pacman does not answer. Only what needs
|
||||
// pacman fails, with the probe's words; the file is written and recorded. Written only with what the
|
||||
// node-engine had before the fix, so it lays over the commit before it, where it fails.
|
||||
func TestReplay233(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
dir := t.TempDir()
|
||||
opts := options{state: filepath.Join(dir, "state.json")}
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||
|
||||
report := applyAndKeep(context.Background(), opts, raw, nil, nil, nil)
|
||||
|
||||
if report.Refused != "" {
|
||||
t.Fatalf("the declaration that restores the package manager's configuration was refused whole: %s", report.Refused)
|
||||
}
|
||||
if got, err := os.ReadFile(conf); err != nil || string(got) != "[options]\n" {
|
||||
t.Fatalf("the file that restores the package manager was not written: %q, %v", got, err)
|
||||
}
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := known.Find("pacman.conf"); !ok {
|
||||
t.Error("the file written was not recorded, so the next apply would not know it is the mesh's")
|
||||
}
|
||||
words := strings.Join(failedWords(report.Failed), "\n")
|
||||
if !strings.Contains(words, "pacman does not answer") || !strings.Contains(words, "could not be read") {
|
||||
t.Errorf("the report does not say the package manager did not answer, in the probe's words: %q", words)
|
||||
}
|
||||
if !strings.Contains(words, "pacman.base") {
|
||||
t.Errorf("the report does not name the package that was not applied: %q", words)
|
||||
}
|
||||
}
|
||||
|
||||
func failedWords(failed map[string]string) []string {
|
||||
var out []string
|
||||
for id, why := range failed {
|
||||
out = append(out, id+": "+why)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Defends novox/hq issue 233 at the machine: `mesh-host apply` applies what does not need the package
|
||||
// manager, and says the apply was partial with the probe's words — a non-zero exit, never "applied".
|
||||
func TestApplyByHandAppliesWhatDoesNotNeedAPackageManagerThatDoesNotAnswer(t *testing.T) {
|
||||
unansweringPacman(t)
|
||||
dir := t.TempDir()
|
||||
out := &bytes.Buffer{}
|
||||
opts := options{state: filepath.Join(dir, "state.json"), out: out}
|
||||
conf := filepath.Join(dir, "pacman.conf")
|
||||
raw := []byte(`{"declaration":1,"resources":[` +
|
||||
`{"id":"pacman.conf","type":"file","path":"` + conf + `","content":"[options]\n"},` +
|
||||
`{"id":"pacman.base","type":"package","package":"base"}]}`)
|
||||
d, err := declaration.ParseFileTrusted(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = runApply(context.Background(), opts, d, raw, fromFile)
|
||||
|
||||
if _, statErr := os.Stat(conf); statErr != nil {
|
||||
t.Fatalf("the file was not written: %v (the apply said %v)", statErr, err)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("an apply whose package manager did not answer said it applied")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "pacman does not answer") || !strings.Contains(err.Error(), "pacman.base") {
|
||||
t.Errorf("the failure does not name the package or the probe's words: %v", err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "pacman does not answer") {
|
||||
t.Errorf("what was said before applying does not say the package manager does not answer:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/accounts"
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/liveness"
|
||||
"github.com/novox/mesh-host/internal/units"
|
||||
@@ -87,7 +87,13 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
|
||||
Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
|
||||
}}
|
||||
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
|
||||
if h.Contract != link.RootContract || link.RootContract != 3 {
|
||||
// At RootContract or later: an engine that states its sealing key to the vault says NodeKeyContract (ADR 0372, issue 786), one above
|
||||
// VaultBoxContract (it opens the vault's box for the core's credentials, ADR 0349 §16, issue 755), one above HandedContract (design 57 P7), one above ProvisionerContract (novox/hq ADR
|
||||
// 0349 §4), one above ADR 0349 §5's OwnershipContract.
|
||||
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.NodeKeyContract ||
|
||||
link.NodeKeyContract != link.VaultBoxContract+1 || link.VaultBoxContract != link.HandedContract+1 ||
|
||||
link.HandedContract != link.ProvisionerContract+1 || link.ProvisionerContract != link.OwnershipContract+1 || link.OwnershipContract != link.BusyContract+1 ||
|
||||
link.BusyContract != link.OwnGroupContract+1 {
|
||||
t.Fatalf("contract %d", h.Contract)
|
||||
}
|
||||
got := map[string]string{}
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The last apply's report, kept beside the state until the mesh takes it (novox/hq issue 264).
|
||||
|
||||
// What a host that died between applying and reporting kept is read back by the next one exactly —
|
||||
// the digest and the outcome, never a new account of the machine.
|
||||
func TestTheKeptReportSurvivesTheHost(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
made := link.Report{Declared: "sha256:d1", Applied: []string{"a"}, Failed: map[string]string{"b": "no room"},
|
||||
Host: "v2"}
|
||||
if err := unsaidBeside(state).Keep(made); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
kept, ok, err := unsaidBeside(state).Pending()
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("nothing kept: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(kept, made) {
|
||||
t.Fatalf("kept %+v, made %+v", kept, made)
|
||||
}
|
||||
|
||||
// Taken about another declaration, it stays; taken about this one, it goes.
|
||||
if err := unsaidBeside(state).Said("sha256:other"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, _ := unsaidBeside(state).Pending(); !ok {
|
||||
t.Fatal("a report about another declaration cleared this one")
|
||||
}
|
||||
if err := unsaidBeside(state).Said("sha256:d1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(store.UnsaidPath(state)); !os.IsNotExist(err) {
|
||||
t.Fatalf("the report was taken and is still kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A node that never applied anything has nothing to say again; and an apply refused before its
|
||||
// declaration was read clears an older report, which would otherwise be re-said after it.
|
||||
func TestNothingKeptWhenNothingWasApplied(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
if _, ok, err := unsaidBeside(state).Pending(); ok || err != nil {
|
||||
t.Fatalf("a node that applied nothing has a report to say: %v %v", ok, err)
|
||||
}
|
||||
if err := unsaidBeside(state).Keep(link.Report{Declared: "sha256:d1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := unsaidBeside(state).Keep(link.Report{Refused: "forged"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok, _ := unsaidBeside(state).Pending(); ok {
|
||||
t.Fatal("an older report is kept after a later declaration was refused")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
"github.com/novox/mesh-host/internal/userlist"
|
||||
)
|
||||
|
||||
// The bridge of novox/hq ADR 0547 §8, wired: the bus node's engine writes the bus's user list alone from the newest
|
||||
// composition of the bus module, under the apply's two locks as a filled file is written (filling.go), and says what
|
||||
// it wrote in the bus module's have.
|
||||
|
||||
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
|
||||
// agent trusts, the write alone, and the have.
|
||||
func userListFollower(opts options, session *link.Session, composedBucket string, trusted func() []ed25519.PublicKey,
|
||||
say link.Announce, wrote func()) *userlist.Follower {
|
||||
return userlist.New(userlist.Env{
|
||||
Node: session.Membership.Node,
|
||||
Composed: bucketWatch{session, composedBucket},
|
||||
Trusted: trusted,
|
||||
Resolve: func(ctx context.Context, raw json.RawMessage) ([]byte, error) {
|
||||
return resolveUserList(session.WithLink(ctx), raw)
|
||||
},
|
||||
OnDisk: userListOnDisk,
|
||||
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
|
||||
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
|
||||
if changed && wrote != nil {
|
||||
wrote()
|
||||
}
|
||||
return digest, changed, err
|
||||
},
|
||||
Report: func(ctx context.Context, module string, w userlist.Written) {
|
||||
session.NoteUserList(ctx, module, w)
|
||||
},
|
||||
// From take-over, the grants part's list (ADR 0547 clauses 7, 8): its one key watched, its object read by name and
|
||||
// opened with this machine's sealing key, the key's public half said in the have so the grants part seals to it.
|
||||
Grants: bucketWatch{session, delivery.GrantsUserListBucket},
|
||||
Fetch: func(ctx context.Context, name string) ([]byte, error) {
|
||||
return session.GrantsObject(ctx, name, grantsListLimit)
|
||||
},
|
||||
Open: func(b seal.Box) ([]byte, error) {
|
||||
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return key.OpenVaultBox(b)
|
||||
},
|
||||
Key: func() string {
|
||||
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
pem, err := key.PublicPEM()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return pem
|
||||
},
|
||||
// The grants part's key as last heard, kept beside the node's state, followed while it cannot be watched; its
|
||||
// watch tried again as a passing failure of the bus is (ADR 0547 clause 8).
|
||||
Kept: func() ([]byte, uint64) { return keptGrantsKey(opts) },
|
||||
Keep: func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) },
|
||||
Backoff: link.Backoff,
|
||||
// The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547
|
||||
// clause 9).
|
||||
VaultKey: vaultKey.get,
|
||||
Say: announceOr(say),
|
||||
})
|
||||
}
|
||||
|
||||
// grantsListLimit is the largest sealed list the bus node reads: the live list is about a hundred kilobytes.
|
||||
const grantsListLimit = 16 << 20
|
||||
|
||||
// userListFile is a user-list resource as the file it declares.
|
||||
func userListFile(raw []byte) (*declaration.File, error) {
|
||||
r, err := declaration.ParseResource(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the user list is not a resource: %w", err)
|
||||
}
|
||||
f, ok := r.(*declaration.File)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the user list is a %s, not a file", r.Kind())
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// resolveUserList is the list a user-list resource holds: its content, or the object its digest names in this node's
|
||||
// file bucket, fetched and verified (apply.ResolveFileReferences, ADR 0486 §1).
|
||||
func resolveUserList(ctx context.Context, raw []byte) ([]byte, error) {
|
||||
f, err := userListFile(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &declaration.Declaration{Version: declaration.Version, Resources: []declaration.Resource{f}}
|
||||
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
|
||||
return nil, fmt.Errorf("the user list could not be had by its digest: %w", err)
|
||||
}
|
||||
return []byte(d.Resources[0].(*declaration.File).Content), nil
|
||||
}
|
||||
|
||||
// userListOnDisk is what the user list's file holds on this machine now; nil when there is none.
|
||||
func userListOnDisk(raw json.RawMessage) ([]byte, error) {
|
||||
f, err := userListFile(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := os.ReadFile(f.Path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return body, err
|
||||
}
|
||||
|
||||
// writeUserListAlone writes one user-list resource alone (apply.WriteAlone) with body as its content, the node's state
|
||||
// read and saved under the apply's locks, and answers the digest of the file on the machine after.
|
||||
func writeUserListAlone(ctx context.Context, opts options, raw []byte, body []byte, say link.Announce) (string, bool, error) {
|
||||
declared, err := userListFile(raw)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
file := *declared
|
||||
file.Content, file.Digest, file.Size = string(body), "", 0
|
||||
|
||||
applying.Lock()
|
||||
defer applying.Unlock()
|
||||
unlock, err := store.Lock(opts.state, nil)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
defer unlock()
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
built, err := system.For(builtFor)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
out, updated, writeErr := apply.WriteAlone(ctx, built, &file, known, apply.ExecRunner, announceOr(say))
|
||||
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
|
||||
if err := store.Save(opts.state, updated); err != nil {
|
||||
writeErr = errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+err.Error()))
|
||||
}
|
||||
digest := ""
|
||||
if body, err := os.ReadFile(file.Path); err == nil {
|
||||
digest = userlist.Digest(body)
|
||||
}
|
||||
return digest, writeErr == nil && out.Action != "unchanged", writeErr
|
||||
}
|
||||
|
||||
// grantsKeyPath is where the grants part's user-list key last heard is kept, beside the node's state.
|
||||
func grantsKeyPath(opts options) string {
|
||||
return filepath.Join(filepath.Dir(opts.state), "grants-user-list.json")
|
||||
}
|
||||
|
||||
type keptKey struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
|
||||
func keptGrantsKey(opts options) ([]byte, uint64) {
|
||||
raw, err := os.ReadFile(grantsKeyPath(opts))
|
||||
if err != nil {
|
||||
return nil, 0
|
||||
}
|
||||
var k keptKey
|
||||
if json.Unmarshal(raw, &k) != nil || len(k.Value) == 0 {
|
||||
return nil, 0
|
||||
}
|
||||
return k.Value, k.Revision
|
||||
}
|
||||
|
||||
func keepGrantsKey(opts options, value []byte, rev uint64, say link.Announce) {
|
||||
path := grantsKeyPath(opts)
|
||||
if value == nil {
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
announceOr(say)("the grants part's user-list key kept on this machine could not be taken away: " + err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(keptKey{Revision: rev, Value: value})
|
||||
tmp := path + ".new"
|
||||
err := os.WriteFile(tmp, raw, 0o600)
|
||||
if err == nil {
|
||||
err = os.Rename(tmp, path)
|
||||
}
|
||||
if err != nil {
|
||||
announceOr(say)("the grants part's user-list key could not be kept on this machine: " + err.Error())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
// Issue 755's proof on a real bus (novox/hq ADR 0349 §16): a core credential the vault sealed to this machine crosses
|
||||
// the bus inside the declaration, which the controller carries and cannot open, and this engine opens it with its
|
||||
// sealing key and writes the value. A listener on every subject keeps every byte that crossed; the value is never
|
||||
// among them. Run against the throwaway bus the build module starts for the integration level (MESH_TEST_NATS).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
const coreCredential = "the-provisioner-clients-bus-password-2025"
|
||||
|
||||
func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
// This machine's sealing key, where the engine reads it.
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
key, err := identity.GenerateSealingKey()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Every byte that crosses.
|
||||
tap, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tap.Close()
|
||||
var mu sync.Mutex
|
||||
var crossed []string
|
||||
if _, err := tap.Subscribe(">", func(m *nats.Msg) {
|
||||
mu.Lock()
|
||||
crossed = append(crossed, m.Subject+" "+string(m.Data))
|
||||
mu.Unlock()
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = tap.Flush()
|
||||
|
||||
// The engine hears the composition on a subject of the test (the node agent reads it from its bucket).
|
||||
const subject = "mesh.test.vaultbox.composition"
|
||||
engine, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer engine.Close()
|
||||
heard, err := engine.SubscribeSync(subject)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = engine.Flush()
|
||||
|
||||
// The vault's answer to the controller: the value in the ADR 0206 box, sealed to this machine's key. The
|
||||
// controller carries it in the declaration as it is.
|
||||
pub, err := key.PublicPEM()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
box, err := seal.Seal(coreCredential, pub)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "broker")
|
||||
body, _ := json.Marshal(map[string]any{"declaration": 1, "resources": []map[string]any{
|
||||
{"id": "need-provisioner-broker", "type": "file", "path": path, "vault-sealed": box}}})
|
||||
controller, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer controller.Close()
|
||||
if err := controller.Publish(subject, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = controller.Flush()
|
||||
|
||||
msg, err := heard.NextMsg(5 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("the declaration did not reach the engine: %v", err)
|
||||
}
|
||||
d, err := declaration.Parse(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sys, err := system.For("arch")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := apply.Apply(context.Background(), sys, d, store.State{}, store.OriginCarried,
|
||||
func(context.Context, string, ...string) (string, error) { return "", nil }, nil, sealOpener(state)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
on, err := os.ReadFile(path)
|
||||
if err != nil || string(on) != coreCredential {
|
||||
t.Fatalf("the engine wrote %q, %v", on, err)
|
||||
}
|
||||
|
||||
_ = tap.Flush()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
mu.Lock()
|
||||
all := strings.Join(crossed, "\n")
|
||||
mu.Unlock()
|
||||
if !strings.Contains(all, subject) || !strings.Contains(all, "vault-sealed") {
|
||||
t.Fatalf("the listener heard nothing of the declaration:\n%s", all)
|
||||
}
|
||||
for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} {
|
||||
if strings.Contains(all, form) {
|
||||
t.Fatal("the core credential crossed the bus in the clear")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The engine opens a value the controller sealed (the NaCl box) as before, and the vault's box, with one key.
|
||||
func TestTheEnginesOpenerOpensBothBoxes(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
key, _ := identity.GenerateSealingKey()
|
||||
if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open := sealOpener(state)
|
||||
nacl, _ := identity.Seal(key.Public, []byte("old"))
|
||||
if got, err := open(nacl); err != nil || string(got) != "old" {
|
||||
t.Errorf("the controller's box opened to %q, %v", got, err)
|
||||
}
|
||||
pub, _ := key.PublicPEM()
|
||||
box, _ := seal.Seal("new", pub)
|
||||
raw, _ := json.Marshal(box)
|
||||
if got, err := open(string(raw)); err != nil || string(got) != "new" {
|
||||
t.Errorf("the vault's box opened to %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
func aFileOf(module, id string) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id,
|
||||
"content": "x", "module": module})
|
||||
return raw
|
||||
}
|
||||
|
||||
// What runs is the agent's compositions and the kept declaration's resources for each module the agent does not hold;
|
||||
// the node's own parts are the kept declaration's until the agent holds the node's composition (ADR 0539 §9). One module
|
||||
// under the agent must not make the rest invisible to the liveness, units, accounts, libraries, filling and the take act.
|
||||
func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) {
|
||||
kept := &declaration.Declaration{Identities: []declaration.Identity{{Module: "db", Name: "mesh-db"}},
|
||||
Libraries: []declaration.Library{{Library: "media"}}}
|
||||
everything := []delivery.Composed{
|
||||
{Node: "n1", Module: "web", Resources: []json.RawMessage{aFileOf("web", "a")}},
|
||||
{Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}}, // still the kept declaration's
|
||||
}
|
||||
d, err := whatRuns(everything, false, func() (*declaration.Declaration, error) { return kept, nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.Resources) != 2 || len(d.Identities) != 1 || len(d.Libraries) != 1 {
|
||||
t.Fatalf("%d resources, %d identities, %d libraries: the kept modules and the node's kept parts are missing",
|
||||
len(d.Resources), len(d.Identities), len(d.Libraries))
|
||||
}
|
||||
|
||||
// Once the agent holds the node's composition, its parts are the node's.
|
||||
node := delivery.Composed{Node: "n1", Own: delivery.NodeOwn{Identities: []declaration.Identity{{Module: "web", Name: "mesh-web"}}}}
|
||||
d, err = whatRuns(append(everything, node), true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.Identities) != 1 || d.Identities[0].Name != "mesh-web" || len(d.Libraries) != 0 {
|
||||
t.Fatalf("the node's parts: %+v %+v", d.Identities, d.Libraries)
|
||||
}
|
||||
}
|
||||
|
||||
// What the take act, the judgement of directories and the filling read is what runs, not the frozen kept declaration.
|
||||
func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
|
||||
setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) {
|
||||
return &declaration.Declaration{Version: 1, Resources: nil, Provisioner: "from-agent"}, nil
|
||||
})
|
||||
defer setRunsHere(nil)
|
||||
d, err := declaredNow("/nowhere", nil)
|
||||
if err != nil || d.Provisioner != "from-agent" {
|
||||
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
|
||||
// the module's: the declaration joined of both names each file once, so the filling writes it; the node's own resources
|
||||
// no module declares stay. (Live from 2026-10-25 12:43: "2 resources declare X" on every node.)
|
||||
func TestTheNodesOwnCopyOfAModulesResourceIsLeftOut(t *testing.T) {
|
||||
own := delivery.Composed{Node: "n1", Resources: []json.RawMessage{aFileOf("zsh", "rc"), aFileOf("mesh-wireguard", "conf")},
|
||||
Filled: []string{"/tmp/r6/zsh/rc"}}
|
||||
zsh := delivery.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")},
|
||||
Filled: []string{"/tmp/r6/zsh/rc"}}
|
||||
d, err := declarationOf(own, zsh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := map[string]int{}
|
||||
for _, r := range d.Resources {
|
||||
at[r.Target()]++
|
||||
}
|
||||
if at["/tmp/r6/zsh/rc"] != 1 || at["/tmp/r6/mesh-wireguard/conf"] != 1 || len(d.Resources) != 2 {
|
||||
t.Errorf("the joined declaration declares %v, want each path once", at)
|
||||
}
|
||||
if !d.Fills("/tmp/r6/zsh/rc") {
|
||||
t.Errorf("the filled path is no longer named")
|
||||
}
|
||||
|
||||
// The same at the node's own apply: what the others declare is theirs.
|
||||
trimmed, dropped := ownWithout(own, []delivery.Composed{zsh})
|
||||
if len(trimmed.Resources) != 1 || len(dropped) != 1 || dropped[0] != "zsh.rc (zsh)" {
|
||||
t.Errorf("the node's own without zsh's: %d resources, dropped %v", len(trimmed.Resources), dropped)
|
||||
}
|
||||
if len(own.Resources) != 2 {
|
||||
t.Errorf("ownWithout changed its argument")
|
||||
}
|
||||
// A node's own composition alone is all of it.
|
||||
if d, err := declarationOf(own); err != nil || len(d.Resources) != 2 {
|
||||
t.Errorf("the node's own alone: %v (%v)", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC18 (unit): the kept declaration's filled paths go to the module whose file is at each path, the node's own to
|
||||
// the node's; and what runs from it, joined with the node's composition, names them filled.
|
||||
func TestTheKeptDeclarationsFilledPathsGoToTheirModules(t *testing.T) {
|
||||
raw := []byte(`{"resources":[
|
||||
{"id":"claude-code.settings","type":"file","module":"claude-code","path":"/etc/cc.json"},
|
||||
{"id":"screen-lock.conf","type":"file","module":"screen-lock","path":"/etc/lock.conf"},
|
||||
{"id":"host.motd","type":"file","path":"/etc/motd"},
|
||||
{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc"}],
|
||||
"filled_files":["/etc/cc.json","/etc/lock.conf","/etc/motd"]}`)
|
||||
by, filled := splitKept(raw)
|
||||
if len(by["claude-code"]) != 1 || len(by[agent.NodeModule]) != 1 {
|
||||
t.Fatalf("resources by module: %v", by)
|
||||
}
|
||||
if strings.Join(filled["claude-code"], ",") != "/etc/cc.json" || strings.Join(filled["screen-lock"], ",") != "/etc/lock.conf" ||
|
||||
strings.Join(filled[agent.NodeModule], ",") != "/etc/motd" || len(filled["zsh"]) != 0 {
|
||||
t.Fatalf("filled by module: %v", filled)
|
||||
}
|
||||
cc := delivery.Composed{Node: "n1", Module: "claude-code", Resources: by["claude-code"], Filled: filled["claude-code"]}
|
||||
node := delivery.Composed{Node: "n1"}
|
||||
d, err := whatRuns([]delivery.Composed{cc, node}, true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") })
|
||||
if err != nil || !d.Fills("/etc/cc.json") {
|
||||
t.Fatalf("what runs does not name claude-code's file filled: %v (%v)", d, err)
|
||||
}
|
||||
if splitBy, _ := splitKept([]byte("not json")); splitBy != nil {
|
||||
t.Errorf("a kept declaration that does not read split into %v", splitBy)
|
||||
}
|
||||
}
|
||||
|
||||
// 1945/AC17 (unit, review): a resource of the node's own that shares only its target with a module's (another id: a
|
||||
// unit or a container of the same name) is not the module's, and is neither dropped nor left to be removed.
|
||||
func TestASharedTargetWithAnotherIDIsNotDropped(t *testing.T) {
|
||||
ownFile, _ := json.Marshal(map[string]any{"id": "host.zshrc", "type": "file", "path": "/tmp/r6/zsh/rc", "content": "x"})
|
||||
own := delivery.Composed{Node: "n1", Resources: []json.RawMessage{ownFile}}
|
||||
zsh := delivery.Composed{Node: "n1", Module: "zsh", FP: "src2:aa", Resources: []json.RawMessage{aFileOf("zsh", "rc")}}
|
||||
trimmed, dropped := ownWithout(own, []delivery.Composed{zsh})
|
||||
if len(dropped) != 0 || len(trimmed.Resources) != 1 {
|
||||
t.Errorf("a resource sharing only its target was dropped: %v", dropped)
|
||||
}
|
||||
}
|
||||
+10
-11
@@ -41,8 +41,9 @@ func rolledBackHosts(statePath string) []string {
|
||||
return upgrade.RolledBackVersions(verdicts)
|
||||
}
|
||||
|
||||
// proving is this host waiting for the mesh to take a report it made under its own build — the
|
||||
// evidence its launcher's trial waits for — and acting on it once.
|
||||
// proving is this host waiting to write a have key under its own build — the evidence its launcher's trial waits for
|
||||
// (novox/hq ADR 0539 §9, ADR 0227 §8: the component replaced is never the only witness of its successor) — and acting
|
||||
// on it once.
|
||||
type proving struct {
|
||||
statePath string
|
||||
version string
|
||||
@@ -56,16 +57,14 @@ func unproved(statePath, version string) bool {
|
||||
return err != nil || known != version
|
||||
}
|
||||
|
||||
// heard is told every account of the machine the mesh has taken. The first one about a declaration,
|
||||
// made by this build, proves it.
|
||||
func (p *proving) heard(r link.Report) {
|
||||
if r.Declared == "" || r.Refused != "" || r.Host != p.version {
|
||||
return
|
||||
}
|
||||
// haveWritten is told when this build has written a have key, or found the bucket already holding what it would write.
|
||||
// The first proves it. **A declaration's report does not**: a node-engine that receives none is not on trial for it, and
|
||||
// one that cannot write its haves is what the launcher keeps rolling back.
|
||||
func (p *proving) haveWritten() {
|
||||
p.once.Do(func() {
|
||||
retired, err := upgrade.Proved(p.statePath, upgrade.VersionsDir(""), p.version)
|
||||
if err != nil {
|
||||
p.say(fmt.Sprintf("this host reported under its own build %s, and proving it was not complete: %v", p.version, err))
|
||||
p.say(fmt.Sprintf("this host wrote its first have under its own build %s, and proving it was not complete: %v", p.version, err))
|
||||
}
|
||||
if len(retired) > 0 {
|
||||
p.say(fmt.Sprintf("host %s is proved; retired the host version(s) %s", p.version, strings.Join(retired, ", ")))
|
||||
@@ -146,7 +145,7 @@ func launcherReplaced(pid int, proc, path string) bool {
|
||||
}
|
||||
|
||||
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
||||
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue, say func(string)) {
|
||||
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
|
||||
host, _ := os.Hostname()
|
||||
w := &witness.Watcher{
|
||||
Root: apply.DaemonRoot(), Node: node, Host: host,
|
||||
@@ -159,7 +158,7 @@ func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Queue
|
||||
},
|
||||
// Said now, not at the next report: the verdict is on every report from here, and the mesh
|
||||
// is asked for one at once.
|
||||
Concluded: func(link.Rollback) { queue.ReportAsked() },
|
||||
Concluded: func(link.Rollback) { queue.FactsAsked(ctx) },
|
||||
Say: say,
|
||||
}
|
||||
w.Watch(ctx)
|
||||
|
||||
@@ -13,9 +13,9 @@ import (
|
||||
"github.com/novox/mesh-host/internal/upgrade"
|
||||
)
|
||||
|
||||
// The first account the mesh takes from this build, about a declaration, proves it to the launcher;
|
||||
// nothing else does — a refusal, a report about no declaration, another build's report (to-be 45 §8).
|
||||
func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) {
|
||||
// The first have this build writes proves it to the launcher; nothing else does — not a report about a declaration, not
|
||||
// a facts event, not the haves of another build (novox/hq ADR 0539 §9, ADR 0227 §8). R6-A AC7.
|
||||
func TestOnlyAHaveUnderThisBuildProvesIt(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state.json")
|
||||
t.Setenv("MESH_HOST_LIBEXEC", t.TempDir())
|
||||
// Never the machine's own engine placed by hand: proving renews it (issue 356).
|
||||
@@ -27,22 +27,20 @@ func TestOnlyAnAccountUnderThisBuildProvesIt(t *testing.T) {
|
||||
var said []string
|
||||
p := &proving{statePath: state, version: "2.0", say: func(s string) { said = append(said, s) }}
|
||||
|
||||
for _, r := range []link.Report{
|
||||
{Host: "2.0", Refused: "no"},
|
||||
{Host: "2.0"},
|
||||
{Host: "1.0", Declared: "abc"},
|
||||
} {
|
||||
p.heard(r)
|
||||
if !unproved(state, "2.0") {
|
||||
t.Fatalf("%+v proved this build", r)
|
||||
}
|
||||
if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" {
|
||||
t.Fatalf("%+v renewed the engine placed by hand", r)
|
||||
}
|
||||
// Facts said, a link opened, a declaration of the old kind reported: none of it is a have.
|
||||
if !unproved(state, "2.0") {
|
||||
t.Fatal("a build that wrote no have is proved")
|
||||
}
|
||||
p.heard(link.Report{Host: "2.0", Declared: "abc"})
|
||||
if got, _ := os.ReadFile(fallback); string(got) != "a stale engine placed by hand" {
|
||||
t.Fatal("the engine placed by hand was renewed before any have")
|
||||
}
|
||||
|
||||
// The session tells the proof when it has a have key saying what it would write.
|
||||
session := &link.Session{HaveWritten: p.haveWritten}
|
||||
_ = session
|
||||
p.haveWritten()
|
||||
if unproved(state, "2.0") {
|
||||
t.Fatalf("an account of a declaration under this build did not prove it (%v)", said)
|
||||
t.Fatalf("the first have under this build did not prove it (%v)", said)
|
||||
}
|
||||
if got, _ := upgrade.ReadKnownGood(upgrade.KnownGoodPath(state)); got != "2.0" {
|
||||
t.Fatalf("known-good is %q", got)
|
||||
@@ -78,12 +76,6 @@ func TestAStandingRollbackIsSaidOnEveryReport(t *testing.T) {
|
||||
got[0].Outcome != link.RolledBack {
|
||||
t.Fatalf("what the report says is %+v", got)
|
||||
}
|
||||
if !worthSaying(link.Report{Rollbacks: got}) {
|
||||
t.Fatal("a reconcile with a rollback standing does not say it")
|
||||
}
|
||||
if adoptionFingerprint(link.Report{Rollbacks: got}) == adoptionFingerprint(link.Report{}) {
|
||||
t.Fatal("a rollback reached or ended is not news to the reconcile")
|
||||
}
|
||||
}
|
||||
|
||||
// The launcher running this host is known to have been replaced only when it certainly was: a shell
|
||||
|
||||
+26
-33
@@ -2,51 +2,44 @@
|
||||
|
||||
## `foundation-first-node.lock`
|
||||
|
||||
What a machine must be before a mesh exists — the bootstrap in
|
||||
[novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole:
|
||||
The one genesis bundle (novox/hq ADR 0331): what a machine must be before a mesh exists — the
|
||||
bootstrap in [novox/hq `07-the-foundation.md`](https://git.novox.be/novox/hq), whole, on the bus the
|
||||
mesh runs on:
|
||||
|
||||
```
|
||||
0 a container runtime
|
||||
0 a container runtime, and a filter before anything listens
|
||||
1 the store runs
|
||||
2 a database per context `inventory` and `identity`
|
||||
2 a database per context `inventory`, `identity` and `licences`
|
||||
3 those contexts' schemas mesh-controller migrate
|
||||
4 the broker runs with a certificate it generated itself
|
||||
5 the control plane runs mesh-controller serve
|
||||
4 the bus's certificate mesh-controller broker certificate
|
||||
5 the bus runs NATS as mesh-broker-nats, where the nats module keeps it
|
||||
(ADR 0333), with the controller's account as its first user
|
||||
6 the control plane runs mesh-controller serve, given the bus (MESH_BUS_NATS)
|
||||
```
|
||||
|
||||
**A machine that applies this is a mesh** — one node, with nothing joined to it yet, which is
|
||||
exactly what the first node is (novox/hq ADR 0004). From here it hands out tokens and everything
|
||||
else joins the ordinary way.
|
||||
|
||||
This file said it stopped at step 4 for longer than that was true, which is its own small lesson:
|
||||
a comment about what something does not do is a comment nobody updates.
|
||||
It is a **template**: `mesh-bootstrap` reads it, makes the store's superuser and the controller's
|
||||
account on the bus, and writes them where the template says `bootstrap` (internal/bootstrap,
|
||||
rootsecrets.go). A template whose bus connection it does not recognise is refused at preflight,
|
||||
naming the placeholder it looked for. The bundle that raised the previous broker was deleted with
|
||||
ADR 0331; there is one bundle, and every lab bed names it.
|
||||
|
||||
Build a host carrying it:
|
||||
**The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333): the container
|
||||
`mesh-broker-nats`, its configuration and user list in `/var/lib/nats-module/conf`, its streams in
|
||||
`/var/lib/mesh-broker-nats`, its certificate in `/var/lib/mesh-broker-tls`, published on the node's bus
|
||||
port. At the installer's broker step the module adopts it: its own image replaces the upstream one, once,
|
||||
on the same streams, users and certificate. `internal/bootstrap` holds the bundle to the module's manifest.
|
||||
|
||||
Build an installer carrying a builder, and point it at this:
|
||||
|
||||
```
|
||||
make host SYSTEM=arch BUNDLE=examples/foundation-first-node.lock
|
||||
make bootstrap IMAGE=<builder image>
|
||||
mesh-bootstrap --bundle examples/foundation-first-node.lock …
|
||||
```
|
||||
|
||||
**The registry address and digests have to be replaced before this is useful.** They are written
|
||||
as `192.0.2.250:5000/…@sha256:…` because a digest belongs to whatever registry serves it — here,
|
||||
one a lab scenario raises, which reports its digests when it comes up. That is not a placeholder
|
||||
to be tidied away: a bundle is built *for a target*, and which registry that target pulls from is
|
||||
part of the target.
|
||||
|
||||
### What was verified, and how
|
||||
|
||||
On a lab machine confirmed sealed — `curl https://example.com` times out, the lab registry answers
|
||||
200 — the whole bundle applied from bare: eight resources, `inventory` created and `mesh` nowhere,
|
||||
the `node` table present with its indexes, the migration recorded, and LavinMQ answering
|
||||
`lavinmqctl status` with AMQP listening on 5672.
|
||||
|
||||
Three consecutive reconciles after that: **already matches — 8 resource(s) checked**, each time.
|
||||
|
||||
Then the machine was **rebooted**, and everything came back: docker from `boot: enabled`, both
|
||||
containers because the host creates every container `--restart unless-stopped`
|
||||
(`internal/apply/apply.go`), the schema intact in its named volume, and reconcile still finding
|
||||
nothing to do.
|
||||
|
||||
The reboot is worth doing rather than assuming. Nothing in the declaration asks for a container to
|
||||
return, so that it does is a property of the host, and the only way to know it holds is to take
|
||||
the machine away and give it back.
|
||||
**The registry address is the lab's.** The images are written as `192.0.2.250:5000/…@sha256:…`; the
|
||||
lab rewrites the third-party ones to upstream references by the same digest, and the installer
|
||||
replaces the control plane's with the image it built.
|
||||
|
||||
+32
-2
@@ -7,11 +7,12 @@ package examples
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
func bundle(t *testing.T) *declaration.Declaration {
|
||||
@@ -48,7 +49,7 @@ func TestTheBundleCarriesTheFoundationAndTheControlPlaneAndNothingElse(t *testin
|
||||
}
|
||||
sort.Strings(images)
|
||||
|
||||
want := []string{"lavinmq", "mesh-controller", "postgres"}
|
||||
want := []string{"mesh-controller", "nats", "pgvector"}
|
||||
if strings.Join(images, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("the bundle carries %v; expected exactly %v.\n\n"+
|
||||
"Adding one is a change to what every first node becomes, and to ADR 0006's "+
|
||||
@@ -67,3 +68,32 @@ func TestTheControlPlaneIsCarriedToo(t *testing.T) {
|
||||
t.Fatal("nothing in the bundle starts the control plane, so the machine would raise a " +
|
||||
"foundation and stop")
|
||||
}
|
||||
|
||||
// One genesis bundle, on the bus the mesh runs on (novox/hq ADR 0331 §2). Two bundles let every bed
|
||||
// and README name the one that could not raise a mesh, for weeks (issue 146).
|
||||
func TestThereIsOneGenesisBundle(t *testing.T) {
|
||||
bundles, err := filepath.Glob("*.lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(bundles) != 1 || bundles[0] != "foundation-first-node.lock" {
|
||||
t.Fatalf("examples/ holds %v; it holds one genesis bundle, foundation-first-node.lock", bundles)
|
||||
}
|
||||
}
|
||||
|
||||
// The control plane the bundle raises is given the bus, with the account the installer fills and
|
||||
// the certificate it pins (ADR 0331 §4); without it, it crash-loops for ever (issue 146).
|
||||
func TestTheControlPlaneIsGivenTheBus(t *testing.T) {
|
||||
for _, r := range bundle(t).Resources {
|
||||
c, ok := r.(*declaration.Container)
|
||||
if !ok || !strings.Contains(c.Image, "mesh-controller") {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(c.Env["MESH_BUS_NATS"], "nats://controller:bootstrap@") || c.Env["MESH_BROKER_CERTIFICATE"] == "" {
|
||||
t.Fatalf("the control plane is given the bus %q and the certificate %q",
|
||||
c.Env["MESH_BUS_NATS"], c.Env["MESH_BROKER_CERTIFICATE"])
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("nothing in the bundle starts the control plane")
|
||||
}
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
// foundation-first-node-nats.lock — what a machine must be before a mesh exists, on the bus being
|
||||
// built (novox/hq ADR 0106, design 25).
|
||||
//
|
||||
// The same twelve steps as foundation-first-node.lock, with one difference that matters: **the mesh
|
||||
// writes its own user list, and at genesis there is no mesh yet to write it.** So this carries the
|
||||
// first one — the controller's own account, at a well-known bootstrap password, exactly as the store
|
||||
// is reached at `postgres:bootstrap` and the old bus at `guest:guest`. It is rotated with those, and
|
||||
// from the controller's first composition onward the file is the controller's to write.
|
||||
//
|
||||
// The accounts file is its own file beside the server's configuration, because the server's own
|
||||
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
||||
// one directory, of necessity: an include path is resolved relative to the including file's own
|
||||
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
||||
// start.
|
||||
//
|
||||
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
||||
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
||||
// authenticates with a password (ADR 0004, design 25 §4).
|
||||
|
||||
{
|
||||
"declaration": 1,
|
||||
"resources": [
|
||||
{
|
||||
"id": "container-runtime",
|
||||
"type": "package",
|
||||
"package": "docker"
|
||||
},
|
||||
{
|
||||
"id": "container-runtime-running",
|
||||
"type": "service",
|
||||
"unit": "docker.service",
|
||||
"state": "running",
|
||||
"boot": "enabled"
|
||||
},
|
||||
// **A filter before anything listens** (novox/hq issue 054, ADR 0088). The store and the
|
||||
// broker are adopted as modules later and so bind to every interface from the moment they
|
||||
// start; the packet filter that governs who may reach them is a module too, installed a
|
||||
// dozen steps later. Between the two, a control-node facing the network had its store and
|
||||
// its bus open to anyone who could reach the machine. So the foundation carries a filter of
|
||||
// its own — the same table the filter module will replace wholesale once it can derive one:
|
||||
// drop by default, keep loopback, replies, ssh and the mesh's own ports (the bus a node
|
||||
// enrols over, the registry a node pulls from), and let the container runtime's own
|
||||
// networks through the forward chain so containers keep working. A published container port
|
||||
// is forwarded, never input (issue 047), which is why the forward chain is where the store's
|
||||
// and broker's ports are refused from outside — and a container on this machine dialling a
|
||||
// port this machine publishes reaches it through the runtime's proxy, which IS input, which
|
||||
// is why the bus and the registry are opened in both chains, exactly as the derived ruleset
|
||||
// does.
|
||||
{
|
||||
"id": "base-filter-package",
|
||||
"type": "package",
|
||||
"package": "nftables"
|
||||
},
|
||||
{
|
||||
"id": "base-filter",
|
||||
"type": "file",
|
||||
"path": "/etc/nftables.conf",
|
||||
"mode": "0644",
|
||||
"content": "#!/usr/sbin/nft -f\n# the foundation's own filter, until the mesh derives one (novox/hq issue 054)\ntable inet mesh {}\ndelete table inet mesh\n\ntable inet mesh {\n\tchain input {\n\t\ttype filter hook input priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\tiif lo accept\n\t\ticmp type echo-request accept\n\t\ticmpv6 type { echo-request, nd-neighbor-solicit, nd-neighbor-advert, nd-router-advert } accept\n\t\t# ssh, from anywhere — never closed\n\t\ttcp dport 22 accept\n\t\t# the mesh's own, from anywhere: the bus a node enrols over and a container on this machine reaches through the proxy, the registry a node pulls from\n\t\ttcp dport 5671 accept\n\t\ttcp dport 5000 accept\n\t}\n\tchain output {\n\t\ttype filter hook output priority filter; policy accept;\n\t}\n\tchain forward {\n\t\ttype filter hook forward priority filter; policy drop;\n\t\tct state established,related accept\n\t\tct state invalid drop\n\t\t# the container runtime's bridge networks, and the networks its compose files are given\n\t\tip saddr 172.16.0.0/12 accept\n\t\tip saddr 192.168.128.0/17 accept\n\t\t# the mesh's own: the bus a node enrols over, the registry a node pulls from\n\t\tct original proto-dst 5671 accept\n\t\tct original proto-dst 5000 accept\n\t}\n}\n"
|
||||
},
|
||||
{
|
||||
"id": "base-filter-loaded",
|
||||
"type": "service",
|
||||
"unit": "nftables.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": ["base-filter"]
|
||||
},
|
||||
{
|
||||
"id": "store",
|
||||
"type": "container",
|
||||
"name": "mesh-store",
|
||||
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
||||
"env": {
|
||||
"POSTGRES_PASSWORD": "bootstrap",
|
||||
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
||||
},
|
||||
"ports": ["5432:5432"],
|
||||
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
||||
},
|
||||
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
||||
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
||||
// exits happy, and the verify a moment later lands in the gap and fails. The action and its
|
||||
// verify must ask the same question, or the action can succeed into a state verify rejects.
|
||||
{
|
||||
"id": "store-ready",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
||||
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
||||
},
|
||||
{
|
||||
"id": "inventory-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
||||
},
|
||||
{
|
||||
"id": "identity-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
||||
},
|
||||
// Each context owns its own database (novox/hq ADR 0008). A third one is a third database,
|
||||
// created the same way and named the same way — which is the whole of adding a context to the
|
||||
// bootstrap, and is why the count is not something the foundation has an opinion about.
|
||||
{
|
||||
"id": "licences-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
||||
},
|
||||
{
|
||||
"id": "context-schemas",
|
||||
"type": "action",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
||||
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"migrate"],
|
||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
},
|
||||
{
|
||||
"id": "bus-certificate",
|
||||
"type": "action",
|
||||
// **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
|
||||
// broker's image while the broker was one that carried it; the bus that replaced it has a
|
||||
// shell and no openssl, and no other image the bundle names has one either. So the program
|
||||
// that needs the certificate writes it — already on this machine, since the schema step ran
|
||||
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
||||
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
||||
// to ask an authority of.
|
||||
// `--user 0:0` because the volume is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh volume.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--into", "/tls"],
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--check", "--into", "/tls"]
|
||||
},
|
||||
{
|
||||
"id": "bus-conf-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-bus-conf",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-conf",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/nats.conf",
|
||||
"mode": "0644",
|
||||
"content": "port: 4222\nhttp: 127.0.0.1:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
},
|
||||
{
|
||||
"id": "bus-accounts",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-bus-conf/accounts.conf",
|
||||
"mode": "0600",
|
||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||
},
|
||||
{
|
||||
"id": "broker",
|
||||
"type": "container",
|
||||
"name": "mesh-broker",
|
||||
"image": "192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927",
|
||||
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["mesh-broker-data:/data", "mesh-broker-tls:/tls:ro", "/var/lib/mesh-bus-conf:/etc/nats:ro"],
|
||||
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||
},
|
||||
{
|
||||
"id": "broker-ready",
|
||||
"type": "action",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh 192.0.2.250:5000/nats@sha256:b83efabe3e7def1e0a4a31ec6e078999bb17c80363f881df35edc70fcb6bb927 -c 'nc -z 127.0.0.1 5671'"]
|
||||
},
|
||||
{
|
||||
"id": "control-plane",
|
||||
"type": "container",
|
||||
"name": "mesh-controller",
|
||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"network": "host",
|
||||
"args": ["serve"],
|
||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
||||
"env": {
|
||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
||||
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
||||
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
||||
}
|
||||
}
|
||||
|
||||
]
|
||||
}
|
||||
@@ -1,36 +1,40 @@
|
||||
// foundation-first-node.lock — what a machine must be before a mesh exists.
|
||||
// foundation-first-node.lock — what a machine must be before a mesh exists: the one genesis bundle
|
||||
// (novox/hq ADR 0331 §2), on the bus the mesh runs on (ADR 0106, ADR 0131, design 25).
|
||||
//
|
||||
// The whole bootstrap (novox/hq 03-DESIGN/01-to-be/07-the-foundation.md): a container runtime, a
|
||||
// store, a database per context, those contexts' schemas, the broker, and the control plane
|
||||
// running on top of them.
|
||||
// filter, a store, a database per context, those contexts' schemas, the bus with its certificate
|
||||
// and its first user list, and the control plane running on top of them.
|
||||
//
|
||||
// It stopped before the control plane once, and this comment said so for longer than it was true.
|
||||
// **The mesh writes its own user list, and at genesis there is no mesh yet to write it.** So this
|
||||
// carries the first one — the controller's own account — and the control plane's address names the
|
||||
// same account. Both say `bootstrap`, and both are placeholders: the installer makes the account's
|
||||
// password, puts it in the address and its hash in the list (mesh-host internal/bootstrap
|
||||
// rootsecrets.go), and refuses a bundle whose bus connection it does not recognise. Until the first
|
||||
// node has enrolled the installer places the list the controller composes; from the bus's adoption
|
||||
// as the `nats` module on, the module receives it like any module (ADR 0331 §1, §5).
|
||||
//
|
||||
// The broker generates its OWN certificate, in its own image, into a volume it then mounts read
|
||||
// only. Self-signed, because at this moment there is no mesh to issue one and no public name to
|
||||
// obtain one for -- and it does not matter, because what a joining node checks is the fingerprint
|
||||
// pinned in its token, not a chain or a name (novox/hq ADR 0004). The subject is decoration.
|
||||
// The accounts file is its own file beside the server's configuration, because the server's own
|
||||
// settings belong to whoever raises it and the users belong to the mesh (design 25 §4). Both live in
|
||||
// one directory, of necessity: an include path is resolved relative to the including file's own
|
||||
// directory, so a server given an absolute one looks for it underneath that directory and refuses to
|
||||
// start.
|
||||
//
|
||||
// No `verify` on the TLS block, deliberately — that setting makes the server demand a *client*
|
||||
// certificate, and nothing in the mesh presents one: a host pins this server's exact certificate and
|
||||
// authenticates with a password (ADR 0004, design 25 §4). The control plane pins it too, through
|
||||
// MESH_BROKER_CERTIFICATE, and dials the bus on the machine's own loopback.
|
||||
//
|
||||
// PINNED BY DIGEST, and the digest is not decoration: a tag can be made to point at a different
|
||||
// image, and this file is applied on a machine with no mesh to ask about anything. These digests
|
||||
// belong to the registry the lab raises, which is what a real node pulls from anyway — what is
|
||||
// required is a reference that is exact and cannot move (novox/hq ADR 0006).
|
||||
// image, and this file is applied on a machine with no mesh to ask about anything (ADR 0006). The
|
||||
// registry prefix is the lab's; the digests are upstream's.
|
||||
//
|
||||
// The store waits up to three minutes rather than one. A machine that has just pulled the
|
||||
// image and is running initdb for the first time can take longer than sixty seconds, and it
|
||||
// failed that way three times in the lab -- a flaky bootstrap that a second run always fixed,
|
||||
// which is the worst kind because it teaches people to run things twice.
|
||||
//
|
||||
// It failed a fourth time on 2026-08-30, on a loaded machine, and the report was `docker exited
|
||||
// 1:` with nothing after the colon. Raising the timeout again would treat the symptom; what makes
|
||||
// a retry the only available response is a timeout that reports nothing. So the wait now says
|
||||
// what it saw before giving up.
|
||||
//
|
||||
// The store's data is a NAMED VOLUME, not a directory on the machine. A directory the host
|
||||
// creates is owned by root, and the database runs as somebody else inside the container — so it
|
||||
// could not write, and the container crash-looped. A named volume lets the image set up its own
|
||||
// ownership, and outlives the container, which is what you want for the thing holding the mesh's
|
||||
// state.
|
||||
// **The store is raised where the `postgres` module keeps it** (novox/hq ADR 0338): the container
|
||||
// `postgres`, on the module's pgvector image, its data in the directory /var/lib/mesh-store owned by
|
||||
// the image's postgres user, its superuser password read from the module's own-secret file — so the
|
||||
// store step adopts it as it runs. Pinned upstream by the module's own digest, not through the lab's
|
||||
// registry prefix, because the two must be the same reference (mesh-host internal/bootstrap holds
|
||||
// them to each other against the module's manifest).
|
||||
|
||||
{
|
||||
"declaration": 1,
|
||||
"resources": [
|
||||
@@ -80,17 +84,28 @@
|
||||
"boot": "enabled",
|
||||
"restart-on": ["base-filter"]
|
||||
},
|
||||
// The store's data is a directory owned by the image's postgres user (999), made before the
|
||||
// container starts: a directory the runtime made would be root's, and the database could not
|
||||
// write it. Measured on the module's image 2026-10-13: it initialises a new store on a directory
|
||||
// made so (novox/hq ADR 0338 §5).
|
||||
{
|
||||
"id": "store-data",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-store",
|
||||
"mode": "0700",
|
||||
"owner": "999:70"
|
||||
},
|
||||
{
|
||||
"id": "store",
|
||||
"type": "container",
|
||||
"name": "mesh-store",
|
||||
"image": "192.0.2.250:5000/postgres@sha256:7abf537131b66ed5af448d90653abf1679b0c7e9a1f07efdd4c3108a401b259a",
|
||||
"name": "postgres",
|
||||
"image": "pgvector/pgvector@sha256:cf134a767f474095eeba57e0117be8e568e011a63f33fbf252f14c9b760f8e6f",
|
||||
"env": {
|
||||
"POSTGRES_PASSWORD": "bootstrap",
|
||||
"PGDATA": "/var/lib/postgresql/data/pgdata"
|
||||
},
|
||||
"ports": ["5432:5432"],
|
||||
"volumes": ["mesh-store-data:/var/lib/postgresql/data"]
|
||||
"volumes": ["/var/lib/mesh-store:/var/lib/postgresql/data"]
|
||||
},
|
||||
// Over TCP, not the socket. While the store initialises it runs a temporary server on the
|
||||
// socket ONLY, then stops it and starts the real one — so a socket check passes, the action
|
||||
@@ -99,21 +114,21 @@
|
||||
{
|
||||
"id": "store-ready",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 180); do pg_isready -h 127.0.0.1 -U postgres >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'the store did not answer within 180s; its own last words follow'; pg_isready -h 127.0.0.1 -U postgres; tail -n 20 /var/lib/postgresql/data/log/*.log 2>/dev/null; exit 1"],
|
||||
"verify": ["pg_isready", "-h", "127.0.0.1", "-U", "postgres"]
|
||||
},
|
||||
{
|
||||
"id": "inventory-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE inventory'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw inventory"]
|
||||
},
|
||||
{
|
||||
"id": "identity-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE identity'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw identity"]
|
||||
},
|
||||
@@ -123,46 +138,112 @@
|
||||
{
|
||||
"id": "licences-database",
|
||||
"type": "action",
|
||||
"in": "mesh-store",
|
||||
"in": "postgres",
|
||||
"command": ["sh", "-c", "psql -U postgres -c 'CREATE DATABASE licences'"],
|
||||
"verify": ["sh", "-c", "psql -U postgres -lqt | cut -d'|' -f1 | grep -qw licences"]
|
||||
},
|
||||
{
|
||||
"id": "context-schemas",
|
||||
"type": "action",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:mesh-store",
|
||||
"command": ["docker", "run", "--rm", "--network", "container:postgres",
|
||||
"-e", "MESH_STORE_INVENTORY=postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"-e", "MESH_STORE_IDENTITY=postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"-e", "MESH_STORE_LICENCES=postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"migrate"],
|
||||
"verify": ["sh", "-c", "docker exec mesh-store psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec mesh-store psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec mesh-store psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
"verify": ["sh", "-c", "docker exec postgres psql -U postgres -d inventory -tAc \"select to_regclass('public.node')\" | grep -qx node && docker exec postgres psql -U postgres -d identity -tAc \"select to_regclass('public.signing_key')\" | grep -qx signing_key && docker exec postgres psql -U postgres -d licences -tAc \"select to_regclass('public.licence')\" | grep -qx licence"]
|
||||
},
|
||||
// **The bus is raised where the `nats` module keeps it** (novox/hq ADR 0333 §1): the container
|
||||
// `mesh-broker-nats`, its configuration and user list in /var/lib/nats-module/conf, its streams in
|
||||
// /var/lib/mesh-broker-nats and its certificate in /var/lib/mesh-broker-tls — the names and paths the
|
||||
// catalogue's module declares. So adopting it as that module at the broker step keeps it: the
|
||||
// module's built image takes the place of the upstream one below, once, on the same streams, users
|
||||
// and certificate, and nothing else moves. mesh-host's internal/bootstrap holds the two to each other
|
||||
// against the module's own manifest (busLayout), at the broker step and in its tests.
|
||||
//
|
||||
// Directories on the machine, not volumes the runtime names: the module mounts these paths, and the
|
||||
// control plane, once it runs as the mesh's process, reads the certificate at its path on the machine.
|
||||
// The certificate's directory is readable by everyone because the certificate is public and the control
|
||||
// plane runs as nobody; its key is written 0600 by the program that makes it.
|
||||
{
|
||||
"id": "bus-tls-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-tls",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "broker-certificate",
|
||||
"type": "action",
|
||||
"command": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
||||
"-c", "test -f /tls/tls.crt || (openssl req -x509 -newkey rsa:2048 -nodes -keyout /tls/tls.key -out /tls/tls.crt -days 3650 -subj '/CN=mesh-broker' >/dev/null 2>&1 && chmod 644 /tls/tls.crt && chmod 600 /tls/tls.key)"],
|
||||
"verify": ["docker", "run", "--rm", "--entrypoint", "sh", "-v", "mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
||||
"-c", "test -s /tls/tls.crt && openssl x509 -in /tls/tls.crt -noout"]
|
||||
"id": "bus-data-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-broker-nats",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-certificate",
|
||||
"type": "action",
|
||||
// **The mesh makes its own** (novox/hq 04-ISSUES/146). This ran `openssl` inside the
|
||||
// broker's image while the broker was one that carried it; the bus that replaced it has a
|
||||
// shell and no openssl, and no other image the bundle names has one either. So the program
|
||||
// that needs the certificate writes it — already on this machine, since the schema step ran
|
||||
// it, and asking nothing of the image it writes into. Self-signed on purpose: a host pins
|
||||
// this server's exact certificate (novox/hq ADR 0004), and at this moment there is no mesh
|
||||
// to ask an authority of.
|
||||
// `--user 0:0` because the directory is root's and this image runs as nobody, which is right
|
||||
// for the long-running control plane and wrong for a one-shot writing into a fresh directory.
|
||||
"command": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--into", "/tls"],
|
||||
"verify": ["docker", "run", "--rm", "--user", "0:0", "-v", "/var/lib/mesh-broker-tls:/tls",
|
||||
"192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"broker", "certificate", "--check", "--into", "/tls"]
|
||||
},
|
||||
// **Seeds, written only where nothing is** (create-once). The bus's configuration and user list are
|
||||
// the bundle's until the bus is adopted, and the module's from then on (ADR 0333 §4): an installer
|
||||
// run again after the adoption applies this bundle again, and an ordinary file would put the
|
||||
// bundle's one user back over every user the mesh composed, and leave the node unable to reach its
|
||||
// bus. Until the adoption the installer places the composed list over the seed (enrol.go).
|
||||
{
|
||||
"id": "bus-conf-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/nats-module/conf",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "bus-conf",
|
||||
"type": "file",
|
||||
"path": "/var/lib/nats-module/conf/nats.conf",
|
||||
"mode": "0644",
|
||||
"create-once": true,
|
||||
"content": "port: 4222\nhttp: 0.0.0.0:8222\n\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\ninclude accounts.conf\n"
|
||||
},
|
||||
{
|
||||
"id": "bus-accounts",
|
||||
"type": "file",
|
||||
"path": "/var/lib/nats-module/conf/accounts.conf",
|
||||
"mode": "0600",
|
||||
"create-once": true,
|
||||
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.mesh-controller_predefined.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.predefined-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
|
||||
},
|
||||
// The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
|
||||
// mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned
|
||||
// step (ADR 0236). The release is the one the module's image is built on (its NATS_BASE, 2.11.17),
|
||||
// pinned upstream by that same digest: the adoption changes no server release, so it can be undone
|
||||
// by putting the old image back and is said reversible. The published port is the
|
||||
// node's bus port, 5671 unless given, which every enrolment token names; the installer gives the
|
||||
// module the setting that maps its declared 4222 to it (ADR 0333 §2).
|
||||
{
|
||||
"id": "broker",
|
||||
"type": "container",
|
||||
"name": "mesh-broker",
|
||||
"image": "192.0.2.250:5000/cloudamqp/lavinmq@sha256:b117c254e6e269a29db479e6b410ca4e46e035b4981e49d24b159673ef09d336",
|
||||
"ports": ["5671:5671", "5672:5672", "127.0.0.1:15672:15672"],
|
||||
"volumes": ["mesh-broker-data:/var/lib/lavinmq", "mesh-broker-tls:/tls:ro"],
|
||||
"args": ["--amqps-port=5671", "--cert=/tls/tls.crt", "--key=/tls/tls.key"]
|
||||
"name": "mesh-broker-nats",
|
||||
"image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00",
|
||||
"ports": ["5671:4222", "127.0.0.1:8222:8222"],
|
||||
"volumes": ["/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "/var/lib/mesh-broker-tls:/tls:ro"],
|
||||
"args": ["-c", "/etc/nats/nats.conf", "-js"]
|
||||
},
|
||||
{
|
||||
"id": "broker-ready",
|
||||
"type": "action",
|
||||
"in": "mesh-broker",
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do lavinmqctl status >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["lavinmqctl", "status"]
|
||||
"command": ["sh", "-c", "for i in $(seq 1 60); do docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671' >/dev/null 2>&1 && exit 0; sleep 1; done; exit 1"],
|
||||
"verify": ["sh", "-c", "docker run --rm --network host --entrypoint sh nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00 -c 'nc -z 127.0.0.1 5671'"]
|
||||
},
|
||||
{
|
||||
"id": "control-plane",
|
||||
@@ -171,13 +252,12 @@
|
||||
"image": "192.0.2.250:5000/mesh-controller@sha256:c67db38439ff0aee242b467486765467bb95801f52175fc5727cc4e437338ace",
|
||||
"network": "host",
|
||||
"args": ["serve"],
|
||||
"volumes": ["mesh-broker-tls:/broker-tls:ro"],
|
||||
"volumes": ["/var/lib/mesh-broker-tls:/broker-tls:ro"],
|
||||
"env": {
|
||||
"MESH_STORE_INVENTORY": "postgres://postgres:bootstrap@127.0.0.1:5432/inventory?sslmode=disable",
|
||||
"MESH_STORE_IDENTITY": "postgres://postgres:bootstrap@127.0.0.1:5432/identity?sslmode=disable",
|
||||
"MESH_STORE_LICENCES": "postgres://postgres:bootstrap@127.0.0.1:5432/licences?sslmode=disable",
|
||||
"MESH_BROKER_AMQP": "amqp://guest:guest@127.0.0.1:5672/",
|
||||
"MESH_BROKER_MANAGEMENT": "http://guest:guest@127.0.0.1:15672",
|
||||
"MESH_BUS_NATS": "nats://controller:bootstrap@127.0.0.1:5671",
|
||||
"MESH_BROKER_ADDRESS": "192.0.2.10:5671",
|
||||
"MESH_BROKER_CERTIFICATE": "/broker-tls/tls.crt"
|
||||
}
|
||||
|
||||
@@ -3,14 +3,20 @@ module github.com/novox/mesh-host
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
golang.org/x/sys v0.48.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,11 +1,29 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366 h1:B7oQ82ar2McqZ9OrbJfyvG+50Ol6fBUoK3eYE2uTPvo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
@@ -14,5 +32,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
|
||||
@@ -45,7 +45,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// The states, in the words liveness says them (the controller reads them alike).
|
||||
@@ -65,9 +65,11 @@ const ReasonRoot = "can become root without a person"
|
||||
// RootGroups are the groups whose members become root by membership alone, without a sudo rule naming
|
||||
// them: the administrators' groups a distribution's own rules or polkit treat as root, the container
|
||||
// runtime's socket (a container with the host's root mounted), the raw disks, and the virtualisation
|
||||
// daemons that start a guest with the host's devices. A closed list, so what the judge calls "grants root"
|
||||
// is written down and reviewable rather than guessed per machine.
|
||||
var RootGroups = []string{"root", "wheel", "sudo", "admin", "docker", "disk", "lxd", "incus-admin", "libvirt"}
|
||||
// daemons that start a guest with the host's devices — and the kernel's memory (kmem) and the password hashes
|
||||
// (shadow), either of which is root a step away (novox/hq ADR 0312 §4 review). A closed list, so what the judge
|
||||
// calls "grants root" is written down and reviewable rather than guessed per machine.
|
||||
var RootGroups = []string{"root", "wheel", "sudo", "admin", "docker", "disk", "lxd", "incus-admin", "libvirt",
|
||||
"kmem", "shadow"}
|
||||
|
||||
// Account is one user resource of a module that declares groups.
|
||||
type Account struct {
|
||||
@@ -117,7 +119,7 @@ func secretsOf(d *declaration.Declaration, account string) []string {
|
||||
var out []string
|
||||
for _, r := range d.Resources {
|
||||
f, ok := r.(*declaration.File)
|
||||
if !ok || f.Path == "" || (f.Sealed == "" && len(f.Secrets) == 0) || f.Owner == account {
|
||||
if !ok || f.Path == "" || (!f.Secret() && len(f.Secrets) == 0) || f.Owner == account {
|
||||
continue
|
||||
}
|
||||
out = append(out, f.Path)
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// The account judge (novox/hq ADR 0252, "how it is checked"): an account in a group its running session
|
||||
|
||||
@@ -285,3 +285,32 @@ func (e Exec) heldBy(pid string) (map[string]bool, error) {
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// IDs is `getent passwd`: an account's uid and primary gid, and whether the user database names it at all. A
|
||||
// database that does not answer is an error, never "no such account".
|
||||
func (e Exec) IDs(ctx context.Context, account string) (uid, gid int, found bool, err error) {
|
||||
out, err := e.Run(ctx, "getent", "passwd", account)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "exited 2") || strings.Contains(err.Error(), "exit status 2") {
|
||||
return 0, 0, false, nil
|
||||
}
|
||||
return 0, 0, false, fmt.Errorf("the user database did not answer about %q: %w", account, err)
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(out), ":")
|
||||
if len(fields) < 4 {
|
||||
return 0, 0, false, fmt.Errorf("the user database gave %q for %q, which is not a passwd entry",
|
||||
strings.TrimSpace(out), account)
|
||||
}
|
||||
// getent reads an all-digit key as a uid: "1000" answers with whoever has uid 1000. Only the account named is
|
||||
// the one asked for.
|
||||
if fields[0] != account {
|
||||
return 0, 0, false, fmt.Errorf("the user database answered %q with the account %q: an account is named, "+
|
||||
"never numbered", account, fields[0])
|
||||
}
|
||||
uid, uerr := strconv.Atoi(fields[2])
|
||||
gid, gerr := strconv.Atoi(fields[3])
|
||||
if uerr != nil || gerr != nil {
|
||||
return 0, 0, false, fmt.Errorf("the user database gave no numbers for %q", account)
|
||||
}
|
||||
return uid, gid, true, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package accounts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// An account is read by its name (novox/hq ADR 0325 §4): getent reads an all-digit key as a uid, so "1000" would
|
||||
// answer with whoever has uid 1000 — refused, never taken for the account named.
|
||||
func TestAnAccountIsReadByItsNameNeverItsNumber(t *testing.T) {
|
||||
db := map[string]string{
|
||||
"nox": "nox:x:1003:1003::/home/nox:/bin/sh",
|
||||
"1000": "jochen:x:1000:1000::/home/jochen:/bin/zsh", // what getent answers for a uid
|
||||
}
|
||||
e := Exec{Run: func(_ context.Context, name string, args ...string) (string, error) {
|
||||
if line, ok := db[args[len(args)-1]]; ok {
|
||||
return line + "\n", nil
|
||||
}
|
||||
return "", errors.New("getent exited 2")
|
||||
}}
|
||||
if uid, gid, found, err := e.IDs(context.Background(), "nox"); err != nil || !found || uid != 1003 || gid != 1003 {
|
||||
t.Fatalf("nox: %d %d %v %v", uid, gid, found, err)
|
||||
}
|
||||
if _, _, found, err := e.IDs(context.Background(), "1000"); found || err == nil || !strings.Contains(err.Error(), "jochen") {
|
||||
t.Fatalf("an all-digit name was read as a uid: found=%v err=%v", found, err)
|
||||
}
|
||||
if _, _, found, err := e.IDs(context.Background(), "ghost"); found || err != nil {
|
||||
t.Fatalf("an absent account: found=%v err=%v", found, err)
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// An account declared never to become root without a person (novox/hq ADR 0266, "how it is checked"): each
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,6 @@ package apply
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
@@ -32,28 +31,6 @@ func TestAnAccessPresentIsConfirmedAndNothingIsChanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Absent is refused, not created. A bind mount whose source does not exist is made by the
|
||||
// container runtime as root, with whatever mode it picks — the silent wrong-ownership
|
||||
// 04-ISSUES/026 records. So the host says plainly that the operator must provide the path, rather
|
||||
// than conjuring a directory it does not own.
|
||||
func TestAnAccessThatIsAbsentIsRefusedClearlyAndNotCreated(t *testing.T) {
|
||||
missing := t.TempDir() + "/media/library" // named, never created
|
||||
d := declare(t, `{"id":"lib","type":"access","path":"`+missing+`"}`)
|
||||
|
||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{},
|
||||
store.OriginDeclared, noServices, nil, nil)
|
||||
if err == nil {
|
||||
t.Fatal("an absent operator-owned path was accepted, and would be created as root by the runtime")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "the operator must provide") ||
|
||||
!strings.Contains(err.Error(), "does not own") {
|
||||
t.Fatalf("the refusal does not say whose the path is: %v", err)
|
||||
}
|
||||
if _, statErr := os.Stat(missing); statErr == nil {
|
||||
t.Fatal("the host created the path it does not own")
|
||||
}
|
||||
}
|
||||
|
||||
// Undeclaring an access never removes the path. Unassigning the module that reached the media
|
||||
// library must not delete the library — that is the data loss ADR 0030 exists to prevent, on a
|
||||
// directory the mesh never made. The record is dropped; the operator's data is left exactly as it
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// AgentHomeRefusedError is an archive or a written-into file the engine will not place below an agent's home.
|
||||
|
||||
@@ -6,7 +6,7 @@ package apply
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
func TestNothingIsUnpackedOrWrittenIntoBelowAnAgentsHome(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// WriteAlone writes one file a module's composition declares, through the apply's own file step, and records it as an
|
||||
// apply would; nothing else of the module is applied, and nothing that reads the file is moved (novox/hq ADR 0547 §8).
|
||||
//
|
||||
// **For a file whose reader takes the change itself.** The bus's user list is the one it is for: the bus module's
|
||||
// entrypoint watches the file and signals the server to reload in place, so the list is in force without a want of the
|
||||
// bus, which nobody writes (ADR 0539 §15), and without the container being touched. A file that a service or a container
|
||||
// must be moved to read is not written alone: it goes through the module's apply, which moves what reads it.
|
||||
//
|
||||
// The file is written only when its content differs from what is there (the file step's own rule), read back, and
|
||||
// recorded under the module whose resource it is, so the module's next apply finds it as its own and writes nothing.
|
||||
// The caller holds the apply's lock. log is said the path, the action and the digest, never the content.
|
||||
func WriteAlone(ctx context.Context, sys system.System, file *declaration.File, known store.State, run Runner,
|
||||
log func(string)) (Outcome, store.State, error) {
|
||||
if log == nil {
|
||||
log = func(string) {}
|
||||
}
|
||||
if file.Into != "" || file.CreateOnce || file.Sealed != "" || file.VaultSealed != nil || len(file.Secrets) > 0 {
|
||||
return Outcome{}, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is not a whole file of plain content, "+
|
||||
"and only such a file is written alone; nothing was written", file.Path)}
|
||||
}
|
||||
if file.IsReference() {
|
||||
return Outcome{}, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is given by digest %s and was not "+
|
||||
"resolved to its content; it is not written empty", file.Path, file.Digest)}
|
||||
}
|
||||
if held, isHeld := known.HeldAt(file.ID); isHeld {
|
||||
return Outcome{}, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
|
||||
"yet taken (%s), so nothing was written", file.Path, held.Why)}
|
||||
}
|
||||
was, _ := known.Find(file.ID)
|
||||
if was.ID == "" {
|
||||
// Only over a file the mesh wrote: a file this node holds no record of is the module's apply's to take, with
|
||||
// the original kept, never this write's. A record of the path under another id (the declaration before the
|
||||
// switch named it so) is the one written to, so no second record of the file is left for an apply to remove.
|
||||
rec, ok := known.At(string(declaration.TypeFile), file.Path)
|
||||
if !ok {
|
||||
return Outcome{}, known, &Error{Resource: file.ID, Err: fmt.Errorf("this node holds no record of %s; it "+
|
||||
"is written by the module's apply first, never alone", file.Path)}
|
||||
}
|
||||
renamed := *file
|
||||
renamed.ID = rec.ID
|
||||
file, was = &renamed, rec
|
||||
}
|
||||
in := inputs{declares: map[string]string{file.Identity(): declaredDigest(file)}, known: &known,
|
||||
groups: groupsWanted([]declaration.Resource{file}), handed: Handed{}, dirsBefore: map[string]bool{},
|
||||
files: map[string]bool{file.Path: true}}
|
||||
outcome, err := applyOne(ctx, sys, file, run, map[string]bool{}, in, was, nil, nil)
|
||||
if err != nil {
|
||||
log(fmt.Sprintf(" failed %s (%s), alone: %v", file.ID, file.Path, err))
|
||||
return outcome, known, &Error{Resource: file.ID, Err: err}
|
||||
}
|
||||
recordKeeping(&known, file, outcome, was)
|
||||
if moved(outcome) {
|
||||
log(fmt.Sprintf(" %s %s (%s), alone: %s", outcome.Action, file.ID, file.Path, outcome.wrote))
|
||||
}
|
||||
return outcome, known, nil
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0547 §8: the bus node writes the bus's user list alone, with no want, through the apply's own
|
||||
// file step: only when it differs, mode 0600, read back and recorded as the module's, and nothing else moves.
|
||||
func TestAFileWrittenAloneMovesNothingElse(t *testing.T) {
|
||||
const was, now = "user a\n", "user a\nuser b\n"
|
||||
for _, c := range []struct {
|
||||
name, fields, before string
|
||||
recorded bool
|
||||
recordedAs string // the id the record has, when not the composition's
|
||||
action, refused string
|
||||
after string
|
||||
}{
|
||||
{name: "no record of it", fields: `"content":%q,"mode":"0600"`, before: was, refused: "holds no record", after: was},
|
||||
{name: "changed", fields: `"content":%q,"mode":"0600"`, before: was, recorded: true, action: "updated", after: now},
|
||||
{name: "the same", fields: `"content":%q,"mode":"0600"`, before: now, recorded: true, action: "unchanged", after: now},
|
||||
{name: "recorded under the id from before the switch", fields: `"content":%q,"mode":"0600"`, before: was,
|
||||
recorded: true, recordedAs: "bus-users", action: "updated", after: now},
|
||||
{name: "absent with a record", fields: `"content":%q,"mode":"0600"`, recorded: true, action: "created", after: now},
|
||||
{name: "written into", fields: `"content":%q,"into":"block","owner-at-runtime":"managed"`, before: was,
|
||||
recorded: true, refused: "only such a file", after: was},
|
||||
{name: "sealed", fields: `"sealed":"c2VhbGVk"`, before: was, recorded: true,
|
||||
refused: "only such a file", after: was},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "accounts.conf")
|
||||
if c.before != "" {
|
||||
if err := os.WriteFile(path, []byte(c.before), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
fields := strings.Replace(c.fields, "%q", fmt.Sprintf("%q", now), 1)
|
||||
d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"nats.bus-users","module":"nats",
|
||||
"type":"file","path":%q,`+fields+`}]}`, path))
|
||||
file := d.Resources[0].(*declaration.File)
|
||||
known := store.State{}
|
||||
id := file.ID
|
||||
if c.recordedAs != "" {
|
||||
id = c.recordedAs
|
||||
}
|
||||
if c.recorded {
|
||||
known.Record(store.Applied{ID: id, Type: string(declaration.TypeFile), Module: "nats", Target: path,
|
||||
Origin: store.OriginDeclared, Wrote: digestOf(c.before)})
|
||||
}
|
||||
calls := 0
|
||||
counting := func(context.Context, string, ...string) (string, error) { calls++; return "", nil }
|
||||
out, after, err := WriteAlone(context.Background(), archHost(t), file, known, counting, nil)
|
||||
if c.refused != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.refused) {
|
||||
t.Fatalf("not refused with %q: %v", c.refused, err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if out.Action != c.action {
|
||||
t.Errorf("action %q, want %q", out.Action, c.action)
|
||||
}
|
||||
if got := readText(t, path); got != c.after {
|
||||
t.Errorf("file holds %q, want %q", got, c.after)
|
||||
}
|
||||
if calls != 0 {
|
||||
t.Errorf("%d commands were run: a file written alone moves nothing", calls)
|
||||
}
|
||||
if c.refused != "" {
|
||||
return
|
||||
}
|
||||
if m := modeAt(t, path); m != 0o600 {
|
||||
t.Errorf("mode %o", m)
|
||||
}
|
||||
rec, ok := after.Find(id)
|
||||
if !ok || rec.Wrote != digestOf(now) || rec.Target != path {
|
||||
t.Errorf("record %+v (found %v), want it to say %s was written", rec, ok, digestOf(now))
|
||||
}
|
||||
if n := len(after.Resources); n != 1 {
|
||||
t.Errorf("%d records of one file", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+772
-175
File diff suppressed because it is too large
Load Diff
@@ -9,7 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335).
|
||||
//
|
||||
// **Asked, never waited on.** The apply asks the container once, and while it says busy leaves it running
|
||||
// as it is and goes on with everything else of the declaration: the machine's lock is held for one ask,
|
||||
// never for the wait. The wait is carried between passes in the container's record (store.Applied's
|
||||
// BusySince), so the next pass — the node-engine asks again sooner than its reconcile while anything waits
|
||||
// — asks again, and so does a node-engine started again in between: the bound runs from the first apply
|
||||
// that would have replaced the container, whoever applied it.
|
||||
|
||||
// waitsBusy is the action of a container left running because it said busy (novox/hq ADR 0335 §2). Its own
|
||||
// word rather than "unchanged": its declaration moved, and the machine is not yet what was declared.
|
||||
const waitsBusy = "waits-busy"
|
||||
|
||||
// What a report says of a busy container (novox/hq ADR 0335 §2, §3).
|
||||
const (
|
||||
// BusyWaiting is a container left running because it said busy, waiting to be replaced.
|
||||
BusyWaiting = "waiting"
|
||||
// BusyReplacedIdle is a container replaced on the first pass it said idle, after it had waited.
|
||||
BusyReplacedIdle = "replaced-when-idle"
|
||||
// BusyReplacedAtBound is a container still busy at the bound, replaced anyway.
|
||||
BusyReplacedAtBound = "replaced-at-bound"
|
||||
)
|
||||
|
||||
// BusyWait is one container a busy command held, as a report says it.
|
||||
type BusyWait struct {
|
||||
// ID is the resource's id (`<module>.<resource>`), Container its runtime name.
|
||||
ID string `json:"id"`
|
||||
Container string `json:"container"`
|
||||
// State is BusyWaiting, BusyReplacedIdle or BusyReplacedAtBound.
|
||||
State string `json:"state"`
|
||||
// Since is when the first apply that would have replaced it ran; Until when the bound replaces it.
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
// Said is the last output its busy command gave.
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// Words is the wait in a sentence, for an outcome's detail and the console.
|
||||
func (b BusyWait) Words() string {
|
||||
since := b.Since.UTC().Format(time.RFC3339)
|
||||
said := ""
|
||||
if b.Said != "" {
|
||||
said = "; its busy command last said: " + b.Said
|
||||
}
|
||||
switch b.State {
|
||||
case BusyReplacedAtBound:
|
||||
return fmt.Sprintf("replaced while still busy: it said busy since %s and the bound of %s ran out "+
|
||||
"(novox/hq ADR 0335 §3)%s", since, declaration.BusyBound, said)
|
||||
case BusyReplacedIdle:
|
||||
return fmt.Sprintf("replaced once it said idle, after waiting since %s (novox/hq ADR 0335 §2)%s", since, said)
|
||||
}
|
||||
return fmt.Sprintf("busy: left running and replaced on the first pass it says idle, or at %s at the latest; "+
|
||||
"waiting since %s (novox/hq ADR 0335 §2)%s", b.Until.UTC().Format(time.RFC3339), since, said)
|
||||
}
|
||||
|
||||
// owedSinceBusy is the reason a container still owes the replacement it was left running for, or empty: its
|
||||
// record keeps when its busy wait began, under the same name.
|
||||
func owedSinceBusy(r *declaration.Container, previous store.Applied) string {
|
||||
if previous.BusySince == nil || previous.Target != r.Name {
|
||||
return ""
|
||||
}
|
||||
return "the replacement owed since its busy wait began at " + previous.BusySince.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// busyClock is the clock the bound is read on. A test moves it.
|
||||
var busyClock = time.Now
|
||||
|
||||
// busySaidMost is how much of a busy command's output a report carries: its last lines, enough to say what
|
||||
// it held.
|
||||
const busySaidMost = 400
|
||||
|
||||
// askBeforeReplacing asks a running container that declares busy whether it may be replaced now, and says
|
||||
// in out what came of it. True when it waits: out is then the whole outcome of this pass. False when the
|
||||
// caller replaces it — idle, or past the bound — with out carrying the wait it ended, if there was one.
|
||||
func askBeforeReplacing(ctx context.Context, r *declaration.Container, run Runner, cri string,
|
||||
previous store.Applied, out *Outcome) bool {
|
||||
now := busyClock().UTC()
|
||||
busy, said := askBusy(ctx, r, run, cri)
|
||||
var since time.Time
|
||||
waited := previous.BusySince != nil && previous.Target == r.Name
|
||||
if waited {
|
||||
since = previous.BusySince.UTC()
|
||||
// A wait kept from a clock that has since gone back reads as starting now, never as later: the bound
|
||||
// holds from no later than this pass.
|
||||
if since.After(now) {
|
||||
since = now
|
||||
}
|
||||
}
|
||||
wait := BusyWait{ID: r.ID, Container: r.Name, Since: since, Said: said}
|
||||
switch {
|
||||
case !busy && !waited:
|
||||
// Idle at the first ask: replaced at once, as a container that declares nothing is.
|
||||
return false
|
||||
case !busy:
|
||||
wait.State, wait.Until = BusyReplacedIdle, since.Add(declaration.BusyBound)
|
||||
out.busy = &wait
|
||||
return false
|
||||
}
|
||||
if !waited {
|
||||
since = now
|
||||
wait.Since = now
|
||||
}
|
||||
wait.Until = since.Add(declaration.BusyBound)
|
||||
if !now.Before(wait.Until) {
|
||||
wait.State = BusyReplacedAtBound
|
||||
out.busy = &wait
|
||||
return false
|
||||
}
|
||||
wait.State = BusyWaiting
|
||||
out.Action, out.Detail = waitsBusy, wait.Words()
|
||||
out.busySince, out.busy = &since, &wait
|
||||
return true
|
||||
}
|
||||
|
||||
// askBusy runs a container's busy command inside it, by its shell, as an exec health check is run: busy is
|
||||
// an exit of 0 and nothing else. An ask that could not be made, or did not answer within its timeout, is
|
||||
// not busy: the bound is what keeps a container whose command never answers from being held.
|
||||
func askBusy(ctx context.Context, r *declaration.Container, run Runner, cri string) (bool, string) {
|
||||
ask, cancel := context.WithTimeout(ctx, r.Busy.Within())
|
||||
defer cancel()
|
||||
output, err := run(ask, cri, "exec", r.Name, "/bin/sh", "-c", r.Busy.Command)
|
||||
said := strings.TrimSpace(output)
|
||||
if err != nil && said == "" {
|
||||
said = err.Error()
|
||||
}
|
||||
return err == nil, lastOf(said, busySaidMost)
|
||||
}
|
||||
|
||||
// lastOf is the end of a text, at most n bytes, on one line.
|
||||
func lastOf(s string, n int) string {
|
||||
s = strings.Join(strings.Fields(s), " ")
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
from := len(s) - n
|
||||
for from < len(s) && !utf8.RuneStart(s[from]) {
|
||||
from++
|
||||
}
|
||||
return "…" + s[from:]
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A container that says it is busy is replaced once it is idle, within a bound (novox/hq ADR 0335). The
|
||||
// machine of these tests runs two containers, a build agent that declares `busy` and a web server that
|
||||
// does not; every call that replaces a container is counted.
|
||||
|
||||
type aBusyMachine struct {
|
||||
mu sync.Mutex
|
||||
// containers is each container's name and the spec it was created with; running whether it runs.
|
||||
containers map[string]string
|
||||
busy bool // what the agent's busy command answers
|
||||
said string // what it prints
|
||||
asked int
|
||||
calls []string
|
||||
}
|
||||
|
||||
func newBusyMachine() *aBusyMachine {
|
||||
return &aBusyMachine{containers: map[string]string{}}
|
||||
}
|
||||
|
||||
func (m *aBusyMachine) run(_ context.Context, _ string, args ...string) (string, error) {
|
||||
args = asideShape(args)
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
switch args[0] {
|
||||
case "info", "image":
|
||||
return "27.0\n", nil
|
||||
case "container": // inspect
|
||||
name := args[len(args)-1]
|
||||
spec, there := m.containers[name]
|
||||
if !there {
|
||||
return "", errors.New("no such container")
|
||||
}
|
||||
return "true\t" + spec, nil
|
||||
case "exec":
|
||||
m.asked++
|
||||
m.calls = append(m.calls, "ask "+args[1])
|
||||
if strings.Join(args[2:], " ") != "/bin/sh -c mesh-builder busy" {
|
||||
return "", errors.New("asked something else: " + strings.Join(args[2:], " "))
|
||||
}
|
||||
if m.busy {
|
||||
return m.said + "\n", nil
|
||||
}
|
||||
return m.said + "\n", errors.New("exited 1")
|
||||
case "rm":
|
||||
name := args[len(args)-1]
|
||||
m.calls = append(m.calls, "rm "+name)
|
||||
delete(m.containers, name)
|
||||
case "run":
|
||||
name := args[slices.Index(args, "--name")+1]
|
||||
m.calls = append(m.calls, "create "+name)
|
||||
for i, a := range args {
|
||||
if a == "--label" && strings.HasPrefix(args[i+1], specLabel+"=") {
|
||||
m.containers[name] = strings.TrimPrefix(args[i+1], specLabel+"=")
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (m *aBusyMachine) mark() int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return len(m.calls)
|
||||
}
|
||||
|
||||
// since is every call since mark that replaced a container, or asked one.
|
||||
func (m *aBusyMachine) since(mark int) []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return append([]string(nil), m.calls[mark:]...)
|
||||
}
|
||||
|
||||
func (m *aBusyMachine) answer(busy bool, said string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.busy, m.said = busy, said
|
||||
}
|
||||
|
||||
// twoContainers is the declaration at a version: v moves the agent's image tag through its env, w the web's.
|
||||
func twoContainers(t *testing.T, agent, web string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
return parse(t, `{"declaration":1,"resources":[
|
||||
{"id":"build-agent.server","type":"container","name":"mesh-build-agent","image":"`+pinned+`",
|
||||
"env":{"V":"`+agent+`"},"busy":{"command":"mesh-builder busy","timeout":"5s"}},
|
||||
{"id":"web.server","type":"container","name":"web","image":"`+pinned+`","env":{"V":"`+web+`"}}
|
||||
]}`)
|
||||
}
|
||||
|
||||
// atClock moves the bound's clock for one test.
|
||||
func atClock(t *testing.T, at *time.Time) {
|
||||
t.Helper()
|
||||
was := busyClock
|
||||
busyClock = func() time.Time { return *at }
|
||||
t.Cleanup(func() { busyClock = was })
|
||||
}
|
||||
|
||||
func busyOf(r Report, id string) (BusyWait, bool) {
|
||||
for _, b := range r.Busy {
|
||||
if b.ID == id {
|
||||
return b, true
|
||||
}
|
||||
}
|
||||
return BusyWait{}, false
|
||||
}
|
||||
|
||||
func applyBusy(t *testing.T, m *aBusyMachine, d *declaration.Declaration, known store.State) (Report, store.State) {
|
||||
t.Helper()
|
||||
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("the apply failed: %v", err)
|
||||
}
|
||||
return report, state
|
||||
}
|
||||
|
||||
// The decision's first check: an apply that would replace a container whose busy command exits 0 leaves it
|
||||
// running, applies the rest, and replaces it on the first pass the command exits non-zero.
|
||||
func TestABusyContainerIsLeftRunningAndReplacedOnTheFirstPassItSaysIdle(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
|
||||
// A merge moves both: the agent holds a build, the web server holds nothing.
|
||||
m.answer(true, "building novox/mesh-controller (check of #280) since 00:58")
|
||||
mark := m.mark()
|
||||
report, state := applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm web", "create web"}) {
|
||||
t.Fatalf("a busy agent was not left running while the rest applied: %v", got)
|
||||
}
|
||||
o := outcomeOf(report, "build-agent.server")
|
||||
if o.Action != waitsBusy {
|
||||
t.Fatalf("a busy container was reported %q, want %q: %+v", o.Action, waitsBusy, o)
|
||||
}
|
||||
for _, says := range []string{"busy", "first pass it says idle", "check of #280", "ADR 0335"} {
|
||||
if !strings.Contains(o.Detail, says) {
|
||||
t.Errorf("the outcome does not say %q: %s", says, o.Detail)
|
||||
}
|
||||
}
|
||||
if (Report{Outcomes: []Outcome{o}}).Changed() {
|
||||
t.Errorf("a container left running reads as a change: %+v", o)
|
||||
}
|
||||
b, said := busyOf(report, "build-agent.server")
|
||||
if !said || b.State != BusyWaiting || !b.Since.Equal(now) || !b.Until.Equal(now.Add(15*time.Minute)) ||
|
||||
b.Container != "mesh-build-agent" || !strings.Contains(b.Said, "check of #280") {
|
||||
t.Fatalf("the report does not say the agent waits, since when and until when: %+v", report.Busy)
|
||||
}
|
||||
if !report.Waiting() {
|
||||
t.Error("a report with a container waiting does not say so")
|
||||
}
|
||||
if rec, _ := state.Find("build-agent.server"); rec.BusySince == nil || !rec.BusySince.Equal(now) {
|
||||
t.Fatalf("the wait is not kept in the container's record: %+v", rec)
|
||||
}
|
||||
|
||||
// Asked again a minute later, still busy: still left running, the wait counted from the first apply.
|
||||
now = now.Add(time.Minute)
|
||||
mark = m.mark()
|
||||
report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent"}) {
|
||||
t.Fatalf("a second pass while busy did more than ask: %v", got)
|
||||
}
|
||||
if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-time.Minute)) {
|
||||
t.Fatalf("the second pass does not keep the wait from the first apply: %+v", report.Busy)
|
||||
}
|
||||
|
||||
// The build ends: the next pass replaces it, and says it waited.
|
||||
now = now.Add(time.Minute)
|
||||
m.answer(false, "idle")
|
||||
mark = m.mark()
|
||||
report, state = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("the agent was not replaced on the first pass it said idle: %v", got)
|
||||
}
|
||||
if o := outcomeOf(report, "build-agent.server"); o.Action != "updated" || !strings.Contains(o.Detail, "once it said idle") {
|
||||
t.Errorf("the replacement does not say it waited: %+v", o)
|
||||
}
|
||||
if b, _ := busyOf(report, "build-agent.server"); b.State != BusyReplacedIdle {
|
||||
t.Errorf("the report does not say the agent was replaced when idle: %+v", report.Busy)
|
||||
}
|
||||
if report.Waiting() {
|
||||
t.Error("a report with nothing waiting says something waits")
|
||||
}
|
||||
if rec, _ := state.Find("build-agent.server"); rec.BusySince != nil {
|
||||
t.Errorf("the wait is still kept after the replacement: %+v", rec)
|
||||
}
|
||||
|
||||
// And the pass after that has nothing to say about it.
|
||||
report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
if len(report.Busy) != 0 {
|
||||
t.Errorf("a container replaced is still said: %+v", report.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// An idle container is replaced at once, as one that declares nothing is: asked once, nothing waits.
|
||||
func TestAnIdleContainerIsReplacedAtOnce(t *testing.T) {
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(false, "")
|
||||
mark := m.mark()
|
||||
report, _ := applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("an idle agent was not replaced at once: %v", got)
|
||||
}
|
||||
if len(report.Busy) != 0 {
|
||||
t.Errorf("an idle container replaced at once is said as a wait: %+v", report.Busy)
|
||||
}
|
||||
// And a container whose declaration did not move is never asked.
|
||||
mark = m.mark()
|
||||
applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
if got := m.since(mark); slices.Contains(got, "ask mesh-build-agent") {
|
||||
t.Errorf("a container nothing replaces was asked: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Past the bound it is replaced anyway, the report carrying the command's last output — and the bound holds
|
||||
// across a restart of the node-engine: the wait is read back from the state on disk.
|
||||
func TestABusyContainerIsReplacedAtTheBoundEvenAfterARestart(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(true, "building novox/mesh-host (a check that never ends)")
|
||||
_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
|
||||
// The node-engine stops and starts again: what it knows is what it saved.
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
if err := store.Save(path, state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloaded, err := store.Load(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now = now.Add(14 * time.Minute)
|
||||
mark := m.mark()
|
||||
report, reloaded := applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
|
||||
if got := m.since(mark); slices.Contains(got, "rm mesh-build-agent") {
|
||||
t.Fatalf("a container busy within the bound was replaced after a restart: %v", got)
|
||||
}
|
||||
if b, _ := busyOf(report, "build-agent.server"); b.State != BusyWaiting || !b.Since.Equal(now.Add(-14*time.Minute)) {
|
||||
t.Fatalf("a restart started the wait again: %+v", report.Busy)
|
||||
}
|
||||
|
||||
now = now.Add(time.Minute)
|
||||
mark = m.mark()
|
||||
report, _ = applyBusy(t, m, twoContainers(t, "2", "1"), reloaded)
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("a container still busy at the bound was not replaced: %v", got)
|
||||
}
|
||||
b, _ := busyOf(report, "build-agent.server")
|
||||
if b.State != BusyReplacedAtBound || !strings.Contains(b.Said, "never ends") {
|
||||
t.Fatalf("the report does not say it replaced a busy container, with its last output: %+v", report.Busy)
|
||||
}
|
||||
if o := outcomeOf(report, "build-agent.server"); !strings.Contains(o.Detail, "still busy") ||
|
||||
!strings.Contains(o.Detail, "never ends") {
|
||||
t.Errorf("the outcome does not say a busy container was replaced, with what it said: %s", o.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// While one container waits, a send that changes another module on the same machine is applied at once: the
|
||||
// apply returns after one ask, and holds nothing for the wait.
|
||||
func TestASendToAnotherModuleIsAppliedWhileOneWaits(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(true, "building")
|
||||
_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
|
||||
// The next send moves only the web server.
|
||||
mark := m.mark()
|
||||
done := make(chan struct{})
|
||||
var report Report
|
||||
go func() {
|
||||
defer close(done)
|
||||
report, _ = applyBusy(t, m, twoContainers(t, "2", "2"), state)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("an apply while a container waits did not return")
|
||||
}
|
||||
got := m.since(mark)
|
||||
if !slices.Contains(got, "create web") || slices.Contains(got, "rm mesh-build-agent") {
|
||||
t.Fatalf("the other module's send was not applied at once beside the wait: %v", got)
|
||||
}
|
||||
if o := outcomeOf(report, "web.server"); o.Action != "updated" {
|
||||
t.Errorf("the web server was reported %q: %+v", o.Action, o)
|
||||
}
|
||||
}
|
||||
|
||||
// A container's busy is held to its shape where it arrives.
|
||||
func TestBusyIsReadAndHeldToItsShape(t *testing.T) {
|
||||
d := twoContainers(t, "1", "1")
|
||||
c := d.Resources[0].(*declaration.Container)
|
||||
if c.Busy == nil || c.Busy.Command != "mesh-builder busy" || c.Busy.Within() != 5*time.Second {
|
||||
t.Fatalf("busy was not read: %+v", c.Busy)
|
||||
}
|
||||
for name, raw := range map[string]string{
|
||||
"no command": `"busy":{"command":" "}`,
|
||||
"a long timeout": `"busy":{"command":"x","timeout":"5m"}`,
|
||||
"not a duration": `"busy":{"command":"x","timeout":"soon"}`,
|
||||
"an unknown key": `"busy":{"command":"x","every":"1m"}`,
|
||||
"on a step": `"run-once":true,"busy":{"command":"x"}`,
|
||||
} {
|
||||
_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"a.b","type":"container",` +
|
||||
`"name":"b","image":"` + pinned + `",` + raw + `}]}`))
|
||||
if err == nil {
|
||||
t.Errorf("%s: a busy of that shape was accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A replacement owed to a restart-on trigger is still owed after the wait: a trigger that is no file or
|
||||
// directory (a service, an identity's or a library's change) is in no container's spec and counts as changed only
|
||||
// in the pass it changed, so the container must not read unchanged at the next pass and drop from the report as if
|
||||
// replaced (novox/hq ADR 0335).
|
||||
func TestAReplacementOwedToARestartTriggerIsMadeOnceIdle(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
d := parse(t, `{"declaration":1,"resources":[
|
||||
{"id":"build-agent.server","type":"container","name":"mesh-build-agent","image":"`+pinned+`",
|
||||
"restart-on":["identity:build-agent"],"busy":{"command":"mesh-builder busy"}}
|
||||
]}`)
|
||||
r := d.Resources[0].(*declaration.Container)
|
||||
in := inputs{declares: map[string]string{}}
|
||||
m := newBusyMachine()
|
||||
m.containers["mesh-build-agent"] = containerSpec(r, in)
|
||||
ctx := context.Background()
|
||||
|
||||
// The trigger moved this pass, and the agent holds a build: left running.
|
||||
m.answer(true, "building")
|
||||
out, err := applyContainer(ctx, r, m.run, map[string]bool{"identity:build-agent": true}, in, store.Applied{})
|
||||
if err != nil || out.Action != waitsBusy || out.busySince == nil {
|
||||
t.Fatalf("a busy container was not left running for its trigger: %+v %v", out, err)
|
||||
}
|
||||
record := store.Applied{ID: r.ID, Target: r.Name, BusySince: out.busySince}
|
||||
|
||||
// The next pass: nothing triggers now, and the replacement is still owed.
|
||||
now = now.Add(time.Minute)
|
||||
m.answer(false, "idle")
|
||||
mark := m.mark()
|
||||
out, err = applyContainer(ctx, r, m.run, map[string]bool{}, in, record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("the replacement owed to the trigger was dropped: %v (%+v)", got, out)
|
||||
}
|
||||
if out.busy == nil || out.busy.State != BusyReplacedIdle || out.busySince != nil {
|
||||
t.Fatalf("the replacement does not say it was made once idle: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
// A wait kept from a clock that later went back is never longer than the bound from now.
|
||||
func TestAWaitFromAClockThatWentBackIsClampedToNow(t *testing.T) {
|
||||
now := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
atClock(t, &now)
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(true, "building")
|
||||
_, state = applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
now = now.Add(-time.Hour) // the clock went back an hour
|
||||
report, _ := applyBusy(t, m, twoContainers(t, "2", "1"), state)
|
||||
b, _ := busyOf(report, "build-agent.server")
|
||||
if b.State != BusyWaiting || b.Since.After(now) || !b.Until.Equal(now.Add(15*time.Minute)) {
|
||||
t.Fatalf("a wait from a later clock is not clamped to now: %+v (now %s)", b, now)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask that fails — no shell in the image, a timeout, a paused container — is not busy: replaced at once.
|
||||
func TestAnAskThatFailsIsIdle(t *testing.T) {
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
failing := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
if args[0] == "exec" {
|
||||
m.mu.Lock()
|
||||
m.calls = append(m.calls, "ask "+args[1])
|
||||
m.mu.Unlock()
|
||||
return "", errors.New("OCI runtime exec failed: exec: \"/bin/sh\": no such file or directory")
|
||||
}
|
||||
return m.run(ctx, name, args...)
|
||||
}
|
||||
mark := m.mark()
|
||||
report, _, err := Apply(context.Background(), archHost(t), twoContainers(t, "2", "1"), state,
|
||||
store.OriginDeclared, failing, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := m.since(mark); !slices.Equal(got, []string{"ask mesh-build-agent", "rm mesh-build-agent", "create mesh-build-agent"}) {
|
||||
t.Fatalf("an ask that failed held the container: %v", got)
|
||||
}
|
||||
if len(report.Busy) != 0 {
|
||||
t.Errorf("an ask that failed is said as a wait: %+v", report.Busy)
|
||||
}
|
||||
}
|
||||
|
||||
// A stopped container holds no work: it is replaced without being asked.
|
||||
func TestAStoppedContainerIsNotAsked(t *testing.T) {
|
||||
m := newBusyMachine()
|
||||
_, state := applyBusy(t, m, twoContainers(t, "1", "1"), store.State{})
|
||||
m.answer(true, "would say busy")
|
||||
stopped := func(ctx context.Context, name string, args ...string) (string, error) {
|
||||
if args[0] == "container" && args[len(args)-1] == "mesh-build-agent" {
|
||||
out, err := m.run(ctx, name, args...)
|
||||
return strings.Replace(out, "true\t", "false\t", 1), err
|
||||
}
|
||||
return m.run(ctx, name, args...)
|
||||
}
|
||||
mark := m.mark()
|
||||
if _, _, err := Apply(context.Background(), archHost(t), twoContainers(t, "2", "1"), state,
|
||||
store.OriginDeclared, stopped, nil, nil); err != nil && !strings.Contains(err.Error(), "not running") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := m.since(mark)
|
||||
if slices.Contains(got, "ask mesh-build-agent") || !slices.Contains(got, "rm mesh-build-agent") {
|
||||
t.Fatalf("a stopped container was asked, or not replaced: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// Issue 1637 (novox/hq ADR 0480 clause 7): a container's supplementary groups reach the runtime as --group-add.
|
||||
|
||||
func groupsContainer(groups ...int64) *declaration.Container {
|
||||
return &declaration.Container{ID: "n8n.app", Name: "n8n", Image: pinned, Groups: groups}
|
||||
}
|
||||
|
||||
// 1637/AC1: both run commands carry one --group-add per group, and none without groups.
|
||||
func TestAContainerIsAddedItsSupplementaryGroups(t *testing.T) {
|
||||
with := groupsContainer(2000, 2001)
|
||||
for name, args := range map[string][]string{
|
||||
"detached": detachedRunArgs(with, "x"),
|
||||
"foreground": foregroundRunArgs(with, "x"),
|
||||
} {
|
||||
got := " " + strings.Join(args, " ") + " "
|
||||
if !strings.Contains(got, " --group-add 2000 ") || !strings.Contains(got, " --group-add 2001 ") {
|
||||
t.Errorf("1637/AC1: the %s run does not add both groups: %s", name, got)
|
||||
}
|
||||
}
|
||||
without := groupsContainer()
|
||||
for name, args := range map[string][]string{
|
||||
"detached": detachedRunArgs(without, "x"),
|
||||
"foreground": foregroundRunArgs(without, "x"),
|
||||
} {
|
||||
if strings.Contains(strings.Join(args, " "), "--group-add") {
|
||||
t.Errorf("1637/AC1: the %s run adds a group none was declared", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1637/AC2: a changed set of groups recreates the container; no groups leaves the spec as it was.
|
||||
func TestAChangeOfGroupsRecreatesTheContainer(t *testing.T) {
|
||||
none, a, b := groupsContainer(), groupsContainer(2000), groupsContainer(2000, 2001)
|
||||
if containerSpec(none, inputs{}) == containerSpec(a, inputs{}) || containerSpec(a, inputs{}) == containerSpec(b, inputs{}) {
|
||||
t.Error("1637/AC2: a change of groups does not change the container's spec")
|
||||
}
|
||||
same := groupsContainer(2000)
|
||||
if containerSpec(a, inputs{}) != containerSpec(same, inputs{}) {
|
||||
t.Error("1637/AC2: the same groups changed the spec")
|
||||
}
|
||||
plain := *none
|
||||
plain.Groups = nil
|
||||
if containerSpec(none, inputs{}) != containerSpec(&plain, inputs{}) {
|
||||
t.Error("1637/AC2: a container with no groups has its spec moved")
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
@@ -20,6 +20,9 @@ func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) {
|
||||
if name != "docker" {
|
||||
return "", errors.New("not installed")
|
||||
}
|
||||
if len(args) > 0 && args[0] == "container" && strings.HasSuffix(args[len(args)-1], asideSuffix) {
|
||||
return "", errors.New("no such container") // nothing was left set aside
|
||||
}
|
||||
ran = append(ran, strings.Join(args, " "))
|
||||
switch args[0] {
|
||||
case "info":
|
||||
@@ -75,69 +78,6 @@ func TestAContainerJoinsTheNetworksItKeeps(t *testing.T) {
|
||||
|
||||
var specOfLast string
|
||||
|
||||
// A module the mesh left out of a declaration is not a module the mesh removed (novox/hq ADR 0163,
|
||||
// rule 6): what the host wrote for it stays, recorded and said; what it holds for it stays held.
|
||||
// A module simply absent is removed as it always was.
|
||||
func TestALeftOutModuleIsNeitherRemovedNorForgotten(t *testing.T) {
|
||||
var removed []string
|
||||
gone := map[string]bool{}
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
args = oldShape(name, args)
|
||||
if name != "docker" {
|
||||
return "", nil
|
||||
}
|
||||
switch args[0] {
|
||||
case "info":
|
||||
return "29.0.0\n", nil
|
||||
case "rm":
|
||||
removed = append(removed, args[len(args)-1])
|
||||
gone[args[len(args)-1]] = true
|
||||
case "container":
|
||||
if gone[args[len(args)-1]] {
|
||||
return "", errors.New("no such container")
|
||||
}
|
||||
return "true\tspec", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
known := store.State{
|
||||
Resources: []store.Applied{
|
||||
{ID: "web.server", Type: "container", Target: "web", Origin: store.OriginDeclared},
|
||||
{ID: "old.server", Type: "container", Target: "old", Origin: store.OriginDeclared},
|
||||
},
|
||||
Held: []store.Held{{ID: "web.page", Module: "web", Kind: "file", Target: "/srv/web/index.html"}},
|
||||
}
|
||||
d := parse(t, `{"declaration":1,"left_out":["web"],"resources":[
|
||||
{"id":"notes.conf","type":"file","path":"`+t.TempDir()+`/notes.conf","content":"x"}
|
||||
]}`)
|
||||
report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(removed) != 1 || removed[0] != "old" {
|
||||
t.Fatalf("removed %v; only the module that is absent goes", removed)
|
||||
}
|
||||
if _, kept := state.At("container", "web"); !kept {
|
||||
t.Fatal("the left-out module's record was forgotten")
|
||||
}
|
||||
if _, held := state.HeldAt("web.page"); !held {
|
||||
t.Fatal("the left-out module's hold was released")
|
||||
}
|
||||
said := false
|
||||
for _, o := range report.Outcomes {
|
||||
if o.ID == "web.server" && o.Action == "unchanged" && strings.Contains(o.Detail, "web was left out of this declaration by the mesh") {
|
||||
said = true
|
||||
}
|
||||
if o.ID == "web.server" && o.Action != "unchanged" {
|
||||
t.Errorf("the left-out module's container was %s", o.Action)
|
||||
}
|
||||
}
|
||||
if !said {
|
||||
t.Fatalf("keeping the left-out module's container was not said: %+v", report.Outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
// A container's capabilities reach the runtime and are part of its spec (novox/hq ADR 0170).
|
||||
func TestACapabilityReachesTheRuntimeAndTheSpec(t *testing.T) {
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
@@ -0,0 +1,158 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// A module that declares its own system account and a .ssh in that account's home is placed (novox/hq issue 1542):
|
||||
// the account is not in the user database yet when the declaration is judged, and its home is under /var/lib.
|
||||
func declaredHomeFixture(t *testing.T) (home string, user *declaration.User) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
passwd := filepath.Join(dir, "passwd")
|
||||
if err := os.WriteFile(passwd, []byte("root:x:0:0::/root:/bin/sh\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := passwdFile
|
||||
passwdFile = passwd
|
||||
t.Cleanup(func() { passwdFile = before })
|
||||
home = filepath.Join(dir, "lib", "mesh-build-go")
|
||||
return home, &declaration.User{ID: "mesh-build-go.account", Type: declaration.TypeUser, Name: "mesh-build-go",
|
||||
Shell: "/usr/bin/nologin", Home: home}
|
||||
}
|
||||
|
||||
// 1542/AC1: the .ssh of a declared account, owned by it, is refused by nothing.
|
||||
func TestIssue1542ADeclaredAccountsSSHIsPlaced(t *testing.T) {
|
||||
home, user := declaredHomeFixture(t)
|
||||
ssh := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
|
||||
Mode: "0700", Owner: "mesh-build-go"}
|
||||
refused := refusedPlaces([]declaration.Resource{user, ssh})
|
||||
if len(refused) != 0 {
|
||||
t.Fatalf("a declared account's own .ssh was refused: %v", refused)
|
||||
}
|
||||
if err := (inputs{refused: refused}).refused[filepath.Clean(ssh.Path)]; err != nil {
|
||||
t.Fatalf("the apply-time check refuses it: %v", err)
|
||||
}
|
||||
if line := JudgeDirectories(&declaration.Declaration{Resources: []declaration.Resource{user, ssh}}); strings.Contains(line, "refused already by the placement guard: ") {
|
||||
t.Fatalf("the read-only judgement refuses it: %s", line)
|
||||
}
|
||||
}
|
||||
|
||||
// 1542/AC2: another account's directory below the declared home is refused as below any home, and a .ssh below
|
||||
// no home of the machine or the declaration is refused with today's words.
|
||||
func TestIssue1542OnlyTheDeclaredAccountsOwnIsPlaced(t *testing.T) {
|
||||
home, user := declaredHomeFixture(t)
|
||||
other := &declaration.Directory{ID: "notes.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
|
||||
Mode: "0700", Owner: "intruder"}
|
||||
refused := refusedPlaces([]declaration.Resource{user, other})
|
||||
err := refused[filepath.Clean(other.Path)]
|
||||
if err == nil || !strings.Contains(err.Error(), "below mesh-build-go's home and declared intruder's") {
|
||||
t.Fatalf("another account's directory below a declared home: %v", err)
|
||||
}
|
||||
stray := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory,
|
||||
Path: filepath.Join(filepath.Dir(home), "elsewhere", ".ssh"), Mode: "0700", Owner: "mesh-build-go"}
|
||||
refused = refusedPlaces([]declaration.Resource{user, stray})
|
||||
err = refused[filepath.Clean(stray.Path)]
|
||||
if err == nil || !strings.Contains(err.Error(), "a .ssh directory is placed only below its account's home, as that account's") {
|
||||
t.Fatalf("a .ssh below no home: %v", err)
|
||||
}
|
||||
// Not a .ssh: a root-owned directory below the declared home is placed exactly as on main.
|
||||
data := &declaration.Directory{ID: "mesh-build-go.data", Type: declaration.TypeDirectory, Path: home + "/data",
|
||||
Mode: "0755", Owner: "root"}
|
||||
if refused := refusedPlaces([]declaration.Resource{user, data}); len(refused) != 0 {
|
||||
t.Fatalf("a root-owned directory below a declared home, not a .ssh, was refused: %v", refused)
|
||||
}
|
||||
// And without the account declared, the same .ssh is refused as it was.
|
||||
own := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
|
||||
Mode: "0700", Owner: "mesh-build-go"}
|
||||
if refusedPlaces([]declaration.Resource{own})[filepath.Clean(own.Path)] == nil {
|
||||
t.Fatal("a .ssh below an account nobody makes was placed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1542/AC3: a declared account is no home to anyone but the directory placement guard.
|
||||
func TestIssue1542DeclaredAccountsAreNotHomesToTheOtherGuards(t *testing.T) {
|
||||
home, user := declaredHomeFixture(t)
|
||||
_ = refusedPlaces([]declaration.Resource{user})
|
||||
if _, ok := accountsOfHomes()[home]; ok {
|
||||
t.Fatal("a declared account became a home of the machine")
|
||||
}
|
||||
if homeAbove(home+"/.ssh") != "" {
|
||||
t.Fatal("a declared account became a home for the links and the move-out")
|
||||
}
|
||||
}
|
||||
|
||||
// A declared account is added as a home and never takes one over (reviewer, issue 1542): the person's own .ssh
|
||||
// stays refused to every other account, whatever a declaration says about a user.
|
||||
func TestIssue1542ADeclaredAccountTakesNoExistingHome(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
passwd := filepath.Join(dir, "passwd")
|
||||
body := "root:x:0:0::/root:/bin/sh\njochen:x:1000:1000::/home/jochen:/bin/sh\n" +
|
||||
"svc:x:998:998::/var/lib/svc:/usr/bin/nologin\n"
|
||||
if err := os.WriteFile(passwd, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := passwdFile
|
||||
passwdFile = passwd
|
||||
t.Cleanup(func() { passwdFile = before })
|
||||
newHome := filepath.Join(dir, "lib", "mesh-build-go")
|
||||
ssh := func(path, owner string) *declaration.Directory {
|
||||
return &declaration.Directory{ID: "x.ssh", Type: declaration.TypeDirectory, Path: path, Mode: "0700", Owner: owner}
|
||||
}
|
||||
user := func(name, home string) *declaration.User {
|
||||
return &declaration.User{ID: "x.user", Type: declaration.TypeUser, Name: name, Home: home}
|
||||
}
|
||||
// A declared user over a person's home, whoever it is named for.
|
||||
for _, name := range []string{"evil", "postgres", "root"} {
|
||||
d := ssh("/home/jochen/.ssh", name)
|
||||
if refusedPlaces([]declaration.Resource{user(name, "/home/jochen"), d})[d.Path] == nil {
|
||||
t.Errorf("%s, declared over jochen's home, was given jochen's .ssh", name)
|
||||
}
|
||||
}
|
||||
// A declared home below a person's home.
|
||||
d := ssh("/home/jochen/sub/.ssh", "evil")
|
||||
if refusedPlaces([]declaration.Resource{user("evil", "/home/jochen/sub"), d})[d.Path] == nil {
|
||||
t.Error("a home declared below jochen's home was counted")
|
||||
}
|
||||
// A name the machine has already, declared with another home: that is not its home.
|
||||
d = ssh(newHome+"/e/.ssh", "svc")
|
||||
if refusedPlaces([]declaration.Resource{user("svc", newHome+"/e"), d})[d.Path] == nil {
|
||||
t.Error("an existing account's declared home was counted")
|
||||
}
|
||||
// A number is no name: chown -1 leaves the owner as it was.
|
||||
for _, owner := range []string{"-1", "-1:0", "0", "1001"} {
|
||||
d := ssh(newHome+"/.ssh", owner)
|
||||
if refusedPlaces([]declaration.Resource{user("mesh-build-go", newHome), d})[d.Path] == nil {
|
||||
t.Errorf("owner %q was placed below a declared home", owner)
|
||||
}
|
||||
}
|
||||
// And the account that is new, by name, still is.
|
||||
d = ssh(newHome+"/.ssh", "mesh-build-go")
|
||||
if got := refusedPlaces([]declaration.Resource{user("mesh-build-go", newHome), d}); len(got) != 0 {
|
||||
t.Errorf("a new account's own .ssh was refused: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The account the mesh made, as declared, is judged the same at the next apply (issue 1542): once it is in the
|
||||
// user database with exactly the declared home, its .ssh is still placed.
|
||||
func TestIssue1542TheAccountTheMeshMadeIsStillPlacedAtTheNextApply(t *testing.T) {
|
||||
home, user := declaredHomeFixture(t)
|
||||
ssh := &declaration.Directory{ID: "mesh-build-go.ssh-dir", Type: declaration.TypeDirectory, Path: home + "/.ssh",
|
||||
Mode: "0700", Owner: "mesh-build-go"}
|
||||
resources := []declaration.Resource{user, ssh}
|
||||
if got := refusedPlaces(resources); len(got) != 0 {
|
||||
t.Fatalf("first apply, the account not yet made: %v", got)
|
||||
}
|
||||
body := "root:x:0:0::/root:/bin/sh\nmesh-build-go:x:998:998::" + home + ":/usr/bin/nologin\n"
|
||||
if err := os.WriteFile(passwdFile, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := refusedPlaces(resources); len(got) != 0 {
|
||||
t.Fatalf("second apply, the account made as declared: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A command killed because its context ended says so, and why: "docker exited -1:" said nothing while novox's bus
|
||||
// container was half replaced (issue 1945, live on 2026-10-25 at 11:23).
|
||||
func TestACommandKilledByItsContextSaysWhy(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() { time.Sleep(50 * time.Millisecond); cancel() }()
|
||||
_, err := ExecRunner(ctx, "sleep", "5")
|
||||
if err == nil || !strings.Contains(err.Error(), "killed") || !strings.Contains(err.Error(), "context canceled") {
|
||||
t.Fatalf("a command its context killed said: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// WriteFilled writes what the provisioner's client filled into one file the declaration names as filled (novox/hq
|
||||
// ADR 0349 §4), and makes what reads the file take the change, exactly as an apply that changed it would.
|
||||
//
|
||||
// **The declared file says everything but the content.** Its path, mode, owner and who owns it at runtime are the
|
||||
// declaration's; the client's request carries the content alone. So the file is applied as the declared one with
|
||||
// that content in place of the declaration's: an owned file whole through applyFile, a seed only when absent, a
|
||||
// managed file's keys through applyInto or its block through applyBlock — the same guards on where nothing is
|
||||
// placed, the same Keep of an original found there, the same record. What the declaration seals or fills with
|
||||
// `${secret:}` the client has already filled, so the content is written as it came; a file that carried a secret
|
||||
// keeps the mode a secret is written with when the declaration gives none.
|
||||
//
|
||||
// **Then what reads it, in the apply's order** (afterWhatTheyRead): a service naming it under restart-on is
|
||||
// restarted and under reload-on reloaded, a process or container naming it under restart-on is restarted, and a
|
||||
// running container that reads it at creation — an env-file, a file mounted into it — is recreated, because what
|
||||
// it read is recorded by digest and this write changed the digest. Each of those is applied and recorded as an
|
||||
// apply does, and what moves goes on moving what names it. Nothing else of the declaration is touched.
|
||||
//
|
||||
// The caller holds the apply's lock: a write and an apply never write one file at once. log is said the path, the
|
||||
// action and the digest, never the content.
|
||||
//
|
||||
// **It does not refresh the runtimes' data roots.** A filled write is one file, not an apply; the placement guard
|
||||
// judges it with the roots the last apply read (runtimeDataRoots), falling back to the runtimes' configuration
|
||||
// files when no apply has read them yet — the runtimes themselves are asked once, at an apply's start, not once
|
||||
// per file (novox/hq issue 1138).
|
||||
func WriteFilled(
|
||||
ctx context.Context,
|
||||
sys system.System,
|
||||
d *declaration.Declaration,
|
||||
known store.State,
|
||||
path string,
|
||||
content []byte,
|
||||
run Runner,
|
||||
log func(string),
|
||||
keep Keep,
|
||||
windows *Windows,
|
||||
) (report Report, _ store.State, _ error) {
|
||||
if log == nil {
|
||||
log = func(string) {}
|
||||
}
|
||||
if !d.Fills(path) {
|
||||
return report, known, &Error{Resource: path, Err: fmt.Errorf("this node's declaration does not name %s "+
|
||||
"as a file the provisioner fills, so nothing was written", path)}
|
||||
}
|
||||
var declared []*declaration.File
|
||||
for _, r := range d.Resources {
|
||||
if f, ok := r.(*declaration.File); ok && f.Path == path {
|
||||
declared = append(declared, f)
|
||||
}
|
||||
}
|
||||
if len(declared) != 1 {
|
||||
// The parser refuses a filled path no file declares; two at one path would need the request to say whose
|
||||
// keys or block it carries, and it carries the content only.
|
||||
return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
|
||||
"is written for exactly one, so nothing was written", len(declared), path)}
|
||||
}
|
||||
file := filledAs(declared[0], content)
|
||||
if held, isHeld := known.HeldAt(file.ID); isHeld {
|
||||
return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
|
||||
"yet taken (%s), so nothing was written", path, held.Why)}
|
||||
}
|
||||
|
||||
in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
|
||||
groups: groupsWanted(d.Resources), handed: Handed{}}
|
||||
in.refused = refusedPlaces(d.Resources)
|
||||
in.agentHomes = rootNeverHomes(d)
|
||||
in.dirsBefore = map[string]bool{}
|
||||
in.files = declaredFiles(d.Resources)
|
||||
// A container that reads the file and mounts a library whose disk or mount is not there is held, as a full apply
|
||||
// holds it: not created over the missing disk (novox/hq ADR 0328 §9).
|
||||
in.held = heldLibraries(ctx, d.Libraries)
|
||||
for _, r := range d.Resources {
|
||||
in.declares[r.Identity()] = declaredDigest(r)
|
||||
}
|
||||
changed := map[string]bool{}
|
||||
|
||||
// The file, as the apply loop applies it.
|
||||
was, _ := known.Find(file.ID)
|
||||
var keepFound Keep
|
||||
if (was.ID == "" || was.Target != file.Path) && !known.Recorded(string(declaration.TypeFile), file.Path) {
|
||||
keepFound = keep
|
||||
}
|
||||
outcome, err := applyOne(ctx, sys, file, run, changed, in, was, nil, keepFound)
|
||||
if err != nil {
|
||||
log(fmt.Sprintf(" failed %s (%s): %v", file.ID, file.Path, err))
|
||||
return report, known, &Error{Resource: file.ID, Err: err, Done: report}
|
||||
}
|
||||
recordKeeping(&known, file, outcome, was)
|
||||
report.Outcomes = append(report.Outcomes, outcome)
|
||||
if !moved(outcome) {
|
||||
return report, known, nil
|
||||
}
|
||||
changed[file.ID] = true
|
||||
line := fmt.Sprintf(" %s %s (%s), filled: %s", outcome.Action, file.ID, file.Path, outcome.wrote)
|
||||
if outcome.Detail != "" {
|
||||
line += ": " + outcome.Detail
|
||||
}
|
||||
log(line)
|
||||
|
||||
// What reads it, in the apply's order, each moving what names it in turn.
|
||||
var failures []*Error
|
||||
for _, r := range afterWhatTheyRead(d.Resources) {
|
||||
if r.Identity() == file.ID || !readsChange(r, path, changed) {
|
||||
continue
|
||||
}
|
||||
if _, isHeld := known.HeldAt(r.Identity()); isHeld {
|
||||
log(fmt.Sprintf(" held %s (%s): held as found for a module not yet taken, so it was not restarted "+
|
||||
"for %s", r.Identity(), r.Target(), file.ID))
|
||||
continue
|
||||
}
|
||||
was, _ := known.Find(r.Identity())
|
||||
o, err := applyOne(ctx, sys, r, run, changed, in, was, nil, nil)
|
||||
if err != nil {
|
||||
if o.found != nil {
|
||||
known.KeepFound(r.Identity(), store.OriginDeclared, *o.found)
|
||||
}
|
||||
failures = append(failures, &Error{Resource: r.Identity(), Err: err})
|
||||
log(fmt.Sprintf(" failed %s (%s): %v", r.Identity(), o.Target, err))
|
||||
continue
|
||||
}
|
||||
if o.Action == "waiting" {
|
||||
if o.found != nil {
|
||||
known.KeepFound(r.Identity(), store.OriginDeclared, *o.found)
|
||||
}
|
||||
report.Outcomes = append(report.Outcomes, o)
|
||||
log(fmt.Sprintf(" waiting %s (%s): %s", o.ID, o.Target, o.Detail))
|
||||
continue
|
||||
}
|
||||
recordKeeping(&known, r, o, was)
|
||||
report.Outcomes = append(report.Outcomes, o)
|
||||
if o.busy != nil {
|
||||
report.Busy = append(report.Busy, *o.busy)
|
||||
}
|
||||
if moved(o) {
|
||||
changed[r.Identity()] = true
|
||||
line := fmt.Sprintf(" %s %s (%s)", o.Action, o.ID, o.Target)
|
||||
if o.Detail != "" {
|
||||
line += ": " + o.Detail
|
||||
}
|
||||
log(line)
|
||||
} else if o.Action != "unchanged" {
|
||||
log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail))
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
first := failures[0]
|
||||
first.Done = report
|
||||
first.Others = len(failures) - 1
|
||||
return report, known, first
|
||||
}
|
||||
return report, known, nil
|
||||
}
|
||||
|
||||
// filledAs is the declared file with the filled content in place of the declaration's.
|
||||
func filledAs(declared *declaration.File, content []byte) *declaration.File {
|
||||
f := *declared
|
||||
f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
|
||||
if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) {
|
||||
// Written as the declaration would have written it: a file carrying a credential is not world-readable.
|
||||
f.Mode = "0600"
|
||||
}
|
||||
return &f
|
||||
}
|
||||
|
||||
// readsChange reports whether applying r is what makes it take a change of what changed: a service's restart-on or
|
||||
// reload-on, a process's or container's restart-on, or a running container that reads path at creation.
|
||||
func readsChange(r declaration.Resource, path string, changed map[string]bool) bool {
|
||||
switch res := r.(type) {
|
||||
case *declaration.Service:
|
||||
return len(restartedBy(res.RestartOn, changed)) > 0 || len(restartedBy(res.ReloadOn, changed)) > 0
|
||||
case *declaration.Process:
|
||||
return len(restartedBy(res.RestartOn, changed)) > 0
|
||||
case *declaration.Container:
|
||||
if len(restartedBy(res.RestartOn, changed)) > 0 {
|
||||
return true
|
||||
}
|
||||
if res.RunOnce || res.Schedule != "" {
|
||||
return false
|
||||
}
|
||||
if slices.Contains(res.EnvFile, path) {
|
||||
return true
|
||||
}
|
||||
for _, v := range res.Volumes {
|
||||
if mountSource(v) == path {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// recordKeeping records r as applied, carrying where the original it replaced was kept from its record when this
|
||||
// write kept none — for as long as the resource is recorded at the same target, as the apply loop carries it.
|
||||
func recordKeeping(known *store.State, r declaration.Resource, o Outcome, was store.Applied) {
|
||||
kept, keptMode, keptOwner := o.kept, o.keptMode, o.keptOwner
|
||||
if kept == "" && was.Target == o.Target {
|
||||
kept, keptMode, keptOwner = was.Kept, was.KeptMode, was.KeptOwner
|
||||
}
|
||||
recordApplied(known, store.OriginDeclared, r, o, kept, keptMode, keptOwner)
|
||||
}
|
||||
|
||||
// leftToTheClient is a filled file as an apply leaves it (novox/hq ADR 0349 §5, design 57 P7): untouched, and said
|
||||
// so. A file the mesh wrote at the same path under another id, going in this apply, is handed to it as an apply hands
|
||||
// a whole file (novox/hq ADR 0223): its record — what the mesh wrote there, and the original it kept — is the filled
|
||||
// file's now, so the verb judges what is there against it, and the file is neither removed nor written.
|
||||
func leftToTheClient(f *declaration.File, known *store.State, handedFiles map[string]store.Applied, report *Report,
|
||||
log func(string)) Outcome {
|
||||
if from, handed := handedFiles[f.ID]; handed {
|
||||
if _, recorded := known.Find(f.ID); !recorded {
|
||||
rec := from
|
||||
rec.ID = f.ID
|
||||
known.Record(rec)
|
||||
}
|
||||
delete(handedFiles, f.ID)
|
||||
known.Forget(from.ID)
|
||||
detail := "no longer declared; " + f.ID + " declares the file now, filled by the provisioner's client, so " +
|
||||
"it was handed to it, not removed"
|
||||
report.Outcomes = append(report.Outcomes, Outcome{ID: from.ID, Type: from.Type, Target: from.Target,
|
||||
Action: "forgotten", Detail: detail})
|
||||
log(fmt.Sprintf(" forgotten %s (%s): %s", from.ID, from.Target, detail))
|
||||
}
|
||||
return Outcome{ID: f.ID, Type: string(declaration.TypeFile), Target: f.Path, Action: "unchanged",
|
||||
Detail: "filled by the provisioner's client, which alone has it written"}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0349 §5, one writer per file (design 57 P7, issue 673): a file the declaration names as
|
||||
// filled is written only through the write verb (WriteFilled), on the provisioner client's word. An apply — the
|
||||
// controller's send and the five-minute re-apply alike — never writes it, never records it, and removes it as an
|
||||
// orphan only when the verb placed it.
|
||||
|
||||
// An apply leaves a filled file alone, whatever who owns it at runtime says: absent stays absent and is not
|
||||
// recorded, and what the verb wrote stays as it wrote it through every re-apply.
|
||||
func TestAnApplyNeverWritesAFilledFile(t *testing.T) {
|
||||
for _, c := range []struct{ name, fields, content string }{
|
||||
{"owned", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n"},
|
||||
{"seed", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n"},
|
||||
{"managed block", `"into":"block","content":"","owner-at-runtime":"managed"`, "10.42.0.1 ace\n"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "the-file")
|
||||
d := filledDecl(t, path, c.fields, "")
|
||||
report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
|
||||
noServices, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
t.Fatalf("an apply wrote a filled file the client has not filled: %q", readText(t, path))
|
||||
}
|
||||
if _, ok := state.Find("m.conf"); ok {
|
||||
t.Errorf("an apply recorded a filled file it did not write")
|
||||
}
|
||||
if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" || !strings.Contains(o.Detail, "provisioner") {
|
||||
t.Errorf("an apply did not say it left the file to the provisioner's client: %+v", o)
|
||||
}
|
||||
|
||||
_, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte(c.content),
|
||||
noServices, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, _ := state.Find("m.conf")
|
||||
want, wantMode := readText(t, path), modeAt(t, path)
|
||||
|
||||
// The re-apply, twice over: nothing written, the verb's record kept as it wrote it.
|
||||
for i := 0; i < 2; i++ {
|
||||
var said []string
|
||||
report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
|
||||
noServices, func(line string) { said = append(said, line) }, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := readText(t, path); got != want || modeAt(t, path) != wantMode {
|
||||
t.Fatalf("re-apply %d wrote the filled file again: %q, want %q", i+1, got, want)
|
||||
}
|
||||
if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" {
|
||||
t.Errorf("re-apply %d: %+v", i+1, o)
|
||||
}
|
||||
if rec, _ := state.Find("m.conf"); rec.Wrote != placed.Wrote || rec.Target != path {
|
||||
t.Errorf("re-apply %d changed the verb's record: %+v, was %+v", i+1, rec, placed)
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Errorf("re-apply %d said something of an untouched file: %q", i+1, said)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// What reads a filled file is moved by the verb's write, once, never again by an apply: a service under
|
||||
// restart-on is not restarted and a container reading it is not recreated by the re-apply.
|
||||
func TestAnApplyMovesNothingForAFilledFile(t *testing.T) {
|
||||
env := filepath.Join(t.TempDir(), "forge.env")
|
||||
d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`,
|
||||
`{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`)
|
||||
m := &machine{containers: map[string]*fakeContainer{}}
|
||||
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"),
|
||||
m.run, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.asked = nil
|
||||
if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil,
|
||||
nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.removed("forge") || m.did("docker run") {
|
||||
t.Errorf("the re-apply recreated the container reading the filled file: %v", m.asked)
|
||||
}
|
||||
if got := readText(t, env); got != "PORT=5432\n" {
|
||||
t.Errorf("the re-apply wrote the filled file: %q", got)
|
||||
}
|
||||
|
||||
conf := filepath.Join(t.TempDir(), "daemon.conf")
|
||||
d = filledDecl(t, conf, `"content":"","owner-at-runtime":"owned"`,
|
||||
`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`)
|
||||
var commands []string
|
||||
_, state, err = WriteFilled(context.Background(), archHost(t), d, store.State{}, conf, []byte("level = 1\n"),
|
||||
recordingServices(&commands), nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
commands = nil
|
||||
if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
|
||||
recordingServices(&commands), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := count(commands, "systemctl stop m.service"); n != 0 {
|
||||
t.Errorf("the re-apply restarted what reads the filled file: %v", commands)
|
||||
}
|
||||
}
|
||||
|
||||
// A filled file the verb never placed is not the mesh's to remove: one found there, or put there by hand, stays
|
||||
// when the file leaves the declaration.
|
||||
func TestAFilledFileTheVerbNeverPlacedIsNotRemoved(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "the-file")
|
||||
if err := os.WriteFile(path, []byte("the machine's own\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
|
||||
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices,
|
||||
nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gone := elsewhere(t)
|
||||
report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices,
|
||||
nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := readText(t, path); got != "the machine's own\n" {
|
||||
t.Errorf("a filled file the verb never placed was touched when it left: %q", got)
|
||||
}
|
||||
if o := outcomeOf(report, "m.conf"); o.Action == "removed" {
|
||||
t.Errorf("a filled file the verb never placed was removed: %+v", o)
|
||||
}
|
||||
if _, ok := state.Find("m.conf"); ok {
|
||||
t.Errorf("a record of the filled file was left")
|
||||
}
|
||||
}
|
||||
|
||||
// What the verb placed is the mesh's: when the file leaves the declaration it is removed as an orphan, as a file
|
||||
// an apply wrote is.
|
||||
func TestAFilledFileTheVerbPlacedIsRemovedWhenItLeaves(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "the-file")
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
|
||||
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices,
|
||||
nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte("port = 8080\n"),
|
||||
noServices, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gone := elsewhere(t)
|
||||
report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices,
|
||||
nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
t.Errorf("a filled file the verb placed stayed when it left the declaration: %q", readText(t, path))
|
||||
}
|
||||
if o := outcomeOf(report, "m.conf"); o.Action != "removed" {
|
||||
t.Errorf("not removed: %+v", o)
|
||||
}
|
||||
if _, ok := state.Find("m.conf"); ok {
|
||||
t.Errorf("its record stayed")
|
||||
}
|
||||
}
|
||||
|
||||
// A file the mesh wrote the old way and that is now filled keeps its record for the verb to judge against, and is
|
||||
// not written over by the apply that hands it to the client; a file at the same path recorded under another id
|
||||
// is handed to the filled one, not removed.
|
||||
func TestAFileThatMovesToTheClientIsHandedNotRewritten(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "the-file")
|
||||
old := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.old","type":"file","path":%q,`+
|
||||
`"content":"port = 1\n"}]}`, path))
|
||||
_, state, err := Apply(context.Background(), archHost(t), old, store.State{}, store.OriginDeclared, noServices,
|
||||
nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was, _ := state.Find("m.old")
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
|
||||
_, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := readText(t, path); got != "port = 1\n" {
|
||||
t.Errorf("the file handed to the client was rewritten or removed: %q", got)
|
||||
}
|
||||
if _, ok := state.Find("m.old"); ok {
|
||||
t.Errorf("the record going was kept beside the filled file's")
|
||||
}
|
||||
if rec, ok := state.Find("m.conf"); !ok || rec.Wrote != was.Wrote || rec.Target != path {
|
||||
t.Errorf("the filled file did not take the record of what the mesh wrote there: %+v", rec)
|
||||
}
|
||||
}
|
||||
|
||||
// elsewhere is a declaration with nothing at the filled path: one unrelated file.
|
||||
func elsewhere(t *testing.T) *declaration.Declaration {
|
||||
t.Helper()
|
||||
return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"n.other","type":"file","path":%q,`+
|
||||
`"content":"x\n"}]}`, filepath.Join(t.TempDir(), "other")))
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A container that reads a filled file and mounts a library whose disk is not there is held by the write verb as a
|
||||
// full apply holds it (novox/hq ADR 0328 §9, issue 668): never created or recreated over the missing disk.
|
||||
func TestAWriteDoesNotRecreateAContainerHeldForItsLibrary(t *testing.T) {
|
||||
// No folder is given the group: the library's root is absent, and nothing is made.
|
||||
gid := 61000
|
||||
libraries(t, nil, nil)
|
||||
base := t.TempDir()
|
||||
root := filepath.Join(base, "gone", "media")
|
||||
env := filepath.Join(base, "radarr.env")
|
||||
raw := fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":[%q],
|
||||
"groups":[{"name":"mesh-media","gid":%d}],"libraries":[%s],"resources":[
|
||||
{"id":"radarr.env","type":"file","path":%q,"content":"","mode":"0600","owner-at-runtime":"owned"},
|
||||
{"id":"radarr.app","type":"container","name":"radarr","image":%q,"env-file":[%q],"volumes":[%q]}]}`,
|
||||
env, gid, libJSON("media", root, gid, true, false, "movies"), env, pinned, env,
|
||||
filepath.Join(root, "movies")+":/movies")
|
||||
d, err := declaration.Parse([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
return "", nil
|
||||
}
|
||||
var state store.State
|
||||
for _, content := range []string{"PORT=1\n", "PORT=2\n"} {
|
||||
var report Report
|
||||
report, state, err = WriteFilled(context.Background(), archHost(t), d, state, env, []byte(content), run,
|
||||
nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" {
|
||||
t.Errorf("after writing %q the container is %+v, not held for its library", content, o)
|
||||
}
|
||||
}
|
||||
for _, line := range ran {
|
||||
if strings.HasPrefix(line, "docker run") || strings.HasPrefix(line, "docker rm") {
|
||||
t.Errorf("the container was created or removed over a missing disk: %v", ran)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// A filled write is one file, not an apply: it asks no container runtime, and judges where nothing is placed
|
||||
// (issue 339) with the roots the last apply read, falling back to the runtimes' configuration files when no apply
|
||||
// has read them yet (novox/hq issue 1138). These tests set the package's runtimeRoots and AskRuntimes, so none
|
||||
// runs in parallel with another that reads or sets them, and each restores what it changed.
|
||||
|
||||
// setRuntimeRoots makes runtimeDataRoots answer roots, as if an apply had just read them (or, with nil, as if none
|
||||
// ever had), restoring what a test before it left there once this one is done.
|
||||
func setRuntimeRoots(t *testing.T, roots []string) {
|
||||
t.Helper()
|
||||
runtimeRootsMu.Lock()
|
||||
before := runtimeRoots
|
||||
runtimeRoots = roots
|
||||
runtimeRootsMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
runtimeRootsMu.Lock()
|
||||
runtimeRoots = before
|
||||
runtimeRootsMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
// askNoRuntime makes AskRuntimes record the name of every runtime it is asked for, answering as one that is not
|
||||
// running does (nothing, no error): were it consulted, that is what would say so. Restored once the test is done.
|
||||
func askNoRuntime(t *testing.T, asked *[]string) {
|
||||
t.Helper()
|
||||
before := AskRuntimes
|
||||
AskRuntimes = func(_ context.Context, name string, _ ...string) (string, error) {
|
||||
*asked = append(*asked, name)
|
||||
return "", nil
|
||||
}
|
||||
t.Cleanup(func() { AskRuntimes = before })
|
||||
}
|
||||
|
||||
// aRuntimeConfig points the runtimes' configuration at a fresh, unmounted root below a temp directory, restoring
|
||||
// the engine's real configuration once the test is done.
|
||||
func aRuntimeConfig(t *testing.T) (dir, root string) {
|
||||
t.Helper()
|
||||
dir = t.TempDir()
|
||||
root = filepath.Join(dir, "docker")
|
||||
daemon := filepath.Join(dir, "daemon.json")
|
||||
if err := os.WriteFile(daemon, []byte(`{"data-root":"`+root+`"}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := runtimeConfigs
|
||||
runtimeConfigs = runtimeFiles{daemonJSON: daemon, storageConf: filepath.Join(dir, "none.conf")}
|
||||
t.Cleanup(func() { runtimeConfigs = before })
|
||||
return dir, root
|
||||
}
|
||||
|
||||
// refusedAsPlacement fails the test unless err is a PlacementRefusedError.
|
||||
func refusedAsPlacement(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
var refusal *PlacementRefusedError
|
||||
if err == nil || !errors.As(err, &refusal) {
|
||||
t.Fatalf("a filled file below a refused runtime root was not refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1138/AC1: a filled write asks no runtime, and judges with the roots the last apply read.
|
||||
func TestAFilledWriteAsksNoRuntimeAndUsesTheLastApplysRoots(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "docker-root")
|
||||
setRuntimeRoots(t, []string{root}) // as if an apply had just read it
|
||||
|
||||
var asked []string
|
||||
askNoRuntime(t, &asked)
|
||||
|
||||
path := filepath.Join(root, "app.conf")
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
|
||||
_, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("x\n"), noServices,
|
||||
nil, nil, nil)
|
||||
|
||||
refusedAsPlacement(t, err)
|
||||
if len(asked) != 0 {
|
||||
t.Errorf("a filled write asked the container runtimes: %v", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1138/AC2: given a real apply that read a runtime data root, a filled write below it is refused as today, with
|
||||
// no apply run again in between.
|
||||
func TestAFilledWriteBelowARootARealApplyReadIsRefused(t *testing.T) {
|
||||
dir, root := aRuntimeConfig(t)
|
||||
t.Cleanup(func() {
|
||||
runtimeRootsMu.Lock()
|
||||
runtimeRoots = nil
|
||||
runtimeRootsMu.Unlock()
|
||||
})
|
||||
|
||||
// A full apply of something unrelated reads the runtimes' configuration once, at its start.
|
||||
elsewhere := filepath.Join(dir, "elsewhere.conf")
|
||||
if _, _, err := Apply(context.Background(), archHost(t), declare(t,
|
||||
fmt.Sprintf(`{"id":"m.other","type":"file","path":%q,"content":"x\n"}`, elsewhere)),
|
||||
store.State{}, store.OriginDeclared, noServices, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A filled write below the runtime's root, with no apply run since, is still refused.
|
||||
path := filepath.Join(root, "sub", "app.conf")
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
|
||||
_, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("x\n"), noServices,
|
||||
nil, nil, nil)
|
||||
refusedAsPlacement(t, err)
|
||||
}
|
||||
|
||||
// 1138/AC3: with no apply having read the roots yet, a filled write judges by the runtimes' configuration files,
|
||||
// asking no runtime.
|
||||
func TestAFilledWriteWithNoApplyYetUsesTheRuntimesConfigFiles(t *testing.T) {
|
||||
setRuntimeRoots(t, nil) // no apply has read the roots yet
|
||||
_, root := aRuntimeConfig(t)
|
||||
|
||||
var asked []string
|
||||
askNoRuntime(t, &asked)
|
||||
|
||||
path := filepath.Join(root, "sub", "app.conf")
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
|
||||
_, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("x\n"), noServices,
|
||||
nil, nil, nil)
|
||||
|
||||
refusedAsPlacement(t, err)
|
||||
if len(asked) != 0 {
|
||||
t.Errorf("a filled write with no apply yet asked the container runtimes: %v", asked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1138/AC4: a full apply still asks each runtime once, at its start, whatever the number of files it writes.
|
||||
func TestAFullApplyAsksEachRuntimeOnceRegardlessOfFileCount(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var asked []string
|
||||
before := AskRuntimes
|
||||
AskRuntimes = func(_ context.Context, name string, _ ...string) (string, error) {
|
||||
asked = append(asked, name)
|
||||
return "", errors.New("not running in this test")
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
AskRuntimes = before
|
||||
runtimeRootsMu.Lock()
|
||||
runtimeRoots = nil
|
||||
runtimeRootsMu.Unlock()
|
||||
})
|
||||
|
||||
var resources []string
|
||||
for i := 0; i < 3; i++ {
|
||||
resources = append(resources, fmt.Sprintf(`{"id":"m.f%d","type":"file","path":%q,"content":"x\n"}`, i,
|
||||
filepath.Join(dir, fmt.Sprintf("f%d.conf", i))))
|
||||
}
|
||||
if _, _, err := Apply(context.Background(), archHost(t), declare(t, strings.Join(resources, ",")),
|
||||
store.State{}, store.OriginDeclared, noServices, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := count(asked, "docker"); got != 1 {
|
||||
t.Errorf("docker was asked %d times for one apply of 3 files; want 1: %v", got, asked)
|
||||
}
|
||||
if got := count(asked, "podman"); got != 1 {
|
||||
t.Errorf("podman was asked %d times for one apply of 3 files; want 1: %v", got, asked)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0349 §4 (issue 668): the provisioner's client has the node-engine write a file it filled,
|
||||
// and the node-engine writes it as an apply of the declared file with that content would — through applyFile,
|
||||
// applyInto or applyBlock by who owns it at runtime — and makes what reads it take the change as an apply does.
|
||||
|
||||
// filledDecl is a declaration naming path as filled, with the file's fields after id, type and path, and more
|
||||
// resources after it.
|
||||
func filledDecl(t *testing.T, path, fields, more string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
if more != "" {
|
||||
more = "," + more
|
||||
}
|
||||
return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner",
|
||||
"filled_files":[%q],"resources":[{"id":"m.conf","type":"file","path":%q,%s}%s]}`, path, path, fields, more))
|
||||
}
|
||||
|
||||
func TestAWriteOfAPathNotFilledIsRefused(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
filled := filepath.Join(dir, "filled.conf")
|
||||
other := filepath.Join(dir, "other.conf")
|
||||
d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[
|
||||
{"id":"m.conf","type":"file","path":%q,"content":"x\n","owner-at-runtime":"owned"},
|
||||
{"id":"m.other","type":"file","path":%q,"content":"y\n"}
|
||||
]}`, filled, filled, other))
|
||||
for _, path := range []string{other, filepath.Join(dir, "nothing.conf")} {
|
||||
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("evil\n"),
|
||||
noServices, nil, nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "does not name") {
|
||||
t.Errorf("%s: a write of a path the declaration does not fill was not refused: %v", path, err)
|
||||
}
|
||||
if _, err := os.Stat(path); err == nil && path != other {
|
||||
t.Errorf("%s was written", path)
|
||||
}
|
||||
if len(state.Resources) != 0 {
|
||||
t.Errorf("a refused write recorded something: %+v", state.Resources)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(other); err == nil {
|
||||
t.Errorf("%s was written", other)
|
||||
}
|
||||
t.Run("a filled path two resources declare", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "hosts")
|
||||
if err := os.WriteFile(path, []byte(workstationHosts), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[
|
||||
{"id":"a.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.1 ace\n","owner-at-runtime":"managed"},
|
||||
{"id":"b.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.2 g14\n","owner-at-runtime":"managed"}
|
||||
]}`, path, path, path))
|
||||
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("6.6.6.6 evil\n"),
|
||||
noServices, nil, nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "2 resources declare") {
|
||||
t.Fatalf("a path two resources declare was not refused: %v", err)
|
||||
}
|
||||
if got := readText(t, path); got != workstationHosts || len(state.Resources) != 0 {
|
||||
t.Errorf("something was written or recorded: %q %+v", got, state.Resources)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Each ownership is written by the write verb exactly as an apply of the declared file with the same content: the
|
||||
// same action, the same bytes on disk, the same mode.
|
||||
func TestAFilledFileIsWrittenAsItsOwnershipSays(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, fields, content, before, action string
|
||||
}{
|
||||
{"owned, absent", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n", "", "created"},
|
||||
{"owned, over what is there", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n",
|
||||
"port = 1\n", "updated"},
|
||||
{"seed, absent", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n", "", "created"},
|
||||
{"seed, present", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n",
|
||||
"user default on\nuser app-one on\n", "kept"},
|
||||
{"managed json", `"into":"json","content":"{}","owner-at-runtime":"managed"`,
|
||||
`{"insecure-registries":["10.42.0.1:5000"]}`, machinesOwn, "updated"},
|
||||
{"managed block", `"into":"block","content":"x\n","owner-at-runtime":"managed"`, "10.42.0.1 ace\n",
|
||||
workstationHosts, "updated"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
at := func() string {
|
||||
path := filepath.Join(t.TempDir(), "the-file")
|
||||
if c.before != "" {
|
||||
if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
written := at()
|
||||
report, state, err := WriteFilled(context.Background(), archHost(t), filledDecl(t, written, c.fields, ""),
|
||||
store.State{}, written, []byte(c.content), noServices, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applied := at()
|
||||
withContent := strings.Replace(c.fields, fmt.Sprintf(`"content":%q`, contentOf(t, c.fields)),
|
||||
fmt.Sprintf(`"content":%q`, c.content), 1)
|
||||
// The same file declared the old way: an apply writes a filled file only on the client's word (P7).
|
||||
plain := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.conf","type":"file",`+
|
||||
`"path":%q,%s}]}`, applied, withContent))
|
||||
want, _, err := Apply(context.Background(), archHost(t), plain, store.State{}, store.OriginDeclared,
|
||||
noServices, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := report.Outcomes[0].Action; got != c.action || got != want.Outcomes[0].Action {
|
||||
t.Errorf("written %s; an apply of the same content %s; want %s", got, want.Outcomes[0].Action, c.action)
|
||||
}
|
||||
if got, apply := readText(t, written), readText(t, applied); got != apply {
|
||||
t.Errorf("written, the file holds\n%q\nan apply of the same content leaves\n%q", got, apply)
|
||||
}
|
||||
if a, b := modeAt(t, written), modeAt(t, applied); a != b {
|
||||
t.Errorf("written mode %o, an apply's %o", a, b)
|
||||
}
|
||||
if rec, ok := state.Find("m.conf"); !ok || rec.Target != written || rec.Origin != store.OriginDeclared {
|
||||
t.Errorf("the write is not recorded as the file's: %+v", rec)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("a filled file that carried a secret is not world-readable", func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "token.env")
|
||||
d := filledDecl(t, path, `"content":"TOKEN=${secret:token}\n","secrets":{"token":"c2VhbGVk"},"owner-at-runtime":"owned"`, "")
|
||||
if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("TOKEN=abc\n"),
|
||||
noServices, nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := modeAt(t, path); m != 0o600 {
|
||||
t.Errorf("a filled file that carried a secret is mode %o", m)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func contentOf(t *testing.T, fields string) string {
|
||||
t.Helper()
|
||||
var holder struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte("{"+fields+"}"), &holder); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return holder.Content
|
||||
}
|
||||
|
||||
func modeAt(t *testing.T, path string) os.FileMode {
|
||||
t.Helper()
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.Mode().Perm()
|
||||
}
|
||||
|
||||
// A file that carried a secret keeps the mode a secret is written with: the client filled the value, and the file
|
||||
// is no more readable for it.
|
||||
|
||||
func TestAWriteUnderReloadOnReloadsItsProgramOnce(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "daemon.conf")
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`,
|
||||
`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","reload-on":["m.conf"]},
|
||||
{"id":"m.other","type":"service","unit":"other.service","state":"running"}`)
|
||||
var commands []string
|
||||
var said []string
|
||||
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"),
|
||||
recordingServices(&commands), func(line string) { said = append(said, line) }, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := count(commands, "systemctl reload m.service"); n != 1 {
|
||||
t.Errorf("the program was reloaded %d times; commands were %v", n, commands)
|
||||
}
|
||||
joined := strings.Join(commands, "\n")
|
||||
if strings.Contains(joined, "restart m.service") || strings.Contains(joined, "stop m.service") {
|
||||
t.Errorf("the program was restarted, not reloaded: %v", commands)
|
||||
}
|
||||
if strings.Contains(joined, "other.service") {
|
||||
t.Errorf("a service that does not read the file was touched: %v", commands)
|
||||
}
|
||||
if _, ok := state.Find("m.daemon"); !ok {
|
||||
t.Errorf("the reloaded service was not recorded")
|
||||
}
|
||||
// The log says the path and the digest, never the content.
|
||||
for _, line := range said {
|
||||
if strings.Contains(line, "level = 1") {
|
||||
t.Errorf("the content was said: %q", line)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), digestOf("level = 1\n")) {
|
||||
t.Errorf("the digest was not said: %q", said)
|
||||
}
|
||||
|
||||
// The same content again moves nothing, and reloads nothing.
|
||||
commands = nil
|
||||
report, _, err := WriteFilled(context.Background(), archHost(t), d, state, path, []byte("level = 1\n"),
|
||||
recordingServices(&commands), nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Outcomes[0].Action != "unchanged" || count(commands, "reload") != 0 {
|
||||
t.Errorf("an unchanged write reloaded: %v %+v", commands, report.Outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWriteUnderRestartOnRestartsItsProgram(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "daemon.conf")
|
||||
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`,
|
||||
`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`)
|
||||
var commands []string
|
||||
if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"),
|
||||
recordingServices(&commands), nil, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A restart is a stop and a start, once.
|
||||
stops, starts := count(commands, "systemctl stop m.service"), count(commands, "systemctl start m.service")
|
||||
if stops != 1 || starts != 1 {
|
||||
t.Errorf("the program was stopped %d and started %d times; commands were %v", stops, starts, commands)
|
||||
}
|
||||
}
|
||||
|
||||
// A running container that reads the file at creation is recreated, as an apply recreates it for a changed file.
|
||||
func TestAWriteOfAFileAContainerReadsRecreatesIt(t *testing.T) {
|
||||
env := filepath.Join(t.TempDir(), "forge.env")
|
||||
d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`,
|
||||
`{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`)
|
||||
m := &machine{containers: map[string]*fakeContainer{}}
|
||||
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"),
|
||||
m.run, nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.did("docker run") {
|
||||
t.Fatalf("the container was not created: %v", m.asked)
|
||||
}
|
||||
m.asked = nil
|
||||
report, _, err := WriteFilled(context.Background(), archHost(t), d, state, env, []byte("PORT=5433\n"), m.run,
|
||||
nil, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !m.removed("forge") || !m.did("docker run") {
|
||||
t.Fatalf("the container kept what it read before the write: %v", m.asked)
|
||||
}
|
||||
if o := outcomeOf(report, "forge.server"); o.Action != "updated" || !strings.Contains(o.Detail, env) {
|
||||
t.Errorf("the recreate did not say which file changed: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
func count(lines []string, prefix string) int {
|
||||
n := 0
|
||||
for _, l := range lines {
|
||||
if strings.Contains(l, prefix) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Two resources declaring one filled path — two modules' blocks in one file — are refused: the request carries the
|
||||
// content only, so it cannot say whose block it is, and nothing is written.
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
@@ -63,7 +63,7 @@ func (w Wanted) othersAsk(account, group, except string) []string {
|
||||
// applyGroups makes the account be in every group the resource declares, and takes it out of every group
|
||||
// this resource put it in and no longer asks for. What the mesh put the account in is recorded on out.
|
||||
func applyGroups(ctx context.Context, sys system.System, r *declaration.User, run Runner,
|
||||
previous store.Applied, wanted Wanted, out *Outcome) error {
|
||||
previous store.Applied, wanted Wanted, handed Handed, out *Outcome) error {
|
||||
// What this resource put the account in before, for this account only: a declaration that renamed its
|
||||
// user says nothing about the new one's groups.
|
||||
var added []string
|
||||
@@ -83,6 +83,10 @@ func applyGroups(ctx context.Context, sys system.System, r *declaration.User, ru
|
||||
var put []string
|
||||
for _, want := range r.Groups {
|
||||
if already[want] {
|
||||
// A record that went handed it here: the mesh's, and this resource's to take back (novox/hq ADR 0312 §5).
|
||||
if handed.has(r.Name, want) && !contains(added, want) {
|
||||
added = append(added, want)
|
||||
}
|
||||
continue
|
||||
}
|
||||
exists, err := system.GroupExists(ctx, system.Runner(run), want)
|
||||
@@ -139,7 +143,11 @@ func applyGroups(ctx context.Context, sys system.System, r *declaration.User, ru
|
||||
// giveGroupsBack is removeUser's groups: every group the mesh put the account in, taken back unless
|
||||
// another declared resource still asks for it. Never fatal, for the shell's reason: a removal that failed
|
||||
// would stay recorded and fail the same way on every apply after. Empty when there was nothing to say.
|
||||
func giveGroupsBack(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted) (bool, string) {
|
||||
//
|
||||
// A group left because another declared resource still asks for it is handed to that resource (handed), which
|
||||
// records it as the mesh's, so it is taken back once nobody asks (novox/hq ADR 0312 §5).
|
||||
func giveGroupsBack(ctx context.Context, sys system.System, a store.Applied, run Runner, wanted Wanted,
|
||||
handed Handed) (bool, string) {
|
||||
if len(a.Groups) == 0 {
|
||||
return false, ""
|
||||
}
|
||||
@@ -157,6 +165,7 @@ func giveGroupsBack(ctx context.Context, sys system.System, a store.Applied, run
|
||||
}
|
||||
if others := wanted.othersAsk(a.Target, g, a.ID); len(others) > 0 {
|
||||
said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g, strings.Join(others, ", ")))
|
||||
handed.hand(a.Target, g)
|
||||
continue
|
||||
}
|
||||
gone, why := leaveGroup(ctx, sys, a.Target, g, run)
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
@@ -111,7 +111,10 @@ type fakeContainer struct {
|
||||
// options. The host passes `--` before every unit (third review of mesh-catalog #147), which
|
||||
// internal/system's own test holds; the fakes read the unit where they always did.
|
||||
func oldShape(name string, args []string) []string {
|
||||
if name != "systemctl" || len(args) < 2 || args[len(args)-2] != "--" {
|
||||
if name != "systemctl" {
|
||||
return asideShape(args)
|
||||
}
|
||||
if len(args) < 2 || args[len(args)-2] != "--" {
|
||||
return args
|
||||
}
|
||||
v := 0
|
||||
@@ -185,6 +188,13 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
|
||||
return "", errors.New("no such image")
|
||||
}
|
||||
return "", nil
|
||||
case "pull":
|
||||
// A real runtime leaves the image present after a pull.
|
||||
if m.images == nil {
|
||||
m.images = map[string]string{}
|
||||
}
|
||||
m.images[args[len(args)-1]] = "2026-01-01T00:00:00Z"
|
||||
return "", nil
|
||||
case "network":
|
||||
if len(args) > 1 && args[1] == "inspect" {
|
||||
return strings.Join(m.members[args[len(args)-1]], ",") + ",\n", nil
|
||||
@@ -1121,3 +1131,16 @@ func TestAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) {
|
||||
t.Errorf("it was re-moded to %o", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
// asideShape reads the steps of a container set aside while its replacement starts (issue 1945) as the removal the
|
||||
// tests were written against: setting it aside is its removal, and what is done to the set-aside container afterwards
|
||||
// is no step of theirs.
|
||||
func asideShape(args []string) []string {
|
||||
if len(args) == 3 && args[0] == "rename" && args[2] == args[1]+asideSuffix {
|
||||
return []string{"rm", "-f", args[1]}
|
||||
}
|
||||
if len(args) > 0 && strings.HasSuffix(args[len(args)-1], asideSuffix) && (args[0] == "rm" || args[0] == "stop") {
|
||||
return []string{"aside"}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ func chmodUnderAs(home, path string, mode os.FileMode, want int) error {
|
||||
return err
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
if err := unix.Fchmod(fd, uint32(mode.Perm())); err != nil {
|
||||
if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
|
||||
return &os.PathError{Op: "chmod", Path: path, Err: err}
|
||||
}
|
||||
return nil
|
||||
@@ -256,8 +256,8 @@ func giveHandedOver(path string, mode os.FileMode, uid, gid int, chown bool) (be
|
||||
if err := unix.Fstat(fd, &st); err != nil {
|
||||
return 0, 0, 0, &os.PathError{Op: "fstat", Path: path, Err: err}
|
||||
}
|
||||
before, beforeUID, beforeGID = os.FileMode(st.Mode&0o777), int(st.Uid), int(st.Gid)
|
||||
if err := unix.Fchmod(fd, uint32(mode.Perm())); err != nil {
|
||||
before, beforeUID, beforeGID = statDirMode(st.Mode), int(st.Uid), int(st.Gid)
|
||||
if err := unix.Fchmod(fd, unixDirMode(mode)); err != nil {
|
||||
return 0, 0, 0, &os.PathError{Op: "chmod", Path: path, Err: err}
|
||||
}
|
||||
if chown && (beforeUID != uid || beforeGID != gid) {
|
||||
|
||||
@@ -8,6 +8,8 @@ package apply
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
func chmodUnder(home, path string, mode os.FileMode) error {
|
||||
@@ -64,3 +66,19 @@ func giveHandedOver(path string, _ os.FileMode, _, _ int, _ bool) (os.FileMode,
|
||||
return 0, 0, 0, &PlacementRefusedError{Path: path, Why: "a handed-over directory is changed only through a " +
|
||||
"descriptor that followed no link, which this system cannot open"}
|
||||
}
|
||||
|
||||
// judgeMeshDir refuses where there is no openat with O_NOFOLLOW (novox/hq issue 358). The node-engine runs on
|
||||
// Linux.
|
||||
func judgeMeshDir(path string) error {
|
||||
return &PlacementRefusedError{Path: path, Why: "a mesh directory is judged only by a walk that follows no link " +
|
||||
"another account could make, which this system cannot do"}
|
||||
}
|
||||
|
||||
// meshDirsByDescriptor is false where there is no openat with O_NOFOLLOW: a mesh directory is changed by path, as
|
||||
// before novox/hq issue 358. The node-engine runs on Linux.
|
||||
const meshDirsByDescriptor = false
|
||||
|
||||
func applyMeshDirectory(r *declaration.Directory, out Outcome, _ os.FileMode, _ bool) (Outcome, error) {
|
||||
return out, &PlacementRefusedError{Path: r.Path, Why: "a mesh directory is changed only through a descriptor " +
|
||||
"that followed no link another account could make, which this system cannot open"}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// Every module is given an identity of its own (novox/hq ADR 0328 §3, §4, §15, issue 567).
|
||||
//
|
||||
// The controller allocates each module a user and a group number once, from a range of its own, the same on every
|
||||
// machine, and declares them as `mesh-<module>`. The engine makes the group and the account with exactly those
|
||||
// numbers — a system account: no login shell, no home, no password — before anything of the declaration is
|
||||
// applied, so a directory the module owns can be given to it and a container told to run as it.
|
||||
//
|
||||
// **Never root** (§11): a number that is 0, or a gid a root group has here (by its number and by its name), is
|
||||
// refused for the whole declaration before anything is made or started, naming why — for an account and for an
|
||||
// image alike. **Never another's**: a name that exists with other numbers, or numbers another name holds, is
|
||||
// refused for that identity alone, naming whose, and its module is not attempted; the rest of the machine is.
|
||||
//
|
||||
// **What it made, and only that, goes** when the identity is no longer declared: the record says whether the
|
||||
// engine made the account, the group or both, and an account or a group that was there before is never removed.
|
||||
|
||||
// accountsTool is how the engine reads and changes the machine's accounts: the system's own tools through the
|
||||
// runner, which is what a test stands in for.
|
||||
type accountsTool interface {
|
||||
CreateSystemAccount(ctx context.Context, run system.Runner, name string, uid, gid int, groups []string) error
|
||||
ModifyAccount(ctx context.Context, run system.Runner, name string, gid int, groups []string) error
|
||||
CreateSystemGroup(ctx context.Context, run system.Runner, name string, gid int) error
|
||||
RemoveAccount(ctx context.Context, run system.Runner, name string) error
|
||||
RemoveGroup(ctx context.Context, run system.Runner, name string) error
|
||||
}
|
||||
|
||||
// entry is one line the user or group database gives for a key: its name and its numbers.
|
||||
type entry struct {
|
||||
name string
|
||||
id, prim int
|
||||
}
|
||||
|
||||
// lookUp asks the database (passwd or group) for key, a name or a number. Absent is (entry{}, false, nil); a
|
||||
// database that does not answer is an error, never read as absent (system.LookUpUser's rule).
|
||||
func lookUp(ctx context.Context, run Runner, database, key string) (entry, bool, error) {
|
||||
out, err := run(ctx, "getent", database, key)
|
||||
if err != nil {
|
||||
if code, ok := system.ExitCode(err); ok && code == 2 {
|
||||
return entry{}, false, nil
|
||||
}
|
||||
return entry{}, false, fmt.Errorf("the %s database did not answer about %s: %w", database, key, err)
|
||||
}
|
||||
fields := strings.Split(strings.TrimSpace(strings.SplitN(out, "\n", 2)[0]), ":")
|
||||
if len(fields) < 3 {
|
||||
return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
|
||||
}
|
||||
e := entry{name: fields[0], prim: -1}
|
||||
if e.id, err = strconv.Atoi(fields[2]); err != nil {
|
||||
return entry{}, false, fmt.Errorf("the %s database gave %q for %s", database, strings.TrimSpace(out), key)
|
||||
}
|
||||
if database == "passwd" && len(fields) >= 4 {
|
||||
if e.prim, err = strconv.Atoi(fields[3]); err != nil {
|
||||
e.prim = -1
|
||||
}
|
||||
}
|
||||
return e, true, nil
|
||||
}
|
||||
|
||||
// refuseRootIdentities is every identity and every container's run-as that is root's (§11), judged before
|
||||
// anything of the declaration is applied: one is enough to refuse it whole, so nothing is made or started.
|
||||
func refuseRootIdentities(d *declaration.Declaration) error {
|
||||
var problems []string
|
||||
for _, id := range d.Identities {
|
||||
// Its user's number, its own group's (the same number), its primary group and every added group.
|
||||
if err := rootIdentity(id.UID, id.UID); err != nil {
|
||||
problems = append(problems, fmt.Sprintf("the identity %s: %v", id.Name, err))
|
||||
}
|
||||
for _, gid := range append([]int{id.GID}, id.Groups...) {
|
||||
if why := rootGroup(gid); why != "" {
|
||||
problems = append(problems, fmt.Sprintf("the identity %s is given gid %s, and a module's identity "+
|
||||
"is never in a root group (novox/hq ADR 0328 §15)", id.Name, why))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, g := range d.Groups {
|
||||
if why := rootGroup(g.GID); why != "" {
|
||||
problems = append(problems, fmt.Sprintf("the library group %s is gid %s, and a library's group is never a "+
|
||||
"root group (novox/hq ADR 0328 §15)", g.Name, why))
|
||||
}
|
||||
}
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok && c.RunAs != nil {
|
||||
if err := rootIdentity(c.RunAs.UID, c.RunAs.GID); err != nil {
|
||||
problems = append(problems, fmt.Sprintf("the container %s is told to run as uid %d, gid %d: %v",
|
||||
c.ID, c.RunAs.UID, c.RunAs.GID, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(problems)
|
||||
return fmt.Errorf("%s; nothing of this declaration was made or started", strings.Join(problems, "; "))
|
||||
}
|
||||
|
||||
// applyIdentity makes one identity as declared (ADR 0328 §4): its own group `mesh-<module>` with the number of its
|
||||
// uid, and its user with that uid, its primary group gid (its own, or its first library's) and its other libraries'
|
||||
// groups as added groups — or refuses it naming whose the name or the numbers are. A user the engine made is given
|
||||
// a changed primary or added groups; one that was there before is never changed, and is refused if it is not as
|
||||
// declared. What it made is recorded on known; an identity already right is unchanged.
|
||||
func applyIdentity(ctx context.Context, tool accountsTool, id declaration.Identity, names map[int]string,
|
||||
known *store.State, run Runner) (Outcome, error) {
|
||||
out := Outcome{ID: "identity " + id.Name, Type: "identity", Target: id.Name, Action: "unchanged"}
|
||||
user, hasUser, err := lookUp(ctx, run, "passwd", id.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
group, hasGroup, err := lookUp(ctx, run, "group", id.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
byUID, uidUsed, err := lookUp(ctx, run, "passwd", strconv.Itoa(id.UID))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(id.UID))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
rec := madeIdentity(known, id.Name)
|
||||
const never = "; nothing was made, and nothing of the module was attempted (novox/hq ADR 0328 §4)"
|
||||
switch {
|
||||
case hasUser && user.id != id.UID:
|
||||
return out, fmt.Errorf("the account %s is here with uid %d, and the mesh allocated uid %d%s", id.Name,
|
||||
user.id, id.UID, never)
|
||||
case hasUser && user.prim != id.GID && !rec.MadeUser:
|
||||
return out, fmt.Errorf("the account %s is here with group %d, was not made by the mesh, and the mesh "+
|
||||
"gives it group %d%s", id.Name, user.prim, id.GID, never)
|
||||
case hasGroup && group.id != id.UID:
|
||||
return out, fmt.Errorf("the group %s is here with gid %d, and a module's own group has its uid, %d%s",
|
||||
id.Name, group.id, id.UID, never)
|
||||
case uidUsed && byUID.name != id.Name:
|
||||
return out, fmt.Errorf("uid %d is the account %s's here, and the mesh allocated it to %s%s", id.UID,
|
||||
byUID.name, id.Name, never)
|
||||
case gidUsed && byGID.name != id.Name:
|
||||
return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s%s", id.UID,
|
||||
byGID.name, id.Name, never)
|
||||
}
|
||||
added := make([]string, 0, len(id.Groups))
|
||||
for _, gid := range id.Groups {
|
||||
added = append(added, names[gid])
|
||||
}
|
||||
sort.Strings(added)
|
||||
rec.Module, rec.Name, rec.UID, rec.GID = id.Module, id.Name, id.UID, id.GID
|
||||
var made []string
|
||||
if !hasGroup {
|
||||
if err := tool.CreateSystemGroup(ctx, run, id.Name, id.UID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
rec.MadeGroup = true
|
||||
made = append(made, fmt.Sprintf("its own group (gid %d)", id.UID))
|
||||
}
|
||||
if !hasUser {
|
||||
if err := tool.CreateSystemAccount(ctx, run, id.Name, id.UID, id.GID, added); err != nil {
|
||||
recordIdentity(known, rec)
|
||||
return out, err
|
||||
}
|
||||
rec.MadeUser = true
|
||||
made = append(made, fmt.Sprintf("the account (uid %d, group %d%s), with no login, no home and no password",
|
||||
id.UID, id.GID, alsoIn(added)))
|
||||
} else if rec.MadeUser {
|
||||
// The mesh's own account: its primary and added groups follow what is declared.
|
||||
in, err := system.GroupsOf(ctx, system.Runner(run), id.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, g := range in {
|
||||
if g != id.Name && g != names[user.prim] {
|
||||
have[g] = true
|
||||
}
|
||||
}
|
||||
same := user.prim == id.GID && len(have) == len(added)
|
||||
for _, g := range added {
|
||||
same = same && have[g]
|
||||
}
|
||||
if !same {
|
||||
if err := tool.ModifyAccount(ctx, run, id.Name, id.GID, added); err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Action = "updated"
|
||||
out.Detail = fmt.Sprintf("gave the account group %d%s", id.GID, alsoIn(added))
|
||||
}
|
||||
}
|
||||
if rec.At.IsZero() {
|
||||
rec.At = time.Now().UTC()
|
||||
}
|
||||
recordIdentity(known, rec)
|
||||
if len(made) > 0 {
|
||||
out.Action, out.Detail = "created", "made "+strings.Join(made, " and ")
|
||||
} else if out.Detail == "" {
|
||||
out.Detail = "here with its numbers"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func alsoIn(groups []string) string {
|
||||
if len(groups) == 0 {
|
||||
return ""
|
||||
}
|
||||
return ", also in " + strings.Join(groups, ", ")
|
||||
}
|
||||
|
||||
// applyGroup makes one library's group with the number the controller allocated (ADR 0328 §3), or refuses it naming
|
||||
// whose the name or the number is, or that the number is a root group's (§15). What it made is recorded on known.
|
||||
func applyGroup(ctx context.Context, tool accountsTool, g declaration.Group, known *store.State,
|
||||
run Runner) (Outcome, error) {
|
||||
out := Outcome{ID: "group " + g.Name, Type: "group", Target: g.Name, Action: "unchanged"}
|
||||
if why := rootGroup(g.GID); why != "" {
|
||||
return out, fmt.Errorf("its gid is %s, and a library's group is never a root group (novox/hq ADR 0328 §15); "+
|
||||
"nothing was made", why)
|
||||
}
|
||||
byName, hasName, err := lookUp(ctx, run, "group", g.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
byGID, gidUsed, err := lookUp(ctx, run, "group", strconv.Itoa(g.GID))
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
switch {
|
||||
case hasName && byName.id != g.GID:
|
||||
return out, fmt.Errorf("the group %s is here with gid %d, and the mesh allocated gid %d; nothing was made",
|
||||
g.Name, byName.id, g.GID)
|
||||
case gidUsed && byGID.name != g.Name:
|
||||
return out, fmt.Errorf("gid %d is the group %s's here, and the mesh allocated it to %s; nothing was made",
|
||||
g.GID, byGID.name, g.Name)
|
||||
case hasName:
|
||||
out.Detail = "here with its number"
|
||||
return out, nil
|
||||
}
|
||||
if err := tool.CreateSystemGroup(ctx, run, g.Name, g.GID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
known.Groups = append(known.Groups, store.MadeGroup{Name: g.Name, GID: g.GID, At: time.Now().UTC()})
|
||||
out.Action, out.Detail = "created", fmt.Sprintf("made the library's group (gid %d)", g.GID)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// removeGroups removes every library group the engine made that is no longer declared, while it still has the
|
||||
// number the engine gave it. Never fatal: one that could not be removed (still a user's primary group) stays
|
||||
// recorded, said, for the next apply.
|
||||
func removeGroups(ctx context.Context, tool accountsTool, declared []declaration.Group, known *store.State,
|
||||
run Runner) []Outcome {
|
||||
wanted := map[string]bool{}
|
||||
for _, g := range declared {
|
||||
wanted[g.Name] = true
|
||||
}
|
||||
var out []Outcome
|
||||
var keep []store.MadeGroup
|
||||
for _, m := range known.Groups {
|
||||
if wanted[m.Name] {
|
||||
keep = append(keep, m)
|
||||
continue
|
||||
}
|
||||
o := Outcome{ID: "group " + m.Name, Type: "group", Target: m.Name, Action: "removed",
|
||||
Detail: "no longer declared: removed the group"}
|
||||
g, has, err := lookUp(ctx, run, "group", m.Name)
|
||||
switch {
|
||||
case err != nil || (has && g.id != m.GID):
|
||||
o.Action, o.Detail = "forgotten", "no longer declared: the group was left, it no longer has the gid the "+
|
||||
"mesh gave it, or the database did not answer"
|
||||
case has:
|
||||
if err := tool.RemoveGroup(ctx, run, m.Name); err != nil {
|
||||
o.Action, o.Detail = "waiting", "no longer declared: the group could not be removed: "+err.Error()
|
||||
keep = append(keep, m)
|
||||
}
|
||||
default:
|
||||
o.Action, o.Detail = "forgotten", "no longer declared, and no longer here"
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
known.Groups = keep
|
||||
return out
|
||||
}
|
||||
|
||||
// madeIdentity is what the engine recorded of name, or an empty record.
|
||||
func madeIdentity(known *store.State, name string) store.MadeIdentity {
|
||||
for _, m := range known.Identities {
|
||||
if m.Name == name {
|
||||
return m
|
||||
}
|
||||
}
|
||||
return store.MadeIdentity{}
|
||||
}
|
||||
|
||||
// recordIdentity records what was made of one identity, in place of any record before.
|
||||
func recordIdentity(known *store.State, rec store.MadeIdentity) {
|
||||
for i, m := range known.Identities {
|
||||
if m.Name == rec.Name {
|
||||
known.Identities[i] = rec
|
||||
return
|
||||
}
|
||||
}
|
||||
if rec.MadeUser || rec.MadeGroup {
|
||||
known.Identities = append(known.Identities, rec)
|
||||
}
|
||||
}
|
||||
|
||||
// removeIdentities removes what the engine made of every identity no longer declared — the account, then the
|
||||
// group — and only while it still has the numbers the engine gave it. Never fatal: what could not be removed
|
||||
// stays recorded, said, for the next apply.
|
||||
func removeIdentities(ctx context.Context, tool accountsTool, declared []declaration.Identity, known *store.State,
|
||||
run Runner) []Outcome {
|
||||
wanted := map[string]bool{}
|
||||
for _, id := range declared {
|
||||
wanted[id.Name] = true
|
||||
}
|
||||
var out []Outcome
|
||||
var keep []store.MadeIdentity
|
||||
for _, m := range known.Identities {
|
||||
if wanted[m.Name] {
|
||||
keep = append(keep, m)
|
||||
continue
|
||||
}
|
||||
o := Outcome{ID: "identity " + m.Name, Type: "identity", Target: m.Name, Action: "removed"}
|
||||
var said []string
|
||||
if m.MadeUser {
|
||||
if u, has, err := lookUp(ctx, run, "passwd", m.Name); err != nil || (has && u.id != m.UID) {
|
||||
said = append(said, "the account was left: it no longer has the uid the mesh gave it, or the "+
|
||||
"database did not answer")
|
||||
m.MadeUser = false
|
||||
} else if has {
|
||||
if err := tool.RemoveAccount(ctx, run, m.Name); err != nil {
|
||||
said = append(said, "the account could not be removed: "+err.Error())
|
||||
} else {
|
||||
m.MadeUser = false
|
||||
said = append(said, "removed the account")
|
||||
}
|
||||
} else {
|
||||
m.MadeUser = false
|
||||
}
|
||||
}
|
||||
if m.MadeGroup && !m.MadeUser {
|
||||
if g, has, err := lookUp(ctx, run, "group", m.Name); err != nil || (has && g.id != m.GID) {
|
||||
said = append(said, "the group was left: it no longer has the gid the mesh gave it, or the "+
|
||||
"database did not answer")
|
||||
m.MadeGroup = false
|
||||
} else if has {
|
||||
if err := tool.RemoveGroup(ctx, run, m.Name); err != nil {
|
||||
said = append(said, "the group could not be removed: "+err.Error())
|
||||
} else {
|
||||
m.MadeGroup = false
|
||||
said = append(said, "removed the group")
|
||||
}
|
||||
} else {
|
||||
m.MadeGroup = false
|
||||
}
|
||||
}
|
||||
if m.MadeUser || m.MadeGroup {
|
||||
o.Action = "waiting"
|
||||
keep = append(keep, m)
|
||||
}
|
||||
o.Detail = "no longer declared: " + strings.Join(said, "; ")
|
||||
out = append(out, o)
|
||||
}
|
||||
known.Identities = keep
|
||||
return out
|
||||
}
|
||||
|
||||
// toolFor is the accounts tool of a system, or why it has none.
|
||||
func toolFor(sys system.System) (accountsTool, error) {
|
||||
if t, ok := sys.(accountsTool); ok {
|
||||
return t, nil
|
||||
}
|
||||
return nil, errors.New("this system makes no module identity: it has no way to make a system account")
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// The engine's half of ADR 0325 §3 and §11 (issue 567): every module given an account and a group of its own with
|
||||
// the numbers the controller allocated, a container told to run as it, and never a root number.
|
||||
|
||||
// accountsMachine is a user and group database the runner answers from, and every command that changed it.
|
||||
type accountsMachine struct {
|
||||
users, groups map[string][2]int // name → id, primary gid (a group's second is unused)
|
||||
// in is each account's added groups, by name; homes each account's home, "/" when not set.
|
||||
in map[string][]string
|
||||
homes map[string]string
|
||||
ran []string
|
||||
}
|
||||
|
||||
// groupNamed is the name of gid in m, or the number.
|
||||
func (m *accountsMachine) groupNamed(gid int) string {
|
||||
for n, ids := range m.groups {
|
||||
if ids[0] == gid {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return strconv.Itoa(gid)
|
||||
}
|
||||
|
||||
func (m *accountsMachine) run(_ context.Context, name string, args ...string) (string, error) {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
switch name {
|
||||
case "getent":
|
||||
db, key := args[0], args[1]
|
||||
table := m.users
|
||||
if db == "group" {
|
||||
table = m.groups
|
||||
}
|
||||
for n, ids := range table {
|
||||
if n == key || strconv.Itoa(ids[0]) == key {
|
||||
if db == "group" {
|
||||
return n + ":x:" + strconv.Itoa(ids[0]) + ":\n", nil
|
||||
}
|
||||
home := m.homes[n]
|
||||
if home == "" {
|
||||
home = "/"
|
||||
}
|
||||
return n + ":x:" + strconv.Itoa(ids[0]) + ":" + strconv.Itoa(ids[1]) + "::" + home + ":/usr/bin/nologin\n", nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("getent exited 2")
|
||||
case "id":
|
||||
name := args[len(args)-1]
|
||||
u, ok := m.users[name]
|
||||
if !ok {
|
||||
return "", errors.New("id exited 1")
|
||||
}
|
||||
return strings.Join(append([]string{m.groupNamed(u[1])}, m.in[name]...), " ") + "\n", nil
|
||||
case "groupadd":
|
||||
m.groups[args[len(args)-1]] = [2]int{mustAtoi(args[2]), 0}
|
||||
case "useradd":
|
||||
name := args[len(args)-1]
|
||||
m.users[name] = [2]int{mustAtoi(args[2]), mustAtoi(args[4])}
|
||||
if args[5] == "--groups" {
|
||||
m.setIn(name, strings.Split(args[6], ","))
|
||||
}
|
||||
if m.homes == nil {
|
||||
m.homes = map[string]string{}
|
||||
}
|
||||
m.homes[name] = "/nonexistent"
|
||||
case "usermod":
|
||||
name := args[len(args)-1]
|
||||
if args[0] == "--append" {
|
||||
m.setIn(name, append(m.in[name], args[2]))
|
||||
} else if args[0] == "--home" {
|
||||
if m.homes == nil {
|
||||
m.homes = map[string]string{}
|
||||
}
|
||||
m.homes[name] = args[1]
|
||||
} else {
|
||||
u := m.users[name]
|
||||
m.users[name] = [2]int{u[0], mustAtoi(args[1])}
|
||||
m.setIn(name, strings.Split(args[3], ","))
|
||||
}
|
||||
case "gpasswd":
|
||||
var kept []string
|
||||
for _, g := range m.in[args[1]] {
|
||||
if g != args[2] {
|
||||
kept = append(kept, g)
|
||||
}
|
||||
}
|
||||
m.setIn(args[1], kept)
|
||||
case "userdel":
|
||||
delete(m.users, args[0])
|
||||
case "groupdel":
|
||||
delete(m.groups, args[0])
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
m.ran = append(m.ran, line)
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (m *accountsMachine) setIn(name string, groups []string) {
|
||||
if m.in == nil {
|
||||
m.in = map[string][]string{}
|
||||
}
|
||||
var out []string
|
||||
for _, g := range groups {
|
||||
if g != "" {
|
||||
out = append(out, g)
|
||||
}
|
||||
}
|
||||
m.in[name] = out
|
||||
}
|
||||
|
||||
func mustAtoi(s string) int {
|
||||
n, _ := strconv.Atoi(s)
|
||||
return n
|
||||
}
|
||||
|
||||
// withGroups is a declaration with library groups, identities and resources.
|
||||
func withGroups(t *testing.T, groups, identities, resources string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
nothing := ""
|
||||
if resources == "" {
|
||||
nothing = `"owns_nothing":true,`
|
||||
}
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"groups":[` + groups + `],"identities":[` + identities +
|
||||
`],` + nothing + `"resources":[` + resources + `]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func withIdentities(t *testing.T, identities, resources string) *declaration.Declaration {
|
||||
t.Helper()
|
||||
d, err := declaration.Parse([]byte(`{"declaration":1,"identities":[` + identities + `],"resources":[` +
|
||||
resources + `]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Row 3: the account made as a system account with exactly its numbers, recorded, and removed — only what was
|
||||
// made — when no longer declared.
|
||||
func TestAModulesIdentityIsMadeWithItsNumbersAndRemovedWhenUndeclared(t *testing.T) {
|
||||
m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
|
||||
dir := t.TempDir()
|
||||
d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`)
|
||||
report, known, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
"groupadd --system --gid 61001 mesh-radarr",
|
||||
"useradd --system --uid 61001 --gid 61001 --no-create-home --home-dir /nonexistent --shell /usr/bin/nologin mesh-radarr",
|
||||
}
|
||||
if strings.Join(m.ran, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("made with:\n%s\nwant:\n%s", strings.Join(m.ran, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if o := outcomeOf(report, "identity mesh-radarr"); o.Action != "created" {
|
||||
t.Errorf("the identity's outcome: %+v", o)
|
||||
}
|
||||
if len(known.Identities) != 1 || !known.Identities[0].MadeUser || !known.Identities[0].MadeGroup {
|
||||
t.Fatalf("what was made is not recorded: %+v", known.Identities)
|
||||
}
|
||||
|
||||
// Applied again: nothing is made twice.
|
||||
m.ran = nil
|
||||
if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, m.run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.ran) != 0 {
|
||||
t.Fatalf("an identity already here was made again: %v", m.ran)
|
||||
}
|
||||
|
||||
// Undeclared, with a group the machine had before for another identity: only what was made goes.
|
||||
m.groups["mesh-sonarr"] = [2]int{61002, 0}
|
||||
other := withIdentities(t, `{"module":"sonarr","name":"mesh-sonarr","uid":61002,"gid":61002}`,
|
||||
`{"id":"x.dir","type":"directory","path":"`+dir+`/x"}`)
|
||||
if _, known, err = Apply(context.Background(), archHost(t), other, known, store.OriginDeclared, m.run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, has := m.users["mesh-radarr"]; has {
|
||||
t.Error("an identity no longer declared kept its account")
|
||||
}
|
||||
if _, has := m.groups["mesh-radarr"]; has {
|
||||
t.Error("an identity no longer declared kept its group")
|
||||
}
|
||||
if rec := madeIdentity(&known, "mesh-sonarr"); rec.MadeGroup {
|
||||
t.Error("a group that was there before is recorded as the mesh's")
|
||||
}
|
||||
}
|
||||
|
||||
// Row 3: a name that exists with other numbers, or numbers another name holds, is refused naming whose, and the
|
||||
// module is not attempted; the rest of the machine is.
|
||||
func TestAnIdentityWhoseNameOrNumbersAreAnothersIsRefused(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, tc := range []struct {
|
||||
users, groups map[string][2]int
|
||||
says string
|
||||
}{
|
||||
{users: map[string][2]int{"mesh-radarr": {1500, 1500}}, groups: map[string][2]int{}, says: "uid 1500"},
|
||||
{users: map[string][2]int{"someone": {61001, 100}}, groups: map[string][2]int{}, says: "someone"},
|
||||
{users: map[string][2]int{}, groups: map[string][2]int{"media": {61001, 0}}, says: "media"},
|
||||
} {
|
||||
m := &accountsMachine{users: tc.users, groups: tc.groups}
|
||||
d := withIdentities(t, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":61001}`,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/radarr"},`+
|
||||
`{"id":"plex.config","type":"directory","path":"`+dir+`/plex"}`)
|
||||
report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.says) {
|
||||
t.Errorf("not refused naming %s: %v", tc.says, err)
|
||||
}
|
||||
if len(m.ran) != 0 {
|
||||
t.Errorf("something was made for an identity that is another's: %v", m.ran)
|
||||
}
|
||||
if o := outcomeOf(report, "radarr.config"); o.Action != "skipped" {
|
||||
t.Errorf("the module of a refused identity was attempted: %+v", o)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "plex")); err != nil {
|
||||
t.Errorf("the rest of the machine was not applied: %v", err)
|
||||
}
|
||||
os.RemoveAll(filepath.Join(dir, "plex"))
|
||||
}
|
||||
}
|
||||
|
||||
// Row 11: a declaration naming a root group for an account or an image is refused with its reason, and nothing is
|
||||
// made or started. A number 0 is refused on arrival, by the parser.
|
||||
func TestNoModuleIdentityIsRoot(t *testing.T) {
|
||||
was := RootGroupIDs
|
||||
RootGroupIDs = func() map[int]string { return map[int]string{10: "wheel"} }
|
||||
defer func() { RootGroupIDs = was }()
|
||||
dir := t.TempDir()
|
||||
for _, d := range []*declaration.Declaration{
|
||||
withGroups(t, `{"name":"mesh-media","gid":10}`, `{"module":"radarr","name":"mesh-radarr","uid":61001,"gid":10}`,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`),
|
||||
withGroups(t, `{"name":"mesh-media","gid":10}`, ``,
|
||||
`{"id":"radarr.config","type":"directory","path":"`+dir+`/config"}`),
|
||||
withIdentities(t, ``, `{"id":"radarr.config","type":"directory","path":"`+dir+`/config"},`+
|
||||
`{"id":"radarr.app","type":"container","name":"radarr","image":"`+pinned+`",`+
|
||||
`"identity":{"mode":"run","uid":61001,"gid":10}}`),
|
||||
} {
|
||||
m := &accountsMachine{users: map[string][2]int{}, groups: map[string][2]int{}}
|
||||
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, m.run, nil, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "wheel") || !strings.Contains(err.Error(), "never") {
|
||||
t.Errorf("a root group was not refused naming why: %v", err)
|
||||
}
|
||||
if len(m.ran) != 0 {
|
||||
t.Errorf("something was made or started: %v", m.ran)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "config")); err == nil {
|
||||
t.Error("a resource of a declaration naming a root group was applied")
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{
|
||||
`{"declaration":1,"identities":[{"module":"m","name":"mesh-m","uid":0,"gid":61001}],"resources":[]}`,
|
||||
`{"declaration":1,"resources":[{"id":"m.app","type":"container","name":"m","image":"` + pinned + `",` +
|
||||
`"identity":{"mode":"env","user_env":"PUID","group_env":"PGID","uid":61001,"gid":0}}]}`,
|
||||
} {
|
||||
if _, err := declaration.Parse([]byte(raw)); err == nil || !strings.Contains(err.Error(), "never root") {
|
||||
t.Errorf("a number 0 was not refused on arrival: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Row 3: a container declaring identity is told it — by the variables its image reads, or the runtime's user
|
||||
// option — and a changed identity is a container recreated.
|
||||
func TestAContainerIsToldItsModulesIdentity(t *testing.T) {
|
||||
env := &declaration.Container{ID: "radarr.app", Name: "radarr", Image: pinned, RunAs: &declaration.ContainerIdentity{
|
||||
Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID", UID: 61001, GID: 61001}}
|
||||
run := &declaration.Container{ID: "plex.app", Name: "plex", Image: pinned, RunAs: &declaration.ContainerIdentity{
|
||||
Mode: declaration.IdentityByRun, UID: 61002, GID: 61002}}
|
||||
if got := strings.Join(foregroundRunArgs(env, "x"), " "); !strings.Contains(got, "--env PUID=61001 --env PGID=61001") {
|
||||
t.Errorf("an image told by environment was not: %s", got)
|
||||
}
|
||||
if got := strings.Join(foregroundRunArgs(run, "x"), " "); !strings.Contains(got, "--user 61002:61002") {
|
||||
t.Errorf("an image told by the runtime's user option was not: %s", got)
|
||||
}
|
||||
plain := *env
|
||||
plain.RunAs = nil
|
||||
moved := *env
|
||||
moved.RunAs = &declaration.ContainerIdentity{Mode: declaration.IdentityByEnv, UserEnv: "PUID", GroupEnv: "PGID",
|
||||
UID: 61009, GID: 61009}
|
||||
if containerSpec(env, inputs{}) == containerSpec(&plain, inputs{}) ||
|
||||
containerSpec(env, inputs{}) == containerSpec(&moved, inputs{}) {
|
||||
t.Error("an identity told, or renumbered, does not recreate the container")
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0325 §5 supersedes ADR 0312's group grant: what a grant put an account in is taken back on the first apply of
|
||||
// the access, and nothing is granted again.
|
||||
func TestWhatAGroupGrantPutAnAccountInIsTakenBack(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
groups := "nox media"
|
||||
var ran []string
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
ran = append(ran, name+" "+strings.Join(args, " "))
|
||||
switch name {
|
||||
case "id":
|
||||
return groups + "\n", nil
|
||||
case "gpasswd":
|
||||
groups = "nox"
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
|
||||
Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
|
||||
d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write","grant":{"account":"nox"}}`)
|
||||
report, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if groups != "nox" || !strings.Contains(strings.Join(ran, "\n"), "gpasswd --delete nox media") {
|
||||
t.Fatalf("the grant's group was not taken back: %v", ran)
|
||||
}
|
||||
if o := outcomeOf(report, "importer.books"); o.Action != "updated" || !strings.Contains(o.Detail, "taken back") {
|
||||
t.Errorf("the take-back is not said: %+v", o)
|
||||
}
|
||||
if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 {
|
||||
t.Errorf("the record still holds the grant's groups: %+v", rec)
|
||||
}
|
||||
}
|
||||
|
||||
// Review L7: a group the grant put an account in that could not be left stays recorded, and is tried again.
|
||||
func TestAGrantsGroupThatCouldNotBeLeftIsTriedAgain(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
refuse := true
|
||||
groups := "nox media"
|
||||
run := func(_ context.Context, name string, args ...string) (string, error) {
|
||||
switch name {
|
||||
case "id":
|
||||
return groups + "\n", nil
|
||||
case "gpasswd":
|
||||
if refuse {
|
||||
return "", errors.New("gpasswd exited 1: locked")
|
||||
}
|
||||
groups = "nox"
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
known := store.State{Resources: []store.Applied{{ID: "importer.books", Type: "access", Target: dir,
|
||||
Origin: store.OriginDeclared, User: "nox", Groups: []string{"media"}}}}
|
||||
d := declare(t, `{"id":"importer.books","type":"access","path":"`+dir+`","mode":"read-write"}`)
|
||||
_, known, err := Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec, _ := known.Find("importer.books"); rec.User != "nox" || len(rec.Groups) != 1 {
|
||||
t.Fatalf("a group that could not be left was forgotten: %+v", rec)
|
||||
}
|
||||
refuse = false
|
||||
if _, known, err = Apply(context.Background(), archHost(t), d, known, store.OriginDeclared, run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rec, _ := known.Find("importer.books"); len(rec.Groups) != 0 || groups != "nox" {
|
||||
t.Fatalf("the group was not taken back on the next apply: %+v (%s)", rec, groups)
|
||||
}
|
||||
}
|
||||
|
||||
// Every system the engine runs on that makes accounts makes module identities: arch and alpine alike.
|
||||
func TestArchAndAlpineMakeModuleIdentities(t *testing.T) {
|
||||
for _, name := range []string{"arch", "alpine"} {
|
||||
sys, err := system.For(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := toolFor(sys); err != nil {
|
||||
t.Errorf("%s makes no module identity: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The read-only judgement of a node's directories (novox/hq issue 358): one line each — placed, would be refused
|
||||
// naming the link and who could replace it, or not there yet (placed: an apply makes it) — and nothing made or
|
||||
// changed. Who could replace the link is "every account", "the account X" or "the group X", by the account and the
|
||||
// directories the test runs as: the checker's isolated step runs as the account nobody, which owns / there (issue 1919,
|
||||
// R4g), so the test demands a who in any of the forms couldReplace and linkReplacer produce, not "every account" alone.
|
||||
func TestJudgeDirectoriesSaysWhatTheWalkWouldRefuseAndChangesNothing(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
plain := filepath.Join(base, "plain")
|
||||
if err := os.Mkdir(plain, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
elsewhere := filepath.Join(base, "elsewhere", "state")
|
||||
if err := os.MkdirAll(elsewhere, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(filepath.Join(base, "elsewhere"), filepath.Join(base, "linked")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
missing := filepath.Join(base, "missing", "deeper")
|
||||
d := declare(t, `{"id":"a.plain","type":"directory","path":"`+plain+`","mode":"0700"},`+
|
||||
`{"id":"a.linked","type":"directory","path":"`+filepath.Join(base, "linked", "state")+`","mode":"0700"},`+
|
||||
`{"id":"a.missing","type":"directory","path":"`+missing+`","mode":"0700"}`)
|
||||
|
||||
said := JudgeDirectories(d)
|
||||
lines := strings.Split(said, "\n")
|
||||
if len(lines) != 4 || !strings.HasPrefix(lines[0], "3 directories declared: 2 placed, 1 would be refused") {
|
||||
t.Fatalf("the judgement: %q", said)
|
||||
}
|
||||
if lines[1] != plain+": placed" || lines[3] != missing+": placed" {
|
||||
t.Errorf("a directory the walk reaches is not said placed: %q", said)
|
||||
}
|
||||
if !strings.Contains(lines[2], "would be refused") || !strings.Contains(lines[2], filepath.Join(base, "linked")) ||
|
||||
!(strings.Contains(lines[2], "every account (") || strings.Contains(lines[2], "the account ") ||
|
||||
strings.Contains(lines[2], "the group ")) {
|
||||
t.Errorf("the linked directory is not said refused naming the link and who: %q", lines[2])
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(base, "missing")); !os.IsNotExist(err) {
|
||||
t.Error("the judgement made a directory")
|
||||
}
|
||||
for _, p := range []string{plain, elsewhere} {
|
||||
if info, _ := os.Stat(p); info.Mode().Perm() != 0o755 {
|
||||
t.Errorf("the judgement changed %s to %o", p, info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A link only root could have made is followed by the judgement as by the apply.
|
||||
func TestJudgeDirectoriesFollowsARootOnlyLink(t *testing.T) {
|
||||
wasDirs, wasLinks := couldReplace, linkReplacer
|
||||
couldReplace = func(string, uint32, uint32, uint32) string { return "" }
|
||||
linkReplacer = func(string, uint32) string { return "" }
|
||||
t.Cleanup(func() { couldReplace, linkReplacer = wasDirs, wasLinks })
|
||||
base := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(base, "real", "state"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink("real", filepath.Join(base, "run")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(base, "run", "state")
|
||||
if said := JudgeDirectories(declare(t, `{"id":"a.s","type":"directory","path":"`+path+`"}`)); !strings.Contains(said, path+": placed") {
|
||||
t.Fatalf("a directory behind a root-only link is not said placed: %q", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// An access is an operator's path a module mounts and the mesh does not own (novox/hq ADR 0051): mounted when
|
||||
// present, refused when absent, nothing changed on it. ADR 0325 kept a shared folder through ACL entries; ADR 0328
|
||||
// supersedes it with libraries (library.go), and an access is again exactly what it was before: an access's `users`
|
||||
// and `taken` are accepted and not acted on.
|
||||
|
||||
// applyAccess is one access, applied as ADR 0051 has it; what ADR 0312's grant put an account in is taken back
|
||||
// first, so no account is in a group for data but a library's (ADR 0328 §11).
|
||||
func applyAccess(ctx context.Context, sys system.System, r *declaration.Access, run Runner, previous store.Applied,
|
||||
in inputs) (Outcome, error) {
|
||||
gave, left, said := takeBackGranted(ctx, sys, previous, run, in.groups, in.handed)
|
||||
out, err := legacyAccess(r)
|
||||
if len(left) > 0 {
|
||||
out.user, out.groups = previous.User, left
|
||||
}
|
||||
if said != "" {
|
||||
out.Detail = joinDetail(out.Detail, said)
|
||||
if gave && out.Action == "unchanged" {
|
||||
// Updated, so what names the access under restart-on — the tool runner — holds the groups it has now.
|
||||
out.Action = "updated"
|
||||
}
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// takeBackGranted takes an account out of every group ADR 0312's grant put it in for an access, as removeUser takes
|
||||
// back a user's (ADR 0252 §3): unless another declared resource still asks for it, which is then handed the group.
|
||||
// Whether any was taken back, the groups still to take back — one whose leaving failed stays recorded, so the next
|
||||
// apply tries again — and what was said; nothing for an access that granted nothing.
|
||||
func takeBackGranted(ctx context.Context, sys system.System, previous store.Applied, run Runner, wanted Wanted,
|
||||
handed Handed) (bool, []string, string) {
|
||||
if previous.User == "" || len(previous.Groups) == 0 {
|
||||
return false, nil, ""
|
||||
}
|
||||
account := previous.User
|
||||
in, err := system.GroupsOf(ctx, system.Runner(run), account)
|
||||
if err != nil {
|
||||
return false, previous.Groups, fmt.Sprintf("the groups ADR 0312's grant put %s in (%s) were not taken "+
|
||||
"back, and are tried again: %v", account, strings.Join(previous.Groups, ", "), err)
|
||||
}
|
||||
already := setOf(in)
|
||||
gave := false
|
||||
var left, said []string
|
||||
for _, g := range previous.Groups {
|
||||
switch {
|
||||
case !already[g]:
|
||||
continue
|
||||
case len(wanted.othersAsk(account, g, previous.ID)) > 0:
|
||||
said = append(said, fmt.Sprintf("left in %s, which %s still asks for", g,
|
||||
strings.Join(wanted.othersAsk(account, g, previous.ID), ", ")))
|
||||
handed.hand(account, g)
|
||||
continue
|
||||
}
|
||||
gone, why := leaveGroup(ctx, sys, account, g, run)
|
||||
if !gone {
|
||||
left = append(left, g)
|
||||
why += "; tried again on the next apply"
|
||||
}
|
||||
gave = gave || gone
|
||||
said = append(said, why)
|
||||
}
|
||||
return gave, left, "the group grant ADR 0312 made is taken back: " + account + " " + strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// legacyAccess is an access from a controller older than ADR 0325, applied exactly as before it (ADR 0051 §7).
|
||||
//
|
||||
// **Absent is refused, not created.** A bind mount whose source does not exist is made for you by the container
|
||||
// runtime, as root, with whatever mode it picks — which is exactly the silent wrong-ownership 04-ISSUES/026
|
||||
// records. So the host checks first and says plainly that the operator must provide the path, rather than
|
||||
// conjuring a directory it does not own and cannot give the right owner. Nothing is written, so this never reports
|
||||
// a change: the machine did not move, the host merely confirmed a fact about it.
|
||||
func legacyAccess(r *declaration.Access) (Outcome, error) {
|
||||
out := begin(r)
|
||||
info, err := os.Stat(r.Path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return out, fmt.Errorf(
|
||||
"%s is not there, and the mesh does not own it — the operator must provide it. It is "+
|
||||
"shared, pre-existing data (novox/hq ADR 0051): the host mounts it and creates "+
|
||||
"nothing, so a missing one is said here rather than made as root by the container "+
|
||||
"runtime", r.Path)
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return out, fmt.Errorf(
|
||||
"%s is not a directory, and an access is a shared directory the operator provides",
|
||||
r.Path)
|
||||
}
|
||||
mode := declaration.AccessRead
|
||||
if r.Mode != "" {
|
||||
mode = r.Mode
|
||||
}
|
||||
out.Action = "unchanged"
|
||||
out.Detail = "operator-owned; present, " + mode + ", nothing managed"
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PlacedID names the filesystem at mount for a wait, with no path in it: the controller asks again when the
|
||||
// folder's filesystem is not the one a warrant was given for. Empty when the filesystem cannot be named, and a folder
|
||||
// on it is then never kept (it fails closed, and waits).
|
||||
func PlacedID(mount string) string { return placedFrom(mount, filesystemKey(mount)) }
|
||||
|
||||
// placedFrom is PlacedID from a filesystem key already read.
|
||||
func placedFrom(mount, key string) string {
|
||||
if mount == "" || key == "" {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256([]byte(mount + "\x00" + key))
|
||||
return "fs-" + hex.EncodeToString(sum[:6])
|
||||
}
|
||||
|
||||
// filesystemKey is what names the filesystem at mount across reboots, in this order:
|
||||
//
|
||||
// - a filesystem on a block device: its UUID, found by matching the mount's device against the devices in
|
||||
// /dev/disk/by-uuid (never blkid). xfs's statfs id follows the device number, so the UUID comes first;
|
||||
// - else the kernel's filesystem id (statfs f_fsid) when it is not zero: ZFS's from its dataset, btrfs's from its
|
||||
// UUID;
|
||||
// - else mountinfo's source with the filesystem's type: an NFS export's server:/export, a FUSE filesystem's own
|
||||
// source string (both report f_fsid 0).
|
||||
//
|
||||
// Never the mount point and type alone, and never a device's name, which a reboot may renumber. Empty when none of
|
||||
// these answers within statBound: then nothing is kept there. Read once per mount at a time.
|
||||
func filesystemKey(mount string) string {
|
||||
if mount == "" {
|
||||
return ""
|
||||
}
|
||||
v, ok := boundedOnce("fs "+mount, func() any { return readFilesystemKey(mount) })
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return v.(string)
|
||||
}
|
||||
|
||||
func readFilesystemKey(mount string) string {
|
||||
if u := MountUUID(mount); u != "" {
|
||||
return "uuid:" + u
|
||||
}
|
||||
if id := MountFsid(mount); strings.Trim(id, "0") != "" {
|
||||
return "fsid:" + id
|
||||
}
|
||||
if source, fstype := mountSourceOf(mount); source != "" && source != "none" {
|
||||
return "source:" + fstype + ":" + source
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// statBound is how long a read of a folder may take before it is said not to answer: a hard network mount whose
|
||||
// server is down blocks a stat for ever, and one access must never hold the apply.
|
||||
var statBound = 5 * time.Second
|
||||
|
||||
// errNoAnswer is a folder whose read did not return within statBound.
|
||||
var errNoAnswer = errors.New("it does not answer: a read of it did not return within its bound (a mount that is down)")
|
||||
|
||||
// lstatWithin is os.Lstat within statBound, and statWithin os.Stat. A read that hangs is left to return on its own.
|
||||
|
||||
func lstatWithin(path string) (os.FileInfo, error) { return within("lstat", os.Lstat, path) }
|
||||
func statWithin(path string) (os.FileInfo, error) { return within("stat", os.Stat, path) }
|
||||
|
||||
// flight is one read in flight, shared by everyone who asks while it is: a read that hangs on a dead mount holds
|
||||
// one goroutine per path, never one per apply or look.
|
||||
type flight struct {
|
||||
done chan struct{}
|
||||
v any
|
||||
}
|
||||
|
||||
var inFlight = struct {
|
||||
sync.Mutex
|
||||
reads map[string]*flight
|
||||
}{reads: map[string]*flight{}}
|
||||
|
||||
// boundedOnce is read's answer within statBound, sharing a read of key already in flight; false when it did not
|
||||
// answer in time.
|
||||
func boundedOnce(key string, read func() any) (any, bool) {
|
||||
inFlight.Lock()
|
||||
f, running := inFlight.reads[key]
|
||||
if !running {
|
||||
f = &flight{done: make(chan struct{})}
|
||||
inFlight.reads[key] = f
|
||||
go func() {
|
||||
f.v = read()
|
||||
inFlight.Lock()
|
||||
delete(inFlight.reads, key)
|
||||
inFlight.Unlock()
|
||||
close(f.done)
|
||||
}()
|
||||
}
|
||||
inFlight.Unlock()
|
||||
select {
|
||||
case <-f.done:
|
||||
return f.v, true
|
||||
case <-time.After(statBound):
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
func within(op string, stat func(string) (os.FileInfo, error), path string) (os.FileInfo, error) {
|
||||
type read struct {
|
||||
info os.FileInfo
|
||||
err error
|
||||
}
|
||||
v, ok := boundedOnce(op+" "+path, func() any {
|
||||
info, err := stat(path)
|
||||
return read{info, err}
|
||||
})
|
||||
if !ok {
|
||||
return nil, errNoAnswer
|
||||
}
|
||||
r := v.(read)
|
||||
return r.info, r.err
|
||||
}
|
||||
|
||||
// absPath is an absolute path in words: what pathless takes out.
|
||||
var absPath = regexp.MustCompile(`(^|[\s"'(=])/[^\s"',;:)]*`)
|
||||
|
||||
// Pathless is an error in words with no path (ADR 0234 §6): a path error's operation and cause, and any other
|
||||
// path said as "a path".
|
||||
func Pathless(err error) string { return pathless(err) }
|
||||
|
||||
func pathless(err error) string {
|
||||
var pe *os.PathError
|
||||
if errors.As(err, &pe) {
|
||||
return pe.Op + ": " + pe.Err.Error()
|
||||
}
|
||||
return absPath.ReplaceAllString(err.Error(), "${1}a path")
|
||||
}
|
||||
@@ -0,0 +1,569 @@
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// A library is kept usable by its group (novox/hq ADR 0328, issue 567).
|
||||
//
|
||||
// A library is a named tree of shared data with one home (§1). On its home the engine keeps its root and every
|
||||
// sub-path its modules ask for — those folders themselves, never what is in them — in the library's group with
|
||||
// group read, write and enter and the setgid bit, so what is made beneath keeps the group (§6); a missing sub-path
|
||||
// is made with the root's owner and the library's group (§9). The owner, the owner's bits and others' bits are
|
||||
// never changed. What the root already holds joins only on the operator's word (§7): until then the library waits
|
||||
// with reason keep and nothing under it is walked. A root that is absent, or whose disk is not there, makes
|
||||
// nothing, holds every container that mounts the library, and waits with reason disk (§9). Off its home a library is
|
||||
// a mount the mounts module makes: nothing is kept there, and a mount that is not there holds the containers too.
|
||||
//
|
||||
// **When a root is the mesh's to keep without asking.** When it holds nothing yet, or is already in the library's
|
||||
// group with group write and setgid (kept so before, by hand or by the mesh). Every other root waits for the take
|
||||
// act, which walks it whole (TakeLibrary). Kept, it stays kept on the filesystem it was kept on — named by UUID,
|
||||
// filesystem id or source (PlacedID) — and another filesystem at its root is judged afresh.
|
||||
|
||||
// libraryRecords is where the engine keeps what it knows of each library it is home to: the file beside the state
|
||||
// in the engine, memory in a test.
|
||||
type libraryRecords interface {
|
||||
All() (map[string]store.LibraryRecord, error)
|
||||
Update(name string, change func(r *store.LibraryRecord, had bool) bool) error
|
||||
}
|
||||
|
||||
// Libraries is the engine's record of the libraries it is home to. The engine sets the file beside its state.
|
||||
var Libraries libraryRecords = NewLibraryMemory()
|
||||
|
||||
type libraryMemory struct {
|
||||
mu sync.Mutex
|
||||
m map[string]store.LibraryRecord
|
||||
}
|
||||
|
||||
// NewLibraryMemory is an empty record held in memory: a test's, or a one-shot apply's.
|
||||
func NewLibraryMemory() libraryRecords { return &libraryMemory{m: map[string]store.LibraryRecord{}} }
|
||||
|
||||
func (k *libraryMemory) All() (map[string]store.LibraryRecord, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
out := map[string]store.LibraryRecord{}
|
||||
for n, r := range k.m {
|
||||
out[n] = r
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *libraryMemory) Update(name string, change func(r *store.LibraryRecord, had bool) bool) error {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
r, had := k.m[name]
|
||||
r.Library = name
|
||||
if change(&r, had) {
|
||||
k.m[name] = r
|
||||
} else {
|
||||
delete(k.m, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// libraryRecord is the record of one library, and whether there is one. A record that cannot be read is none: the
|
||||
// library is judged afresh, which waits rather than keeps.
|
||||
func libraryRecord(name string) (store.LibraryRecord, bool) {
|
||||
all, err := Libraries.All()
|
||||
if err != nil {
|
||||
return store.LibraryRecord{}, false
|
||||
}
|
||||
r, had := all[name]
|
||||
return r, had
|
||||
}
|
||||
|
||||
// libraryLocks is one lock per library, held by every change to its root and every walk under it.
|
||||
var libraryLocks sync.Map
|
||||
|
||||
func libraryLock(name string) *sync.Mutex {
|
||||
l, _ := libraryLocks.LoadOrStore(name, &sync.Mutex{})
|
||||
return l.(*sync.Mutex)
|
||||
}
|
||||
|
||||
// libraryWalks is every library a take walk is running under now, with the filesystem it walks: set once the walk
|
||||
// has been judged and started, cleared when it ends.
|
||||
var libraryWalks sync.Map
|
||||
|
||||
// LibraryAccess is the access a library's wait and its take act name.
|
||||
func LibraryAccess(name string) string { return declaration.LibraryAccessPrefix + name }
|
||||
|
||||
// KeepWords and DiskWords are what a library's wait says the operator gives (ADR 0328 §8, §9): never a path.
|
||||
func KeepWords(name string) string { return "let the mesh keep the library " + name }
|
||||
func DiskWords(name string) string { return "mount the disk that holds the library " + name }
|
||||
|
||||
// The judgements of a library.
|
||||
const (
|
||||
// libraryKeep is a library the mesh keeps: taken, empty when first seen, or found in shape.
|
||||
libraryKeep = "keep"
|
||||
// libraryWaitKeep is a root that holds data never taken: it waits for the operator's word.
|
||||
libraryWaitKeep = "wait-keep"
|
||||
// libraryWaitDisk is a root that is absent, or whose disk (or, off its home, whose mount) is not there.
|
||||
libraryWaitDisk = "wait-disk"
|
||||
// libraryUnknown is a root whose filesystem could not be named now: left as it is until it is.
|
||||
libraryUnknown = "unknown"
|
||||
// libraryMounted is a library off its home whose mount is there: nothing to keep.
|
||||
libraryMounted = "mounted"
|
||||
)
|
||||
|
||||
// libraryVerdict is the judgement of one library, with what it read.
|
||||
type libraryVerdict struct {
|
||||
kind string
|
||||
// how is why a kept library is the mesh's: store.KeptByWarrant, store.KeptEmpty or store.KeptInShape.
|
||||
how string
|
||||
// mount is the mount point that holds the root; placed and fs name its filesystem.
|
||||
mount, placed, fs string
|
||||
// moved is that another filesystem than the one the library was kept on is at its root.
|
||||
moved bool
|
||||
why string
|
||||
err error
|
||||
}
|
||||
|
||||
// assessLibrary judges a library as an apply, a look and the take act all do (ADR 0328 §7, §9, §15): its group
|
||||
// first, then its disk, then whether the mesh may keep it. Its words carry no path: they reach the operator's
|
||||
// channel. Reads only.
|
||||
func assessLibrary(ctx context.Context, l declaration.Library, rec store.LibraryRecord) libraryVerdict {
|
||||
if why := rootGroup(l.GID); why != "" {
|
||||
return libraryVerdict{err: fmt.Errorf("its group is %s, and a library's group is never a root group "+
|
||||
"(novox/hq ADR 0328 §15); nothing was made or changed", why)}
|
||||
}
|
||||
// Nowhere the machine's own trees are, as for an access (novox/hq issue 339): a library at /etc would be
|
||||
// the machine's configuration given a group.
|
||||
if err := refusePath(l.Root, placingAccess, "", ""); err != nil {
|
||||
return libraryVerdict{err: errors.New(pathless(err) + "; nothing was made or changed")}
|
||||
}
|
||||
if !l.Home {
|
||||
// Off its home the mounts module mounts it: there when its root is a mount point now.
|
||||
for _, p := range MountPoints() {
|
||||
if p == l.Root {
|
||||
return libraryVerdict{kind: libraryMounted, mount: p}
|
||||
}
|
||||
}
|
||||
return libraryVerdict{kind: libraryWaitDisk, why: "its mount is not there"}
|
||||
}
|
||||
info, err := lstatWithin(l.Root)
|
||||
switch {
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
return libraryVerdict{kind: libraryWaitDisk, why: "its root is not there"}
|
||||
case errors.Is(err, errNoAnswer):
|
||||
return libraryVerdict{kind: libraryUnknown, why: "its root does not answer: a read of it did not return " +
|
||||
"within its bound"}
|
||||
case err != nil:
|
||||
return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
|
||||
case !info.IsDir():
|
||||
return libraryVerdict{err: errors.New("its root is not a folder (a file, or a link), and a library's root " +
|
||||
"is a folder the engine reaches through no link")}
|
||||
}
|
||||
pl := placedOn(ctx, l.Root, l.Root, rec.Mount, recordedLibraryMounts(l.Library))
|
||||
if pl.Away {
|
||||
return libraryVerdict{kind: libraryWaitDisk, mount: pl.Mount, why: "the disk that holds its root is not there"}
|
||||
}
|
||||
fs := filesystemKey(pl.Mount)
|
||||
here := placedFrom(pl.Mount, fs)
|
||||
// **A filesystem that cannot be named now is unknown, never a move**: the library holds what it is — nothing
|
||||
// walked, nothing newly kept, nothing reset — and is judged again when it is named. One never kept is so never
|
||||
// kept: it fails closed.
|
||||
if here == "" {
|
||||
return libraryVerdict{kind: libraryUnknown, mount: pl.Mount,
|
||||
why: "its filesystem could not be named this time (no UUID, filesystem id or source answered)"}
|
||||
}
|
||||
moved := pl.Moved || (rec.Fs != "" && rec.Mount == pl.Mount && rec.Placed != here)
|
||||
v := libraryVerdict{mount: pl.Mount, placed: here, fs: fs, moved: moved}
|
||||
// The controller's word that the library was taken holds for the filesystem the warrant was carried out on.
|
||||
takenHere := l.Taken && (rec.KeptOn == here || (rec.KeptOn == "" && !moved))
|
||||
switch {
|
||||
case rec.Kept && rec.KeptOn == here && !moved && rec.How == store.KeptByWarrant, takenHere:
|
||||
// Taken whole on the operator's word: every sub-path is the library's.
|
||||
v.kind, v.how = libraryKeep, store.KeptByWarrant
|
||||
return v
|
||||
case rec.Kept && rec.KeptOn == here && !moved:
|
||||
// Kept without a warrant (empty, or in shape): a sub-path asked for since that holds data not in the group
|
||||
// waits for the operator's word as the root would (§7).
|
||||
v.kind, v.how = libraryKeep, rec.How
|
||||
if !pathsInShape(l, rec.Paths) {
|
||||
v.kind, v.how = libraryWaitKeep, ""
|
||||
}
|
||||
return v
|
||||
}
|
||||
facts, err := readFolder(l.Root)
|
||||
if err != nil {
|
||||
return libraryVerdict{err: errors.New("its root cannot be read: " + pathless(err))}
|
||||
}
|
||||
switch {
|
||||
case facts.Empty:
|
||||
// Nothing there, so no sub-path either: every one is made.
|
||||
v.kind, v.how = libraryKeep, store.KeptEmpty
|
||||
case int(facts.GID) == l.GID && facts.Mode&0o2070 == 0o2070 && sampleInShape(l.Root, l.GID) &&
|
||||
pathsInShape(l, nil):
|
||||
// The root and a bounded sample beneath it already in the group with group write and setgid: data that
|
||||
// is the library's already, kept without asking. One entry of the sample out of shape asks (§7).
|
||||
v.kind, v.how = libraryKeep, store.KeptInShape
|
||||
default:
|
||||
// What is there holds data the mesh never took — a disk mounted over a root kept while it was empty below
|
||||
// it included (a move): it waits for the operator's word, and nothing under it is walked.
|
||||
v.kind = libraryWaitKeep
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// recordedLibraryMounts is every mount point recorded for another library: where a disk was once seen.
|
||||
func recordedLibraryMounts(except string) []string {
|
||||
all, err := Libraries.All()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for name, r := range all {
|
||||
if name != except && r.Mount != "" && r.Mount != "/" {
|
||||
out = append(out, r.Mount)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// applyLibrary is one library, applied (ADR 0328 §6, §7, §9): kept on its home when the mesh may keep it, waiting
|
||||
// otherwise, and failing for a root group. Nothing is changed on the root but the group, group rwx and setgid of
|
||||
// the root and its requested sub-paths, and the sub-paths made.
|
||||
func applyLibrary(ctx context.Context, l declaration.Library) (Outcome, libraryVerdict, error) {
|
||||
out := Outcome{ID: LibraryAccess(l.Library), Type: "library", Target: l.Root}
|
||||
lock := libraryLock(l.Library)
|
||||
if !lock.TryLock() {
|
||||
// **A take walk is running (up to its bound, half an hour): the apply does not wait for it.** The library is
|
||||
// the walk's until it is through; it keeps it, and its containers are not held for it. Only a walk that
|
||||
// has started, on the filesystem it judged: anything else holding the lock (a take act about to be
|
||||
// refused, say) is short, and the apply waits for it and judges the library as always.
|
||||
if on, walking := libraryWalks.Load(l.Library); walking {
|
||||
out.Action = "unchanged"
|
||||
out.Detail = "the take act is setting its group under its root now; this apply leaves the library to it"
|
||||
return out, libraryVerdict{kind: libraryKeep, how: store.KeptByWarrant, placed: on.(string)}, nil
|
||||
}
|
||||
lock.Lock()
|
||||
}
|
||||
defer lock.Unlock()
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
v := assessLibrary(ctx, l, rec)
|
||||
if v.err != nil {
|
||||
return out, v, v.err
|
||||
}
|
||||
switch v.kind {
|
||||
case libraryMounted:
|
||||
out.Action, out.Detail = "unchanged", "not its home: mounted, and nothing is kept here"
|
||||
return out, v, nil
|
||||
case libraryWaitDisk:
|
||||
out.Action = "waiting"
|
||||
out.Detail = v.why + ": nothing is made, and no container that mounts it is created until it is there " +
|
||||
"(novox/hq ADR 0328 §9); waits for the operator to " + DiskWords(l.Library)
|
||||
return out, v, nil
|
||||
case libraryUnknown:
|
||||
out.Action = "waiting"
|
||||
out.Detail = v.why + ": the library is left exactly as it is, and judged again on the next apply"
|
||||
return out, v, nil
|
||||
case libraryWaitKeep:
|
||||
_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, had bool) bool {
|
||||
if !had || v.moved {
|
||||
r.Root, r.Mount, r.Placed, r.Fs = l.Root, v.mount, v.placed, v.fs
|
||||
}
|
||||
if v.moved {
|
||||
// Kept on another filesystem: what was kept there no longer holds here.
|
||||
r.Kept, r.How, r.Cursor = false, "", nil
|
||||
}
|
||||
return true
|
||||
})
|
||||
out.Action = "waiting"
|
||||
out.Detail = "its root holds data the mesh has not taken: nothing is changed under it until the operator's " +
|
||||
"warrant (novox/hq ADR 0328 §7); waits for the operator to " + KeepWords(l.Library)
|
||||
return out, v, nil
|
||||
}
|
||||
changed, made, err := keepFolders(l.Root, l.Paths, l.GID)
|
||||
if err != nil {
|
||||
return out, v, err
|
||||
}
|
||||
_ = Libraries.Update(l.Library, func(r *store.LibraryRecord, _ bool) bool {
|
||||
if !r.Kept || r.KeptOn != v.placed {
|
||||
r.How, r.At = v.how, time.Now().UTC()
|
||||
}
|
||||
r.Root, r.Mount, r.Placed, r.Fs, r.GID = l.Root, v.mount, v.placed, v.fs, l.GID
|
||||
r.Kept, r.KeptOn = true, v.placed
|
||||
r.Paths = unionPaths(r.Paths, l.Paths)
|
||||
return true
|
||||
})
|
||||
out.Action = "unchanged"
|
||||
if changed+made > 0 {
|
||||
out.Action = "updated"
|
||||
}
|
||||
out.Detail = fmt.Sprintf("kept in %s: its root and %d sub-path(s) with group rwx and setgid, %d made; owners "+
|
||||
"and everyone else's rights unchanged (novox/hq ADR 0328 §6)", l.Group, len(l.Paths), made)
|
||||
return out, v, nil
|
||||
}
|
||||
|
||||
// heldRoots is every library root whose containers are held: its disk, or its mount, is not there (§9).
|
||||
func heldRoots(verdicts map[string]libraryVerdict, libs []declaration.Library) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, l := range libs {
|
||||
if v, ok := verdicts[l.Library]; ok && v.kind == libraryWaitDisk {
|
||||
out[l.Root] = l.Library
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldLibraries is the root of every library whose disk or mount is not there, with its name, judged now and read
|
||||
// only: what a full apply holds its containers for (ADR 0328 §9), for one that judges without applying libraries.
|
||||
func heldLibraries(ctx context.Context, libs []declaration.Library) map[string]string {
|
||||
verdicts := map[string]libraryVerdict{}
|
||||
for _, l := range libs {
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
verdicts[l.Library] = assessLibrary(ctx, l, rec)
|
||||
}
|
||||
return heldRoots(verdicts, libs)
|
||||
}
|
||||
|
||||
// mountsHeld is the library a container waits for: one whose root it mounts, or mounts below, is held.
|
||||
func mountsHeld(c *declaration.Container, held map[string]string) string {
|
||||
for _, v := range c.Volumes {
|
||||
src := mountSource(v)
|
||||
if !strings.HasPrefix(src, "/") {
|
||||
continue
|
||||
}
|
||||
for root, name := range held {
|
||||
if atOrBelow(filepath.Clean(src), root) {
|
||||
return name
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// WaitingOnLibrary is every container of the declaration that waits for a library whose disk or mount is not
|
||||
// there, read now: not created by the apply, so not judged alive or dead — the library says why.
|
||||
func WaitingOnLibrary(d *declaration.Declaration) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if d == nil || len(d.Libraries) == 0 {
|
||||
return out
|
||||
}
|
||||
held := heldLibraries(context.Background(), d.Libraries)
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok && mountsHeld(c, held) != "" {
|
||||
out[c.ID] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The judgements of a library as a look says them.
|
||||
const (
|
||||
VerdictKeep = "keep"
|
||||
VerdictMounted = "mounted"
|
||||
VerdictWaitKeep = "wait-keep"
|
||||
VerdictWaitDisk = "wait-disk"
|
||||
VerdictUnknown = "unknown"
|
||||
VerdictBroken = "broken"
|
||||
)
|
||||
|
||||
// LibraryJudgement is a look's judgement of a library, and why, in words with no path; Placed names its root's
|
||||
// filesystem for a keep wait, with no path (PlacedID).
|
||||
type LibraryJudgement struct {
|
||||
Kind, Why, Placed string
|
||||
// Act is the last walk under its root.
|
||||
Act *store.KeptAct
|
||||
}
|
||||
|
||||
// JudgeLibrary judges a library for a look, read only.
|
||||
func JudgeLibrary(ctx context.Context, l declaration.Library) LibraryJudgement {
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
v := assessLibrary(ctx, l, rec)
|
||||
j := LibraryJudgement{Act: rec.Act}
|
||||
switch {
|
||||
case v.err != nil:
|
||||
j.Kind, j.Why = VerdictBroken, pathless(v.err)
|
||||
case v.kind == libraryWaitKeep:
|
||||
j.Kind, j.Placed = VerdictWaitKeep, v.placed
|
||||
case v.kind == libraryWaitDisk:
|
||||
j.Kind, j.Why = VerdictWaitDisk, v.why
|
||||
case v.kind == libraryUnknown:
|
||||
j.Kind, j.Why = VerdictUnknown, v.why
|
||||
case v.kind == libraryMounted:
|
||||
j.Kind = VerdictMounted
|
||||
default:
|
||||
j.Kind = VerdictKeep
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
// joinRunner puts a library's tool runner's account in its group (ADR 0328 §11).
|
||||
func joinRunner(ctx context.Context, sys system.System, l declaration.Library, known *store.State,
|
||||
run Runner) (bool, string, error) {
|
||||
if l.Runner == "" {
|
||||
return false, "", nil
|
||||
}
|
||||
return joinGroup(ctx, sys, l.Runner, l.Group, store.Membership{Library: l.Library}, known, run)
|
||||
}
|
||||
|
||||
// IdentityChange is what a tool runner joining a module's own group changes (ADR 0334 §2): `identity:<module>`,
|
||||
// which the tool runner's unit names under restart-on, as it names `library:<name>`.
|
||||
func IdentityChange(module string) string { return "identity:" + module }
|
||||
|
||||
// joinGroup puts a tool runner's account in a group (ADR 0328 §11, ADR 0334 §2), as ADR 0252 puts an account in a
|
||||
// group: recorded only when the mesh put it there, so one it was in before is never taken back. The account must be
|
||||
// here and never root's: uid 0, or a root group as its primary (§15). Whether it moved, and what was said.
|
||||
func joinGroup(ctx context.Context, sys system.System, account, group string, of store.Membership, known *store.State,
|
||||
run Runner) (bool, string, error) {
|
||||
who, found, err := lookUp(ctx, run, "passwd", account)
|
||||
switch {
|
||||
case err != nil:
|
||||
return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
|
||||
case !found:
|
||||
return false, "", fmt.Errorf("the tool runner's account %s is not on this machine, so it joins no group", account)
|
||||
}
|
||||
if err := rootIdentity(who.id, who.prim); err != nil {
|
||||
return false, "", fmt.Errorf("the tool runner's account %s joins no group: %w", account, err)
|
||||
}
|
||||
in, err := system.GroupsOf(ctx, system.Runner(run), account)
|
||||
if err != nil {
|
||||
return false, "", fmt.Errorf("the tool runner's account %s could not be read: %w", account, err)
|
||||
}
|
||||
if setOf(in)[group] {
|
||||
return false, "", nil
|
||||
}
|
||||
if err := sys.AddUserToGroup(ctx, system.Runner(run), account, group); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
for _, m := range known.Members {
|
||||
if m.Account == account && m.Group == group {
|
||||
return true, fmt.Sprintf("put the tool runner's %s back in %s", account, group), nil
|
||||
}
|
||||
}
|
||||
of.Account, of.Group = account, group
|
||||
known.Members = append(known.Members, of)
|
||||
return true, fmt.Sprintf("put the tool runner's %s in %s; it holds it once restarted", account, group), nil
|
||||
}
|
||||
|
||||
// leaveRunners takes every tool runner's account the mesh put in a library's group, or a module's own group, out of
|
||||
// it once no declared library or identity puts it there and no declared user resource asks for it (ADR 0252 §3). Never fatal: what could not be
|
||||
// left stays recorded, said, for the next apply.
|
||||
func leaveRunners(ctx context.Context, sys system.System, libs []declaration.Library, ids []declaration.Identity,
|
||||
known *store.State, run Runner, wanted Wanted) []Outcome {
|
||||
declared := map[string]bool{}
|
||||
for _, l := range libs {
|
||||
if l.Runner != "" {
|
||||
declared[l.Runner+" "+l.Group] = true
|
||||
}
|
||||
}
|
||||
for _, id := range ids {
|
||||
if id.Runner != "" {
|
||||
declared[id.Runner+" "+id.Name] = true
|
||||
}
|
||||
}
|
||||
var out []Outcome
|
||||
var keep []store.Membership
|
||||
for _, m := range known.Members {
|
||||
if declared[m.Account+" "+m.Group] {
|
||||
keep = append(keep, m)
|
||||
continue
|
||||
}
|
||||
o := Outcome{ID: LibraryAccess(m.Library) + " runner", Type: "library", Target: m.Account,
|
||||
changes: LibraryAccess(m.Library)}
|
||||
if m.Module != "" {
|
||||
o.ID, o.Type = "identity "+declaration.IdentityPrefix+m.Module+" runner", "identity"
|
||||
o.changes = IdentityChange(m.Module)
|
||||
}
|
||||
if others := wanted.othersAsk(m.Account, m.Group, ""); len(others) > 0 {
|
||||
o.Action, o.Detail = "forgotten", fmt.Sprintf("left %s in %s, which %s asks for", m.Account, m.Group,
|
||||
strings.Join(others, ", "))
|
||||
out = append(out, o)
|
||||
continue
|
||||
}
|
||||
gone, why := leaveGroup(ctx, sys, m.Account, m.Group, run)
|
||||
o.Action, o.Detail = "restored", m.Account+" "+why
|
||||
if !gone {
|
||||
o.Action = "waiting"
|
||||
keep = append(keep, m)
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
known.Members = keep
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out
|
||||
}
|
||||
|
||||
// pathsInShape is whether every requested sub-path not kept before is absent (it is made), empty, or already in the
|
||||
// group with group write and setgid, it and a bounded sample beneath it: what may be kept without the operator's
|
||||
// word (§7). One that cannot be read is not.
|
||||
func pathsInShape(l declaration.Library, kept []string) bool {
|
||||
done := map[string]bool{}
|
||||
for _, p := range kept {
|
||||
done[p] = true
|
||||
}
|
||||
for _, p := range l.Paths {
|
||||
if done[p] {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(l.Root, p)
|
||||
if _, err := lstatWithin(path); errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
facts, err := readFolder(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if facts.Empty {
|
||||
continue
|
||||
}
|
||||
if int(facts.GID) != l.GID || facts.Mode&0o2070 != 0o2070 || !sampleInShape(path, l.GID) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// unionPaths is a with every path of b not in it.
|
||||
func unionPaths(a, b []string) []string {
|
||||
have := map[string]bool{}
|
||||
for _, p := range a {
|
||||
have[p] = true
|
||||
}
|
||||
for _, p := range b {
|
||||
if !have[p] {
|
||||
a = append(a, p)
|
||||
have[p] = true
|
||||
}
|
||||
}
|
||||
sort.Strings(a)
|
||||
return a
|
||||
}
|
||||
|
||||
// RunnerWarning is a tool runner's account that joins no group (missing, or root's): said as a resource of its own,
|
||||
// `runner:<module>` or `runner:library:<name>`, unhealthy with words with no path — never a failure of the apply.
|
||||
type RunnerWarning struct {
|
||||
Resource, Module, Why string
|
||||
}
|
||||
|
||||
var runnerWarnings struct {
|
||||
sync.Mutex
|
||||
list []RunnerWarning
|
||||
}
|
||||
|
||||
// setRunnerWarnings is the warnings the last apply found, in place of the ones before.
|
||||
func setRunnerWarnings(ws []RunnerWarning) {
|
||||
runnerWarnings.Lock()
|
||||
defer runnerWarnings.Unlock()
|
||||
runnerWarnings.list = append([]RunnerWarning(nil), ws...)
|
||||
}
|
||||
|
||||
// RunnerWarnings is every tool runner's account the last apply could not put in a group.
|
||||
func RunnerWarnings() []RunnerWarning {
|
||||
runnerWarnings.Lock()
|
||||
defer runnerWarnings.Unlock()
|
||||
return append([]RunnerWarning(nil), runnerWarnings.list...)
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//go:build linux
|
||||
|
||||
package apply
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The take act on a library, and the walk that goes on with one stopped at its bound (novox/hq ADR 0328 §7).
|
||||
|
||||
// actRecord is a walk's counts as the record keeps them.
|
||||
func actRecord(warrant, by, on string, started time.Time, w *treeWalk, err error) *store.KeptAct {
|
||||
c := w.counts
|
||||
rec := &store.KeptAct{Act: "take", Warrant: warrant, By: by, On: on, State: "done", Started: started,
|
||||
Ended: time.Now().UTC(), Folders: c.Folders, Files: c.Files, Kept: c.Kept, Links: c.Links, Other: c.Other,
|
||||
Mounts: c.Mounts, Shared: c.Shared, Linked: c.Linked, Stripped: c.Stripped}
|
||||
if c.Shared+c.Linked+c.Stripped > 0 {
|
||||
rec.Why = fmt.Sprintf("%d file(s) with another name given the group, %d root's left as they are, %d whose "+
|
||||
"setuid or setgid bit the group change took", c.Linked, c.Shared, c.Stripped)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, ErrWalkBound):
|
||||
rec.State, rec.Why = "stopped", "it met its bound of entries or time; it goes on from where it stopped"
|
||||
case err != nil:
|
||||
rec.State, rec.Why = "failed", err.Error()
|
||||
case w.Failed != "":
|
||||
rec.State = "failed"
|
||||
rec.Why = fmt.Sprintf("%d entries could not be given the group: %s; walked again on the next look", c.Failed,
|
||||
w.Failed)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
// TakeLibrary is the take act (ADR 0328 §7): on the operator's warrant, the library's group, group read and write,
|
||||
// and on folders group enter and setgid, set on its root and everything under it — no link followed, no other
|
||||
// filesystem entered, the owner and the owner's and others' bits unchanged, a file with another name elsewhere left
|
||||
// as it is. It judges the ask against the declaration last applied and the root as it is, and starts the walk: what
|
||||
// it answers at once, and a channel closed when the walk is through or stopped at its bound. Refused, nothing is
|
||||
// changed.
|
||||
func TakeLibrary(ctx context.Context, d *declaration.Declaration, access, path, warrant, by string) (<-chan struct{},
|
||||
string, error) {
|
||||
if d == nil {
|
||||
return nil, "", errors.New("this engine has applied no declaration, and takes no library it was not told of")
|
||||
}
|
||||
var lib *declaration.Library
|
||||
for i := range d.Libraries {
|
||||
if LibraryAccess(d.Libraries[i].Library) == access {
|
||||
lib = &d.Libraries[i]
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case lib == nil:
|
||||
return nil, "", fmt.Errorf("the access %s names no library declared on this machine", access)
|
||||
case !lib.Home:
|
||||
return nil, "", fmt.Errorf("this machine is not the home of the library %s, and a library is taken on its "+
|
||||
"home", lib.Library)
|
||||
case lib.Root != path:
|
||||
return nil, "", fmt.Errorf("the library %s is declared at another root than the one asked", lib.Library)
|
||||
}
|
||||
lock := libraryLock(lib.Library)
|
||||
if !lock.TryLock() {
|
||||
return nil, "", errors.New("a walk of the library is running; asked again once it is through, the act goes on")
|
||||
}
|
||||
rec, _ := libraryRecord(lib.Library)
|
||||
v := assessLibrary(ctx, *lib, rec)
|
||||
var refused error
|
||||
switch {
|
||||
case v.err != nil:
|
||||
refused = v.err
|
||||
case v.kind == libraryWaitDisk:
|
||||
refused = errors.New(v.why)
|
||||
case v.kind == libraryUnknown:
|
||||
refused = errors.New(v.why + ", so a warrant could name no filesystem it was given for")
|
||||
}
|
||||
if refused != nil {
|
||||
lock.Unlock()
|
||||
return nil, "", refused
|
||||
}
|
||||
if _, _, err := keepFolders(lib.Root, lib.Paths, lib.GID); err != nil {
|
||||
lock.Unlock()
|
||||
return nil, "", err
|
||||
}
|
||||
started := time.Now().UTC()
|
||||
_ = Libraries.Update(lib.Library, func(r *store.LibraryRecord, _ bool) bool {
|
||||
r.Root, r.Mount, r.Placed, r.Fs, r.GID = lib.Root, v.mount, v.placed, v.fs, lib.GID
|
||||
r.Kept, r.KeptOn, r.How, r.At = true, v.placed, store.KeptByWarrant, started
|
||||
r.Warrant, r.By, r.Cursor = warrant, by, nil
|
||||
r.Act = &store.KeptAct{Act: "take", Warrant: warrant, By: by, On: v.placed, State: "running", Started: started}
|
||||
return true
|
||||
})
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer lock.Unlock()
|
||||
walkLibrary(lib.Library, lib.Root, lib.GID, warrant, by, v.placed, nil)
|
||||
}()
|
||||
return done, fmt.Sprintf("the take act started under the warrant %s: %s's group, group read and write, and "+
|
||||
"setgid on its folders are set on everything under its root, following no link and entering no other "+
|
||||
"filesystem", warrant, lib.Library), nil
|
||||
}
|
||||
|
||||
// walkLibrary walks a library's root from resume with its group, and records what it did: a walk stopped at its
|
||||
// bound keeps its cursor for the next. The caller holds the library's lock.
|
||||
func walkLibrary(name, root string, gid int, warrant, by, on string, resume []string) {
|
||||
libraryWalks.Store(name, on)
|
||||
defer libraryWalks.Delete(name)
|
||||
started := time.Now().UTC()
|
||||
w, err := walkRoot(root, ActLimits, machineStats, resume, groupVisit(gid))
|
||||
rec := actRecord(warrant, by, on, started, w, err)
|
||||
_ = Libraries.Update(name, func(r *store.LibraryRecord, _ bool) bool {
|
||||
r.Act = rec
|
||||
r.Cursor = nil
|
||||
if errors.Is(err, ErrWalkBound) {
|
||||
r.Cursor = w.Cursor
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
// ReconcileLibraries goes on with every take act that stopped at its bound or left entries it could not change,
|
||||
// from where it stopped — only while the library is still kept on the filesystem it was taken on (judged as an apply
|
||||
// judges it). What it did, in words, for the engine to say.
|
||||
func ReconcileLibraries(ctx context.Context, d *declaration.Declaration) []string {
|
||||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
var said []string
|
||||
for _, l := range d.Libraries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if !l.Home {
|
||||
continue
|
||||
}
|
||||
rec, _ := libraryRecord(l.Library)
|
||||
if rec.Act == nil || (rec.Act.State != "stopped" && rec.Act.State != "failed") || rec.How != store.KeptByWarrant {
|
||||
continue
|
||||
}
|
||||
lock := libraryLock(l.Library)
|
||||
if !lock.TryLock() {
|
||||
continue
|
||||
}
|
||||
v := assessLibrary(ctx, l, rec)
|
||||
if v.err != nil || v.kind != libraryKeep || v.placed != rec.KeptOn || v.moved {
|
||||
lock.Unlock()
|
||||
continue
|
||||
}
|
||||
walkLibrary(l.Library, l.Root, l.GID, rec.Warrant, rec.By, rec.KeptOn, rec.Cursor)
|
||||
lock.Unlock()
|
||||
after, _ := libraryRecord(l.Library)
|
||||
said = append(said, fmt.Sprintf("the library %s: the take act went on, %s (%d folders and %d files changed)",
|
||||
l.Library, after.Act.State, after.Act.Folders, after.Act.Files))
|
||||
}
|
||||
return said
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,7 +5,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
@@ -270,6 +270,7 @@ func newRegistryMachine(spec string) *aRegistryMachine {
|
||||
}
|
||||
|
||||
func (m *aRegistryMachine) run(_ context.Context, _ string, args ...string) (string, error) {
|
||||
args = asideShape(args)
|
||||
m.mu.Lock()
|
||||
switch args[0] {
|
||||
case "info", "image":
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package apply
|
||||
|
||||
// A mesh directory outside a home is given its mode and owner only through a descriptor (novox/hq issue 358).
|
||||
//
|
||||
// Below a home every component is opened without following a link (ADR 0266, home_links.go), and a directory the
|
||||
// operator handed to the mesh is changed through one descriptor opened from / with no link followed (issue 356).
|
||||
// Every other directory the mesh owns — a module's state under /var/lib, a drop-in directory under /etc — was
|
||||
// given its mode and owner by path, with chmod and chown, which follow a link anywhere in the path: whoever could
|
||||
// turn a parent into a link between two applies directed root's chmod and chown at the link's target.
|
||||
//
|
||||
// The danger is a link another account can make or replace, not a link as such: /var/run is a link to /run on
|
||||
// every node, and a disk is sometimes reached through one. So (the decision recorded on issue 358, 2026-10-11):
|
||||
//
|
||||
// - **the directory is opened component by component from /**, each with O_DIRECTORY|O_NOFOLLOW, and its mode
|
||||
// and owner are changed with fchmod and fchown on that descriptor; a directory the walk makes is made in its
|
||||
// parent's descriptor. A link put in place after any check is refused by the kernel at the moment of use;
|
||||
// - **a link in a parent is followed only when root alone could have put it there**: the link is root's, and
|
||||
// every directory from / down to the one holding it is root's and writable by no other account (no write for
|
||||
// others, no write for a group other than root's). Its target is then walked the same way, from /;
|
||||
// - **any other link refuses that directory**, naming the link and the account that could replace it; nothing
|
||||
// is changed there, and the rest of the declaration applies;
|
||||
// - **the directory itself is never a link** (issue 339, applyDirectory).
|
||||
//
|
||||
// A directory below a home keeps the walk of ADR 0266, which follows no link below the home at all.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
osuser "os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
)
|
||||
|
||||
// LinkInPathError is a mesh directory refused because a link in its path is one another account could replace.
|
||||
type LinkInPathError struct {
|
||||
// Path is the directory declared, Link the link in its path, Who the account that could replace it.
|
||||
Path, Link, Who string
|
||||
}
|
||||
|
||||
func (e *LinkInPathError) Error() string { return e.Path + " " + e.why() }
|
||||
|
||||
// why is the refusal without the path it refuses, as a PlacementRefusedError says it.
|
||||
func (e *LinkInPathError) why() string {
|
||||
return fmt.Sprintf("is reached through the link %s, which %s could replace: the node-engine runs as root and "+
|
||||
"follows a link to a directory it gives a mode and owner only when root alone could have made it (novox/hq "+
|
||||
"issue 358). Make %s a directory, or make it and every directory above it root's and writable by no other "+
|
||||
"account", e.Link, e.Who, e.Link)
|
||||
}
|
||||
|
||||
// couldReplace says who besides root could replace an entry of a directory with this owner, group and mode: ""
|
||||
// when nobody can. A variable so a test, which runs below a directory every account may write (/tmp), can name
|
||||
// which of its own directories count as root's alone.
|
||||
var couldReplace = func(path string, uid, gid, mode uint32) string {
|
||||
if uid != 0 {
|
||||
return "the account " + accountNamed(uid) + " (it owns " + path + ")"
|
||||
}
|
||||
if mode&0o002 != 0 {
|
||||
return "every account (" + path + " is writable by all)"
|
||||
}
|
||||
if mode&0o020 != 0 && gid != 0 {
|
||||
return "the group " + groupNamed(gid) + " (it may write " + path + ")"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// linkReplacer says who besides root owns a link: "" when root does. A variable for the same reason.
|
||||
var linkReplacer = func(link string, uid uint32) string {
|
||||
if uid != 0 {
|
||||
return "the account " + accountNamed(uid) + " (it owns the link)"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func accountNamed(uid uint32) string {
|
||||
if u, err := osuser.LookupId(strconv.FormatUint(uint64(uid), 10)); err == nil {
|
||||
return u.Username
|
||||
}
|
||||
return "with uid " + strconv.FormatUint(uint64(uid), 10)
|
||||
}
|
||||
|
||||
func groupNamed(gid uint32) string {
|
||||
if g, err := osuser.LookupGroupId(strconv.FormatUint(uint64(gid), 10)); err == nil {
|
||||
return g.Name
|
||||
}
|
||||
return "with gid " + strconv.FormatUint(uint64(gid), 10)
|
||||
}
|
||||
|
||||
// JudgeDirectories is the walk's judgement over every directory a declaration names, read-only (novox/hq issue
|
||||
// 358): nothing is made, chmodded or chowned. One line per directory, in the declaration's order, after one line
|
||||
// that counts them: placed, would be refused and why, refused already by the placement guard (issue 339), or
|
||||
// below a home, where ADR 0266's walk judges it as before.
|
||||
func JudgeDirectories(d *declaration.Declaration) string {
|
||||
var lines []string
|
||||
var placed, refused, guarded, homed int
|
||||
declared := declaredHomes(d.Resources)
|
||||
for _, r := range d.Resources {
|
||||
dir, ok := r.(*declaration.Directory)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
path := filepath.Clean(dir.Path)
|
||||
if err := refusePathWith(path, placingDirectory, moduleOfID(dir.ID), dir.Owner, declared); err != nil {
|
||||
guarded++
|
||||
lines = append(lines, path+": refused already by the placement guard: "+err.Error())
|
||||
continue
|
||||
}
|
||||
if home := homeAbove(path); home != "" {
|
||||
homed++
|
||||
lines = append(lines, path+": below the home "+home+", walked without following any link as before (ADR 0266)")
|
||||
continue
|
||||
}
|
||||
if err := judgeMeshDir(path); err != nil {
|
||||
refused++
|
||||
lines = append(lines, path+": would be refused: "+err.Error())
|
||||
continue
|
||||
}
|
||||
placed++
|
||||
lines = append(lines, path+": placed")
|
||||
}
|
||||
head := fmt.Sprintf("%d directories declared: %d placed, %d would be refused, %d refused already by the "+
|
||||
"placement guard, %d below a home (novox/hq issue 358; judged read-only, nothing changed)",
|
||||
placed+refused+guarded+homed, placed, refused, guarded, homed)
|
||||
return strings.Join(append([]string{head}, lines...), "\n")
|
||||
}
|
||||
|
||||
// meshDataRoot is the mesh's own tree, where a module placed "mesh" keeps its directory (<data root>/mesh/<module>):
|
||||
// the engine's, 0755, so only the engine's account makes a directory in it (novox/hq issue 1873). A variable so a
|
||||
// test can place it.
|
||||
var meshDataRoot = "/var/lib/mesh"
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user