Compare commits
298
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bc16740feb | ||
|
|
312878befb | ||
|
|
bfc3883af9 | ||
|
|
19c1d35939 | ||
|
|
bc0c25e510 | ||
|
|
8c6e18e822 | ||
|
|
d9fa5536a8 | ||
|
|
dd0d156b9e | ||
|
|
51bc6a65b6 | ||
|
|
66bf26c0d7 | ||
|
|
34f210e5eb | ||
|
|
4e0036cc95 | ||
|
|
4784762a70 | ||
|
|
668fc2a722 | ||
|
|
20d18eb4f5 | ||
|
|
e1f393bbb1 | ||
|
|
35e86b00d3 | ||
|
|
53a1de2aaa | ||
|
|
403a5d72ab | ||
|
|
1ff6d33978 | ||
|
|
a2c813a6b4 | ||
|
|
a9560890ee | ||
|
|
bec9585d50 | ||
|
|
fc4f57c4b3 | ||
|
|
88fa74ce5e | ||
|
|
9205222a98 | ||
|
|
285566eed1 | ||
|
|
51756ee9a4 | ||
|
|
d7375c9811 | ||
|
|
232ded26ea | ||
|
|
691c97ec31 | ||
|
|
ba092d543f | ||
|
|
c3915ffb11 | ||
|
|
34338df950 | ||
|
|
07462c4caa | ||
|
|
6003da0a58 | ||
|
|
3e5d4c30ff | ||
|
|
1672f40233 | ||
|
|
e1dca0a39f | ||
|
|
4b503f8d0d | ||
|
|
8cd22892a3 | ||
|
|
1f6fc027c0 | ||
|
|
f625d98713 | ||
|
|
c2967432e6 | ||
|
|
69041c5302 | ||
|
|
792fb2ccfd | ||
|
|
89aa6fc17c | ||
|
|
a1d73bdcf0 | ||
|
|
d9384bcb23 | ||
|
|
b9b9c5120c | ||
|
|
37eff6fe5c | ||
|
|
687d715e99 | ||
|
|
ec2bd3b990 | ||
|
|
2248b985bf | ||
|
|
780be2fae5 | ||
|
|
5ab7ccf47e | ||
|
|
6147a20454 | ||
|
|
7aacbc1620 | ||
|
|
22b70c82b0 | ||
|
|
6aa8e869a3 | ||
|
|
104533e8c0 | ||
|
|
259bfd0c2b | ||
|
|
875c680d98 | ||
|
|
cdfc63cf10 | ||
|
|
d1af87a83a | ||
|
|
e76a5a31a0 | ||
|
|
8892a11768 | ||
|
|
4fac7810a1 | ||
|
|
b07e741c41 | ||
|
|
72a477eb1c | ||
|
|
8c3bfad86b | ||
|
|
9cb4d59093 | ||
|
|
fd2ae83431 | ||
|
|
4dd967e62f | ||
|
|
ab2ca6e3a7 | ||
|
|
a90f523edc | ||
|
|
fc7c4e97e6 | ||
|
|
f7cd91e49d | ||
|
|
3b3cabd2b0 | ||
|
|
f8b8f3a666 | ||
|
|
1f4ff20488 | ||
|
|
39a2144cd6 | ||
|
|
2aab86c5cb | ||
|
|
33387ac3e5 | ||
|
|
57d2f31f05 | ||
|
|
bb8b3bb43b | ||
|
|
d13bc6ecc6 | ||
|
|
ad1f6e6ac2 | ||
|
|
97895851f8 | ||
|
|
d14a6d301c | ||
|
|
7cf73ed76b | ||
|
|
d32743499c | ||
|
|
8d65cb3016 | ||
|
|
0c8d0f73fa | ||
|
|
dfc2c858d9 | ||
|
|
06fe1ee8b3 | ||
|
|
bd7731f358 | ||
|
|
c154bdedd3 | ||
|
|
5ea6c215a4 | ||
|
|
2b8ada5eb9 | ||
|
|
5652312f09 | ||
|
|
076e1f7947 | ||
|
|
26517ac00b | ||
|
|
637c33b7f8 | ||
|
|
c61212eba7 | ||
|
|
c6e669f71e | ||
|
|
580da61a91 | ||
|
|
78364e318b | ||
|
|
afbe371003 | ||
|
|
9aaa5c80ac | ||
|
|
07a65927c3 | ||
|
|
92fdc8c86a | ||
|
|
0730ef6092 | ||
|
|
4495a16608 | ||
|
|
b944d07158 | ||
|
|
6d0c2076a3 | ||
|
|
4375ed8b2c | ||
|
|
c0bbff3b4b | ||
|
|
725590e06e | ||
|
|
5792c3b959 | ||
|
|
d4deecdb38 | ||
|
|
216d2b4411 | ||
|
|
86f9a1a5c5 | ||
|
|
36d9eba51d | ||
|
|
90b20f5a3b | ||
|
|
85b5a2687e | ||
|
|
b5d8eb03ce | ||
|
|
863a981e01 | ||
|
|
b68fc75589 | ||
|
|
6c857e1c02 | ||
|
|
f8a9420952 | ||
|
|
f0f66d5b8d | ||
|
|
66e8fdaa62 | ||
|
|
88375c5786 | ||
|
|
757509366e | ||
|
|
c8e35097e8 | ||
|
|
ddfa119720 | ||
|
|
a86d607e48 | ||
|
|
86ac7596c9 | ||
|
|
cb54756c38 | ||
|
|
89dd57bed0 | ||
|
|
465beb4ea5 | ||
|
|
c9f978d330 | ||
|
|
efd3d224e1 | ||
|
|
081dc616ff | ||
|
|
b7b81e040f | ||
|
|
66b052aeee | ||
|
|
ea1b2ab395 | ||
|
|
3558903ff9 | ||
|
|
7bf4a8053a | ||
|
|
36d6f7745c | ||
|
|
1572e50e0e | ||
|
|
51e517f055 | ||
|
|
df9a2bba8c | ||
|
|
8406bb3a77 | ||
|
|
0b990a9c31 | ||
|
|
9e296295c0 | ||
|
|
512472c02a | ||
|
|
a776ca525a | ||
|
|
43b6361d09 | ||
|
|
927852fda8 | ||
|
|
4d4112a008 | ||
|
|
a51dc144a1 | ||
|
|
c5705ed551 | ||
|
|
4024909a29 | ||
|
|
6f9a83646a | ||
|
|
31c3b6cedc | ||
|
|
ab0c16ddad | ||
|
|
c020c303ff | ||
|
|
527840af9b | ||
|
|
bd5feec19b | ||
|
|
d1886bbd7b | ||
|
|
8a4381d0c6 | ||
|
|
781ba6aeb4 | ||
|
|
013d6231ed | ||
|
|
a706310d34 | ||
|
|
e58761afce | ||
|
|
14fd5cb68f | ||
|
|
075bf1e1fe | ||
|
|
cf965b93c2 | ||
|
|
8f0150bba2 | ||
|
|
0729556ba3 | ||
|
|
28d4b9eb0d | ||
|
|
3eadff23eb | ||
|
|
892a3955b7 | ||
|
|
0864ae5e47 | ||
|
|
66da19827c | ||
|
|
12319ce9dd | ||
|
|
57d852e7da | ||
|
|
6be65acb57 | ||
|
|
c1ba7bc1ae | ||
|
|
f7cf488ee7 | ||
|
|
bd9ff3797d | ||
|
|
b7e1c22512 | ||
|
|
aecb52c1af | ||
|
|
7823403869 | ||
|
|
d3436abd82 | ||
|
|
084aa4ae75 | ||
|
|
fa08c5c308 | ||
|
|
7ba9bcd873 | ||
|
|
80fa1a0551 | ||
|
|
cb1b7a6f5d | ||
|
|
3cd2bd8104 | ||
|
|
9e2dc045b6 | ||
|
|
a79f737fc7 | ||
|
|
16ce926406 | ||
|
|
5e4fcc6db7 | ||
|
|
aa70feb9b5 | ||
|
|
be506bb502 | ||
|
|
1d6a0e892e | ||
|
|
88581d0aca | ||
|
|
e5e3e62f28 | ||
|
|
3387abf50e | ||
|
|
33b87b794c | ||
|
|
21cccc6a8d | ||
|
|
d4967f186e | ||
|
|
9b4c2ba591 | ||
|
|
1bbf75ae19 | ||
|
|
605eca2c0a | ||
|
|
0cdbcc6019 | ||
|
|
ba684e5497 | ||
|
|
5786ba190c | ||
|
|
839238fec9 | ||
|
|
58373d65b4 | ||
|
|
6079ddfe84 | ||
|
|
2f1454699a | ||
|
|
44d15fa0be | ||
|
|
a6dd698114 | ||
|
|
1536dfa745 | ||
|
|
84a7643723 | ||
|
|
8360a3bba2 | ||
|
|
2db8ef5079 | ||
|
|
ba3aa8f907 | ||
|
|
91763dd70e | ||
|
|
eaec209a37 | ||
|
|
4580d80a32 | ||
|
|
9ebfb87b5d | ||
|
|
9ade199740 | ||
|
|
0596f44414 | ||
|
|
d792646110 | ||
|
|
732238cb6a | ||
|
|
845c8b0e56 | ||
|
|
aa68b67389 | ||
|
|
a39661aa36 | ||
|
|
662fb0a57c | ||
|
|
0bc5659076 | ||
|
|
c23833d3e5 | ||
|
|
0ae600dc97 | ||
|
|
15af148520 | ||
|
|
6d620f24b7 | ||
|
|
34392e7491 | ||
|
|
cfa20f05ad | ||
|
|
f1438dff6a | ||
|
|
1ffbf9ea5b | ||
|
|
e05e7130ae | ||
|
|
80d3975762 | ||
|
|
f73ec7457a | ||
|
|
94702bcd79 | ||
|
|
de2d944c2a | ||
|
|
0796341c8d | ||
|
|
96155dea23 | ||
|
|
1361c1b456 | ||
|
|
c8bd137082 | ||
|
|
9908faaa02 | ||
|
|
238440d6f8 | ||
|
|
953446cc8f | ||
|
|
f692fe7da8 | ||
|
|
2b420603fe | ||
|
|
f70a99dbf4 | ||
|
|
a4e7f480b3 | ||
|
|
d7cac79585 | ||
|
|
377fd6d67d | ||
|
|
bb7c82cac6 | ||
|
|
0db956dd83 | ||
|
|
40f3658557 | ||
|
|
219bedbdd2 | ||
|
|
1bc5719b8b | ||
|
|
a2d2af00dc | ||
|
|
5933eb757d | ||
|
|
f4745447ba | ||
|
|
9588871d66 | ||
|
|
a0fc82fdcd | ||
|
|
d49d3a14d8 | ||
|
|
c1ae30b0a2 | ||
|
|
67bfa22132 | ||
|
|
4dc5f29d3e | ||
|
|
9747f3e1b1 | ||
|
|
d41d255f28 | ||
|
|
5794ccd569 | ||
|
|
eaed934629 | ||
|
|
4cf45b5ca0 | ||
|
|
d9e24e95a2 | ||
|
|
ee1e1a8303 | ||
|
|
24f5ee81bd | ||
|
|
b4900f52b8 | ||
|
|
19cfe0ef03 | ||
|
|
33917f91ac | ||
|
|
622b6b4f55 |
@@ -1,2 +1,3 @@
|
||||
node_modules/
|
||||
dist/
|
||||
vendor/
|
||||
|
||||
@@ -20,7 +20,13 @@ Not the manifest schema (the control plane owns and parses that, in Go). The sha
|
||||
contains — the contract both a provider and a consumer conform to), and the tool and event types.
|
||||
These change rarely and deliberately; when one does, a rebuild of everything is *correct*.
|
||||
|
||||
### `provisioner` — the reconcile harness every provider shares
|
||||
### `provider-loop` — the reconcile harness every provider on the grant files shares
|
||||
Called `provisioner` until design 57 (novox/hq issue 678); that word is mesh-provisioner's now. The old path and
|
||||
names (`/provisioner`, `runProvisioner`, `provisionerTools`) stay aliases until the providers importing them move to
|
||||
the Go helper that answers consumers over the bus (`go/provider`, design 57 B2 and B3) and this loop retires with the
|
||||
grant files. Its events and tools keep their names (`provisioner.failing`, `provisioner_retirement`, …): the
|
||||
controller and the operator read them.
|
||||
|
||||
The loop is identical for postgres, redis, minio and umami: watch the grants directory, for each
|
||||
requested grant call the provider's adapter, write the sealed credential, handle withdrawal, and
|
||||
keep converging (`watch`). That loop lives here. A **module provides only the adapter** — "create
|
||||
@@ -39,11 +45,21 @@ never called to retire**; it is the delete of an adapter without `delete`, reach
|
||||
person's `provisioner_delete`. A stable set of more than 3, or more than half of those held where more
|
||||
than one is held, is not retired without a person: the provider waits for
|
||||
`provisioner_retire_approve` or `provisioner_retire_reject`, and says `provisioner.retirement`
|
||||
(`waiting`) through the `announce` option. Asked for again, a retired consumer is created — re-enabled
|
||||
(`waiting`). Asked for again, a retired consumer is created — re-enabled
|
||||
— at once. Given `module`, the four tools (`provisioner_retirement`, `provisioner_retire_approve`,
|
||||
`provisioner_retire_reject`, `provisioner_delete`) are registered for it; `provisionerTools(handle)`
|
||||
builds them otherwise. Additions and changes are never delayed.
|
||||
|
||||
**A provider that keeps failing a consumer says so on the bus** (novox/hq ADR 0224). A consumer whose
|
||||
create, check (`holds`) or secret has failed for five minutes without one success between
|
||||
(`failingAfterMs`) is announced `provisioner.failing` — the consumer, its machine, the class of error
|
||||
(`credentials-rejected`, `unreachable`, `secret-unreadable`, `refused`; an adapter's `classify` is
|
||||
believed over the words), the error with the password removed, since when and how many attempts — and
|
||||
again every fifteen minutes while it lasts. The first success after it is `provisioner.recovered`, as is
|
||||
a consumer no longer asked for or retired, and the first success for each consumer since the process
|
||||
started. Every event goes through the `announce` option, which defaults to the SDK's `emit`: a provider
|
||||
says them without asking, and a failed publish is said in its journal, never failing the act.
|
||||
|
||||
### `tools` — the tool-serving harness
|
||||
`registerModuleTools`, the `ToolDefinition` type, and the registry/worker that loads a module's
|
||||
tools and exposes them through the mesh's command surface. *How* a tool is declared and served is
|
||||
@@ -136,12 +152,12 @@ import { UmamiClient } from "../client.js"; // the client lives in the module, n
|
||||
registerModuleTools("umami", (env) => getUmamiTools(new UmamiClient(env.UMAMI_URL)));
|
||||
```
|
||||
|
||||
A provider's provisioner:
|
||||
A provider's loop:
|
||||
|
||||
```ts
|
||||
import { runProvisioner, Grant, Credential } from "@novox/mesh-sdk/provisioner";
|
||||
import { runProviderLoop, Grant, Credential } from "@novox/mesh-sdk/provider-loop";
|
||||
|
||||
runProvisioner("analytics", {
|
||||
runProviderLoop("analytics", {
|
||||
async create(grant: Grant): Promise<Credential> { /* create a umami site, return the grant */ },
|
||||
async remove(grant: Grant): Promise<void> { /* delete the site */ },
|
||||
});
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Release 57: config and provisioning
|
||||
|
||||
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
|
||||
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
|
||||
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
|
||||
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
|
||||
+5
-3
@@ -2,8 +2,8 @@
|
||||
|
||||
`stdio.Serve("", tools)` speaks MCP over stdio to the node's tool runtime, as the module the runtime
|
||||
names (novox/hq ADR 0188, ADR 0193). A `Tool` is a name, a description, an argument schema and a
|
||||
function; calls run concurrently. A tool named `<seat>.<verb>` is the seat's implementation.
|
||||
`stdio.Emit(type, body)` publishes an event as the module, `stdio.Subscribe(pattern, handler)` receives the module's events (taken when the handler returns nil, offered again when it returns an error — the one rule is in the repository README, "Taking an event") and `stdio.Ask(key, body)` calls a tool, and `stdio.State(name)` gets, puts, deletes, lists and watches the module's declared state — all through the runtime (novox/hq ADR 0193, 0198, 0201). Standard library only.
|
||||
function; calls run concurrently. `stdio.CallerOf(args)` inside a tool is who called it: the bus user the runtime took from the subject the call arrived on, named in the call's `_meta`, never read from the arguments — or `stdio.Unproven` when the runtime named none (novox/hq issue 365). A tool named `<seat>.<verb>` is the seat's implementation.
|
||||
`stdio.Emit(type, body)` publishes an event as the module, `stdio.Subscribe(pattern, handler)` receives the module's events (taken when the handler returns nil, offered again when it returns an error — the one rule is in the repository README, "Taking an event") and `stdio.Ask(key, body)` calls a tool, and `stdio.State(name)` gets, puts, deletes, lists, watches and reads the kept history of the module's declared state, and `stdio.Log(name)` appends to, reads, and measures a log of operations the module declared — all through the runtime (novox/hq ADR 0193, 0198, 0201, 0297, 0307). `stdio.Config(key)` and `stdio.Bound(provision, key)` read, and `stdio.WatchConfig` and `stdio.WatchBound` watch, the module's own config values and the non-secret values of what it is bound to, for what its manifest declares: the runtime asks its machine's provisioner client as the module it launched, never a module the code names. `stdio.Secret(name)` is refused while the module's code runs in the tool runner; its secrets reach it through its files (novox/hq ADR 0349 §14, ADR 0370). `provider` is a providing module's half of a binding (novox/hq ADR 0349 §12, issue 678): it sees each consumer that binds the provider in the plans of the assignments mirror, writes one entry per consumer to `<module>_bound` (`<node>.<module>.<provision>[.<local>]`: served values composed with the consumer's identity, the login, and the name of the vault's credential, never a secret), hands the provider's adapter the vault's value (`provision/<provider>/<provision>` shared, `pair/<provider>/<provision>/<node>/<module>[/<local>]` per consumer), refuses a derived value to a consumer bound under a local name, naming both ends, and removes the entry of one that unbinds; the bus and the vault are interfaces the provider wires, the vault as `provider.ConsumersFile`, the file its machine's provisioner client fills from `${consumers:<provision>}` with each bound consumer's credential (novox/hq ADR 0373), since a provider in the tool runner never asks the vault itself. Standard library only.
|
||||
|
||||
## Installing
|
||||
|
||||
@@ -16,4 +16,6 @@ at the same version as the TypeScript package.
|
||||
|
||||
What it holds, under ADR 0039's test: the stdio loop and the tool type, and `cli`, the local protocol
|
||||
`mesh-cli` speaks to the node-engine on its machine (novox/hq ADR 0272): the socket's path and one request and one
|
||||
answer. Nothing else — no bus, no client, nothing of any module's. `go build ./example` and pipe a `tools/list` request into it.
|
||||
answer; and `seal`, the mesh's sealed box (novox/hq ADR 0206: X25519, HKDF-SHA256, AES-256-GCM, `{v, eph, iv, tag, ct}`), byte for byte the box the licence manager made before it lived here and the TypeScript `@novox/mesh-sdk/seal` makes. Nothing else — no bus, no client, nothing of any module's. `go build ./example` and pipe a `tools/list` request into it.
|
||||
|
||||
From release 4a of design 47 (novox/hq ADR 0453 clause 3, ADR 0446 clause 4) it also holds the composition of a machine's declaration, over a plain input and with nothing behind it: `declaration` and `validate`, the node-engine's parser and its validator, copied from mesh-host; `catalogue`, the manifests, their resolution and rendering, and `overlay`, the private network's generator, copied from mesh-controller; `compose`, the controller's planFor, renderingFor and declarationWith as functions of an `Input` (the composition never allocates, makes, keeps or writes: what a send would do it names as foreseen or returns); `sdkversion`, which mesh-sdk version a program was built with; and, in `delivery`, the layouts of the keys the builder, the checker and the indexer write (the source fingerprint's new form, the builds, the verdicts and kept verdicts, the manifests by digest). None of them holds a bus, a store or a vault client, which `go list -deps` of `compose` shows.
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// Package announce is how a module says on the bus which states it keeps and which of them others may read (novox/hq
|
||||
// ADR 0495 clauses 2 and 3; issue 1790): the subject it says it on, the value it says, and where the controller keeps
|
||||
// the last one said by each module. What a module announced is the truth of what it keeps; a reader is granted a read
|
||||
// from it, never from a list or from the owner's manifest.
|
||||
//
|
||||
// A module announces from its code (process.Conn.Keeps, stdio.Keeps, the TypeScript SDK's keeps), never from its
|
||||
// manifest, which only the core reads (clause 1). A module that keeps no state on the bus announces nothing.
|
||||
package announce
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Version is the version of the value this package writes.
|
||||
const Version = 1
|
||||
|
||||
// Event is the event type of an announcement, under the module's own namespace.
|
||||
const Event = "states"
|
||||
|
||||
// Subject is where module announces what it keeps: its own event subject, mesh.mod.<module>.event.states, which its
|
||||
// account may already publish to and the events stream keeps.
|
||||
func Subject(module string) string { return "mesh.mod." + module + ".event." + Event }
|
||||
|
||||
// SubjectAll is every module's announcement, what the controller consumes.
|
||||
const SubjectAll = "mesh.mod.*.event." + Event
|
||||
|
||||
// KeptBucket is the controller's bucket of the last announcement of each module, by Key.
|
||||
const KeptBucket = "mesh-controller_states-announced"
|
||||
|
||||
var name = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// Key is module's key in KeptBucket: the module itself, which must be one plain name, since a module's name is part of
|
||||
// its buckets' names (catalogue.StateProblems).
|
||||
func Key(module string) (string, error) {
|
||||
if !name.MatchString(module) {
|
||||
return "", fmt.Errorf("announce: %q is not a module that keeps state: lower-case letters, digits and hyphens", module)
|
||||
}
|
||||
return module, nil
|
||||
}
|
||||
|
||||
// State is one state a module keeps, by its local name, and whether others may read it. Others never write it: a
|
||||
// state another module writes is the owner's own code's business.
|
||||
type State struct {
|
||||
Name string `json:"name"`
|
||||
Readable bool `json:"readable,omitempty"`
|
||||
}
|
||||
|
||||
// States is an announcement: the module, the machine whose instance said it, and every state the module keeps, sorted
|
||||
// by name. An empty list says the module keeps nothing.
|
||||
type States struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node,omitempty"`
|
||||
States []State `json:"states"`
|
||||
}
|
||||
|
||||
// Keeps says whether the module announced keeping the state.
|
||||
func (s States) Keeps(state string) bool {
|
||||
for _, k := range s.States {
|
||||
if k.Name == state {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Readable says whether the module announced the state readable by others.
|
||||
func (s States) Readable(state string) bool {
|
||||
for _, k := range s.States {
|
||||
if k.Name == state {
|
||||
return k.Readable
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *States) check() error {
|
||||
if s.Module == "" {
|
||||
return fmt.Errorf("announce: an announcement names no module")
|
||||
}
|
||||
if _, err := Key(s.Module); err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, k := range s.States {
|
||||
if !name.MatchString(k.Name) {
|
||||
return fmt.Errorf("announce: %s announces the state %q, and a state is named locally: lower-case letters, digits and hyphens", s.Module, k.Name)
|
||||
}
|
||||
if seen[k.Name] {
|
||||
return fmt.Errorf("announce: %s announces the state %q twice", s.Module, k.Name)
|
||||
}
|
||||
seen[k.Name] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Encode is s as JSON at Version, its states sorted; it refuses a module or a state no bucket could be named by and a
|
||||
// state announced twice, naming it.
|
||||
func Encode(s States) ([]byte, error) {
|
||||
if err := s.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.V = Version
|
||||
s.States = append([]State{}, s.States...)
|
||||
sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name })
|
||||
return json.Marshal(s)
|
||||
}
|
||||
|
||||
// Decode reads an announcement; it refuses another version and what Encode refuses.
|
||||
func Decode(data []byte) (States, error) {
|
||||
var s States
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
return States{}, fmt.Errorf("announce: the announcement is not JSON: %w", err)
|
||||
}
|
||||
if s.V != Version {
|
||||
return States{}, fmt.Errorf("announce: the announcement is version %d, and this reads version %d", s.V, Version)
|
||||
}
|
||||
if err := s.check(); err != nil {
|
||||
return States{}, err
|
||||
}
|
||||
sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name })
|
||||
return s, nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package announce
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module says on the bus which states it keeps and which of them others may read (novox/hq ADR 0495 clause 2), on its
|
||||
// own event subject, in a versioned value the controller keeps by module.
|
||||
func TestAnAnnouncementRoundTripsSortedAndSaysWhatIsReadable(t *testing.T) {
|
||||
data, err := Encode(States{Module: "mesh-controller", Node: "novox", States: []State{
|
||||
{Name: "plans-building", Readable: true}, {Name: "assignments", Readable: true}, {Name: "own"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
back, err := Decode(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if back.V != Version || back.Module != "mesh-controller" || back.Node != "novox" {
|
||||
t.Fatalf("decoded %+v", back)
|
||||
}
|
||||
var names []string
|
||||
for _, s := range back.States {
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
if strings.Join(names, ",") != "assignments,own,plans-building" {
|
||||
t.Fatalf("the states are not sorted: %v", names)
|
||||
}
|
||||
if !back.Readable("plans-building") || back.Readable("own") || back.Readable("absent") {
|
||||
t.Fatal("Readable does not say what was announced readable")
|
||||
}
|
||||
if !back.Keeps("own") || back.Keeps("absent") {
|
||||
t.Fatal("Keeps does not say what was announced")
|
||||
}
|
||||
if Subject("mesh-controller") != "mesh.mod.mesh-controller.event.states" || SubjectAll != "mesh.mod.*.event.states" {
|
||||
t.Fatalf("the subjects are %q and %q", Subject("mesh-controller"), SubjectAll)
|
||||
}
|
||||
if KeptBucket != "mesh-controller_states-announced" {
|
||||
t.Fatalf("the kept bucket is %q", KeptBucket)
|
||||
}
|
||||
if k, err := Key("mesh-delivery-builder"); err != nil || k != "mesh-delivery-builder" {
|
||||
t.Fatalf("Key = %q, %v", k, err)
|
||||
}
|
||||
if nothing, err := Encode(States{Module: "quiet"}); err != nil || !strings.Contains(string(nothing), `"states":[]`) {
|
||||
t.Fatalf("a module keeping nothing: %s, %v", nothing, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnnouncementRefusesWhatNoBucketCouldBe(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
s States
|
||||
says string
|
||||
}{
|
||||
{States{Module: ""}, "no module"},
|
||||
{States{Module: "a.b"}, `"a.b"`},
|
||||
{States{Module: "a", States: []State{{Name: "x_y"}}}, `"x_y"`},
|
||||
{States{Module: "a", States: []State{{Name: ""}}}, `""`},
|
||||
{States{Module: "a", States: []State{{Name: "x"}, {Name: "x", Readable: true}}}, "twice"},
|
||||
} {
|
||||
if _, err := Encode(c.s); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%+v: %v, wanted a refusal saying %s", c.s, err, c.says)
|
||||
}
|
||||
}
|
||||
if _, err := Decode([]byte(`{"v":2,"module":"a","states":[]}`)); err == nil {
|
||||
t.Error("an unknown version was read")
|
||||
}
|
||||
if _, err := Key("a.b"); err == nil {
|
||||
t.Error("a module with a dot became a key")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Package announcetest is the test each module runs with its own checks to hold that it announces on the bus every
|
||||
// state it keeps (novox/hq ADR 0495 clause 5; issue 1790): it starts the module against a bus of the test's own and
|
||||
// compares what the module announced with what its code declares. With the bind guard of process.Conn.Keeps,
|
||||
// stdio.Keeps and the TypeScript keeps, which refuses a state the code did not announce, announcement and code agree.
|
||||
//
|
||||
// It replaces the merge gate's check of one module's reads against another's manifest (clause 5); a test using it
|
||||
// carries `//go:build integration`, since it starts a bus.
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
// Wait is how long Check waits for the module's announcement once it started it.
|
||||
var Wait = 10 * time.Second
|
||||
|
||||
// Check starts a bus of the test's own, with a stream taking module's events, calls start with its address (start
|
||||
// runs the module, or its part that connects, against it, and answers how to stop it), waits for the module's
|
||||
// announcement and fails the test unless it is module's and names exactly want (in any order), readable as want says.
|
||||
// It answers the announcement.
|
||||
func Check(t testing.TB, module string, want []announce.State, start func(url string) (stop func())) announce.States {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: connecting to the test bus: %v", err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), Wait)
|
||||
defer cancel()
|
||||
stream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ANNOUNCETEST", Subjects: []string{"mesh.mod." + module + ".event.>"}})
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: making the events stream: %v", err)
|
||||
}
|
||||
stop := start(url)
|
||||
if stop != nil {
|
||||
defer stop()
|
||||
}
|
||||
got, err := waitFor(ctx, stream, module)
|
||||
if err != nil {
|
||||
t.Fatalf("announcetest: %v", err)
|
||||
}
|
||||
if problem := differs(got, module, want); problem != "" {
|
||||
t.Fatalf("announcetest: %s", problem)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
// waitFor reads the stream until module's announcement is in it.
|
||||
func waitFor(ctx context.Context, stream jetstream.Stream, module string) (announce.States, error) {
|
||||
for {
|
||||
msg, err := stream.GetLastMsgForSubject(ctx, announce.Subject(module))
|
||||
if err == nil {
|
||||
return announce.Decode(msg.Data)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return announce.States{}, fmt.Errorf("%s announced nothing on %s within %s", module, announce.Subject(module), Wait)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// differs says how got is not module's announcement of want, or "".
|
||||
func differs(got announce.States, module string, want []announce.State) string {
|
||||
if got.Module != module {
|
||||
return fmt.Sprintf("the announcement names the module %q, not %q", got.Module, module)
|
||||
}
|
||||
w := append([]announce.State{}, want...)
|
||||
sort.Slice(w, func(i, j int) bool { return w[i].Name < w[j].Name })
|
||||
g := append([]announce.State{}, got.States...)
|
||||
if !reflect.DeepEqual(g, w) && !(len(g) == 0 && len(w) == 0) {
|
||||
return fmt.Sprintf("%s announced %v, and its code declares %v", module, g, w)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//go:build integration
|
||||
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// 1790/S11: a module started against the test bus announces through process.Conn.Keeps, and Check reads that
|
||||
// announcement back against what the code declares.
|
||||
func TestCheckReadsTheModulesAnnouncementOffARealBus(t *testing.T) {
|
||||
declared := []announce.State{{Name: "accounts", Readable: true}, {Name: "drafts"}}
|
||||
got := Check(t, "mesh-mail", declared, func(url string) func() {
|
||||
conn, err := process.Connect(process.Account{URL: url, Module: "mesh-mail", Node: "novox", User: "u", Password: "p"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := conn.Keeps(declared...); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return conn.Close
|
||||
})
|
||||
if got.Node != "novox" {
|
||||
t.Fatalf("announced from %q", got.Node)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package announcetest
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
)
|
||||
|
||||
// The comparison is exact up to order: a state missing, one extra, a readability that differs or another module's
|
||||
// announcement each fail, naming what differs.
|
||||
func TestAnAnnouncementDiffersFromTheCodeByNameAndReadability(t *testing.T) {
|
||||
got := announce.States{Module: "m", States: []announce.State{{Name: "a", Readable: true}, {Name: "b"}}}
|
||||
if p := differs(got, "m", []announce.State{{Name: "b"}, {Name: "a", Readable: true}}); p != "" {
|
||||
t.Fatalf("an equal announcement differs: %s", p)
|
||||
}
|
||||
for _, want := range [][]announce.State{
|
||||
{{Name: "a", Readable: true}},
|
||||
{{Name: "a", Readable: true}, {Name: "b"}, {Name: "c"}},
|
||||
{{Name: "a"}, {Name: "b"}},
|
||||
} {
|
||||
if p := differs(got, "m", want); p == "" || !strings.Contains(p, "declares") {
|
||||
t.Errorf("want %v: %q", want, p)
|
||||
}
|
||||
}
|
||||
if p := differs(got, "other", nil); !strings.Contains(p, `"other"`) {
|
||||
t.Errorf("another module's announcement: %q", p)
|
||||
}
|
||||
if p := differs(announce.States{Module: "m"}, "m", nil); p != "" {
|
||||
t.Errorf("a module keeping nothing: %q", p)
|
||||
}
|
||||
}
|
||||
+69
-5
@@ -170,10 +170,18 @@ func (a Ask) Digest() string {
|
||||
canonical(&b, v)
|
||||
}
|
||||
}
|
||||
if a.Whole != "" || a.Details != "" {
|
||||
// Whole and Details are written whenever either or WholeDetails is given, and WholeDetails after them whenever
|
||||
// it is given (novox/hq ADR 0347): the fields after the options are then always one of three fixed shapes —
|
||||
// none, the two, or the three — and the count of values the sequence holds tells them apart, so no two asks
|
||||
// encode the same (the review of mesh-sdk #25). An ask with none of them digests as before they existed, and
|
||||
// one without WholeDetails as before it existed.
|
||||
if a.Whole != "" || a.Details != "" || a.WholeDetails != "" {
|
||||
canonical(&b, a.Whole)
|
||||
canonical(&b, a.Details)
|
||||
}
|
||||
if a.WholeDetails != "" {
|
||||
canonical(&b, a.WholeDetails)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(b.String()))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -202,9 +210,59 @@ type Ask struct {
|
||||
// The asker withholds a value shaped like a secret in it, and the router refuses the ask when one is left;
|
||||
// Explanation stays under the whole content rule everywhere else. Empty, Explanation is shown everywhere.
|
||||
Whole string `json:"whole,omitempty"`
|
||||
// Details is what the Details answer on the ask's message shows, line by line under the content rule: a
|
||||
// fingerprint, how to read the proposal whole at the terminal. Empty, an ask's own message offers no Details.
|
||||
// Details are the ask's secondary words, line by line under the content rule: who proposed it, a
|
||||
// fingerprint, how to read it whole. The router shows them in the note of the ask's own message
|
||||
// (Message.Note), after what the ask's options do and when it expires — never in its body (novox/hq
|
||||
// issue 639).
|
||||
Details string `json:"details,omitempty"`
|
||||
// WholeDetails are the Details with their exact values whole, shown in place of Details where Whole is shown
|
||||
// in place of the explanation (a kind that proves who answers, given the ask's answers): every value an
|
||||
// approval sets, whole, so the person reads exactly what they approve (novox/hq issue 383). Held, like
|
||||
// Whole, to the secret rule alone; the router refuses the ask when a secret's shape is left in it. Empty,
|
||||
// Details are shown everywhere.
|
||||
WholeDetails string `json:"whole-details,omitempty"`
|
||||
}
|
||||
|
||||
// What an ask's explanation is held to, so it reads at a glance on a phone (novox/hq issue 639): a summary
|
||||
// of two or three short sentences on its first line — what changes, why, what it costs or risks — then at
|
||||
// most three points, a line each, only where they add something. Whole is held to the same. Who asked, when,
|
||||
// the fingerprint and how to read it whole are Details, never the explanation.
|
||||
const (
|
||||
SummaryLength = 300
|
||||
MostPoints = 3
|
||||
PointLength = 160
|
||||
)
|
||||
|
||||
// Glance says why an explanation does not read at a glance, or nil: its first line, the summary, longer than
|
||||
// SummaryLength; more than MostPoints lines after it; a point longer than PointLength. Blank lines are not
|
||||
// counted. An asker holds its asks to it in its own tests; the router says, without refusing, an ask that
|
||||
// does not hold to it.
|
||||
func Glance(explanation string) error {
|
||||
var lines []string
|
||||
for _, l := range strings.Split(explanation, "\n") {
|
||||
if strings.TrimSpace(l) != "" {
|
||||
lines = append(lines, strings.TrimSpace(l))
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
switch {
|
||||
case len(lines) == 0:
|
||||
problems = append(problems, "it has no summary")
|
||||
case len([]rune(lines[0])) > SummaryLength:
|
||||
problems = append(problems, fmt.Sprintf("its summary is %d characters, more than %d", len([]rune(lines[0])), SummaryLength))
|
||||
}
|
||||
if len(lines) > 1+MostPoints {
|
||||
problems = append(problems, fmt.Sprintf("it has %d points after its summary, more than %d", len(lines)-1, MostPoints))
|
||||
}
|
||||
for i, l := range lines {
|
||||
if i > 0 && len([]rune(l)) > PointLength {
|
||||
problems = append(problems, fmt.Sprintf("its point %d is %d characters, more than %d", i, len([]rune(l)), PointLength))
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return errors.New("the explanation does not read at a glance: " + strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The bounds of an ask (novox/hq ADR 0234 §8, ADR 0259 §4).
|
||||
@@ -435,8 +493,14 @@ type Message struct {
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
Buttons []Button `json:"buttons,omitempty"`
|
||||
Urgent bool `json:"urgent,omitempty"`
|
||||
Silent bool `json:"silent,omitempty"`
|
||||
// Note is the message's secondary words, after its body and its buttons: who asked and when, what each
|
||||
// answer does, when it expires, and the ask's Details. A channel shows it smaller, collapsed or set apart
|
||||
// where it can, and after the body where it cannot (novox/hq issue 639). A channel that bounds a message's
|
||||
// length cuts the body first and the note only when the title and the note alone do not fit, saying each
|
||||
// cut (novox/hq ADR 0347 §5).
|
||||
Note string `json:"note,omitempty"`
|
||||
Urgent bool `json:"urgent,omitempty"`
|
||||
Silent bool `json:"silent,omitempty"`
|
||||
// Reply is the Choice or LinkAsked this answers, by its ID: the channel shows it where that was made.
|
||||
Reply string `json:"reply,omitempty"`
|
||||
// To is the account linked as the operator on this kind, for a channel that verifies its sender: where
|
||||
|
||||
+88
-2
@@ -253,8 +253,8 @@ func TestDigestsAreCanonicalOverNamedFields(t *testing.T) {
|
||||
t.Error("changing Whole beside Details left the digest the same")
|
||||
}
|
||||
// A field added to Ask or Option without being added to Digest fails here.
|
||||
if n := reflect.TypeOf(Ask{}).NumField(); n != 11 {
|
||||
t.Errorf("Ask has %d fields, and Digest covers 11: add the new one to Digest and to this test", n)
|
||||
if n := reflect.TypeOf(Ask{}).NumField(); n != 12 {
|
||||
t.Errorf("Ask has %d fields, and Digest covers 12: add the new one to Digest and to this test", n)
|
||||
}
|
||||
if n := reflect.TypeOf(Option{}).NumField(); n != 5 {
|
||||
t.Errorf("Option has %d fields, and Digest covers 5: add the new one to Digest and to this test", n)
|
||||
@@ -279,3 +279,89 @@ func TestAWarrantForAnAskWithNoExpiryIsRefused(t *testing.T) {
|
||||
t.Error("a warrant for an ask with no expiry was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// An explanation that reads at a glance passes; a wall of text says each way it is one (novox/hq issue 639).
|
||||
func TestAnExplanationReadsAtAGlance(t *testing.T) {
|
||||
if err := Glance("Your agents' memory follows you to every machine. A file changed on two keeps both.\n\n• Only the memory folders are carried\n• A conflict reaches you as a warning"); err != nil {
|
||||
t.Fatalf("a summary and two points: %v", err)
|
||||
}
|
||||
var wall strings.Builder
|
||||
wall.WriteString("ADR 0343 was proposed by fix-367 and approved by the knowledge base's own review.")
|
||||
for i := 1; i <= 13; i++ {
|
||||
wall.WriteString("\n" + strings.Repeat("A clause lead. ", 3))
|
||||
}
|
||||
err := Glance(wall.String())
|
||||
if err == nil || !strings.Contains(err.Error(), "13 points") {
|
||||
t.Fatalf("thirteen clause lines pass as a glance: %v", err)
|
||||
}
|
||||
if err := Glance(strings.Repeat("word ", 70)); err == nil || !strings.Contains(err.Error(), "summary is 349") {
|
||||
t.Fatalf("a long summary passes: %v", err)
|
||||
}
|
||||
if err := Glance("short.\n" + strings.Repeat("p", PointLength+1)); err == nil || !strings.Contains(err.Error(), "point 1") {
|
||||
t.Fatalf("a long point passes: %v", err)
|
||||
}
|
||||
if err := Glance(" \n"); err == nil {
|
||||
t.Fatal("an empty explanation passes")
|
||||
}
|
||||
}
|
||||
|
||||
// A message's note travels beside its body, and is left out of the encoding when empty, so a channel older
|
||||
// than it reads every message it read before.
|
||||
func TestAMessageCarriesItsNoteApart(t *testing.T) {
|
||||
raw, _ := json.Marshal(Message{Handle: "h", Title: "t", Body: "b", Note: "Asked by x."})
|
||||
if !strings.Contains(string(raw), `"note":"Asked by x."`) {
|
||||
t.Fatalf("the note is not encoded: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(Message{Handle: "h", Title: "t"})
|
||||
if strings.Contains(string(raw), "note") {
|
||||
t.Fatalf("an empty note is encoded: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// WholeDetails is digested when given, and an ask without it digests as before it existed: a router and an asker
|
||||
// of different builds still agree on every such ask.
|
||||
func TestWholeDetailsAreDigestedOnlyWhenGiven(t *testing.T) {
|
||||
a := Ask{ID: "a", Headline: "h", Explanation: "e", Details: "d"}
|
||||
before := a.Digest()
|
||||
a.WholeDetails = "+ shares: media=/storage/media"
|
||||
if a.Digest() == before {
|
||||
t.Fatal("the whole details are not in the digest")
|
||||
}
|
||||
b := a
|
||||
b.WholeDetails = "+ shares: media=/storage/other"
|
||||
if a.Digest() == b.Digest() {
|
||||
t.Fatal("two whole details digest the same")
|
||||
}
|
||||
a.WholeDetails = ""
|
||||
if a.Digest() != before {
|
||||
t.Fatal("an ask without whole details digests otherwise than before")
|
||||
}
|
||||
}
|
||||
|
||||
// No two asks encode the same (the review of mesh-sdk #25): whole details given beside nothing else are not read as
|
||||
// a Whole and Details that happen to spell the same values, nor as any other split of the three fields.
|
||||
func TestWholeDetailsCollideWithNoOtherAsk(t *testing.T) {
|
||||
base := Ask{ID: "a", Headline: "h", Explanation: "e"}
|
||||
shapes := []Ask{}
|
||||
for _, f := range []struct{ whole, details, wholeDetails string }{
|
||||
{"", "", "+ x: 1"},
|
||||
{"whole-details", "+ x: 1", ""},
|
||||
{"", "+ x: 1", ""},
|
||||
{"+ x: 1", "", ""},
|
||||
{"", "", "whole-details"},
|
||||
{"", "+ x: 1", "+ x: 1"},
|
||||
{"+ x: 1", "+ x: 1", ""},
|
||||
{"", "", ""},
|
||||
} {
|
||||
a := base
|
||||
a.Whole, a.Details, a.WholeDetails = f.whole, f.details, f.wholeDetails
|
||||
shapes = append(shapes, a)
|
||||
}
|
||||
seen := map[string]int{}
|
||||
for i, a := range shapes {
|
||||
if j, ok := seen[a.Digest()]; ok {
|
||||
t.Errorf("asks %d and %d digest the same: %+v / %+v", j, i, shapes[j], a)
|
||||
}
|
||||
seen[a.Digest()] = i
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ConditionSubject is where a module says a condition to the operator: a by-caller accept of the operator-channel
|
||||
// seat, `mesh.seat.operator-channel.accept.condition.<publisher>` (novox/hq ADR 0437 §9). The router reads the
|
||||
// verb at the fifth part and the caller at the sixth, so the publisher is one token, the module's own name; it is
|
||||
// refused when empty or when it holds a dot, a wildcard or whitespace.
|
||||
func ConditionSubject(publisher string) (string, error) {
|
||||
if publisher == "" {
|
||||
return "", errors.New("asks: the condition's publisher is empty")
|
||||
}
|
||||
if strings.ContainsAny(publisher, ".*> \t\r\n") || strings.ContainsFunc(publisher, func(r rune) bool { return r <= ' ' }) {
|
||||
return "", fmt.Errorf("asks: the condition's publisher %q is not one token: it holds a dot, a wildcard or whitespace", publisher)
|
||||
}
|
||||
return "mesh.seat." + Seat + ".accept.condition." + publisher, nil
|
||||
}
|
||||
|
||||
// ConditionVersion is the version of the condition value this package writes and the only one it reads.
|
||||
const ConditionVersion = 1
|
||||
|
||||
// ErrUnknownConditionVersion is what DecodeCondition returns for a value whose version is missing or not
|
||||
// ConditionVersion. Test it with errors.Is.
|
||||
var ErrUnknownConditionVersion = errors.New("asks: unknown condition version")
|
||||
|
||||
// The words a condition's explanation opens with (novox/hq ADR 0253 §1).
|
||||
const (
|
||||
NothingToDo = "Nothing for you to do."
|
||||
NeedsYou = "Needs you: "
|
||||
)
|
||||
|
||||
// The severities and states of a condition.
|
||||
const (
|
||||
SeverityUrgent = "urgent"
|
||||
SeverityWarning = "warning"
|
||||
StateRaised = "raised"
|
||||
StateCleared = "cleared"
|
||||
)
|
||||
|
||||
// Scope is what a condition is about, under the controller's names.
|
||||
type Scope struct {
|
||||
Scope string `json:"scope"`
|
||||
ID string `json:"id"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
}
|
||||
|
||||
// Action is one answer a notification offers, under the controller's names.
|
||||
type Action struct {
|
||||
Label string `json:"label"`
|
||||
Verb string `json:"verb"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
Arguments map[string]string `json:"arguments,omitempty"`
|
||||
Level Level `json:"level,omitempty"`
|
||||
}
|
||||
|
||||
// Condition is a condition a module says to the operator, under the names of the controller's conditions: its key,
|
||||
// severity, subject, state (raised or cleared), summary and the plain words of ADR 0253. A cleared condition needs
|
||||
// only its key, subject and state. The router alone chooses the channels.
|
||||
type Condition struct {
|
||||
V int `json:"v"`
|
||||
Key string `json:"key"`
|
||||
Severity string `json:"severity,omitempty"`
|
||||
Subject Scope `json:"subject"`
|
||||
State string `json:"state"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Headline string `json:"headline,omitempty"`
|
||||
Needs string `json:"needs,omitempty"`
|
||||
Explanation string `json:"explanation,omitempty"`
|
||||
Resolved string `json:"resolved,omitempty"`
|
||||
Actions []Action `json:"actions,omitempty"`
|
||||
}
|
||||
|
||||
func (c Condition) check() error {
|
||||
if c.Key == "" {
|
||||
return errors.New("asks: a condition has no key")
|
||||
}
|
||||
if c.Subject.Scope == "" || c.Subject.ID == "" {
|
||||
return errors.New("asks: a condition's subject needs its scope and id")
|
||||
}
|
||||
switch c.State {
|
||||
case StateCleared:
|
||||
return nil
|
||||
case StateRaised:
|
||||
default:
|
||||
return fmt.Errorf("asks: the condition's state %q is not raised or cleared", c.State)
|
||||
}
|
||||
switch c.Severity {
|
||||
case SeverityUrgent, SeverityWarning:
|
||||
default:
|
||||
return fmt.Errorf("asks: the condition's severity %q is not urgent or warning", c.Severity)
|
||||
}
|
||||
if c.Headline == "" {
|
||||
return errors.New("asks: a raised condition has no headline")
|
||||
}
|
||||
if n := len([]rune(c.Headline)); n > HeadlineLength {
|
||||
return fmt.Errorf("asks: the condition's headline is %d characters, more than %d", n, HeadlineLength)
|
||||
}
|
||||
verdict := NothingToDo
|
||||
if c.Needs != "" {
|
||||
verdict = NeedsYou + c.Needs
|
||||
}
|
||||
if !strings.HasPrefix(c.Explanation, verdict) {
|
||||
return fmt.Errorf("asks: the condition's explanation must open with %q", verdict)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeCondition is c as JSON at ConditionVersion; it refuses what ADR 0253 and ADR 0437 §9 refuse.
|
||||
func EncodeCondition(c Condition) ([]byte, error) {
|
||||
if err := c.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.V = ConditionVersion
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
// DecodeCondition reads a condition; it refuses an unknown version (ErrUnknownConditionVersion) and what
|
||||
// EncodeCondition refuses.
|
||||
func DecodeCondition(data []byte) (Condition, error) {
|
||||
var probe struct {
|
||||
V *int `json:"v"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &probe); err != nil {
|
||||
return Condition{}, fmt.Errorf("asks: the condition is not JSON: %w", err)
|
||||
}
|
||||
if probe.V == nil {
|
||||
return Condition{}, fmt.Errorf("%w: no version in the value, this reader reads %d", ErrUnknownConditionVersion, ConditionVersion)
|
||||
}
|
||||
if *probe.V != ConditionVersion {
|
||||
return Condition{}, fmt.Errorf("%w: the value is version %d, this reader reads %d", ErrUnknownConditionVersion, *probe.V, ConditionVersion)
|
||||
}
|
||||
var c Condition
|
||||
if err := json.Unmarshal(data, &c); err != nil {
|
||||
return Condition{}, fmt.Errorf("asks: the condition does not fit its shape: %w", err)
|
||||
}
|
||||
if err := c.check(); err != nil {
|
||||
return Condition{}, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// SameCondition says whether two conditions are the same value.
|
||||
func SameCondition(a, b Condition) bool { return reflect.DeepEqual(a, b) }
|
||||
@@ -0,0 +1,143 @@
|
||||
package asks
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConditionSubject(t *testing.T) { // 1202/AC1
|
||||
s, err := ConditionSubject("mesh-delivery-teller")
|
||||
if err != nil || s != "mesh.seat.operator-channel.accept.condition.mesh-delivery-teller" {
|
||||
t.Fatalf("subject = %q, %v", s, err)
|
||||
}
|
||||
// The router splits an accept subject in six parts and reads the verb at 4 and the caller at 5.
|
||||
parts := strings.Split(s, ".")
|
||||
if len(parts) != 6 || parts[4] != "condition" || parts[5] != "mesh-delivery-teller" {
|
||||
t.Errorf("subject %q does not keep the router's shape: %v", s, parts)
|
||||
}
|
||||
for _, bad := range []string{"", "a.b", "a*", "a>", "a b", "a\tb", "a\nb"} {
|
||||
if _, err := ConditionSubject(bad); err == nil {
|
||||
t.Errorf("publisher %q accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func raisedCondition() Condition {
|
||||
return Condition{
|
||||
Key: "delivery.stuck.mesh-mail", Severity: "warning",
|
||||
Subject: Scope{Scope: "module", ID: "mesh-mail", Machine: "ace"}, State: "raised",
|
||||
Summary: "plan 12 stopped", Headline: "Delivery of mesh-mail is stuck",
|
||||
Needs: "approve the merge.", Explanation: "Needs you: approve the merge. The plan waits.",
|
||||
Resolved: "The delivery of mesh-mail moved on.",
|
||||
Actions: []Action{{Label: "Approve", Verb: "mesh-controller.assign", Machine: "ace",
|
||||
Arguments: map[string]string{"why": ""}, Level: Approve}},
|
||||
}
|
||||
}
|
||||
|
||||
func clearedCondition() Condition {
|
||||
return Condition{Key: "delivery.stuck.mesh-mail", Subject: Scope{Scope: "module", ID: "mesh-mail"}, State: "cleared"}
|
||||
}
|
||||
|
||||
func TestConditionRoundTrip(t *testing.T) { // 1202/AC2
|
||||
for _, c := range []Condition{raisedCondition(), clearedCondition()} {
|
||||
data, err := EncodeCondition(c)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.State, err)
|
||||
}
|
||||
var probe map[string]any
|
||||
if err := json.Unmarshal(data, &probe); err != nil || probe["v"] != float64(1) {
|
||||
t.Fatalf("%s: %s does not carry v:1", c.State, data)
|
||||
}
|
||||
c.V = ConditionVersion
|
||||
got, err := DecodeCondition(data)
|
||||
if err != nil || !SameCondition(got, c) {
|
||||
t.Errorf("%s round trip: %+v %v", c.State, got, err)
|
||||
}
|
||||
}
|
||||
data, _ := EncodeCondition(raisedCondition())
|
||||
var probe map[string]any
|
||||
_ = json.Unmarshal(data, &probe)
|
||||
for _, name := range []string{"key", "severity", "subject", "summary", "headline", "needs", "explanation", "resolved", "actions", "state"} {
|
||||
if _, ok := probe[name]; !ok {
|
||||
t.Errorf("the JSON has no %q: %s", name, data)
|
||||
}
|
||||
}
|
||||
subject := probe["subject"].(map[string]any)
|
||||
for _, name := range []string{"scope", "id", "machine"} {
|
||||
if _, ok := subject[name]; !ok {
|
||||
t.Errorf("the subject has no %q: %s", name, data)
|
||||
}
|
||||
}
|
||||
action := probe["actions"].([]any)[0].(map[string]any)
|
||||
for _, name := range []string{"label", "verb", "machine", "arguments", "level"} {
|
||||
if _, ok := action[name]; !ok {
|
||||
t.Errorf("the action has no %q: %s", name, data)
|
||||
}
|
||||
}
|
||||
for _, in := range []string{`{"key":"k"}`, `{"v":2,"key":"k"}`} {
|
||||
if _, err := DecodeCondition([]byte(in)); !errors.Is(err, ErrUnknownConditionVersion) {
|
||||
t.Errorf("decode of %s = %v, want ErrUnknownConditionVersion", in, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionVerdict(t *testing.T) { // 1202/AC3
|
||||
c := raisedCondition()
|
||||
c.Needs, c.Explanation = "", "The plan waits."
|
||||
_, err := EncodeCondition(c)
|
||||
if err == nil || !strings.Contains(err.Error(), "Nothing for you to do.") {
|
||||
t.Errorf("empty needs without its verdict: %v", err)
|
||||
}
|
||||
c.Explanation = "Nothing for you to do. The plan waits."
|
||||
if _, err := EncodeCondition(c); err != nil {
|
||||
t.Errorf("empty needs with its verdict refused: %v", err)
|
||||
}
|
||||
c = raisedCondition()
|
||||
c.Explanation = "Nothing for you to do. The plan waits."
|
||||
_, err = EncodeCondition(c)
|
||||
if err == nil || !strings.Contains(err.Error(), "Needs you: approve the merge.") {
|
||||
t.Errorf("needs set with the wrong verdict: %v", err)
|
||||
}
|
||||
c.Explanation = "Needs you: something else. The plan waits."
|
||||
if _, err = EncodeCondition(c); err == nil {
|
||||
t.Error("needs set with other needs in the verdict accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionRefused(t *testing.T) { // 1202/AC4
|
||||
mutate := func(f func(*Condition)) error {
|
||||
c := raisedCondition()
|
||||
f(&c)
|
||||
_, err := EncodeCondition(c)
|
||||
return err
|
||||
}
|
||||
for name, f := range map[string]func(*Condition){
|
||||
"severity": func(c *Condition) { c.Severity = "critical" },
|
||||
"state": func(c *Condition) { c.State = "open" },
|
||||
"key": func(c *Condition) { c.Key = "" },
|
||||
"headline": func(c *Condition) { c.Headline = "" },
|
||||
"long headline": func(c *Condition) { c.Headline = strings.Repeat("h", HeadlineLength+1) },
|
||||
} {
|
||||
if err := mutate(f); err == nil {
|
||||
t.Errorf("a condition with a bad %s accepted", name)
|
||||
}
|
||||
}
|
||||
if err := mutate(func(c *Condition) { c.Headline = strings.Repeat("h", HeadlineLength) }); err != nil {
|
||||
t.Errorf("a headline of the greatest length refused: %v", err)
|
||||
}
|
||||
if _, err := EncodeCondition(clearedCondition()); err != nil {
|
||||
t.Errorf("a cleared condition with only key, subject and state refused: %v", err)
|
||||
}
|
||||
c := clearedCondition()
|
||||
c.Key = ""
|
||||
if _, err := EncodeCondition(c); err == nil {
|
||||
t.Error("a cleared condition without its key accepted")
|
||||
}
|
||||
c = clearedCondition()
|
||||
c.State = "gone"
|
||||
if _, err := EncodeCondition(c); err == nil {
|
||||
t.Error("a condition with an unknown state accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **One ask, one whole job** (novox/hq ADR 0525 §3). A build module runs a pulled ask with its own step tools' bodies,
|
||||
// the same ones its request and reply tools run (ADR 0525 §8), in the order the job names: a build ask fetches, then
|
||||
// builds or copies each artifact; a check ask fetches, digests the level, takes the kept verdicts and tests the rest,
|
||||
// the packages whose tests the change touches first. The answers are the tools' own, whole, so the asker judges them as
|
||||
// it judged the tools' answers.
|
||||
|
||||
// StepFunc is one step tool's body: its arguments in, its answer out.
|
||||
type StepFunc func(ctx context.Context, args map[string]any) (any, error)
|
||||
|
||||
// AskSteps are the step tools a build module runs an ask with. Copy is the image module's alone; Digest the Go module's.
|
||||
type AskSteps struct {
|
||||
Fetch, Build, Copy, Digest, Test StepFunc
|
||||
// Toolchain is the toolchain the module builds with now; a build ask naming another is refused. Nil checks nothing
|
||||
// (an image or an archive names none).
|
||||
Toolchain func(ctx context.Context) (delivery.ToolchainUsed, error)
|
||||
// Kept looks up a kept verdict by level and digest (the checker's kept bucket, delivery.KeptKey): whether one is
|
||||
// kept and how many tests it counted. Nil takes none.
|
||||
Kept func(ctx context.Context, level, digest string) (tests int, found bool, err error)
|
||||
// Release removes a workspace once the ask's steps on it ended (the module's release tool's body); nil leaves it to
|
||||
// the module's sweep.
|
||||
Release func(workspace string) error
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// RunAsk runs ask's job with steps and answers a delivery.BuildAnswer or a delivery.CheckAnswer.
|
||||
func RunAsk(ctx context.Context, steps AskSteps, ask delivery.StepAsk) (any, error) {
|
||||
if ask.Build != nil {
|
||||
return steps.build(ctx, *ask.Build)
|
||||
}
|
||||
if ask.Check != nil {
|
||||
return steps.check(ctx, *ask.Check)
|
||||
}
|
||||
return nil, Refuse("the ask %s holds no job", ask.ID)
|
||||
}
|
||||
|
||||
func raw(v any) (json.RawMessage, error) {
|
||||
if r, ok := v.(json.RawMessage); ok {
|
||||
return r, nil
|
||||
}
|
||||
return json.Marshal(v)
|
||||
}
|
||||
|
||||
// fetched is what an ask reads of a fetch's answer.
|
||||
type fetched struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Merge string `json:"merge"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Changed []string `json:"changed"`
|
||||
Modules []string `json:"modules"`
|
||||
}
|
||||
|
||||
func (s AskSteps) fetch(ctx context.Context, args map[string]any) (json.RawMessage, fetched, error) {
|
||||
if s.Fetch == nil {
|
||||
return nil, fetched{}, Refuse("this build module fetches nothing")
|
||||
}
|
||||
out, err := s.Fetch(ctx, args)
|
||||
if err != nil {
|
||||
return nil, fetched{}, err
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return nil, fetched{}, err
|
||||
}
|
||||
var f fetched
|
||||
if err := json.Unmarshal(r, &f); err != nil || f.Workspace == "" {
|
||||
return r, f, fmt.Errorf("the fetch answered no workspace: %s", r)
|
||||
}
|
||||
return r, f, nil
|
||||
}
|
||||
|
||||
func (s AskSteps) build(ctx context.Context, job delivery.BuildJob) (any, error) {
|
||||
answer := delivery.BuildAnswer{}
|
||||
if job.Toolchain != nil && s.Toolchain != nil {
|
||||
tc, err := s.Toolchain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tc.Version != job.Toolchain.Version || tc.Recipe != job.Toolchain.Recipe {
|
||||
return nil, Refuse("the ask is for %s (recipe %s) and this node builds with %s (recipe %s)",
|
||||
job.Toolchain.Version, job.Toolchain.Recipe, tc.Version, tc.Recipe)
|
||||
}
|
||||
answer.Toolchain = &tc
|
||||
}
|
||||
var f fetched
|
||||
defer func() { s.release(f.Workspace) }()
|
||||
for _, a := range job.Artifacts {
|
||||
if a.From == "" && f.Workspace == "" {
|
||||
r, got, err := s.fetch(ctx, job.Fetch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, candidate := job.Fetch["base"]; candidate {
|
||||
// A candidate is built from its head merged onto the trunk; a head that does not merge leaves the tree at the
|
||||
// trunk, and building that would store the trunk's build under the candidate's fingerprint.
|
||||
if len(got.Conflicts) > 0 {
|
||||
return nil, fmt.Errorf("the head does not merge into the trunk: files in conflict: %s", strings.Join(got.Conflicts, ", "))
|
||||
}
|
||||
if got.Merge == "" {
|
||||
return nil, fmt.Errorf("the fetch with a base answered no merge: %s", r)
|
||||
}
|
||||
}
|
||||
f = got
|
||||
}
|
||||
var (
|
||||
out any
|
||||
err error
|
||||
)
|
||||
if a.From != "" {
|
||||
if s.Copy == nil {
|
||||
return nil, Refuse("this build module copies no artifact (%s)", a.Artifact)
|
||||
}
|
||||
out, err = s.Copy(ctx, map[string]any{"module": job.Module, "artifact": a.Artifact, "from": a.From})
|
||||
} else {
|
||||
if s.Build == nil {
|
||||
return nil, Refuse("this build module builds no artifact (%s)", a.Artifact)
|
||||
}
|
||||
args := map[string]any{"workspace": f.Workspace, "module": job.Path, "artifact": a.Artifact}
|
||||
if job.Publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
out, err = s.Build(ctx, args)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", a.Artifact, err)
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var built struct {
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
if err := json.Unmarshal(r, &built); err != nil || built.Digest == "" {
|
||||
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, r)
|
||||
}
|
||||
answer.Artifacts = append(answer.Artifacts, delivery.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
func (s AskSteps) check(ctx context.Context, job delivery.CheckJob) (any, error) {
|
||||
r, f, err := s.fetch(ctx, job.Fetch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer s.release(f.Workspace)
|
||||
answer := delivery.CheckAnswer{Fetch: r}
|
||||
if len(f.Conflicts) > 0 {
|
||||
return answer, nil // the asker reads the conflict from the fetch's answer: a verdict, not an error
|
||||
}
|
||||
if s.Test == nil {
|
||||
return nil, Refuse("this build module tests nothing")
|
||||
}
|
||||
runner := ""
|
||||
if job.Runner != "" && (job.RunnerWhen == "" || changesUnder(f.Changed, job.RunnerWhen)) {
|
||||
runner = job.Runner
|
||||
}
|
||||
for _, level := range job.Levels {
|
||||
la := delivery.LevelAnswer{Level: level}
|
||||
passed, err := s.level(ctx, job, level, f, runner, &la)
|
||||
answer.Levels = append(answer.Levels, la)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !passed {
|
||||
return answer, nil // the first level that did not pass ends the check (ADR 0484 clause 13)
|
||||
}
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// level runs one level of a check ask and says whether it passed.
|
||||
func (s AskSteps) level(ctx context.Context, job delivery.CheckJob, level string, f fetched, runner string, la *delivery.LevelAnswer) (bool, error) {
|
||||
withRunner := func(args map[string]any) map[string]any {
|
||||
if runner != "" {
|
||||
args["runner"] = runner
|
||||
}
|
||||
return args
|
||||
}
|
||||
test := func(args map[string]any) (bool, error) {
|
||||
out, err := s.Test(ctx, args)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
la.Tests = append(la.Tests, r)
|
||||
var t struct {
|
||||
Passed bool `json:"passed"`
|
||||
}
|
||||
_ = json.Unmarshal(r, &t)
|
||||
return t.Passed, nil
|
||||
}
|
||||
// Every test step of the level runs, also after one that did not pass, so one check names every failing test of the
|
||||
// level and not only the first (issue 1921); the level as a whole passes only when each of its steps did.
|
||||
all := true
|
||||
if s.Digest == nil {
|
||||
// A module with no digest (Node) tests each module directory of the tree, those whose tests changed first.
|
||||
for _, priority := range []bool{true, false} {
|
||||
for _, dir := range f.Modules {
|
||||
if changesTestIn(f.Changed, dir) != priority {
|
||||
continue
|
||||
}
|
||||
passed, err := test(withRunner(map[string]any{"workspace": f.Workspace, "module": dir, "level": level}))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the %s level of %s: %w", level, dir, err)
|
||||
}
|
||||
all = all && passed
|
||||
}
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
todo, whole, keepRunner := s.plan(ctx, job, level, f, runner, withRunner, la)
|
||||
if !keepRunner {
|
||||
runner = "" // an installed module that ignores the runner judges the level as it judges any other (ADR 0535 §2)
|
||||
}
|
||||
for _, priority := range []bool{true, false} {
|
||||
args := withRunner(map[string]any{"workspace": f.Workspace, "level": level})
|
||||
if job.Run != "" {
|
||||
args["run"] = job.Run
|
||||
}
|
||||
if whole {
|
||||
if priority {
|
||||
continue // nothing to put first: the level runs whole after
|
||||
}
|
||||
} else {
|
||||
var run []string
|
||||
for _, p := range todo {
|
||||
if changesTestOf(f.Changed, p) == priority {
|
||||
run = append(run, p)
|
||||
}
|
||||
}
|
||||
if len(run) == 0 {
|
||||
continue
|
||||
}
|
||||
args["packages"] = run
|
||||
}
|
||||
passed, err := test(args)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the %s level: %w", level, err)
|
||||
}
|
||||
all = all && passed // the packages left still run (issue 1921)
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// release removes an ask's workspace, saying a failure: the module's sweep removes it later.
|
||||
func (s AskSteps) release(workspace string) {
|
||||
if workspace == "" || s.Release == nil {
|
||||
return
|
||||
}
|
||||
if err := s.Release(workspace); err != nil && s.Logf != nil {
|
||||
s.Logf("the workspace %s is left to the sweep: %v", workspace, err)
|
||||
}
|
||||
}
|
||||
|
||||
// plan digests the level and takes the kept verdicts: the packages left to test, or all when the level runs whole, and
|
||||
// whether the runner stays. A try of some tests runs whole, digesting nothing (a partial run proves no package). Under the
|
||||
// candidate's runner the digest is asked as the candidate: when the candidate answers it, every package runs and nothing
|
||||
// is digested for keeping (ADR 0535 §4); when the installed module answers it (it does not take the runner), the level is
|
||||
// planned as any other and the runner is dropped. The level runs whole and takes nothing when the digest names no package
|
||||
// or does not cover the siblings the fetch placed (ADR 0444, L5).
|
||||
func (s AskSteps) plan(ctx context.Context, job delivery.CheckJob, level string, f fetched, runner string, withRunner func(map[string]any) map[string]any, la *delivery.LevelAnswer) ([]string, bool, bool) {
|
||||
if job.Run != "" {
|
||||
return nil, true, true
|
||||
}
|
||||
out, err := s.Digest(ctx, withRunner(map[string]any{"workspace": f.Workspace, "level": level}))
|
||||
if err != nil {
|
||||
if s.Logf != nil {
|
||||
s.Logf("no digest of the %s level, so it runs whole: %v", level, err)
|
||||
}
|
||||
return nil, true, true
|
||||
}
|
||||
r, err := raw(out)
|
||||
if err != nil {
|
||||
return nil, true, true
|
||||
}
|
||||
var d delivery.LevelDigest // packages kept as a sorted list, never keyed by a package path (issue 1930)
|
||||
if err := json.Unmarshal(r, &d); err != nil {
|
||||
return nil, true, true
|
||||
}
|
||||
if runner != "" {
|
||||
if strings.HasPrefix(d.JudgedBy, "candidate ") {
|
||||
la.Judged = d.JudgedBy
|
||||
return nil, true, true // the candidate decides which verdicts would be taken: all run, nothing kept
|
||||
}
|
||||
runner = ""
|
||||
}
|
||||
if len(d.Packages) == 0 || !slices.Equal(d.Beside, besideOf(job.Fetch)) {
|
||||
return nil, true, runner != "" // a digest that names nothing or not the siblings keys no verdict: the level runs whole
|
||||
}
|
||||
la.Digest = &d
|
||||
var todo []string
|
||||
for _, pd := range d.Packages {
|
||||
p := pd.Package
|
||||
if job.Kept && s.Kept != nil {
|
||||
tests, found, err := s.Kept(ctx, level, pd.Digest)
|
||||
if err != nil && s.Logf != nil {
|
||||
s.Logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, level, err)
|
||||
}
|
||||
if err == nil && found {
|
||||
la.Kept = append(la.Kept, delivery.KeptTaken{Package: p, Digest: pd.Digest, Tests: tests})
|
||||
continue
|
||||
}
|
||||
}
|
||||
todo = append(todo, p)
|
||||
}
|
||||
return todo, false, false
|
||||
}
|
||||
|
||||
// besideOf are the siblings a fetch's arguments place, as a digest echoes them: `<owner/repo> <commit>`, sorted.
|
||||
func besideOf(fetch map[string]any) []string {
|
||||
raw, err := json.Marshal(fetch["beside"])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var refs []struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &refs)
|
||||
var out []string
|
||||
for _, r := range refs {
|
||||
out = append(out, r.Repository+" "+r.Commit)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// changesUnder says whether a path the merge changes is under prefix.
|
||||
func changesUnder(changed []string, prefix string) bool {
|
||||
for _, p := range changed {
|
||||
if strings.HasPrefix(p, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isTestFile says whether a repository-relative path is a test file: Go's _test.go, and a TypeScript module's .test.
|
||||
// and .spec. files and test directories.
|
||||
func isTestFile(p string) bool {
|
||||
switch {
|
||||
case strings.HasSuffix(p, "_test.go"):
|
||||
return true
|
||||
case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."):
|
||||
return true
|
||||
}
|
||||
for _, seg := range strings.Split(path.Dir(p), "/") {
|
||||
if seg == "__tests__" || seg == "test" || seg == "tests" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the file's
|
||||
// directory and the import path agree on their last segments.
|
||||
func changesTestOf(changed []string, pkg string) bool {
|
||||
for _, f := range changed {
|
||||
if !strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
dir := path.Dir(f)
|
||||
if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// changesTestIn says whether the merge changes a test file inside the directory of a Node module.
|
||||
func changesTestIn(changed []string, dir string) bool {
|
||||
for _, f := range changed {
|
||||
if !isTestFile(f) || strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const runCommit = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// recorder is a build module's step tools that record each call.
|
||||
type recorder struct {
|
||||
calls []string
|
||||
fetch map[string]any
|
||||
tests map[string]bool // package or module directory -> passed
|
||||
digests map[string]string
|
||||
}
|
||||
|
||||
func (r *recorder) steps() AskSteps {
|
||||
call := func(name string, answer func(args map[string]any) any) StepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||
b, _ := json.Marshal(args)
|
||||
r.calls = append(r.calls, name+" "+string(b))
|
||||
return answer(args), nil
|
||||
}
|
||||
}
|
||||
return AskSteps{
|
||||
Fetch: call("fetch", func(map[string]any) any { return r.fetch }),
|
||||
Build: call("build", func(args map[string]any) any {
|
||||
return map[string]any{"digest": "sha256:" + args["artifact"].(string), "reference": "artifact-store://" + args["artifact"].(string)}
|
||||
}),
|
||||
Copy: call("copy", func(args map[string]any) any { return map[string]any{"digest": "sha256:copied"} }),
|
||||
Digest: call("digest", func(map[string]any) any { return map[string]any{"level": "unit", "packages": r.digests} }),
|
||||
Test: call("test", func(args map[string]any) any {
|
||||
passed := true
|
||||
if pkgs, ok := args["packages"].([]string); ok {
|
||||
for _, p := range pkgs {
|
||||
passed = passed && r.tests[p]
|
||||
}
|
||||
}
|
||||
if dir, ok := args["module"].(string); ok {
|
||||
passed = r.tests[dir]
|
||||
}
|
||||
return map[string]any{"passed": passed, "tests": 1}
|
||||
}),
|
||||
Kept: func(ctx context.Context, level, digest string) (int, bool, error) { return 7, digest == "d-kept", nil },
|
||||
}
|
||||
}
|
||||
|
||||
func ask(job any) delivery.StepAsk {
|
||||
a := delivery.StepAsk{ID: NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: "t", At: time.Now()}
|
||||
switch j := job.(type) {
|
||||
case delivery.BuildJob:
|
||||
a.Build = &j
|
||||
case delivery.CheckJob:
|
||||
a.Check = &j
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// 1887/A3: a build ask fetches once, builds each artifact from the workspace with publish, copies a pinned one without a
|
||||
// fetch, and answers each artifact's digest and reference.
|
||||
func TestRunAskBuilds(t *testing.T) {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w/1"}}
|
||||
steps := r.steps()
|
||||
var released []string
|
||||
steps.Release = func(ws string) error { released = append(released, ws); return nil }
|
||||
out, err := RunAsk(context.Background(), steps, ask(delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true,
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "registry/b@sha256:00"}, {Artifact: "bundle"}, {Artifact: "lib"}}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(r.calls, "\n")
|
||||
want := `copy {"artifact":"base","from":"registry/b@sha256:00","module":"m"}
|
||||
fetch {"commit":"` + runCommit + `","repository":"novox/x"}
|
||||
build {"artifact":"bundle","module":"modules/m","publish":true,"workspace":"/w/1"}
|
||||
build {"artifact":"lib","module":"modules/m","publish":true,"workspace":"/w/1"}`
|
||||
if got != want {
|
||||
t.Fatalf("calls:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
if len(released) != 1 || released[0] != "/w/1" {
|
||||
t.Errorf("released %v, want the one workspace", released)
|
||||
}
|
||||
a := out.(delivery.BuildAnswer)
|
||||
if len(a.Artifacts) != 3 || a.Artifacts[1].Reference != "artifact-store://bundle" {
|
||||
t.Fatalf("answer %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// A build ask for another toolchain is refused, so its step ends refused and the asker reads why.
|
||||
func TestRunAskRefusesAnotherToolchain(t *testing.T) {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w"}}
|
||||
s := r.steps()
|
||||
s.Toolchain = func(context.Context) (delivery.ToolchainUsed, error) {
|
||||
return delivery.ToolchainUsed{Version: "go1.27", Recipe: "3"}, nil
|
||||
}
|
||||
_, err := RunAsk(context.Background(), s, ask(delivery.BuildJob{Module: "m", Path: ".", Fetch: map[string]any{"repository": "novox/x", "commit": runCommit},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "bundle"}}, Toolchain: &delivery.ToolchainUsed{Version: "go1.26", Recipe: "3"}}))
|
||||
var ref *Refusal
|
||||
if !errors.As(err, &ref) || len(r.calls) != 0 {
|
||||
t.Fatalf("err %v, calls %v", err, r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A3, A5a: a check ask runs the levels of one check after one fetch: it digests the unit level, takes the kept
|
||||
// verdicts, tests the packages whose tests changed first, then the rest also after a failure (issue 1921), and runs no
|
||||
// level after one that did not pass (ADR 0484 clause 13).
|
||||
func TestRunAskChecksTheLevelsOfOneCheck(t *testing.T) {
|
||||
r := &recorder{
|
||||
fetch: map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
|
||||
digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b", "x/pkg/k": "d-kept"},
|
||||
tests: map[string]bool{"x/pkg/a": true, "x/pkg/b": false},
|
||||
}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration}, Kept: true,
|
||||
Runner: "workspace", RunnerWhen: "modules/mesh-build-go/", Fetch: map[string]any{"repository": "novox/x", "commit": runCommit, "base": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := strings.Join(r.calls, "\n")
|
||||
want := `fetch {"base":"` + runCommit + `","commit":"` + runCommit + `","repository":"novox/x"}
|
||||
digest {"level":"unit","workspace":"/w"}
|
||||
test {"level":"unit","packages":["x/pkg/b"],"workspace":"/w"}
|
||||
test {"level":"unit","packages":["x/pkg/a"],"workspace":"/w"}`
|
||||
if got != want {
|
||||
t.Fatalf("calls:\n%s\nwant:\n%s", got, want)
|
||||
}
|
||||
a := out.(delivery.CheckAnswer)
|
||||
if len(a.Levels) != 1 {
|
||||
t.Fatalf("levels %+v, want the unit level alone", a.Levels)
|
||||
}
|
||||
u := a.Levels[0]
|
||||
if u.Level != delivery.LevelUnit || len(u.Kept) != 1 || u.Kept[0].Package != "x/pkg/k" || u.Kept[0].Tests != 7 || len(u.Tests) != 2 || u.Digest == nil {
|
||||
t.Fatalf("the unit level %+v", u)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0535 §2, §4: a merge that changes the build module's own code is judged by the candidate's runner: its digest says
|
||||
// the candidate ran, and every package runs with nothing kept; an installed module that ignores the runner is planned as
|
||||
// any other, the runner dropped. A digest that does not cover the fetch's siblings runs the level whole too.
|
||||
func TestRunAskRunsWholeUnderTheRunnerOrWithoutTheSiblings(t *testing.T) {
|
||||
for _, tc := range []struct{ judged, want string }{
|
||||
{"candidate 999", `digest {"level":"unit","runner":"workspace","workspace":"/w"}
|
||||
test {"level":"unit","runner":"workspace","workspace":"/w"}`},
|
||||
{"installed", `digest {"level":"unit","runner":"workspace","workspace":"/w"}`},
|
||||
} {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w", "changed": []string{"modules/mesh-build-go/x.go"}},
|
||||
digests: map[string]string{"x/pkg/a": "d-kept"}, tests: map[string]bool{}}
|
||||
steps := r.steps()
|
||||
digest := steps.Digest
|
||||
steps.Digest = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
_, _ = digest(ctx, args)
|
||||
return map[string]any{"level": "unit", "packages": r.digests, "judged_by": tc.judged}, nil
|
||||
}
|
||||
out, err := RunAsk(context.Background(), steps, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Kept: true,
|
||||
Runner: "workspace", RunnerWhen: "modules/mesh-build-go/", Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(r.calls[1:], "\n"); got != tc.want {
|
||||
t.Errorf("judged by %s: calls\n%s\nwant\n%s", tc.judged, got, tc.want)
|
||||
}
|
||||
l := out.(delivery.CheckAnswer).Levels[0]
|
||||
if tc.judged == "installed" && (len(l.Kept) != 1 || l.Judged != "") {
|
||||
t.Errorf("an installed module's level %+v, want the kept verdict taken", l)
|
||||
}
|
||||
if tc.judged != "installed" && (len(l.Kept) != 0 || l.Digest != nil || l.Judged != tc.judged) {
|
||||
t.Errorf("the candidate's level %+v, want nothing kept and the program named", l)
|
||||
}
|
||||
}
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w"}, digests: map[string]string{"x/pkg/a": "d-kept"}, tests: map[string]bool{}}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Kept: true,
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit, "beside": []any{map[string]any{"repository": "novox/mesh-sdk", "commit": runCommit}}}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if l := out.(delivery.CheckAnswer).Levels[0]; len(l.Kept) != 0 || l.Digest != nil {
|
||||
t.Fatalf("a digest without the siblings kept %+v", l)
|
||||
}
|
||||
}
|
||||
|
||||
// A Node check tests each module directory, the one whose tests changed first; a conflict ends the ask with the fetch
|
||||
// alone, a verdict for the asker.
|
||||
func TestRunAskChecksNodeModulesAndConflicts(t *testing.T) {
|
||||
r := &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}},
|
||||
tests: map[string]bool{"a": true, "b": true}}
|
||||
s := r.steps()
|
||||
s.Digest = nil
|
||||
if _, err := RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(r.calls[1:]) != `[test {"level":"unit","module":"b","workspace":"/w"} test {"level":"unit","module":"a","workspace":"/w"}]` {
|
||||
t.Fatalf("calls %v", r.calls)
|
||||
}
|
||||
r = &recorder{fetch: map[string]any{"workspace": "/w", "conflicts": []string{"go.mod"}}}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit}, Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil || len(r.calls) != 1 || len(out.(delivery.CheckAnswer).Levels) != 0 {
|
||||
t.Fatalf("a conflict: %v %v", err, r.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1921: a failure in the packages whose tests changed does not stop the level: the other packages still run, so two
|
||||
// failing packages, one changed and one not, are two test results of the one level, and the level does not pass. A Node
|
||||
// check runs every module directory after one fails too.
|
||||
func TestRunAskRunsEveryPackageAfterAFailure(t *testing.T) {
|
||||
r := &recorder{
|
||||
fetch: map[string]any{"workspace": "/w", "changed": []string{"pkg/b/b_test.go"}},
|
||||
digests: map[string]string{"x/pkg/a": "d-a", "x/pkg/b": "d-b"},
|
||||
tests: map[string]bool{"x/pkg/a": false, "x/pkg/b": false},
|
||||
}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelIntegration, delivery.LevelUnit},
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.(delivery.CheckAnswer)
|
||||
if len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 {
|
||||
t.Fatalf("levels %+v, want the one level with both packages' results", a.Levels)
|
||||
}
|
||||
for i, raw := range a.Levels[0].Tests {
|
||||
var res struct{ Passed bool }
|
||||
if err := json.Unmarshal(raw, &res); err != nil || res.Passed {
|
||||
t.Errorf("result %d %s, want a failure", i, raw)
|
||||
}
|
||||
}
|
||||
|
||||
r = &recorder{fetch: map[string]any{"workspace": "/w", "modules": []string{"a", "b"}, "changed": []string{"b/test/x.test.ts"}},
|
||||
tests: map[string]bool{"a": true, "b": false}}
|
||||
s := r.steps()
|
||||
s.Digest = nil
|
||||
out, err = RunAsk(context.Background(), s, ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit, delivery.LevelIntegration},
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := out.(delivery.CheckAnswer); len(r.calls) != 3 || len(a.Levels) != 1 || len(a.Levels[0].Tests) != 2 {
|
||||
t.Fatalf("calls %v, levels %+v: want both directories of the unit level and no integration level", r.calls, a.Levels)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly
|
||||
// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling
|
||||
// to RegistryMostPause. Variables so a test shortens them.
|
||||
var (
|
||||
RegistryWait = 5 * time.Minute
|
||||
RegistryFirstPause = time.Second
|
||||
RegistryMostPause = 10 * time.Second
|
||||
)
|
||||
|
||||
// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from
|
||||
// (novox/hq ADR 0498 clause 1): artifact-store://<module>/<artifact>/blobs/<digest>, the store's address left out
|
||||
// (issue 102).
|
||||
func BlobReference(module, artifact, digest string) string {
|
||||
return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest
|
||||
}
|
||||
|
||||
// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (<module>/<artifact>) by
|
||||
// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498
|
||||
// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST
|
||||
// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other
|
||||
// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not
|
||||
// their digest are refused before anything is sent.
|
||||
func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error {
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest)
|
||||
}
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
base := "http://" + registry + "/v2/" + repository + "/blobs/"
|
||||
var present bool
|
||||
if err := waitForStore(ctx, registry, func() error {
|
||||
var err error
|
||||
present, err = blobPresent(ctx, client, base+digest)
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if present {
|
||||
return nil
|
||||
}
|
||||
var location string
|
||||
if err := waitForStore(ctx, registry, func() error {
|
||||
var err error
|
||||
location, err = startUpload(ctx, client, base+"uploads/")
|
||||
return err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
target, err := url.Parse(base)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
loc, err := target.Parse(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err)
|
||||
}
|
||||
q := loc.Query()
|
||||
q.Set("digest", digest)
|
||||
loc.RawQuery = q.Encode()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("putting %s of %s: %w", digest, repository, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return storeSaid(resp, "putting "+digest+" of "+repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
resp.Body.Close()
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status)
|
||||
}
|
||||
|
||||
func startUpload(ctx context.Context, client *http.Client, at string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
return "", storeSaid(resp, "starting an upload at "+at)
|
||||
}
|
||||
location := resp.Header.Get("Location")
|
||||
if location == "" {
|
||||
return "", fmt.Errorf("starting an upload at %s: the store named no location", at)
|
||||
}
|
||||
return location, nil
|
||||
}
|
||||
|
||||
// storeSaid is the store's refusal: its status and the first of its own words.
|
||||
func storeSaid(resp *http.Response, what string) error {
|
||||
words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words)))
|
||||
}
|
||||
|
||||
// refused is whether an error is a connection refused.
|
||||
func refused(err error) bool {
|
||||
return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
|
||||
}
|
||||
|
||||
// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause
|
||||
// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing
|
||||
// twice.
|
||||
func waitForStore(ctx context.Context, registry string, try func() error) error {
|
||||
err := try()
|
||||
if !refused(err) {
|
||||
return err
|
||||
}
|
||||
started, pause := time.Now(), RegistryFirstPause
|
||||
for {
|
||||
left := RegistryWait - time.Since(started)
|
||||
if left <= 0 {
|
||||
return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w",
|
||||
registry, time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err)
|
||||
case <-time.After(min(pause, left)):
|
||||
}
|
||||
pause = min(pause*2, RegistryMostPause)
|
||||
if err = try(); !refused(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func digestOfBytes(b []byte) string {
|
||||
h := sha256.Sum256(b)
|
||||
return "sha256:" + hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// registry is an artifact store as far as a blob push needs one: HEAD a blob, start an upload, PUT it with its digest.
|
||||
type registry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte
|
||||
calls []string
|
||||
put int // the status a PUT answers, 201 when 0
|
||||
}
|
||||
|
||||
func (r *registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.calls = append(r.calls, req.Method+" "+req.URL.Path)
|
||||
switch {
|
||||
case req.Method == http.MethodHead && strings.Contains(req.URL.Path, "/blobs/sha256:"):
|
||||
if _, ok := r.blobs[req.URL.Path[strings.LastIndex(req.URL.Path, "/")+1:]]; ok {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/blobs/uploads/"):
|
||||
w.Header().Set("Location", strings.TrimSuffix(req.URL.Path, "/")+"/u1?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case req.Method == http.MethodPut && strings.Contains(req.URL.Path, "/blobs/uploads/u1"):
|
||||
if r.put != 0 {
|
||||
w.WriteHeader(r.put)
|
||||
_, _ = io.WriteString(w, `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content"}]}`)
|
||||
return
|
||||
}
|
||||
body, _ := io.ReadAll(req.Body)
|
||||
d := req.URL.Query().Get("digest")
|
||||
if req.URL.Query().Get("state") != "x" || d != digestOfBytes(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
r.blobs[d] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
// A done build's bundle or archive is pushed to the artifact store by its build module (novox/hq ADR 0498 clauses 1
|
||||
// and 2): a blob the store already holds is done with no upload; one it lacks is uploaded by digest; its reference is
|
||||
// the artifact-store form the builder's key and the controller check.
|
||||
func TestABlobIsPushedOnceByItsDigest(t *testing.T) {
|
||||
reg := ®istry{blobs: map[string][]byte{}}
|
||||
srv := httptest.NewServer(reg)
|
||||
defer srv.Close()
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
body := []byte("the packed bundle")
|
||||
d := digestOfBytes(body)
|
||||
if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(reg.blobs[d]) != string(body) {
|
||||
t.Fatalf("the store holds %q", reg.blobs[d])
|
||||
}
|
||||
reg.calls = nil
|
||||
if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(reg.calls) != 1 || !strings.HasPrefix(reg.calls[0], "HEAD ") {
|
||||
t.Fatalf("a blob the store holds was uploaded again: %v", reg.calls)
|
||||
}
|
||||
ref := BlobReference("mail", "code", d)
|
||||
if ref != "artifact-store://mail/code/blobs/"+d {
|
||||
t.Fatalf("BlobReference = %q", ref)
|
||||
}
|
||||
b := delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("4", 64),
|
||||
Artifacts: []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: d, Reference: ref}}}
|
||||
if _, err := delivery.EncodeBuilderBuild(b); err != nil {
|
||||
t.Fatalf("the key refuses the reference BlobReference gives: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A store that refuses the upload fails the push naming the store's own words; bytes that are not the digest are
|
||||
// refused before anything is sent.
|
||||
func TestAPushTheStoreRefusesSaysTheStoresWords(t *testing.T) {
|
||||
reg := ®istry{blobs: map[string][]byte{}, put: http.StatusBadRequest}
|
||||
srv := httptest.NewServer(reg)
|
||||
defer srv.Close()
|
||||
addr := strings.TrimPrefix(srv.URL, "http://")
|
||||
body := []byte("x")
|
||||
err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes(body), body)
|
||||
if err == nil || !strings.Contains(err.Error(), "provided digest did not match") || !strings.Contains(err.Error(), "400") {
|
||||
t.Fatalf("a refused upload said %v", err)
|
||||
}
|
||||
reg.calls = nil
|
||||
err = PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes([]byte("other")), body)
|
||||
if err == nil || len(reg.calls) != 0 {
|
||||
t.Fatalf("bytes that are not their digest: %v, calls %v", err, reg.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A store refusing connections (its nightly collection holds it still) is waited out, up to RegistryWait (issue 457).
|
||||
func TestAPushWaitsOutAStoreRefusingConnections(t *testing.T) {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := l.Addr().String()
|
||||
l.Close() // refusing now
|
||||
reg := ®istry{blobs: map[string][]byte{}}
|
||||
go func() {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
l2, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = http.Serve(l2, reg)
|
||||
}()
|
||||
beforeWait, beforePause := RegistryWait, RegistryFirstPause
|
||||
RegistryWait, RegistryFirstPause = 3*time.Second, 100*time.Millisecond
|
||||
defer func() { RegistryWait, RegistryFirstPause = beforeWait, beforePause }()
|
||||
body := []byte("late")
|
||||
if err := PushBlob(context.Background(), http.DefaultClient, addr, "mail/code", digestOfBytes(body), body); err != nil {
|
||||
t.Fatalf("a store that came back was not waited for: %v", err)
|
||||
}
|
||||
|
||||
l3, _ := net.Listen("tcp", "127.0.0.1:0")
|
||||
gone := l3.Addr().String()
|
||||
l3.Close()
|
||||
RegistryWait = 300 * time.Millisecond
|
||||
err = PushBlob(context.Background(), http.DefaultClient, gone, "mail/code", digestOfBytes(body), body)
|
||||
if err == nil || !strings.Contains(err.Error(), "refused every connection") {
|
||||
t.Fatalf("a store that stayed down said %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// CapacitySetting is the build modules' setting of each node's build capacity (novox/hq ADR 0525 §5): `<node>=<asks at
|
||||
// once>` pairs separated by commas, as the builder's was (ADR 0485 §1). Each build module declares it with
|
||||
// DefaultCapacities as its default, and the controller lays it into the module's settings file.
|
||||
const CapacitySetting = "capacity"
|
||||
|
||||
// DefaultCapacities is the setting's default, the builder's before R4g: the control-node (novox) and the laptop (g14)
|
||||
// build nothing.
|
||||
const DefaultCapacities = "shanks=3,ace=2,g14=0,novox=0"
|
||||
|
||||
// NodeCapacity is node's build capacity, read from the module's settings file at path at every call, so a change takes
|
||||
// effect at the next pull; with no path it reads fallback. A node the setting does not name runs one ask at a time (as
|
||||
// the builder's did). A file or a setting that cannot be read gives 0 and says why: pulling on a setting nobody can read
|
||||
// could build on a node that must not.
|
||||
func NodeCapacity(path, fallback, node string, logf func(string, ...any)) func() int {
|
||||
say := func(format string, args ...any) {
|
||||
if logf != nil {
|
||||
logf(format, args...)
|
||||
}
|
||||
}
|
||||
return func() int {
|
||||
setting := fallback
|
||||
if path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
say("the build capacity cannot be read, so %s pulls nothing: %v", node, err)
|
||||
return 0
|
||||
}
|
||||
var s map[string]any
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
say("the settings file %s is not JSON, so %s pulls nothing: %v", path, node, err)
|
||||
return 0
|
||||
}
|
||||
setting, _ = s[CapacitySetting].(string)
|
||||
}
|
||||
caps, err := buildrun.ParseCapacity(setting)
|
||||
if err != nil {
|
||||
say("%v, so %s pulls nothing", fmt.Errorf("the build capacity: %w", err), node)
|
||||
return 0
|
||||
}
|
||||
return (&buildrun.Places{Capacity: caps}).CapacityOf(node)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1887/A7: the node's build capacity is the build module's setting, read at every pull: the control-node at 0 pulls
|
||||
// nothing, a change takes effect at the next read, and a setting that cannot be read pulls nothing.
|
||||
func TestNodeCapacity(t *testing.T) {
|
||||
if got := NodeCapacity("", DefaultCapacities, "novox", nil)(); got != 0 {
|
||||
t.Errorf("novox = %d by default, want 0", got)
|
||||
}
|
||||
if got := NodeCapacity("", DefaultCapacities, "shanks", nil)(); got != 3 {
|
||||
t.Errorf("shanks = %d by default, want 3", got)
|
||||
}
|
||||
file := filepath.Join(t.TempDir(), "settings.json")
|
||||
os.WriteFile(file, []byte(`{"capacity":"shanks=1,ace=2"}`), 0o600)
|
||||
capacity := NodeCapacity(file, DefaultCapacities, "shanks", nil)
|
||||
if got := capacity(); got != 1 {
|
||||
t.Errorf("shanks = %d from the file, want 1", got)
|
||||
}
|
||||
os.WriteFile(file, []byte(`{"capacity":"shanks=4"}`), 0o600)
|
||||
if got := capacity(); got != 4 {
|
||||
t.Errorf("shanks = %d after the change, want 4", got)
|
||||
}
|
||||
if got := NodeCapacity(file, "", "pi", nil)(); got != 1 {
|
||||
t.Errorf("a node not named = %d, want 1", got)
|
||||
}
|
||||
os.WriteFile(file, []byte(`{"capacity":"shanks=lots"}`), 0o600)
|
||||
if got := capacity(); got != 0 {
|
||||
t.Errorf("an unreadable setting = %d, want 0", got)
|
||||
}
|
||||
if got := NodeCapacity(filepath.Join(t.TempDir(), "none"), DefaultCapacities, "shanks", nil)(); got != 0 {
|
||||
t.Errorf("a missing file = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// credentialField is the node runtime's rule for a state value (mesh-tools node-tools/internal/bus/state.go, novox/hq
|
||||
// ADR 0201), copied whole: the first field anywhere in the value whose name, lower-cased and without - _ and spaces, ends
|
||||
// in a credential word, or "". A step's end is refused when it answers a field.
|
||||
func credentialField(value json.RawMessage) string {
|
||||
endings := []string{"password", "passwd", "secret", "token", "credential", "credentials",
|
||||
"authorization", "apikey", "privatekey", "accesskey", "cookie"}
|
||||
var v any
|
||||
if json.Unmarshal(value, &v) != nil {
|
||||
return ""
|
||||
}
|
||||
var walk func(any) string
|
||||
walk = func(v any) string {
|
||||
switch t := v.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(t))
|
||||
for k := range t {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k))
|
||||
for _, end := range endings {
|
||||
if strings.HasSuffix(norm, end) {
|
||||
return k
|
||||
}
|
||||
}
|
||||
if found := walk(t[k]); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, x := range t {
|
||||
if found := walk(x); found != "" {
|
||||
return found
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return walk(v)
|
||||
}
|
||||
|
||||
// runtimeState is a steps state that refuses a value as the node runtime does, and keeps every value it took.
|
||||
type runtimeState struct {
|
||||
mu sync.Mutex
|
||||
took map[string][]delivery.Step
|
||||
refused int
|
||||
}
|
||||
|
||||
func newRuntimeState() *runtimeState { return &runtimeState{took: map[string][]delivery.Step{}} }
|
||||
|
||||
func (s *runtimeState) put(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
if f := credentialField(raw); f != "" {
|
||||
s.mu.Lock()
|
||||
s.refused++
|
||||
s.mu.Unlock()
|
||||
return 0, fmt.Errorf("steps.%s carries a field %q, which names a credential", key, f)
|
||||
}
|
||||
st, err := delivery.DecodeStep(raw)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.took[key] = append(s.took[key], st)
|
||||
return uint64(len(s.took[key])), nil
|
||||
}
|
||||
|
||||
func (s *runtimeState) last(key string) (delivery.Step, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
t := s.took[key]
|
||||
if len(t) == 0 {
|
||||
return delivery.Step{}, false
|
||||
}
|
||||
return t[len(t)-1], true
|
||||
}
|
||||
|
||||
func fastEnds(t *testing.T) {
|
||||
attempts, retry := EndAttempts, EndRetry
|
||||
EndAttempts, EndRetry = 2, time.Millisecond
|
||||
t.Cleanup(func() { EndAttempts, EndRetry = attempts, retry })
|
||||
}
|
||||
|
||||
// 1930/AC1: given a check of a repository with the package .../internal/token, when its ask's answer is stored as the
|
||||
// step's end, then no field is keyed by a package path: the runtime's credential rule takes the value, and the package
|
||||
// and its digest read back from the stored end.
|
||||
func TestADigestOfAPackageNamedTokenIsStoredAsAList(t *testing.T) {
|
||||
fastEnds(t)
|
||||
r := &recorder{
|
||||
fetch: map[string]any{"workspace": "/w"},
|
||||
digests: map[string]string{"x/internal/token": "d-token", "x/pkg/a": "d-a"},
|
||||
tests: map[string]bool{"x/internal/token": true, "x/pkg/a": true},
|
||||
}
|
||||
out, err := RunAsk(context.Background(), r.steps(), ask(delivery.CheckJob{Levels: []string{delivery.LevelUnit},
|
||||
Fetch: map[string]any{"repository": "novox/x", "commit": runCommit}}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old, _ := json.Marshal(map[string]any{"levels": []any{map[string]any{"level": "unit", "digest": map[string]any{"packages": r.digests}}}})
|
||||
if credentialField(old) != "x/internal/token" {
|
||||
t.Fatalf("the rule does not refuse the form before issue 1930 (%s): the test proves nothing", old)
|
||||
}
|
||||
st := newRuntimeState()
|
||||
if err := finish(st.put, "shanks.a1", delivery.Step{ID: "a1", Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()}, out, nil, t.Logf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, ok := st.last("shanks.a1")
|
||||
if !ok || s.State != delivery.StepPassed || st.refused != 0 {
|
||||
t.Fatalf("the end %+v (written %v, refused %d), want it passed and taken at once", s, ok, st.refused)
|
||||
}
|
||||
var a delivery.CheckAnswer
|
||||
if err := json.Unmarshal(s.Result, &a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(a.Levels) != 1 || a.Levels[0].Digest == nil {
|
||||
t.Fatalf("levels %+v, want the unit level with its digest", a.Levels)
|
||||
}
|
||||
want := []delivery.PackageDigest{{Package: "x/internal/token", Digest: "d-token"}, {Package: "x/pkg/a", Digest: "d-a"}}
|
||||
if got := a.Levels[0].Digest.Packages; fmt.Sprint(got) != fmt.Sprint(want) {
|
||||
t.Errorf("packages %v, want %v", got, want)
|
||||
}
|
||||
if !strings.Contains(string(s.Result), `"package":"x/internal/token"`) {
|
||||
t.Errorf("the stored end does not name the package as a value: %s", s.Result)
|
||||
}
|
||||
}
|
||||
|
||||
// 1930/AC2: given an end the state refuses every time, when finish writes it, then it logs the refusal and writes a small
|
||||
// failed end without the result that names the write's error; and when no end can be written at all it answers an error.
|
||||
func TestARefusedEndIsSaidAndEndsTheStepFailed(t *testing.T) {
|
||||
fastEnds(t)
|
||||
big := errors.New("the value carries a field \"x/internal/token\", which names a credential")
|
||||
var refused int
|
||||
var stored []delivery.Step
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
st, _ := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if len(st.Result) > 0 {
|
||||
refused++
|
||||
return 0, big
|
||||
}
|
||||
stored = append(stored, st)
|
||||
return 1, nil
|
||||
}
|
||||
var logged []string
|
||||
logf := func(format string, args ...any) { logged = append(logged, fmt.Sprintf(format, args...)) }
|
||||
err := finish(put, "shanks.a2", delivery.Step{ID: "a2", Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()}, map[string]any{"levels": 1}, nil, logf)
|
||||
if err != nil {
|
||||
t.Fatalf("finish answered %v, want the small end written", err)
|
||||
}
|
||||
if refused != EndAttempts || len(stored) != 1 {
|
||||
t.Fatalf("refused %d, stored %d, want %d refusals and one small end", refused, len(stored), EndAttempts)
|
||||
}
|
||||
s := stored[0]
|
||||
if s.State != delivery.StepFailed || len(s.Result) != 0 || !strings.Contains(s.Reason, "could not be written") ||
|
||||
!strings.Contains(s.Reason, big.Error()) || !strings.Contains(s.Reason, "passed") {
|
||||
t.Errorf("the small end %+v, want failed, no result, naming the write's error and that it passed", s)
|
||||
}
|
||||
if len(logged) == 0 || !strings.Contains(logged[0], big.Error()) {
|
||||
t.Errorf("logged %q, want the refusal said", logged)
|
||||
}
|
||||
none := func(string, any) (uint64, error) { return 0, errors.New("the bus is gone") }
|
||||
if err := finish(none, "shanks.a3", delivery.Step{ID: "a3", Tool: "check ask", Node: "shanks", Started: time.Now()}, nil, nil, logf); err == nil || !strings.Contains(err.Error(), "the bus is gone") {
|
||||
t.Errorf("finish with no write possible answered %v, want the error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1930/AC2: given an ask whose end the state refuses, when it runs, then the ask is acknowledged only once its small
|
||||
// failed end is written; when no end can be written at all, it is given back, never acknowledged as done.
|
||||
func TestAnAskWhoseEndCannotBeWrittenIsNotAcked(t *testing.T) {
|
||||
fastEnds(t)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
smallOK bool
|
||||
wantAcked bool
|
||||
}{{"the small end written", true, true}, {"no end written", false, false}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
src := NewFakeAsks()
|
||||
a := testAsk(delivery.AskCheck)
|
||||
_ = src.PublishAsk(a)
|
||||
var mu sync.Mutex
|
||||
var ends []delivery.Step
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
st, err := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if st.State != delivery.StepRunning && (len(st.Result) > 0 || !tc.smallOK) {
|
||||
return 0, errors.New("refused")
|
||||
}
|
||||
mu.Lock()
|
||||
ends = append(ends, st)
|
||||
mu.Unlock()
|
||||
return 1, nil
|
||||
}
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Logf: t.Logf,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return map[string]any{"ran": true}, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask's fate", func() bool { ac, tm, nk := src.Fates(); return len(ac)+len(tm)+len(nk) > 0 })
|
||||
cancel()
|
||||
acked, termed, naked := src.Fates()
|
||||
if tc.wantAcked && (len(acked) != 1 || len(naked) != 0) || !tc.wantAcked && (len(acked) != 0 || len(naked) == 0) || len(termed) != 0 {
|
||||
t.Fatalf("acked %v naked %v termed %v", acked, naked, termed)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if tc.smallOK {
|
||||
last := ends[len(ends)-1]
|
||||
if last.State != delivery.StepFailed || !strings.Contains(last.Reason, "refused") {
|
||||
t.Errorf("the end %+v, want the small failed end", last)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Option changes FetchCommit.
|
||||
type Option func(*fetchOptions)
|
||||
|
||||
type fetchOptions struct {
|
||||
allowFile bool
|
||||
mirror string
|
||||
}
|
||||
|
||||
// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production
|
||||
// code does not pass it, and FetchCommit then refuses anything but ssh://.
|
||||
func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } }
|
||||
|
||||
// WithMirror makes FetchCommit (and MergeInto) borrow objects from mirror, a bare repository that UpdateMirror keeps
|
||||
// in the build module's own cache (novox/hq ADR 0536 §2; issue 890): the workspace's .git/objects/info/alternates
|
||||
// names the mirror's objects directory, so git transfers only the objects the mirror lacks. It is a plain file, never
|
||||
// a symlink. mirror must be an absolute path of an existing directory; the workspace never writes to it.
|
||||
func WithMirror(mirror string) Option { return func(o *fetchOptions) { o.mirror = mirror } }
|
||||
|
||||
// useMirror writes the alternates file of the repository in dir, when a mirror is set.
|
||||
func useMirror(o fetchOptions, dir string) error {
|
||||
if o.mirror == "" {
|
||||
return nil
|
||||
}
|
||||
info := filepath.Join(dir, ".git", "objects", "info")
|
||||
if err := os.MkdirAll(info, 0o755); err != nil {
|
||||
return fmt.Errorf("buildstep: the mirror cannot be used: %w", err)
|
||||
}
|
||||
line := filepath.Join(o.mirror, "objects") + "\n"
|
||||
if err := os.WriteFile(filepath.Join(info, "alternates"), []byte(line), 0o644); err != nil {
|
||||
return fmt.Errorf("buildstep: the mirror cannot be used: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`)
|
||||
)
|
||||
|
||||
// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached
|
||||
// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh://<ssh name>/<owner>/<repo>.git, the ssh name being the
|
||||
// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names
|
||||
// the pull request's head (refs/pull/<n>/head) when the commit is only there. Every git command runs through run
|
||||
// as `git -C dir ...` with GIT_TERMINAL_PROMPT=0 and no credential helper, so nothing is asked for and no
|
||||
// credential is used. A remote that is not ssh://, or one with user information, and a commit that is not 40
|
||||
// hex digits are refused before anything runs.
|
||||
func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, opts ...Option) error {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, commit, ref); err != nil {
|
||||
return err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) (string, error) {
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
if _, err := git("init", "-q"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := useMirror(o, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git("remote", "add", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git("fetch", "--no-tags", "--depth", "1", "origin", commit); err != nil {
|
||||
if ref == "" {
|
||||
return err
|
||||
}
|
||||
if _, rerr := git("fetch", "--no-tags", "origin", ref); rerr != nil {
|
||||
return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr)
|
||||
}
|
||||
if _, verr := git("cat-file", "-e", commit+"^{commit}"); verr != nil {
|
||||
return fmt.Errorf("the commit %s is not in %s: %w", commit, ref, verr)
|
||||
}
|
||||
}
|
||||
if _, err := git("checkout", "-q", "--detach", commit); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if head := strings.TrimSpace(out); head != commit {
|
||||
return fmt.Errorf("the checkout is at %s, not at the commit %s", head, commit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateMirror brings the bare repository mirror to commit: it makes the mirror when it is absent (git init --bare,
|
||||
// origin set to remote), then fetches the commit, or the ref when only the ref holds it, with the same rules as
|
||||
// FetchCommit (no prompt, no credential helper, ssh:// or an allowed file:// remote, a 40-hex commit). The mirror
|
||||
// keeps every commit it was brought to, so later fetches through WithMirror transfer only what is new. Callers
|
||||
// serialise updates of one mirror.
|
||||
func UpdateMirror(ctx context.Context, run forgekey.Runner, mirror, remote, commit, ref string, opts ...Option) error {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
if !filepath.IsAbs(mirror) {
|
||||
return fmt.Errorf("buildstep: the mirror %q is not an absolute path", mirror)
|
||||
}
|
||||
if err := checkFetch(o, mirror, remote, commit, ref); err != nil {
|
||||
return err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) error {
|
||||
r := run(ctx, env, "git", append([]string{"-c", "credential.helper="}, args...)...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
in := func(args ...string) error { return git(append([]string{"-C", mirror}, args...)...) }
|
||||
if _, err := os.Stat(filepath.Join(mirror, "HEAD")); err != nil {
|
||||
if err := git("init", "-q", "--bare", mirror); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := in("remote", "add", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := in("remote", "set-url", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := in("fetch", "--no-tags", "origin", commit); err != nil {
|
||||
if ref == "" {
|
||||
return err
|
||||
}
|
||||
if rerr := in("fetch", "--no-tags", "origin", ref); rerr != nil {
|
||||
return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr)
|
||||
}
|
||||
}
|
||||
if err := in("cat-file", "-e", commit+"^{commit}"); err != nil {
|
||||
return fmt.Errorf("the commit %s is not in the mirror: %w", commit, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFetch(o fetchOptions, dir, remote, commit, ref string) error {
|
||||
if dir == "" {
|
||||
return fmt.Errorf("buildstep: no workspace directory")
|
||||
}
|
||||
if o.mirror != "" {
|
||||
if !filepath.IsAbs(o.mirror) {
|
||||
return fmt.Errorf("buildstep: the mirror %q is not an absolute path", o.mirror)
|
||||
}
|
||||
if fi, err := os.Stat(o.mirror); err != nil || !fi.IsDir() {
|
||||
return fmt.Errorf("buildstep: the mirror %q is not an existing directory", o.mirror)
|
||||
}
|
||||
}
|
||||
if !commitOK.MatchString(commit) {
|
||||
return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit)
|
||||
}
|
||||
if ref != "" && !refOK.MatchString(ref) {
|
||||
return fmt.Errorf("buildstep: %q is not a ref (refs/...)", ref)
|
||||
}
|
||||
u, err := url.Parse(remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("buildstep: the remote is not a URL: %w", err)
|
||||
}
|
||||
if u.User != nil {
|
||||
return fmt.Errorf("buildstep: the remote carries user information; the key is the credential")
|
||||
}
|
||||
switch {
|
||||
case u.Scheme == "ssh" && u.Host != "":
|
||||
case u.Scheme == "file" && o.allowFile:
|
||||
default:
|
||||
return fmt.Errorf("buildstep: the remote %q is not ssh://", remote)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
func gitIn(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid"}, args...)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// 1474/AC5: a local bare repository with a commit only under refs/pull/1/head (a file:// remote, allowed by the
|
||||
// test-only option): the workspace's HEAD is the commit. The forge itself is proven live after the build
|
||||
// modules' delivery (ADR 0466 §5).
|
||||
func TestFetchCommitFromAPullRefOnly(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
bare := filepath.Join(root, "r.git")
|
||||
work := filepath.Join(root, "work")
|
||||
for _, d := range []string{bare, work} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gitIn(t, bare, "init", "-q", "--bare")
|
||||
gitIn(t, work, "init", "-q")
|
||||
if err := os.WriteFile(filepath.Join(work, "f"), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gitIn(t, work, "add", "f")
|
||||
gitIn(t, work, "commit", "-q", "-m", "pull request head")
|
||||
commit := gitIn(t, work, "rev-parse", "HEAD")
|
||||
gitIn(t, work, "push", "-q", bare, "HEAD:refs/pull/1/head")
|
||||
|
||||
ws := filepath.Join(root, "ws")
|
||||
if err := os.MkdirAll(ws, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := FetchCommit(context.Background(), forgekey.ExecRunner, ws, "file://"+bare, commit, "refs/pull/1/head", AllowFileRemote())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if head := gitIn(t, ws, "rev-parse", "HEAD"); head != commit {
|
||||
t.Errorf("HEAD = %s, want %s", head, commit)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const sha = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// fakeGit records every command and answers by a rule.
|
||||
type fakeGit struct {
|
||||
calls []string
|
||||
envs [][]string
|
||||
rule func(args string) forgekey.Ran
|
||||
}
|
||||
|
||||
func (f *fakeGit) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
f.calls = append(f.calls, line)
|
||||
f.envs = append(f.envs, env)
|
||||
if f.rule != nil {
|
||||
if r := f.rule(line); r.Status != 0 || r.Stdout != "" || r.Err != "" {
|
||||
return r
|
||||
}
|
||||
}
|
||||
if len(args) > 4 && args[4] == "rev-parse" {
|
||||
return forgekey.Ran{Stdout: sha + "\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
|
||||
const pre = "git -c credential.helper= -C /w "
|
||||
|
||||
// 1474/AC1: the git commands are exactly the sequence, none with a credential or an https remote.
|
||||
func TestFetchCommitRunsTheSequence(t *testing.T) {
|
||||
f := &fakeGit{}
|
||||
if err := FetchCommit(context.Background(), f.run, "/w", "ssh://mesh-forge/novox/mesh-sdk.git", sha, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{
|
||||
pre + "init -q",
|
||||
pre + "remote add origin ssh://mesh-forge/novox/mesh-sdk.git",
|
||||
pre + "fetch --no-tags --depth 1 origin " + sha,
|
||||
pre + "checkout -q --detach " + sha,
|
||||
pre + "rev-parse HEAD",
|
||||
}
|
||||
if strings.Join(f.calls, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("commands:\n%s\nwant:\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
for i, c := range f.calls {
|
||||
if strings.Contains(c, "https://") || strings.Contains(c, "@") || !strings.Contains(c, "credential.helper= ") {
|
||||
t.Errorf("command %q carries a credential or https", c)
|
||||
}
|
||||
if len(f.envs[i]) != 1 || f.envs[i][0] != "GIT_TERMINAL_PROMPT=0" {
|
||||
t.Errorf("env %v", f.envs[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1474/AC2: the depth-1 fetch fails and a pull ref is given: the ref is fetched and the commit verified; with no
|
||||
// ref, the fetch's failure is the answer.
|
||||
func TestFetchCommitFallsBackToThePullRef(t *testing.T) {
|
||||
f := &fakeGit{rule: func(l string) forgekey.Ran {
|
||||
if strings.Contains(l, "--depth 1") {
|
||||
return forgekey.Ran{Status: 128, Stderr: "fatal: couldn't find remote ref " + sha + "\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
if err := FetchCommit(context.Background(), f.run, "/w", "ssh://mesh-forge/o/r.git", sha, "refs/pull/1/head"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{pre + "fetch --no-tags origin refs/pull/1/head", pre + "cat-file -e " + sha + "^{commit}", pre + "checkout -q --detach " + sha}
|
||||
got := strings.Join(f.calls[3:6], "\n")
|
||||
if got != strings.Join(want, "\n") {
|
||||
t.Errorf("after the failed fetch:\n%s\nwant:\n%s", got, strings.Join(want, "\n"))
|
||||
}
|
||||
|
||||
g := &fakeGit{rule: f.rule}
|
||||
err := FetchCommit(context.Background(), g.run, "/w", "ssh://mesh-forge/o/r.git", sha, "")
|
||||
if err == nil || !strings.Contains(err.Error(), "couldn't find remote ref") || len(g.calls) != 3 {
|
||||
t.Errorf("without a ref: %v after %d commands", err, len(g.calls))
|
||||
}
|
||||
|
||||
h := &fakeGit{rule: func(l string) forgekey.Ran {
|
||||
if strings.Contains(l, "--depth 1") || strings.Contains(l, "cat-file") {
|
||||
return forgekey.Ran{Status: 1, Stderr: "no"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
if err := FetchCommit(context.Background(), h.run, "/w", "ssh://mesh-forge/o/r.git", sha, "refs/pull/1/head"); err == nil {
|
||||
t.Error("a commit that is not in the ref was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// 1474/AC3: HEAD differs from the commit after the checkout: an error naming both.
|
||||
func TestFetchCommitAnswersAHeadThatIsNotTheCommit(t *testing.T) {
|
||||
other := strings.Repeat("a", 40)
|
||||
f := &fakeGit{rule: func(l string) forgekey.Ran {
|
||||
if strings.HasSuffix(l, "rev-parse HEAD") {
|
||||
return forgekey.Ran{Stdout: other + "\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
err := FetchCommit(context.Background(), f.run, "/w", "ssh://mesh-forge/o/r.git", sha, "")
|
||||
if err == nil || !strings.Contains(err.Error(), other) || !strings.Contains(err.Error(), sha) {
|
||||
t.Errorf("error = %v, want both shas", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1474/AC4: an https remote, user information, a file remote without the option, or a commit that is not 40
|
||||
// hex digits is refused before anything runs.
|
||||
func TestFetchCommitRefusesBeforeRunningAnything(t *testing.T) {
|
||||
for _, c := range []struct{ remote, commit, ref string }{
|
||||
{"https://forge.example/o/r.git", sha, ""},
|
||||
{"http://forge.example/o/r.git", sha, ""},
|
||||
{"ssh://git@mesh-forge/o/r.git", sha, ""},
|
||||
{"ssh://user:pw@mesh-forge/o/r.git", sha, ""},
|
||||
{"file:///tmp/r.git", sha, ""},
|
||||
{"/tmp/r.git", sha, ""},
|
||||
{"ssh://mesh-forge/o/r.git", "abc123", ""},
|
||||
{"ssh://mesh-forge/o/r.git", strings.ToUpper(sha), ""},
|
||||
{"ssh://mesh-forge/o/r.git", "main", ""},
|
||||
{"ssh://mesh-forge/o/r.git", sha, "--upload-pack=x"},
|
||||
} {
|
||||
f := &fakeGit{}
|
||||
if err := FetchCommit(context.Background(), f.run, "/w", c.remote, c.commit, c.ref); err == nil || len(f.calls) != 0 {
|
||||
t.Errorf("%+v: err %v, %d commands", c, err, len(f.calls))
|
||||
}
|
||||
}
|
||||
f := &fakeGit{}
|
||||
if err := FetchCommit(context.Background(), f.run, "/w", "file:///tmp/r.git", sha, "", AllowFileRemote()); err != nil {
|
||||
t.Errorf("a file remote with the option: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Guard is a build module's check that the toolchain installed on its node is the one its recipe builds with
|
||||
// (novox/hq ADR 0469 §1; issue 1473). The same guard serves ADR 0470 §4's postgres major for the integration
|
||||
// level, with its own Probe and Want.
|
||||
type Guard struct {
|
||||
// Language is the toolchain's key part: go or typescript (delivery.ToolchainKey).
|
||||
Language string
|
||||
// Want is the toolchain version the recipe builds with, and Recipe the recipe's own version. The indexer
|
||||
// compares both texts as they are written here.
|
||||
Want, Recipe string
|
||||
// Probe reads the installed version (go version, node --version).
|
||||
Probe func(ctx context.Context) (installed string, err error)
|
||||
Node string
|
||||
// Put is the toolchains state's Put (stdio.State(delivery.ToolchainsState).Put).
|
||||
Put Put
|
||||
}
|
||||
|
||||
// Check probes the installed version, writes it to <node>.<language> of the toolchains state every time it runs,
|
||||
// and answers nil when it is the recipe's. Otherwise it answers a *Refusal naming both versions; the installed
|
||||
// version is written even then, so the indexer sees that builders disagree. A probe that fails is a *Refusal
|
||||
// naming its failure, and nothing is written. A write that fails is an error that is not a Refusal.
|
||||
func (g Guard) Check(ctx context.Context) error {
|
||||
installed, err := g.Probe(ctx)
|
||||
if err != nil {
|
||||
return &Refusal{Text: fmt.Sprintf("the installed %s version could not be read: %v", g.Language, err), Want: g.Want}
|
||||
}
|
||||
key, err := delivery.ToolchainKey(g.Node, g.Language)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodeToolchain(delivery.Toolchain{Language: g.Language, Version: installed, Recipe: g.Recipe, At: now().UTC()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := g.Put(key, json.RawMessage(raw)); err != nil {
|
||||
return fmt.Errorf("buildstep: writing the toolchain %s: %w", key, err)
|
||||
}
|
||||
if installed != g.Want {
|
||||
return &Refusal{Text: fmt.Sprintf("%s %s installed, the recipe builds with %s %s", g.Language, installed, g.Language, g.Want),
|
||||
Want: g.Want, Have: installed}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
type toolchainKeys map[string]delivery.Toolchain
|
||||
|
||||
func (k toolchainKeys) put(key string, value any) (uint64, error) {
|
||||
tc, err := delivery.DecodeToolchain(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
k[key] = tc
|
||||
return uint64(len(k)), nil
|
||||
}
|
||||
|
||||
func guardFor(k toolchainKeys, installed string, probeErr error) Guard {
|
||||
return Guard{Language: "go", Want: "1.27.0", Recipe: "r7", Node: "ace", Put: k.put,
|
||||
Probe: func(context.Context) (string, error) { return installed, probeErr }}
|
||||
}
|
||||
|
||||
// 1473/AC1: the probe answers the recipe's version: nil, and <node>.go holds that version and the recipe's.
|
||||
func TestCheckPassesAndWritesTheToolchain(t *testing.T) {
|
||||
k := toolchainKeys{}
|
||||
if err := guardFor(k, "1.27.0", nil).Check(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tc, ok := k["ace.go"]
|
||||
if !ok || tc.Version != "1.27.0" || tc.Recipe != "r7" || tc.Language != "go" || tc.At.IsZero() {
|
||||
t.Errorf("keys = %+v", k)
|
||||
}
|
||||
}
|
||||
|
||||
// 1473/AC2: another version: a Refusal naming both, and the key holds the installed version.
|
||||
func TestCheckRefusesAnotherVersionAndWritesTheInstalledOne(t *testing.T) {
|
||||
k := toolchainKeys{}
|
||||
err := guardFor(k, "1.27.1", nil).Check(context.Background())
|
||||
var ref *Refusal
|
||||
if !errors.As(err, &ref) {
|
||||
t.Fatalf("Check = %v, want a Refusal", err)
|
||||
}
|
||||
if ref.Text != "go 1.27.1 installed, the recipe builds with go 1.27.0" || ref.Want != "1.27.0" || ref.Have != "1.27.1" {
|
||||
t.Errorf("refusal = %+v", ref)
|
||||
}
|
||||
if k["ace.go"].Version != "1.27.1" {
|
||||
t.Errorf("the key holds %q, want the installed 1.27.1", k["ace.go"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// 1473/AC3: a probe that fails: a Refusal naming the failure, and nothing put.
|
||||
func TestCheckRefusesAFailedProbeAndPutsNothing(t *testing.T) {
|
||||
k := toolchainKeys{}
|
||||
err := guardFor(k, "", errors.New("go: not found")).Check(context.Background())
|
||||
var ref *Refusal
|
||||
if !errors.As(err, &ref) || !strings.Contains(ref.Text, "go: not found") {
|
||||
t.Fatalf("Check = %v, want a Refusal naming the probe's failure", err)
|
||||
}
|
||||
if len(k) != 0 {
|
||||
t.Errorf("put %v", k)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed write is an error, not a refusal: the step fails instead of refusing.
|
||||
func TestCheckAnswersAFailedWriteAsAnError(t *testing.T) {
|
||||
g := guardFor(toolchainKeys{}, "1.27.0", nil)
|
||||
g.Put = func(string, any) (uint64, error) { return 0, errors.New("no grant") }
|
||||
err := g.Check(context.Background())
|
||||
var ref *Refusal
|
||||
if err == nil || errors.As(err, &ref) {
|
||||
t.Errorf("Check = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs
|
||||
// unprivileged user namespaces (issue 1308).
|
||||
const Bubblewrap = "bwrap"
|
||||
|
||||
// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one
|
||||
// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the
|
||||
// change's code; the run step runs the change's build and tests in bubblewrap with:
|
||||
//
|
||||
// - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but
|
||||
// resolves no name and reaches no socket of the machine;
|
||||
// - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since
|
||||
// none of them is bound; HOME is a directory in the step's own /tmp unless Env names one;
|
||||
// - an environment of exactly Env and the runner's env: nothing of the build module's own passes;
|
||||
// - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries,
|
||||
// certificates, time zone), with a hosts file naming only the loopback;
|
||||
// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step
|
||||
// downloaded) read-only;
|
||||
// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone
|
||||
// (WriteCaches), so a candidate's code never changes what a later build reads.
|
||||
type Isolation struct {
|
||||
Workspace string
|
||||
Writable []string
|
||||
ReadOnly []string
|
||||
Caches []string
|
||||
WriteCaches bool
|
||||
Env []string
|
||||
// Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback.
|
||||
Hosts string
|
||||
}
|
||||
|
||||
// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the
|
||||
// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name.
|
||||
var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf",
|
||||
"/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"}
|
||||
|
||||
// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks
|
||||
// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it.
|
||||
var systemArgs = func() []string {
|
||||
args := []string{"--ro-bind", "/usr", "/usr"}
|
||||
for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} {
|
||||
info, err := os.Lstat(d)
|
||||
switch {
|
||||
case err != nil:
|
||||
case info.Mode()&os.ModeSymlink != 0:
|
||||
if target, err := os.Readlink(d); err == nil {
|
||||
args = append(args, "--symlink", target, d)
|
||||
}
|
||||
case info.IsDir():
|
||||
args = append(args, "--ro-bind", d, d)
|
||||
}
|
||||
}
|
||||
for _, e := range etcNeeded {
|
||||
args = append(args, "--ro-bind-try", e, e)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it.
|
||||
const stepHome = "/tmp/home"
|
||||
|
||||
func checkDir(what, dir string) error {
|
||||
if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" {
|
||||
return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It
|
||||
// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=".
|
||||
func (iso Isolation) Command(name string, args ...string) ([]string, error) {
|
||||
return iso.command(nil, name, args...)
|
||||
}
|
||||
|
||||
func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) {
|
||||
if err := checkDir("workspace", iso.Workspace); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, group := range []struct {
|
||||
what string
|
||||
dirs []string
|
||||
}{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} {
|
||||
for _, d := range group.dirs {
|
||||
if err := checkDir(group.what, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
words := append(append([]string{}, iso.Env...), env...)
|
||||
home := false
|
||||
for _, w := range words {
|
||||
k, _, ok := strings.Cut(w, "=")
|
||||
if !ok || k == "" {
|
||||
return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w)
|
||||
}
|
||||
home = home || k == "HOME"
|
||||
}
|
||||
argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv",
|
||||
"--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome}
|
||||
argv = append(argv, systemArgs()...)
|
||||
for _, d := range iso.ReadOnly {
|
||||
argv = append(argv, "--ro-bind", d, d)
|
||||
}
|
||||
argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...)
|
||||
argv = append(argv, "--bind", iso.Workspace, iso.Workspace)
|
||||
for _, d := range iso.Writable {
|
||||
argv = append(argv, "--bind", d, d)
|
||||
}
|
||||
if iso.Hosts != "" {
|
||||
argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts")
|
||||
}
|
||||
for _, w := range words {
|
||||
k, v, _ := strings.Cut(w, "=")
|
||||
argv = append(argv, "--setenv", k, v)
|
||||
}
|
||||
if !home {
|
||||
argv = append(argv, "--setenv", "HOME", stepHome)
|
||||
}
|
||||
argv = append(argv, "--chdir", iso.Workspace, "--", name)
|
||||
return append(argv, args...), nil
|
||||
}
|
||||
|
||||
// cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other
|
||||
// read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and
|
||||
// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace).
|
||||
func cacheArgs(caches []string, write bool) []string {
|
||||
var argv []string
|
||||
for _, c := range caches {
|
||||
if write {
|
||||
argv = append(argv, "--bind", c, c)
|
||||
continue
|
||||
}
|
||||
argv = append(argv, "--overlay-src", c, "--tmp-overlay", c)
|
||||
}
|
||||
return argv
|
||||
}
|
||||
|
||||
// loopbackHosts is the hosts file of a step: the loopback and nothing else.
|
||||
const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n"
|
||||
|
||||
// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules'
|
||||
// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of
|
||||
// its own, naming only the loopback, removed when it ends.
|
||||
func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
dir, err := os.MkdirTemp("", "step-hosts-")
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
hosts := filepath.Join(dir, "hosts")
|
||||
if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()}
|
||||
}
|
||||
iso := iso
|
||||
iso.Hosts = hosts
|
||||
argv, err := iso.command(env, name, args...)
|
||||
if err != nil {
|
||||
return forgekey.Ran{Status: 1, Err: err.Error()}
|
||||
}
|
||||
return run(ctx, nil, Bubblewrap, argv...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package buildstep
|
||||
|
||||
import "testing"
|
||||
|
||||
// The shared warm caches are read under a throwaway writable layer (novox/hq ADR 0448; issue 1309): a candidate's step
|
||||
// writes into them as a build does, and nothing it writes outlives it or reaches a later build; only a trunk build
|
||||
// writes them.
|
||||
func TestACandidatesStepWritesTheCachesOnlyIntoAThrowawayLayer(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
iso := Isolation{Workspace: "/w", Caches: []string{"/var/cache/go-build", "/var/cache/npm"}}
|
||||
argv, err := iso.Command("go", "test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range iso.Caches {
|
||||
if !holds(argv, "--overlay-src", c, "--tmp-overlay", c) {
|
||||
t.Errorf("the cache %s is not under a throwaway layer: %v", c, argv)
|
||||
}
|
||||
if holds(argv, "--bind", c, c) {
|
||||
t.Errorf("a candidate's step writes the cache %s: %v", c, argv)
|
||||
}
|
||||
}
|
||||
iso.WriteCaches = true
|
||||
argv, _ = iso.Command("go", "build")
|
||||
for _, c := range iso.Caches {
|
||||
if !holds(argv, "--bind", c, c) || holds(argv, "--tmp-overlay", c) {
|
||||
t.Errorf("a trunk build does not write the cache %s: %v", c, argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//go:build integration && isolation
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// 1309/S19, the issue's proof: a run step, in bubblewrap as Isolated runs it, reads no key and no login, resolves no
|
||||
// name, reaches no TCP or unix socket of the machine, finds no home of the account, and leaves the shared cache it
|
||||
// wrote unchanged; the workspace it writes keeps what it wrote. It needs bubblewrap and unprivileged user namespaces
|
||||
// (issue 1308) on the machine that runs it, and fails naming them when they are not there.
|
||||
func TestARunStepReachesNothingAndLeavesTheCachesUnchanged(t *testing.T) {
|
||||
if _, err := exec.LookPath(Bubblewrap); err != nil {
|
||||
t.Fatalf("bubblewrap is not installed here, so the isolation of a run step cannot be proved (issue 1309): %v", err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
account := filepath.Join(root, "account")
|
||||
key, login := filepath.Join(account, ".ssh", "id_ed25519"), filepath.Join(account, "bus-login.json")
|
||||
writeFile(t, filepath.Dir(key), filepath.Base(key), "PRIVATE KEY")
|
||||
writeFile(t, account, "bus-login.json", `{"password":"secret"}`)
|
||||
cacheDir := filepath.Join(root, "cache")
|
||||
writeFile(t, cacheDir, "warm", "warm\n")
|
||||
workspace := filepath.Join(root, "work", "tree")
|
||||
if err := os.MkdirAll(workspace, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
probe := filepath.Join(workspace, "probe")
|
||||
if out, err := exec.Command("go", "build", "-o", probe, "./testdata/isolated-probe").CombinedOutput(); err != nil {
|
||||
t.Fatalf("building the probe: %v\n%s", err, out)
|
||||
}
|
||||
tcp, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tcp.Close()
|
||||
socket := filepath.Join(account, "bus.sock")
|
||||
unix, err := net.Listen("unix", socket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer unix.Close()
|
||||
|
||||
run := Isolated(forgekey.ExecRunnerFor(0), Isolation{Workspace: workspace, Caches: []string{cacheDir}})
|
||||
r := run(context.Background(), []string{"PATH=/usr/bin:/bin"}, probe, key, login, tcp.Addr().String(), socket, "localhost.localdomain.invalid", filepath.Join(cacheDir, "warm"))
|
||||
if r.Status != 0 || r.Err != "" || strings.TrimSpace(r.Stdout) != "shut off" {
|
||||
t.Fatalf("the run step was not shut off (status %d, %s):\n%s%s", r.Status, r.Err, r.Stdout, r.Stderr)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(cacheDir, "warm")); string(b) != "warm\n" {
|
||||
t.Fatalf("the step changed the shared cache: %q", b)
|
||||
}
|
||||
|
||||
// The workspace is the step's own: what it writes there stays.
|
||||
r = run(context.Background(), nil, "/bin/sh", "-c", "echo built > out")
|
||||
if b, err := os.ReadFile(filepath.Join(workspace, "out")); r.Status != 0 || err != nil || string(b) != "built\n" {
|
||||
t.Fatalf("the step's write to its workspace did not stay (status %d, %s%s): %q %v", r.Status, r.Stderr, r.Err, b, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// pairs says whether argv holds the words of want one after another.
|
||||
func holds(argv []string, want ...string) bool {
|
||||
for i := 0; i+len(want) <= len(argv); i++ {
|
||||
match := true
|
||||
for j, w := range want {
|
||||
if argv[i+j] != w {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fixedSystem(t *testing.T) {
|
||||
t.Helper()
|
||||
before := systemArgs
|
||||
systemArgs = func() []string { return []string{"--ro-bind", "/usr", "/usr", "--symlink", "usr/bin", "/bin"} }
|
||||
t.Cleanup(func() { systemArgs = before })
|
||||
}
|
||||
|
||||
// A run step's command runs in bubblewrap (novox/hq ADR 0448; issue 1309): no network, no namespace shared, an
|
||||
// environment of exactly what the step is given, the workspace the one tree it writes, the toolchain read-only, and
|
||||
// nothing else of the machine's: not the account's home, not the forge key or a bus login, not the runtime directory,
|
||||
// and no name service.
|
||||
func TestARunStepRunsShutOffInBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
iso := Isolation{Workspace: "/work/ab12/tree", ReadOnly: []string{"/usr/lib/go"}, Writable: []string{"/work/ab12/out"},
|
||||
Env: []string{"GOFLAGS=-mod=mod", "PATH=/usr/bin"}, Hosts: "/work/ab12/hosts"}
|
||||
argv, err := iso.Command("go", "test", "./...")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range [][]string{
|
||||
{"--unshare-all"}, {"--die-with-parent"}, {"--new-session"}, {"--clearenv"},
|
||||
{"--proc", "/proc"}, {"--dev", "/dev"}, {"--tmpfs", "/tmp"},
|
||||
{"--ro-bind", "/usr", "/usr"}, {"--symlink", "usr/bin", "/bin"},
|
||||
{"--ro-bind", "/usr/lib/go", "/usr/lib/go"},
|
||||
{"--bind", "/work/ab12/tree", "/work/ab12/tree"}, {"--bind", "/work/ab12/out", "/work/ab12/out"},
|
||||
{"--ro-bind", "/work/ab12/hosts", "/etc/hosts"},
|
||||
{"--setenv", "GOFLAGS", "-mod=mod"}, {"--setenv", "PATH", "/usr/bin"}, {"--setenv", "HOME", "/tmp/home"},
|
||||
{"--chdir", "/work/ab12/tree"},
|
||||
{"--", "go", "test", "./..."},
|
||||
} {
|
||||
if !holds(argv, want...) {
|
||||
t.Errorf("the command has no %v: %v", want, argv)
|
||||
}
|
||||
}
|
||||
if holds(argv, "--share-net") || strings.Contains(strings.Join(argv, " "), "resolv.conf") || strings.Contains(strings.Join(argv, " "), "/run") {
|
||||
t.Errorf("the command reaches the network, the name service or the runtime directory: %v", argv)
|
||||
}
|
||||
// The workspace is mounted after /tmp's tmpfs, so a workspace under /tmp is not hidden by it.
|
||||
if strings.Index(strings.Join(argv, "\x00"), "--tmpfs\x00/tmp") > strings.Index(strings.Join(argv, "\x00"), "--bind\x00/work/ab12/tree") {
|
||||
t.Errorf("the tmpfs of /tmp comes after the workspace: %v", argv)
|
||||
}
|
||||
// A HOME the step is given is its own.
|
||||
iso.Env = append(iso.Env, "HOME=/work/ab12/home")
|
||||
argv, _ = iso.Command("true")
|
||||
if holds(argv, "--setenv", "HOME", "/tmp/home") || !holds(argv, "--setenv", "HOME", "/work/ab12/home") {
|
||||
t.Errorf("the step's own HOME was replaced: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIsolationNamesNoPathItCannotPin(t *testing.T) {
|
||||
for _, iso := range []Isolation{
|
||||
{Workspace: ""},
|
||||
{Workspace: "tree"},
|
||||
{Workspace: "/"},
|
||||
{Workspace: "/w", ReadOnly: []string{"relative"}},
|
||||
{Workspace: "/w", Writable: []string{"/"}},
|
||||
{Workspace: "/w", Caches: []string{"/a/../b"}},
|
||||
{Workspace: "/w", Env: []string{"NOEQUALS"}},
|
||||
} {
|
||||
if _, err := iso.Command("true"); err == nil {
|
||||
t.Errorf("%+v was accepted", iso)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Isolated runs every command through bwrap, with the step's environment inside and a hosts file that names only the
|
||||
// loopback, removed after.
|
||||
func TestIsolatedRunsEachCommandThroughBubblewrap(t *testing.T) {
|
||||
fixedSystem(t)
|
||||
var ran []string
|
||||
var hosts string
|
||||
inner := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
ran = append([]string{name}, args...)
|
||||
for i, a := range args {
|
||||
if a == "/etc/hosts" && i > 0 {
|
||||
hosts = args[i-1]
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "ok"}
|
||||
}
|
||||
r := Isolated(inner, Isolation{Workspace: "/w"})(context.Background(), []string{"A=1"}, "go", "build")
|
||||
if r.Stdout != "ok" || ran[0] != Bubblewrap || !holds(ran, "--setenv", "A", "1") || !holds(ran, "--", "go", "build") {
|
||||
t.Fatalf("ran %v: %+v", ran, r)
|
||||
}
|
||||
if hosts == "" {
|
||||
t.Fatalf("no hosts file was bound: %v", ran)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package buildstep
|
||||
|
||||
import "strings"
|
||||
|
||||
// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as
|
||||
// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says.
|
||||
const LauncherSuffix = ".serve.mjs"
|
||||
|
||||
// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript
|
||||
// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other
|
||||
// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller
|
||||
// share (R4e), so the name a build writes is the name a reader expects.
|
||||
func Launchers(language string, entrypoints []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
if language != "typescript" {
|
||||
return out
|
||||
}
|
||||
for _, entry := range entrypoints {
|
||||
if strings.HasSuffix(entry, ".js") {
|
||||
out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A TypeScript bundle's entrypoint x.js is started as x.serve.mjs beside it (novox/hq ADR 0193), the rule mesh-build-node
|
||||
// and build-agent both apply; an entrypoint that is not .js gets none, and another language gets none.
|
||||
func TestATypeScriptEntrypointIsStartedByTheLauncherBesideIt(t *testing.T) {
|
||||
got := Launchers("typescript", []string{"index.js", "tools/x.js", "data.json"})
|
||||
want := map[string]string{"index.js": "index.serve.mjs", "tools/x.js": "tools/x.serve.mjs"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Launchers(typescript) = %v, want %v", got, want)
|
||||
}
|
||||
if got := Launchers("go", []string{"index.js"}); len(got) != 0 {
|
||||
t.Fatalf("a Go bundle got launchers: %v", got)
|
||||
}
|
||||
if got := Launchers("typescript", nil); len(got) != 0 {
|
||||
t.Fatalf("a bundle with no entrypoint got launchers: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// MergeResult is what MergeInto made of a head and a trunk commit: the trunk commit (Base) and the head, whole, and the
|
||||
// merge commit and the paths the merge changes against the trunk commit; or, when the head does not merge cleanly, the
|
||||
// files in conflict, with Merge and Changed empty.
|
||||
type MergeResult struct {
|
||||
Base, Head, Merge string
|
||||
Conflicts []string
|
||||
Changed []string
|
||||
}
|
||||
|
||||
// MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit
|
||||
// base, as the forge's default merge would (a merge commit, never a fast-forward), and leaves the workspace at the merge
|
||||
// (novox/hq ADR 0484 clause 3, ADR 0323). It is the one merge the build modules' fetch steps make, with the identity
|
||||
// and flags of the controller's check merge (mesh-controller internal/builder/check_merge.go): a name nobody owns, no
|
||||
// signature, no hook, nothing of the machine's git configuration that changes a merge. Nothing is pushed.
|
||||
//
|
||||
// A merge needs both histories, so it makes the fetch whole (the shallow clone FetchCommit made is deepened) and
|
||||
// fetches base by its sha. A head that does not merge cleanly is an answer, not an error: the files in conflict, the
|
||||
// merge undone and the workspace left at base. A head already in base answers a Merge equal to Base and no changed
|
||||
// paths. An error is that the merge could not be tried: a base or commit that is not a full sha, no base, a workspace
|
||||
// not at commit, or a fetch or git command that failed. remote and the options are those of FetchCommit; the runner's
|
||||
// bound must allow a whole fetch.
|
||||
func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, commit string, opts ...Option) (MergeResult, error) {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
res := MergeResult{Base: base, Head: commit}
|
||||
if base == "" {
|
||||
return res, fmt.Errorf("buildstep: no base: a merge needs the trunk commit it is made onto")
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, base, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, commit, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if err := useMirror(o, dir); err != nil {
|
||||
return res, err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) (string, error) {
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
at, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if head := strings.TrimSpace(at); head != commit {
|
||||
return res, fmt.Errorf("buildstep: the workspace is at %s, not at the commit %s", head, commit)
|
||||
}
|
||||
shallow, err := git("rev-parse", "--is-shallow-repository")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
fetch := []string{"fetch", "--no-tags", "origin", base}
|
||||
if strings.TrimSpace(shallow) == "true" {
|
||||
fetch = []string{"fetch", "--no-tags", "--unshallow", "origin", base}
|
||||
}
|
||||
if _, err := git(fetch...); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s: %w", base, err)
|
||||
}
|
||||
if _, err := git("checkout", "-q", "--detach", base); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s cannot be checked out: %w", base, err)
|
||||
}
|
||||
message := fmt.Sprintf("Check the merge of %s into %s (made for a check, never pushed)", short(commit), short(base))
|
||||
args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, commit)
|
||||
if _, err := git(args...); err != nil {
|
||||
out, _ := git("diff", "--name-only", "--diff-filter=U")
|
||||
res.Conflicts = strings.Fields(out)
|
||||
_, _ = git("merge", "--abort")
|
||||
if len(res.Conflicts) == 0 {
|
||||
return res, fmt.Errorf("the head %s could not be merged into the trunk commit %s: %w", short(commit), short(base), err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
merge, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the merge cannot be read: %w", err)
|
||||
}
|
||||
res.Merge = strings.TrimSpace(merge)
|
||||
changed, err := git("diff", "--name-only", "--no-renames", base, res.Merge)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err)
|
||||
}
|
||||
res.Changed = strings.Fields(changed)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the
|
||||
// clone's or the build machine's git configuration runs, and no setting there changes the merge (the controller's
|
||||
// check merge, internal/builder/check_merge.go).
|
||||
var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"}
|
||||
|
||||
func short(commit string) string {
|
||||
if len(commit) > 12 {
|
||||
return commit[:12]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// The merge on real git repositories starts git processes, so it is an integration test (the unit level keeps its bound,
|
||||
// issue 895); the unit tests of merge_test.go prove the commands and the refusals with a fake runner.
|
||||
|
||||
func realGit(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, dir, name, content string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// mergeRig is a local bare repository standing in for the forge, a work repository that pushes to it, and an empty
|
||||
// workspace: the fetch step's shape (FetchCommit of the head into the workspace, then MergeInto).
|
||||
type mergeRig struct {
|
||||
t *testing.T
|
||||
bare, work, ws string
|
||||
fork, trunk string // the commit both histories share, and the trunk's head
|
||||
remote string
|
||||
}
|
||||
|
||||
func newMergeRig(t *testing.T) *mergeRig {
|
||||
root := t.TempDir()
|
||||
r := &mergeRig{t: t, bare: filepath.Join(root, "r.git"), work: filepath.Join(root, "work"), ws: filepath.Join(root, "ws")}
|
||||
for _, d := range []string{r.bare, r.work, r.ws} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r.remote = "file://" + r.bare
|
||||
realGit(t, r.bare, "init", "-q", "--bare")
|
||||
realGit(t, r.work, "init", "-q", "-b", "main")
|
||||
writeFile(t, r.work, "a.txt", "one\ntwo\nthree\n")
|
||||
writeFile(t, r.work, "modules/m/module.json", "{}\n")
|
||||
realGit(t, r.work, "add", ".")
|
||||
realGit(t, r.work, "commit", "-q", "-m", "fork point")
|
||||
r.fork = realGit(t, r.work, "rev-parse", "HEAD")
|
||||
return r
|
||||
}
|
||||
|
||||
// head makes a commit on a branch off the fork point, pushes it as the pull request's head ref and answers its sha.
|
||||
func (r *mergeRig) head(files map[string]string) string {
|
||||
realGit(r.t, r.work, "checkout", "-q", "-B", "pr", r.fork)
|
||||
for n, c := range files {
|
||||
writeFile(r.t, r.work, n, c)
|
||||
}
|
||||
realGit(r.t, r.work, "add", ".")
|
||||
realGit(r.t, r.work, "commit", "-q", "-m", "head")
|
||||
h := realGit(r.t, r.work, "rev-parse", "HEAD")
|
||||
realGit(r.t, r.work, "push", "-q", r.bare, "HEAD:refs/pull/1/head")
|
||||
return h
|
||||
}
|
||||
|
||||
// advance moves the trunk past the fork point with a commit changing files, pushes it and answers its sha.
|
||||
func (r *mergeRig) advance(files map[string]string) string {
|
||||
realGit(r.t, r.work, "checkout", "-q", "-B", "main", r.fork)
|
||||
for n, c := range files {
|
||||
writeFile(r.t, r.work, n, c)
|
||||
}
|
||||
realGit(r.t, r.work, "add", ".")
|
||||
realGit(r.t, r.work, "commit", "-q", "-m", "trunk moves")
|
||||
r.trunk = realGit(r.t, r.work, "rev-parse", "HEAD")
|
||||
realGit(r.t, r.work, "push", "-q", "-f", r.bare, "HEAD:refs/heads/main")
|
||||
return r.trunk
|
||||
}
|
||||
|
||||
func (r *mergeRig) fetch(head string) {
|
||||
if err := FetchCommit(context.Background(), forgekey.ExecRunner, r.ws, r.remote, head, "refs/pull/1/head", AllowFileRemote()); err != nil {
|
||||
r.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1674/AC1: a clean merge answers the merge commit and the changed files, and leaves the workspace at the merge, which
|
||||
// holds both sides' changes.
|
||||
func TestMergeIntoACleanMergeAnswersTheMergeCommitAndTheChangedFiles(t *testing.T) {
|
||||
r := newMergeRig(t)
|
||||
head := r.head(map[string]string{"b.txt": "head\n"})
|
||||
base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"})
|
||||
r.fetch(head)
|
||||
res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Base != base || res.Head != head || len(res.Merge) != 40 || res.Merge == base || res.Merge == head || len(res.Conflicts) != 0 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Changed, []string{"b.txt"}) {
|
||||
t.Errorf("changed = %v, want only what the head adds to the trunk: b.txt", res.Changed)
|
||||
}
|
||||
if at := realGit(t, r.ws, "rev-parse", "HEAD"); at != res.Merge {
|
||||
t.Errorf("the workspace is at %s, not at the merge %s", at, res.Merge)
|
||||
}
|
||||
for f, want := range map[string]string{"b.txt": "head\n", "a.txt": "one\ntwo\nthree\nfour\n"} {
|
||||
if got, _ := os.ReadFile(filepath.Join(r.ws, f)); string(got) != want {
|
||||
t.Errorf("%s = %q, want %q", f, got, want)
|
||||
}
|
||||
}
|
||||
if parents := strings.Fields(realGit(t, r.ws, "rev-list", "--parents", "-n", "1", res.Merge)); len(parents) != 3 || parents[1] != base || parents[2] != head {
|
||||
t.Errorf("the merge commit's parents = %v, want the trunk commit and the head", parents)
|
||||
}
|
||||
if author := realGit(t, r.ws, "log", "-1", "--format=%an <%ae>", res.Merge); author != "mesh merge check <merge-check@mesh.invalid>" {
|
||||
t.Errorf("the merge is made as %q", author)
|
||||
}
|
||||
}
|
||||
|
||||
// 1674/AC2: a conflict is an answer: the files in conflict, no merge commit, and the workspace left at the trunk commit.
|
||||
func TestMergeIntoAConflictAnswersTheFilesAndLeavesTheTreeAtTheTrunk(t *testing.T) {
|
||||
r := newMergeRig(t)
|
||||
head := r.head(map[string]string{"a.txt": "one\nHEAD\nthree\n", "c.txt": "c\n"})
|
||||
base := r.advance(map[string]string{"a.txt": "one\nTRUNK\nthree\n"})
|
||||
r.fetch(head)
|
||||
res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote())
|
||||
if err != nil {
|
||||
t.Fatalf("a conflict is an answer, not an error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(res.Conflicts, []string{"a.txt"}) || res.Merge != "" || len(res.Changed) != 0 || res.Base != base || res.Head != head {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if at := realGit(t, r.ws, "rev-parse", "HEAD"); at != base {
|
||||
t.Errorf("the workspace is at %s, not at the trunk commit %s", at, base)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(r.ws, "a.txt")); string(got) != "one\nTRUNK\nthree\n" {
|
||||
t.Errorf("a.txt = %q, want the trunk's", got)
|
||||
}
|
||||
if st := realGit(t, r.ws, "status", "--porcelain"); st != "" {
|
||||
t.Errorf("the tree is not clean after the aborted merge: %q", st)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(r.ws, "c.txt")); err == nil {
|
||||
t.Error("a file of the head is in a tree left at the trunk")
|
||||
}
|
||||
}
|
||||
|
||||
// A head already in the trunk merges into nothing: the merge is the trunk commit and no path changes.
|
||||
func TestMergeIntoAHeadAlreadyInTheTrunkChangesNothing(t *testing.T) {
|
||||
r := newMergeRig(t)
|
||||
head := r.head(map[string]string{"b.txt": "head\n"})
|
||||
r.fork = head // the trunk continues from the head
|
||||
base := r.advance(map[string]string{"a.txt": "one\ntwo\nthree\nfour\n"})
|
||||
r.fetch(head)
|
||||
res, err := MergeInto(context.Background(), forgekey.ExecRunner, r.ws, r.remote, base, head, AllowFileRemote())
|
||||
if err != nil || res.Merge != base || len(res.Changed) != 0 || len(res.Conflicts) != 0 {
|
||||
t.Errorf("result = %+v, %v", res, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// 1674/AC3: no base is refused before anything runs; so is a base or a commit that is not a full sha, a workspace that
|
||||
// is not at the commit, and a remote that is not ssh://.
|
||||
func TestMergeIntoRefusesWhatIsNotAMerge(t *testing.T) {
|
||||
f := &fakeGit{}
|
||||
for name, c := range map[string]struct{ remote, base, commit string }{
|
||||
"no base": {"ssh://mesh-forge/o/r.git", "", sha},
|
||||
"a short base": {"ssh://mesh-forge/o/r.git", "abc123", sha},
|
||||
"a branch as base": {"ssh://mesh-forge/o/r.git", "main", sha},
|
||||
"a short commit": {"ssh://mesh-forge/o/r.git", sha, "abc"},
|
||||
"an https remote": {"https://forge.example/o/r.git", sha, sha},
|
||||
} {
|
||||
_, err := MergeInto(context.Background(), f.run, "/w", c.remote, c.base, c.commit)
|
||||
if err == nil {
|
||||
t.Errorf("%s accepted", name)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Errorf("git ran for a refused merge: %v", f.calls)
|
||||
}
|
||||
g := &fakeGit{rule: func(l string) forgekey.Ran {
|
||||
if strings.Contains(l, "rev-parse HEAD") {
|
||||
return forgekey.Ran{Stdout: strings.Repeat("f", 40) + "\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
_, err := MergeInto(context.Background(), g.run, "/w", "ssh://mesh-forge/o/r.git", strings.Repeat("a", 40), sha)
|
||||
if err == nil || !strings.Contains(err.Error(), "not at the commit") || len(g.calls) != 1 {
|
||||
t.Errorf("a workspace at another commit: %v after %d commands", err, len(g.calls))
|
||||
}
|
||||
}
|
||||
|
||||
// 1674/AC4: the merge is made with the check merge's identity and flags, hooks off, and fetches with no credential.
|
||||
func TestMergeIntoRunsTheCheckMergesCommands(t *testing.T) {
|
||||
base := strings.Repeat("b", 40)
|
||||
f := &fakeGit{rule: func(l string) forgekey.Ran {
|
||||
switch {
|
||||
case strings.HasSuffix(l, "rev-parse HEAD"):
|
||||
return forgekey.Ran{Stdout: sha + "\n"}
|
||||
case strings.Contains(l, "--is-shallow-repository"):
|
||||
return forgekey.Ran{Stdout: "true\n"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}}
|
||||
if _, err := MergeInto(context.Background(), f.run, "/w", "ssh://mesh-forge/o/r.git", base, sha); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := strings.Join(f.calls, "\n")
|
||||
for _, want := range []string{
|
||||
pre + "fetch --no-tags --unshallow origin " + base,
|
||||
pre + "checkout -q --detach " + base,
|
||||
"user.name=mesh merge check", "user.email=merge-check@mesh.invalid", "commit.gpgsign=false", "core.hooksPath=/dev/null",
|
||||
"merge --no-ff --no-edit --no-verify --quiet -m",
|
||||
pre + "diff --name-only --no-renames " + base,
|
||||
} {
|
||||
if !strings.Contains(all, want) {
|
||||
t.Errorf("the commands lack %q:\n%s", want, all)
|
||||
}
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
if strings.Contains(c, "https://") || !strings.Contains(c, "credential.helper= ") {
|
||||
t.Errorf("command %q carries a credential or https", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 890 C1 (AC3): a mirror brought to two commits in turn; a fetch through it checks out each commit, the
|
||||
// alternates name the mirror, and no symlink is in the workspace.
|
||||
func TestMirrorServesTwoCommitsWithoutSymlinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
src := filepath.Join(root, "src.git")
|
||||
work := filepath.Join(root, "work")
|
||||
for _, d := range []string{src, work} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gitIn(t, src, "init", "-q", "--bare")
|
||||
gitIn(t, work, "init", "-q")
|
||||
var commits []string
|
||||
for _, c := range []string{"one", "two"} {
|
||||
if err := os.WriteFile(filepath.Join(work, "f"), []byte(c), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gitIn(t, work, "add", "f")
|
||||
gitIn(t, work, "commit", "-q", "-m", c)
|
||||
commits = append(commits, gitIn(t, work, "rev-parse", "HEAD"))
|
||||
gitIn(t, work, "push", "-q", src, "HEAD:refs/heads/main")
|
||||
}
|
||||
remote := "file://" + src
|
||||
mirror := filepath.Join(root, "mirror.git")
|
||||
for i, commit := range commits {
|
||||
if err := UpdateMirror(context.Background(), forgekey.ExecRunner, mirror, remote, commit, "", AllowFileRemote()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ws := filepath.Join(root, "ws"+string(rune('0'+i)))
|
||||
if err := os.MkdirAll(ws, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := FetchCommit(context.Background(), forgekey.ExecRunner, ws, remote, commit, "", AllowFileRemote(), WithMirror(mirror)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if head := gitIn(t, ws, "rev-parse", "HEAD"); head != commit {
|
||||
t.Errorf("HEAD = %s, want %s", head, commit)
|
||||
}
|
||||
b, _ := os.ReadFile(filepath.Join(ws, ".git", "objects", "info", "alternates"))
|
||||
if !strings.Contains(string(b), filepath.Join(mirror, "objects")) {
|
||||
t.Errorf("alternates %q", b)
|
||||
}
|
||||
_ = filepath.Walk(ws, func(p string, fi os.FileInfo, err error) error {
|
||||
if err == nil && fi.Mode()&os.ModeSymlink != 0 {
|
||||
t.Errorf("symlink %s", p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Issue 890 C1 (ADR 0536 §2): a fetch through a bare mirror names the mirror in .git/objects/info/alternates, a
|
||||
// plain file, and never links it.
|
||||
func TestFetchCommitWithMirrorWritesAlternates(t *testing.T) {
|
||||
mirror := t.TempDir()
|
||||
ws := t.TempDir()
|
||||
f := &fakeGit{}
|
||||
// the fake's init does not create .git; make the objects directory as git init would.
|
||||
if err := os.MkdirAll(filepath.Join(ws, ".git", "objects", "info"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := FetchCommit(context.Background(), f.run, ws, "ssh://mesh-forge/novox/mesh-sdk.git", sha, "", WithMirror(mirror))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := filepath.Join(ws, ".git", "objects", "info", "alternates")
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := strings.TrimSpace(string(b)), filepath.Join(mirror, "objects"); got != want {
|
||||
t.Errorf("alternates %q, want %q", got, want)
|
||||
}
|
||||
if fi, err := os.Lstat(p); err != nil || !fi.Mode().IsRegular() {
|
||||
t.Errorf("alternates is not a plain file: %v %v", fi, err)
|
||||
}
|
||||
// the alternates exist before the fetch runs
|
||||
if len(f.calls) < 3 || !strings.Contains(f.calls[2], "fetch") {
|
||||
t.Errorf("calls %v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithMirrorRefusesARelativeOrMissingMirror(t *testing.T) {
|
||||
for _, m := range []string{"rel/mirror", filepath.Join(t.TempDir(), "absent")} {
|
||||
f := &fakeGit{}
|
||||
err := FetchCommit(context.Background(), f.run, t.TempDir(), "ssh://mesh-forge/novox/x.git", sha, "", WithMirror(m))
|
||||
if err == nil || len(f.calls) != 0 {
|
||||
t.Errorf("mirror %q: err %v, calls %v", m, err, f.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMirrorInitsBareThenFetches(t *testing.T) {
|
||||
mirror := filepath.Join(t.TempDir(), "mirror.git")
|
||||
f := &fakeGit{}
|
||||
if err := UpdateMirror(context.Background(), f.run, mirror, "ssh://mesh-forge/novox/x.git", sha, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pm := "git -c credential.helper= -C " + mirror + " "
|
||||
want := []string{
|
||||
"git -c credential.helper= init -q --bare " + mirror,
|
||||
pm + "remote add origin ssh://mesh-forge/novox/x.git",
|
||||
pm + "fetch --no-tags origin " + sha,
|
||||
pm + "cat-file -e " + sha + "^{commit}",
|
||||
}
|
||||
if strings.Join(f.calls, "\n") != strings.Join(want, "\n") {
|
||||
t.Errorf("commands:\n%s\nwant:\n%s", strings.Join(f.calls, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMirrorKeepsAnExistingMirrorAndFallsBackToTheRef(t *testing.T) {
|
||||
mirror := t.TempDir() // exists: no init
|
||||
if err := os.WriteFile(filepath.Join(mirror, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := &fakeGit{}
|
||||
f.rule = func(args string) (r forgekey.Ran) {
|
||||
if strings.HasSuffix(args, "origin "+sha) {
|
||||
r.Status = 1
|
||||
r.Stderr = "not our ref"
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := UpdateMirror(context.Background(), f.run, mirror, "ssh://mesh-forge/novox/x.git", sha, "refs/pull/3/head"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := strings.Join(f.calls, "\n")
|
||||
if strings.Contains(all, "init") || strings.Contains(all, "remote add") || !strings.Contains(all, "fetch --no-tags origin refs/pull/3/head") {
|
||||
t.Errorf("commands:\n%s", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMirrorRefusesBeforeRunning(t *testing.T) {
|
||||
f := &fakeGit{}
|
||||
for _, c := range []struct{ remote, commit string }{
|
||||
{"https://x/y.git", sha}, {"ssh://u@h/o/r.git", sha}, {"ssh://h/o/r.git", "abc"},
|
||||
} {
|
||||
if err := UpdateMirror(context.Background(), f.run, t.TempDir(), c.remote, c.commit, ""); err == nil {
|
||||
t.Errorf("%v accepted", c)
|
||||
}
|
||||
}
|
||||
if len(f.calls) != 0 {
|
||||
t.Errorf("ran %v", f.calls)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// narrowKept are the files of a module's directory a narrowed build keeps beside its named paths: its manifest and
|
||||
// its lock files, each an input of the module's key (ModuleTree's Manifest and Locks). package.json and go.mod are
|
||||
// not kept: neither is in the named tree, so a build that needs one names it among its artifact's paths, which puts
|
||||
// it in the key (novox/hq ADR 0499 clause 4).
|
||||
var narrowKept = append([]string{"module.json"}, lockFiles...)
|
||||
|
||||
// Narrow leaves in the fetched tree only what a build of an artifact that names its paths may see (novox/hq ADR 0499
|
||||
// clause 4): r.Dir's named paths, its manifest and lock files, and .git; everything else is removed, so a file outside
|
||||
// them cannot change what is built without changing its key. A named path absent at the tree is an error naming it,
|
||||
// and nothing is removed then. With no paths in r it changes nothing. Both build modules' fetch steps call it.
|
||||
func Narrow(tree string, r FetchRequest) error {
|
||||
if len(r.Paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
keep := map[string]bool{".git": true}
|
||||
for _, p := range r.Paths {
|
||||
rel := path.Join(r.Dir, p)
|
||||
if strings.HasPrefix(rel, "../") || rel == ".." || path.IsAbs(p) {
|
||||
return fmt.Errorf("the named path %q leaves the module's directory", p)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(tree, filepath.FromSlash(rel))); err != nil {
|
||||
return fmt.Errorf("the named path %q is not in the fetched tree: %w", p, err)
|
||||
}
|
||||
keep[rel] = true
|
||||
}
|
||||
for _, f := range narrowKept {
|
||||
keep[path.Join(r.Dir, f)] = true
|
||||
}
|
||||
return prune(tree, "", keep)
|
||||
}
|
||||
|
||||
// prune removes from dir (rel within the tree) every entry that is neither kept nor on the way to one.
|
||||
func prune(root, rel string, keep map[string]bool) error {
|
||||
entries, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
p := path.Join(rel, e.Name())
|
||||
if keep[p] {
|
||||
continue
|
||||
}
|
||||
ancestor := false
|
||||
for k := range keep {
|
||||
if strings.HasPrefix(k, p+"/") {
|
||||
ancestor = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ancestor && e.IsDir() {
|
||||
if err := prune(root, p, keep); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(root, filepath.FromSlash(p))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func filesUnder(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
var out []string
|
||||
_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err == nil && !info.IsDir() {
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// A build of an artifact that names its paths sees only those paths, its manifest and its lock files (novox/hq ADR 0499
|
||||
// clause 4): Narrow removes everything else of the fetched tree but .git, package.json and go.mod among it, since
|
||||
// neither is in the named tree nor among the lock files; with no paths it changes nothing.
|
||||
func TestNarrowLeavesOnlyTheNamedPathsTheManifestAndTheLocks(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{".git/HEAD", "module.json", "package.json", "package-lock.json", "go.sum", "src/index.ts",
|
||||
"src/deep/a.ts", "test/a.test.ts", "tsconfig.json", "go/delivery/x.go", "README.md", "other/module.json"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(filesUnder(t, tree)); got != 12 {
|
||||
t.Fatalf("a request without paths changed the tree: %d files", got)
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: ".", Paths: []string{"src", "test", "tsconfig.json"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := ".git/HEAD,go.sum,module.json,package-lock.json,src/deep/a.ts,src/index.ts,test/a.test.ts,tsconfig.json"
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != want {
|
||||
t.Fatalf("the narrowed tree holds %s\nwant %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNarrowKeepsAModuleDirectorysPathsAndFailsOnAnAbsentOne(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{".git/config", "modules/a/module.json", "modules/a/go.mod", "modules/a/src/x.ts",
|
||||
"modules/a/notes.md", "modules/b/module.json", "root.txt"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
err := Narrow(tree, FetchRequest{Dir: "modules/a", Paths: []string{"src", "gone"}})
|
||||
if err == nil || !strings.Contains(err.Error(), `"gone"`) {
|
||||
t.Fatalf("an absent named path gave %v", err)
|
||||
}
|
||||
if len(filesUnder(t, tree)) != 7 {
|
||||
t.Fatal("a refused narrowing removed files")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: "modules/a", Paths: []string{"src"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != ".git/config,modules/a/module.json,modules/a/src/x.ts" {
|
||||
t.Fatalf("the narrowed tree holds %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that needs package.json or go.mod names it among its paths, and so keys on it (ADR 0499 clause 4).
|
||||
func TestNarrowKeepsAManifestOfTheLanguageOnlyWhenNamed(t *testing.T) {
|
||||
tree := t.TempDir()
|
||||
for _, f := range []string{"module.json", "package.json", "go.mod", "src/a.ts"} {
|
||||
writeTreeFile(t, tree, f, "x")
|
||||
}
|
||||
if err := Narrow(tree, FetchRequest{Dir: ".", Paths: []string{"src", "package.json"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(filesUnder(t, tree), ","); got != "module.json,package.json,src/a.ts" {
|
||||
t.Fatalf("the narrowed tree holds %s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Pack tars and gzips a directory the way the build agent does, byte for byte, and answers the archive and its
|
||||
// digest, "sha256:" and the hex of the archive's bytes (issue 1479; novox/hq ADR 0405 §7, ADR 0453 §7). It is a
|
||||
// copy of mesh-controller's internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425), which stays
|
||||
// where it is until the build agent goes: a build module's digest is compared with the build agent's for one
|
||||
// fingerprint, so the bytes must be equal.
|
||||
//
|
||||
// Deterministically: regular files only (a symlink or a device is refused), sorted relative paths with
|
||||
// slashes, each header carrying its name, permission bits and size and nothing else, the gzip default writer.
|
||||
func Pack(root string) (archive []byte, digest string, err error) {
|
||||
raw, err := pack(root)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return raw, "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func pack(root string) ([]byte, error) {
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory", root)
|
||||
}
|
||||
|
||||
var paths []string
|
||||
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
// Only files. A symlink or a device in an archive is refused by the host that unpacks
|
||||
// it, so putting one in would build something that cannot be applied.
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+
|
||||
"only those", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// filepath.Walk walks in lexical order; the sort is kept as the build agent keeps it, since an archive
|
||||
// whose digest changes because the traversal did is a silent failure.
|
||||
sort.Strings(paths)
|
||||
|
||||
var out bytes.Buffer
|
||||
zipped := gzip.NewWriter(&out)
|
||||
writer := tar.NewWriter(zipped)
|
||||
for _, path := range paths {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
relative, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode := int64(info.Mode().Perm())
|
||||
if err := writer.WriteHeader(&tar.Header{
|
||||
Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg,
|
||||
// Everything else left at its zero value on purpose: no times, owners or names.
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := writer.Write(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := zipped.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fixture writes three files in two directories (modes 0644 and 0755).
|
||||
func fixture(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for _, f := range []struct {
|
||||
p string
|
||||
mode os.FileMode
|
||||
body string
|
||||
}{{"a/x.txt", 0o644, "x\n"}, {"a/run.sh", 0o755, "#!/bin/sh\necho hi\n"}, {"b/c/y.txt", 0o644, "why\n"}} {
|
||||
path := filepath.Join(root, f.p)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(f.body), f.mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chmod(path, f.mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
// 1479/AC1: the tar stream inside the archive equals the bytes mesh-controller's pack makes for this tree.
|
||||
// testdata/pack-golden.tar is the uncompressed stream, made once by running mesh-controller
|
||||
// internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425) on this fixture (above, modes set by chmod)
|
||||
// and gunzipping its archive. The gzip layer is not compared byte for byte, since compress/gzip's output differs
|
||||
// between Go versions; its header is checked to carry no name, no comment and no time.
|
||||
func TestPackEqualsTheBuildAgentsArchive(t *testing.T) {
|
||||
golden, err := os.ReadFile("testdata/pack-golden.tar")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive, digest, err := Pack(fixture(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(archive))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if zr.Name != "" || zr.Comment != "" || !zr.ModTime.IsZero() {
|
||||
t.Errorf("gzip header = name %q, comment %q, time %v; want none", zr.Name, zr.Comment, zr.ModTime)
|
||||
}
|
||||
tarBytes, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(tarBytes, golden) {
|
||||
t.Errorf("the tar stream is %d bytes, the build agent's %d, and they differ", len(tarBytes), len(golden))
|
||||
}
|
||||
sum := sha256.Sum256(archive)
|
||||
if digest != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Errorf("digest = %s, not the sha256 of the archive", digest)
|
||||
}
|
||||
}
|
||||
|
||||
// 1479/AC2: touching every file's time changes nothing.
|
||||
func TestPackDigestIgnoresFileTimes(t *testing.T) {
|
||||
root := fixture(t)
|
||||
_, first, err := Pack(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
later := time.Now().Add(48 * time.Hour)
|
||||
filepath.Walk(root, func(p string, _ os.FileInfo, _ error) error { return os.Chtimes(p, later, later) })
|
||||
if _, second, err := Pack(root); err != nil || second != first {
|
||||
t.Errorf("digests %s and %s (%v)", first, second, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
//go:build unix
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1479/AC3: anything but a file or a directory is refused, naming it. The rule of this repository is that no
|
||||
// test makes a symlink, so the non-regular entry is a named pipe: Pack refuses a symlink on the same branch
|
||||
// (filepath.Walk reports it by lstat as not regular).
|
||||
func TestPackRefusesAnEntryThatIsNotAFile(t *testing.T) {
|
||||
root := fixture(t)
|
||||
pipe := filepath.Join(root, "a", "pipe")
|
||||
if err := syscall.Mkfifo(pipe, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := Pack(root)
|
||||
if err == nil || !strings.Contains(err.Error(), pipe) {
|
||||
t.Errorf("Pack = %v, want a refusal naming %s", err, pipe)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The pull loop the build modules share** (novox/hq ADR 0525 §4-§6, ADR 0536 §4; issue 1887). A build module on a node
|
||||
// whose build capacity is above zero pulls the asks of its language's seat: from the delivery worker first, then check,
|
||||
// then recheck, holding at most its capacity at once. Each ask runs as a step under the ask's own id, so its end is the
|
||||
// key <node>.<ask id> in the module's steps state, which the asker watches; the ask is acknowledged when that end is
|
||||
// written, kept alive while it runs by an in-progress signal, and redelivered by the stream to another node if this one
|
||||
// stops. No part keeps a timer for an ask: the pull's own wait and the stream's acknowledgement wait are the bus's.
|
||||
|
||||
// Pulled is one ask a source handed over, with what decides its fate on the bus.
|
||||
type Pulled struct {
|
||||
// Kind is the worker it came from, filled by the puller.
|
||||
Kind string
|
||||
Data []byte
|
||||
// Delivered is how many times the bus has handed it over, this time included.
|
||||
Delivered uint64
|
||||
// Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its
|
||||
// acknowledgement wait. Nak hands it back to be delivered again at once (to this node or another). Term drops it.
|
||||
Ack, InProgress, Nak, Term func() error
|
||||
}
|
||||
|
||||
// AskSource is where a puller takes asks from: the seat's workers on the bus (JetStreamAsks) or a test double
|
||||
// (FakeAsks).
|
||||
type AskSource interface {
|
||||
// Next takes one ask of kind: with wait 0 only one already waiting, else waiting for one at most wait. It answers
|
||||
// nil and no error when none came.
|
||||
Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error)
|
||||
}
|
||||
|
||||
// Defaults of a Puller.
|
||||
var (
|
||||
// DefaultPullWait is how long one pull of the delivery worker waits for an ask while every worker is empty: the
|
||||
// pull's own wait (as process.Consume's), after which the check and recheck workers are looked at again.
|
||||
DefaultPullWait = 5 * time.Second
|
||||
// DefaultHeartbeat is how often a running ask tells the bus it is still being worked; under the seat worker's
|
||||
// acknowledgement wait of 60 s (mesh-controller internal/broker HolderConsumerFor).
|
||||
DefaultHeartbeat = 20 * time.Second
|
||||
)
|
||||
|
||||
// Puller pulls a build seat's asks and runs them.
|
||||
type Puller struct {
|
||||
Source AskSource
|
||||
Language string // the seat's language: delivery.AskGo, AskNode or AskImage
|
||||
Node string // the node this holder runs on, the first part of its step keys
|
||||
Put Put // writes the module's steps state
|
||||
// Capacity is how many asks the node runs at once; read before every pull, so a change takes effect at the next
|
||||
// one. 0 pulls nothing.
|
||||
Capacity func() int
|
||||
// Run runs one ask's job; a *Refusal ends its step refused. Its result is the step's result.
|
||||
Run func(ctx context.Context, ask delivery.StepAsk) (any, error)
|
||||
// PullWait and Heartbeat default to DefaultPullWait and DefaultHeartbeat.
|
||||
PullWait time.Duration
|
||||
Heartbeat time.Duration
|
||||
Logf func(format string, args ...any)
|
||||
// Load reads one key of the module's steps state (nil and no error when it is absent). When set, Pull reads the
|
||||
// node's pause key (delivery.PauseKey) before it pulls, and Pause and Resume write it with Put, so a pause outlives
|
||||
// a restart.
|
||||
Load func(key string) ([]byte, error)
|
||||
|
||||
mu sync.Mutex
|
||||
inHand int
|
||||
running map[string]RunningAsk
|
||||
paused bool
|
||||
wake chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (p *Puller) init() {
|
||||
p.once.Do(func() { p.wake = make(chan struct{}, 1) })
|
||||
}
|
||||
|
||||
func (p *Puller) logf(format string, args ...any) {
|
||||
if p.Logf != nil {
|
||||
p.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Wake makes a waiting puller look again: after a capacity change, a resume, or a step's end.
|
||||
func (p *Puller) Wake() {
|
||||
p.init()
|
||||
select {
|
||||
case p.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// Pause stops the puller taking asks; the asks it holds run to their end. by names who paused it. With Load set the
|
||||
// pause is written on the node's pause key first, and an error there leaves the puller as it was.
|
||||
func (p *Puller) Pause(by string) error { return p.setPaused(true, by) }
|
||||
|
||||
// Resume lets the puller take asks again, written as Pause's is.
|
||||
func (p *Puller) Resume(by string) error { return p.setPaused(false, by) }
|
||||
|
||||
func (p *Puller) setPaused(paused bool, by string) error {
|
||||
if p.Load != nil {
|
||||
key, err := delivery.PauseKey(p.Node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodePause(delivery.Pause{Paused: paused, By: by, At: now()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := p.Put(key, json.RawMessage(raw)); err != nil {
|
||||
return fmt.Errorf("buildstep: writing the pause of %s: %w", p.Node, err)
|
||||
}
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.paused = paused
|
||||
p.mu.Unlock()
|
||||
p.Wake()
|
||||
return nil
|
||||
}
|
||||
|
||||
// restore reads the node's pause key, when Load is set.
|
||||
func (p *Puller) restore() error {
|
||||
if p.Load == nil {
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.PauseKey(p.Node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := p.Load(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("buildstep: reading the pause of %s: %w", p.Node, err)
|
||||
}
|
||||
if raw == nil {
|
||||
return nil
|
||||
}
|
||||
v, err := delivery.DecodePause(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("buildstep: the pause of %s: %w", p.Node, err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.paused = v.Paused
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Paused says whether the puller takes no asks.
|
||||
func (p *Puller) Paused() bool {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.paused
|
||||
}
|
||||
|
||||
// InHand is how many asks run now.
|
||||
func (p *Puller) InHand() int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.inHand
|
||||
}
|
||||
|
||||
// RunningAsk is an ask a puller runs now, as its seat's current verb says it.
|
||||
type RunningAsk struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Started time.Time `json:"started"`
|
||||
}
|
||||
|
||||
// Running are the asks the puller runs now, oldest first.
|
||||
func (p *Puller) Running() []RunningAsk {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make([]RunningAsk, 0, len(p.running))
|
||||
for _, r := range p.running {
|
||||
out = append(out, r)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Started.Before(out[j].Started) })
|
||||
return out
|
||||
}
|
||||
|
||||
// room says whether the puller may take one more ask now.
|
||||
func (p *Puller) room() bool {
|
||||
capacity := 0
|
||||
if p.Capacity != nil {
|
||||
capacity = p.Capacity()
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return !p.paused && p.inHand < capacity
|
||||
}
|
||||
|
||||
// Pull takes and runs asks until ctx ends. An ask still running then is neither ended nor acknowledged: the stream
|
||||
// delivers it again, to another node or to this one restarted.
|
||||
func (p *Puller) Pull(ctx context.Context) error {
|
||||
p.init()
|
||||
if p.Source == nil || p.Put == nil || p.Run == nil {
|
||||
return errors.New("buildstep: a puller needs its source, its steps' put and its run")
|
||||
}
|
||||
if err := p.restore(); err != nil {
|
||||
return err // pulling on a pause that cannot be read could build on a node paused on purpose
|
||||
}
|
||||
wait := p.PullWait
|
||||
if wait <= 0 {
|
||||
wait = DefaultPullWait
|
||||
}
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !p.room() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-p.wake:
|
||||
}
|
||||
continue
|
||||
}
|
||||
msg, err := p.next(ctx, wait)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
p.logf("the %s asks cannot be pulled: %v", p.Language, err)
|
||||
select { // the pull's own wait before the next try, as an empty pull's
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-p.wake:
|
||||
case <-time.After(wait):
|
||||
}
|
||||
continue
|
||||
}
|
||||
if msg == nil {
|
||||
continue
|
||||
}
|
||||
p.take(ctx, msg)
|
||||
}
|
||||
}
|
||||
|
||||
// next is one ask from the workers in their order: one already waiting on delivery, check or recheck, and when none
|
||||
// waits, a pull of the delivery worker that waits for one.
|
||||
func (p *Puller) next(ctx context.Context, wait time.Duration) (*Pulled, error) {
|
||||
for _, kind := range delivery.AskKinds {
|
||||
msg, err := p.Source.Next(ctx, kind, 0)
|
||||
if err != nil || msg != nil {
|
||||
return withKind(msg, kind), err
|
||||
}
|
||||
}
|
||||
msg, err := p.Source.Next(ctx, delivery.AskDelivery, wait)
|
||||
return withKind(msg, delivery.AskDelivery), err
|
||||
}
|
||||
|
||||
func withKind(msg *Pulled, kind string) *Pulled {
|
||||
if msg != nil {
|
||||
msg.Kind = kind
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
// take starts one ask: it decodes it, refuses one that is not of this seat, and runs it as a step under its own id.
|
||||
func (p *Puller) take(ctx context.Context, msg *Pulled) {
|
||||
kind := msg.Kind
|
||||
ask, err := delivery.DecodeStepAsk(msg.Data)
|
||||
if err == nil && (ask.Language != p.Language || (kind != "" && ask.Kind != kind)) {
|
||||
err = fmt.Errorf("the ask %s is a %s ask of %s, pulled from the %s %s worker", ask.ID, ask.Kind, ask.Language, p.Language, kind)
|
||||
}
|
||||
if err != nil {
|
||||
// An ask that is not one runs nowhere: delivering it again would only fail again.
|
||||
p.logf("an ask dropped: %v", err)
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
step, key, err := begin(p.Put, p.Node, ask.Kind+" ask", ask.ID)
|
||||
if err != nil {
|
||||
p.logf("the ask %s cannot start on %s: %v", ask.ID, p.Node, err)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if msg.Delivered > delivery.AskRuns {
|
||||
// Run AskRuns times and never ended: each run took its node down or died with it. It is run no more, and said
|
||||
// (issue 1907): the asker's key fails with why.
|
||||
why := fmt.Errorf("the ask was taken %d times and no run of it ended: each took its build node down or died with it, so it is run no more (poison)", msg.Delivered-1)
|
||||
p.logf("the ask %s is ended on %s: %v", ask.ID, p.Node, why)
|
||||
_ = finish(p.Put, key, step, nil, why, p.Logf) // run no more either way: the asker's bound says it then
|
||||
_ = msg.Term()
|
||||
return
|
||||
}
|
||||
module := ""
|
||||
if ask.Build != nil {
|
||||
module = ask.Build.Module
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.inHand++
|
||||
if p.running == nil {
|
||||
p.running = map[string]RunningAsk{}
|
||||
}
|
||||
p.running[ask.ID] = RunningAsk{ID: ask.ID, Kind: ask.Kind, Module: module, Started: step.Started}
|
||||
p.mu.Unlock()
|
||||
go p.run(ctx, msg, ask, key, step)
|
||||
}
|
||||
|
||||
func (p *Puller) run(ctx context.Context, msg *Pulled, ask delivery.StepAsk, key string, step delivery.Step) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
p.inHand--
|
||||
delete(p.running, ask.ID)
|
||||
p.mu.Unlock()
|
||||
p.Wake()
|
||||
}()
|
||||
beat := p.Heartbeat
|
||||
if beat <= 0 {
|
||||
beat = DefaultHeartbeat
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(beat)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-t.C:
|
||||
if msg.InProgress != nil {
|
||||
_ = msg.InProgress()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
result, err := protect(ctx, func(ctx context.Context) (any, error) { return p.Run(ctx, ask) })
|
||||
close(done)
|
||||
if ctx.Err() != nil {
|
||||
// The process stops: the ask is neither ended nor acknowledged here, so the stream gives it to a node that runs.
|
||||
return
|
||||
}
|
||||
if ferr := finish(p.Put, key, step, result, err, p.Logf); ferr != nil {
|
||||
// No end was written: the ask is not acknowledged as done, so the stream gives it again and a run that can
|
||||
// write its end (or the poison rule, issue 1907) ends it where its asker sees it (issue 1930).
|
||||
p.logf("the ask %s ran on %s and no end of it could be written, so it is given back: %v", ask.ID, p.Node, ferr)
|
||||
_ = msg.Nak()
|
||||
return
|
||||
}
|
||||
if aerr := msg.Ack(); aerr != nil {
|
||||
p.logf("the ask %s ended on %s and its acknowledgement failed: %v", ask.ID, p.Node, aerr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// JetStreamAsks is a build seat's workers on the bus: the shared pull consumer of each kind on the seat's stream
|
||||
// (delivery.AskStream, AskWorker), which the broker makes when the seat is declared; a holder only binds them.
|
||||
type JetStreamAsks struct {
|
||||
JS jetstream.JetStream
|
||||
Language string
|
||||
|
||||
mu sync.Mutex
|
||||
cons map[string]jetstream.Consumer
|
||||
}
|
||||
|
||||
func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Consumer, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if c, ok := s.cons[kind]; ok {
|
||||
return c, nil
|
||||
}
|
||||
stream, err := delivery.AskStream(s.Language)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
worker, err := delivery.AskWorker(s.Language, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := s.JS.Consumer(ctx, stream, worker)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("binding the %s worker %s on %s: %w", kind, worker, stream, err)
|
||||
}
|
||||
if s.cons == nil {
|
||||
s.cons = map[string]jetstream.Consumer{}
|
||||
}
|
||||
s.cons[kind] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Next is AskSource's: a fetch of one with no wait when wait is 0, else one that waits at most wait.
|
||||
func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
c, err := s.consumer(ctx, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var batch jetstream.MessageBatch
|
||||
if wait <= 0 {
|
||||
batch, err = c.FetchNoWait(1)
|
||||
} else {
|
||||
batch, err = c.Fetch(1, jetstream.FetchMaxWait(wait))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case msg, ok := <-batch.Messages():
|
||||
if !ok {
|
||||
return nil, batch.Error()
|
||||
}
|
||||
var delivered uint64 = 1
|
||||
if meta, err := msg.Metadata(); err == nil && meta != nil {
|
||||
delivered = meta.NumDelivered
|
||||
}
|
||||
return &Pulled{Data: append([]byte(nil), msg.Data()...), Delivered: delivered,
|
||||
Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// DeclareAskWorkers makes language's seat stream and its three workers on a bus with no broker: a test's (the build
|
||||
// modules' integration tests start one). On the mesh the broker declares them from the seat, as it declares every
|
||||
// seat's stream (mesh-controller internal/broker SeatStreams); a holder never does.
|
||||
func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language string, ackWait time.Duration) error {
|
||||
stream, err := delivery.AskStream(language)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seat, _ := delivery.BuildSeat(language)
|
||||
if _, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.seat." + seat + ".accept.>"},
|
||||
Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
return fmt.Errorf("the stream %s: %w", stream, err)
|
||||
}
|
||||
for _, kind := range delivery.AskKinds {
|
||||
worker, _ := delivery.AskWorker(language, kind)
|
||||
subject, _ := delivery.AskSubject(language, kind)
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubject: subject,
|
||||
AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil {
|
||||
return fmt.Errorf("the worker %s: %w", worker, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// FakeAsks is a test double of a seat's workers for the build modules' unit tests: Publish puts an ask on a kind's
|
||||
// worker, Next hands them out in their order, and the fates the puller gives them are counted. Safe for concurrent use.
|
||||
type FakeAsks struct {
|
||||
mu sync.Mutex
|
||||
waiting map[string][][]byte
|
||||
added chan struct{}
|
||||
// Acked, Termed and Naked are the asks' ids or data by fate, InProgress how many heartbeats were given.
|
||||
Acked, Termed, Naked []string
|
||||
InProgress int
|
||||
// Delivered, when set, is the delivery count every ask handed over says (1 when 0): a test of a poisoned ask sets it.
|
||||
Delivered uint64
|
||||
}
|
||||
|
||||
// NewFakeAsks is an empty double.
|
||||
func NewFakeAsks() *FakeAsks {
|
||||
return &FakeAsks{waiting: map[string][][]byte{}, added: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// Publish puts an encoded ask on kind's worker.
|
||||
func (f *FakeAsks) Publish(kind string, data []byte) {
|
||||
f.mu.Lock()
|
||||
f.waiting[kind] = append(f.waiting[kind], data)
|
||||
f.mu.Unlock()
|
||||
select {
|
||||
case f.added <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// PublishAsk encodes a and puts it on its kind's worker.
|
||||
func (f *FakeAsks) PublishAsk(a delivery.StepAsk) error {
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.Publish(a.Kind, raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Waiting is how many asks wait on kind's worker.
|
||||
func (f *FakeAsks) Waiting(kind string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.waiting[kind])
|
||||
}
|
||||
|
||||
// Fates is a copy of the acked, termed and naked lists.
|
||||
func (f *FakeAsks) Fates() (acked, termed, naked []string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]string(nil), f.Acked...), append([]string(nil), f.Termed...), append([]string(nil), f.Naked...)
|
||||
}
|
||||
|
||||
// Next is AskSource's.
|
||||
func (f *FakeAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
var deadline <-chan time.Time
|
||||
if wait > 0 {
|
||||
t := time.NewTimer(wait)
|
||||
defer t.Stop()
|
||||
deadline = t.C
|
||||
}
|
||||
for {
|
||||
f.mu.Lock()
|
||||
if q := f.waiting[kind]; len(q) > 0 {
|
||||
data := q[0]
|
||||
f.waiting[kind] = q[1:]
|
||||
f.mu.Unlock()
|
||||
return f.pulled(data), nil
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if wait <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-deadline:
|
||||
return nil, nil
|
||||
case <-f.added:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FakeAsks) pulled(data []byte) *Pulled {
|
||||
id := string(data)
|
||||
if a, err := delivery.DecodeStepAsk(data); err == nil {
|
||||
id = a.ID
|
||||
}
|
||||
fate := func(list *[]string) func() error {
|
||||
return func() error {
|
||||
f.mu.Lock()
|
||||
*list = append(*list, id)
|
||||
f.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
delivered := f.Delivered
|
||||
if delivered == 0 {
|
||||
delivered = 1
|
||||
}
|
||||
return &Pulled{Data: data, Delivered: delivered, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed),
|
||||
InProgress: func() error { f.mu.Lock(); f.InProgress++; f.mu.Unlock(); return nil }}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
)
|
||||
|
||||
func askBus(t *testing.T, ackWait time.Duration) jetstream.JetStream {
|
||||
t.Helper()
|
||||
nc, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := DeclareAskWorkers(context.Background(), js, delivery.AskGo, ackWait); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
|
||||
func publishAsk(t *testing.T, js jetstream.JetStream, a delivery.StepAsk) {
|
||||
t.Helper()
|
||||
raw, err := delivery.EncodeStepAsk(a)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
subject, _ := delivery.AskSubject(a.Language, a.Kind)
|
||||
if _, err := js.Publish(context.Background(), subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2a, 1887/AC-order, 1887/AC-pull on the real server: a check ask published before a delivery ask runs after it;
|
||||
// with capacity 2 and three asks two run, the third waits in the stream and starts when one ends.
|
||||
func TestPullOrderAndCapacityOnTheBus(t *testing.T) {
|
||||
js := askBus(t, 30*time.Second)
|
||||
check, first, second := testAsk(delivery.AskCheck), testAsk(delivery.AskDelivery), testAsk(delivery.AskDelivery)
|
||||
publishAsk(t, js, check)
|
||||
publishAsk(t, js, first)
|
||||
publishAsk(t, js, second)
|
||||
release := make(chan struct{})
|
||||
var mu sync.Mutex
|
||||
var order []string
|
||||
st := newSteps()
|
||||
p := &Puller{Source: &JetStreamAsks{JS: js, Language: delivery.AskGo}, Language: delivery.AskGo, Node: "shanks", Put: st.put,
|
||||
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
mu.Lock()
|
||||
order = append(order, a.ID)
|
||||
mu.Unlock()
|
||||
<-release
|
||||
return nil, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "two in hand", func() bool { return p.InHand() == 2 })
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
mu.Lock()
|
||||
if len(order) != 2 || order[0] != first.ID || order[1] != second.ID {
|
||||
t.Fatalf("ran %v, want the two delivery asks %s %s first", order, first.ID, second.ID)
|
||||
}
|
||||
mu.Unlock()
|
||||
release <- struct{}{}
|
||||
waitFor(t, "the check taken", func() bool { mu.Lock(); defer mu.Unlock(); return len(order) == 3 })
|
||||
close(release)
|
||||
waitFor(t, "every step ended", func() bool {
|
||||
for _, a := range []delivery.StepAsk{check, first, second} {
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
if s, ok := st.get(key); !ok || s.State != delivery.StepPassed {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
info, err := js.Stream(ctx, "SEAT_NODE_BUILD_GO")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, "the stream empty (every ask acknowledged)", func() bool {
|
||||
si, err := info.Info(ctx)
|
||||
return err == nil && si.State.Msgs == 0
|
||||
})
|
||||
}
|
||||
|
||||
// 1887/A2c, ADR 0525 §6: a killed puller's ask is redelivered by the stream's acknowledgement wait to the other node,
|
||||
// whose step key names the reference.
|
||||
func TestPullRedeliversAKilledPullersAsk(t *testing.T) {
|
||||
js := askBus(t, 2*time.Second)
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
publishAsk(t, js, a)
|
||||
st := newSteps()
|
||||
taken := make(chan struct{})
|
||||
dying := &Puller{Source: &JetStreamAsks{JS: js, Language: delivery.AskGo}, Language: delivery.AskGo, Node: "ace", Put: st.put,
|
||||
Capacity: func() int { return 1 }, PullWait: 200 * time.Millisecond, Heartbeat: time.Hour,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
close(taken)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}}
|
||||
kill, killed := context.WithCancel(context.Background())
|
||||
go dying.Pull(kill)
|
||||
<-taken
|
||||
killed()
|
||||
other := &Puller{Source: &JetStreamAsks{JS: js, Language: delivery.AskGo}, Language: delivery.AskGo, Node: "shanks", Put: st.put,
|
||||
Capacity: func() int { return 1 }, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
return delivery.BuildAnswer{Artifacts: []delivery.AskBuilt{{Artifact: "bundle", Digest: "sha256:00", Reference: "artifact-store://x"}}}, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go other.Pull(ctx)
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
if s, ok := st.get(key); ok && s.State == delivery.StepPassed {
|
||||
if want := `artifact-store://x`; !contains(string(s.Result), want) {
|
||||
t.Fatalf("the step's result %s names no %s", s.Result, want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the killed puller's ask was not taken by shanks within 10 s")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, sub string) bool { return strings.Contains(s, sub) }
|
||||
@@ -0,0 +1,44 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// SeatAsks is a build seat's workers as a tools bundle reaches them: through its runtime's seat pull and settle
|
||||
// (stdio.SeatPull, SeatSettle; issue 1887 A11), one message per pull, only from the workers the module's membership
|
||||
// lists.
|
||||
type SeatAsks struct {
|
||||
Language string
|
||||
// Pull and Settle are stdio.SeatPull and stdio.SeatSettle when nil; a test gives its own.
|
||||
Pull func(worker string, wait time.Duration) (*stdio.Pulled, error)
|
||||
Settle func(id, fate string) error
|
||||
}
|
||||
|
||||
// Next is AskSource's.
|
||||
func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) {
|
||||
pull, settle := s.Pull, s.Settle
|
||||
if pull == nil {
|
||||
pull = stdio.SeatPull
|
||||
}
|
||||
if settle == nil {
|
||||
settle = stdio.SeatSettle
|
||||
}
|
||||
worker, err := delivery.AskWorker(s.Language, kind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := pull(worker, wait)
|
||||
if err != nil || p == nil {
|
||||
return nil, err
|
||||
}
|
||||
fate := func(f string) func() error { return func() error { return settle(p.ID, f) } }
|
||||
return &Pulled{Data: p.Body, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress),
|
||||
Nak: fate(stdio.SettleNak), Term: fate(stdio.SettleTerm)}, nil
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1887/A11: a bundle's source pulls the kind's worker by its seat's name and settles each ask by the runtime's id.
|
||||
func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) {
|
||||
var pulled []string
|
||||
var settled []string
|
||||
src := SeatAsks{Language: delivery.AskGo,
|
||||
Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) {
|
||||
pulled = append(pulled, worker)
|
||||
if worker != "SEAT_NODE_BUILD_GO_CHECK_worker" {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.Pulled{ID: "p9", Worker: worker, Body: []byte(`{"v":1}`), Delivered: 2}, nil
|
||||
},
|
||||
Settle: func(id, fate string) error { settled = append(settled, id+" "+fate); return nil }}
|
||||
if p, err := src.Next(context.Background(), delivery.AskDelivery, 0); err != nil || p != nil {
|
||||
t.Fatalf("delivery = %+v, %v", p, err)
|
||||
}
|
||||
p, err := src.Next(context.Background(), delivery.AskCheck, time.Second)
|
||||
if err != nil || p == nil || string(p.Data) != `{"v":1}` || p.Delivered != 2 {
|
||||
t.Fatalf("check = %+v, %v", p, err)
|
||||
}
|
||||
_ = p.InProgress()
|
||||
_ = p.Ack()
|
||||
if len(pulled) != 2 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" {
|
||||
t.Errorf("pulled %v", pulled)
|
||||
}
|
||||
if len(settled) != 2 || settled[0] != "p9 progress" || settled[1] != "p9 ack" {
|
||||
t.Errorf("settled %v", settled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// steps records a steps state's puts: the last value of each key.
|
||||
type steps struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]delivery.Step
|
||||
}
|
||||
|
||||
func newSteps() *steps { return &steps{keys: map[string]delivery.Step{}} }
|
||||
|
||||
func (s *steps) put(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
st, err := delivery.DecodeStep(raw)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.keys[key] = st
|
||||
s.mu.Unlock()
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (s *steps) get(key string) (delivery.Step, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
st, ok := s.keys[key]
|
||||
return st, ok
|
||||
}
|
||||
|
||||
func testAsk(kind string) delivery.StepAsk {
|
||||
id := NewID()
|
||||
a := delivery.StepAsk{ID: id, Kind: kind, Language: delivery.AskGo, Asker: "mesh-delivery-builder", At: time.Now()}
|
||||
fetch := map[string]any{"repository": "novox/x", "commit": "0123456789abcdef0123456789abcdef01234567"}
|
||||
if kind == delivery.AskDelivery {
|
||||
b := buildJobFixture
|
||||
b.Fetch = fetch
|
||||
a.Build = &b
|
||||
} else {
|
||||
a.Check = &delivery.CheckJob{Fetch: fetch, Levels: []string{delivery.LevelUnit}}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
var buildJobFixture = delivery.BuildJob{Module: "m", Path: ".", Artifacts: []delivery.AskArtifact{{Artifact: "bundle"}}}
|
||||
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for !cond() {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("waited 2 s for %s", what)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2a: given a check ask published before a delivery ask and one slot, the delivery ask runs first.
|
||||
func TestPullTakesDeliveryBeforeCheck(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
check, recheck, deliver := testAsk(delivery.AskCheck), testAsk(delivery.AskRecheck), testAsk(delivery.AskDelivery)
|
||||
for _, a := range []delivery.StepAsk{recheck, check, deliver} {
|
||||
if err := src.PublishAsk(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var order []string
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
mu.Lock()
|
||||
order = append(order, a.Kind)
|
||||
mu.Unlock()
|
||||
return map[string]string{"ran": a.ID}, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "three asks acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 })
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if want := []string{delivery.AskDelivery, delivery.AskCheck, delivery.AskRecheck}; len(order) != 3 || order[0] != want[0] || order[1] != want[1] || order[2] != want[2] {
|
||||
t.Fatalf("order = %v, want %v", order, want)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("shanks", deliver.ID)
|
||||
s, ok := st.get(key)
|
||||
if !ok || s.State != delivery.StepPassed || string(s.Result) != `{"ran":"`+deliver.ID+`"}` {
|
||||
t.Fatalf("the step %s = %+v (%v)", key, s, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A2a, 1887/AC-pull: given capacity 2 and three asks waiting, two run and the third waits, and it starts when
|
||||
// one ends, with no timer.
|
||||
func TestPullHoldsAtMostItsCapacity(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
for i := 0; i < 3; i++ {
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
}
|
||||
release := make(chan struct{})
|
||||
var mu sync.Mutex
|
||||
started := 0
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 2 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
mu.Lock()
|
||||
started++
|
||||
mu.Unlock()
|
||||
<-release
|
||||
return nil, nil
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "two in hand", func() bool { return p.InHand() == 2 })
|
||||
time.Sleep(30 * time.Millisecond) // three pull waits: the third must still wait
|
||||
if p.InHand() != 2 || src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatalf("in hand %d, waiting %d; want 2 and 1", p.InHand(), src.Waiting(delivery.AskDelivery))
|
||||
}
|
||||
release <- struct{}{}
|
||||
waitFor(t, "the third taken", func() bool { return src.Waiting(delivery.AskDelivery) == 0 })
|
||||
// Taken from the source comes a moment before its run starts: wait for the start, not only the take.
|
||||
waitFor(t, "the third started", func() bool { mu.Lock(); defer mu.Unlock(); return started == 3 })
|
||||
close(release)
|
||||
waitFor(t, "all acked", func() bool { a, _, _ := src.Fates(); return len(a) == 3 })
|
||||
}
|
||||
|
||||
// An ask that is not one, or not this seat's, is dropped and runs nowhere.
|
||||
func TestPullDropsWhatIsNoAsk(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Publish(delivery.AskDelivery, []byte(`{"v":1}`))
|
||||
other := testAsk(delivery.AskDelivery)
|
||||
other.Language = delivery.AskNode
|
||||
_ = src.PublishAsk(other)
|
||||
ran := false
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { ran = true; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "two termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 2 })
|
||||
if ran {
|
||||
t.Fatal("a wrong ask ran")
|
||||
}
|
||||
}
|
||||
|
||||
// load reads the recorded steps state's raw values: the pause key is not a step, so it is kept apart.
|
||||
type kv struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
}
|
||||
|
||||
func (k *kv) put(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.vals == nil {
|
||||
k.vals = map[string][]byte{}
|
||||
}
|
||||
k.vals[key] = append([]byte(nil), raw...)
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (k *kv) load(key string) ([]byte, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return k.vals[key], nil
|
||||
}
|
||||
|
||||
// 1887/A2b, 1887/AC-zero: at capacity 0 the puller takes nothing; a capacity raised takes effect at the next pull.
|
||||
func TestPullAtCapacityZeroTakesNothing(t *testing.T) {
|
||||
src, st := NewFakeAsks(), &kv{}
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
var mu sync.Mutex
|
||||
capacity := 0
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "novox", Put: st.put, PullWait: 10 * time.Millisecond,
|
||||
Capacity: func() int { mu.Lock(); defer mu.Unlock(); return capacity },
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatal("a node at capacity 0 took an ask")
|
||||
}
|
||||
mu.Lock()
|
||||
capacity = 1
|
||||
mu.Unlock()
|
||||
p.Wake()
|
||||
waitFor(t, "the ask taken at capacity 1", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
|
||||
}
|
||||
|
||||
// 1887/A2b: a paused puller takes nothing, its pause is kept on the pause key, and a puller started again on that state
|
||||
// stays paused until resumed.
|
||||
func TestPullPauseOutlivesARestart(t *testing.T) {
|
||||
src, st := NewFakeAsks(), &kv{}
|
||||
run := func(ctx context.Context, a delivery.StepAsk) (any, error) { return nil, nil }
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Load: st.load, PullWait: 10 * time.Millisecond,
|
||||
Capacity: func() int { return 2 }, Run: run}
|
||||
if err := p.Pause("the operator"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.PauseKey("shanks")
|
||||
raw, _ := st.load(key)
|
||||
if v, err := delivery.DecodePause(raw); err != nil || !v.Paused || v.By != "the operator" {
|
||||
t.Fatalf("the pause key holds %s (%v)", raw, err)
|
||||
}
|
||||
// The restart: a new puller on the same state.
|
||||
again := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Load: st.load, PullWait: 10 * time.Millisecond,
|
||||
Capacity: func() int { return 2 }, Run: run}
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go again.Pull(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if !again.Paused() || src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatalf("restarted: paused %v, waiting %d; want paused and 1", again.Paused(), src.Waiting(delivery.AskDelivery))
|
||||
}
|
||||
if err := again.Resume("the operator"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, "the ask taken after the resume", func() bool { a, _, _ := src.Fates(); return len(a) == 1 })
|
||||
}
|
||||
|
||||
// 1887/A2c: a long step tells the bus it is still being worked; a holder that stops mid-step neither ends nor
|
||||
// acknowledges its ask, so the stream gives it to another node.
|
||||
func TestPullHeartbeatAndStop(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
running := make(chan struct{})
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Heartbeat: 5 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
close(running)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
stopped := make(chan struct{})
|
||||
go func() { p.Pull(ctx); close(stopped) }()
|
||||
<-running
|
||||
waitFor(t, "two heartbeats", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return src.InProgress >= 2 })
|
||||
cancel()
|
||||
<-stopped
|
||||
waitFor(t, "the step's goroutine gone", func() bool { return p.InHand() == 0 })
|
||||
acked, termed, naked := src.Fates()
|
||||
if len(acked)+len(termed)+len(naked) != 0 {
|
||||
t.Fatalf("a stopped holder gave its ask a fate: acked %v termed %v naked %v", acked, termed, naked)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
if s, ok := st.get(key); !ok || s.State != delivery.StepRunning {
|
||||
t.Fatalf("the step %s = %+v, want left running", key, s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1907: an ask taken past AskRuns (each run took its node down) is run no more: its step ends failed saying why, so the
|
||||
// asker's key fails, and it is ended on the stream, not offered to another node.
|
||||
func TestAPoisonedAskIsSaidAndRunNoMore(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
src.Delivered = delivery.AskRuns + 1
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
ran := false
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { ran = true; return nil, nil }}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask termed", func() bool { _, tm, _ := src.Fates(); return len(tm) == 1 })
|
||||
key, _ := delivery.AskStepKey("shanks", a.ID)
|
||||
s, ok := st.get(key)
|
||||
if ran || !ok || s.State != delivery.StepFailed || !strings.Contains(s.Reason, "taken 3 times") || !strings.Contains(s.Reason, "poison") {
|
||||
t.Fatalf("ran %v, step %+v", ran, s)
|
||||
}
|
||||
if acked, _, naked := src.Fates(); len(acked)+len(naked) != 0 {
|
||||
t.Errorf("acked %v naked %v, want it termed alone", acked, naked)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package buildstep
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the
|
||||
// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470
|
||||
// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text.
|
||||
type Refusal struct {
|
||||
// Text is the reason, in words the operator reads.
|
||||
Text string
|
||||
// Want is the version the recipe asks for and Have the one found; either may be empty.
|
||||
Want, Have string
|
||||
}
|
||||
|
||||
func (r *Refusal) Error() string { return r.Text }
|
||||
|
||||
// Refuse is a Refusal with its text formatted.
|
||||
func Refuse(format string, args ...any) *Refusal { return &Refusal{Text: fmt.Sprintf(format, args...)} }
|
||||
@@ -0,0 +1,115 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
requestCommitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
requestRepositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
)
|
||||
|
||||
// FetchRequest is what a build module's fetch tool is asked: the repository and the full commit, the pull request's
|
||||
// ref when only its head carries the commit, the trunk commit to merge onto for a merge preview, and, for a build of a
|
||||
// module whose every artifact names its paths, that module's directory and its named paths (novox/hq ADR 0499 clause 4).
|
||||
//
|
||||
// **The paths come from the builder, not from a manifest** (ADR 0495 clause 1; the lead's decision L8 of R4e): the
|
||||
// builder fills Dir and Paths from the indexer's Module value (Path, and KeptInputs2.Paths' keys), and the build module
|
||||
// gives the build and test steps a workspace holding those paths, the manifest and its lock files. A request without
|
||||
// them builds from the module's directory, as before.
|
||||
type FetchRequest struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Dir string `json:"dir,omitempty"`
|
||||
Paths []string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
// FetchRequestProperties are the JSON schema properties of FetchRequest's dir and paths, for a fetch tool's Input.
|
||||
func FetchRequestProperties() map[string]any {
|
||||
return map[string]any{
|
||||
"dir": map[string]any{"type": "string", "description": "the module's directory in the repository (\".\" for its root), with paths"},
|
||||
"paths": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "the paths the module names, relative to dir, from the indexer's kept inputs: the build sees only these, the manifest and its lock files"},
|
||||
}
|
||||
}
|
||||
|
||||
// Args is r as a tool call's arguments.
|
||||
func (r FetchRequest) Args() map[string]any {
|
||||
args := map[string]any{"repository": r.Repository, "commit": r.Commit}
|
||||
for k, v := range map[string]string{"ref": r.Ref, "base": r.Base, "dir": r.Dir} {
|
||||
if v != "" {
|
||||
args[k] = v
|
||||
}
|
||||
}
|
||||
if len(r.Paths) > 0 {
|
||||
args["paths"] = append([]string(nil), r.Paths...)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// ParseFetchRequest reads a fetch tool's arguments; arguments it does not know (a build module's own) are left to the
|
||||
// caller. It refuses a repository that is not <owner>/<name>, a commit or a base that is not 40 hex digits, paths with no
|
||||
// module directory or a directory with no path, a directory leaving the repository and a path leaving the directory,
|
||||
// naming each.
|
||||
func ParseFetchRequest(args map[string]any) (FetchRequest, error) {
|
||||
var r FetchRequest
|
||||
r.Repository, _ = args["repository"].(string)
|
||||
r.Commit, _ = args["commit"].(string)
|
||||
r.Ref, _ = args["ref"].(string)
|
||||
r.Base, _ = args["base"].(string)
|
||||
r.Dir, _ = args["dir"].(string)
|
||||
switch v := args["paths"].(type) {
|
||||
case nil:
|
||||
case []string:
|
||||
r.Paths = append([]string(nil), v...)
|
||||
case []any:
|
||||
for _, p := range v {
|
||||
s, ok := p.(string)
|
||||
if !ok {
|
||||
return FetchRequest{}, fmt.Errorf("paths is a list of strings")
|
||||
}
|
||||
r.Paths = append(r.Paths, s)
|
||||
}
|
||||
default:
|
||||
return FetchRequest{}, fmt.Errorf("paths is a list of strings")
|
||||
}
|
||||
return r, r.Check()
|
||||
}
|
||||
|
||||
// Check is what ParseFetchRequest refuses.
|
||||
func (r FetchRequest) Check() error {
|
||||
if !requestRepositoryOK.MatchString(r.Repository) || strings.Contains(r.Repository, "..") {
|
||||
return fmt.Errorf("%q is not a repository, <owner>/<name>", r.Repository)
|
||||
}
|
||||
if !requestCommitOK.MatchString(r.Commit) {
|
||||
return fmt.Errorf("%q is not a full commit (40 hex digits)", r.Commit)
|
||||
}
|
||||
if r.Base != "" && !requestCommitOK.MatchString(r.Base) {
|
||||
return fmt.Errorf("%q is not a trunk commit (40 hex digits)", r.Base)
|
||||
}
|
||||
switch {
|
||||
case len(r.Paths) > 0 && r.Dir == "":
|
||||
return fmt.Errorf("the request names paths and names no module directory (dir) they are relative to")
|
||||
case len(r.Paths) == 0 && r.Dir != "":
|
||||
return fmt.Errorf("the request names a module directory and no path")
|
||||
}
|
||||
if r.Dir != "" && (outside(r.Dir) || strings.HasPrefix(r.Dir, "/")) {
|
||||
return fmt.Errorf("the module directory %q leaves the repository", r.Dir)
|
||||
}
|
||||
for _, p := range r.Paths {
|
||||
if p == "" || strings.HasPrefix(p, "/") || outside(p) || path.Clean(p) == "." {
|
||||
return fmt.Errorf("the path %q leaves the module's directory or is not a path in it", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func outside(p string) bool {
|
||||
c := path.Clean(p)
|
||||
return c == ".." || strings.HasPrefix(c, "../")
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A build's fetch is handed the paths of a module that names them (novox/hq ADR 0499 clause 4): the builder fills them
|
||||
// from the indexer's kept inputs and the build module reads them from the call, never from the manifest (ADR 0495
|
||||
// clause 1). The request survives the trip through a tool call's arguments.
|
||||
func TestAFetchRequestCarriesTheNamedPathsThroughTheToolCall(t *testing.T) {
|
||||
req := FetchRequest{Repository: "novox/mesh-sdk", Commit: strings.Repeat("a", 40), Ref: "refs/pull/7/head",
|
||||
Dir: ".", Paths: []string{"package.json", "src"}}
|
||||
args := req.Args()
|
||||
// A tool call's arguments arrive as JSON: lists of any.
|
||||
args["paths"] = []any{"package.json", "src"}
|
||||
got, err := ParseFetchRequest(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, req) {
|
||||
t.Fatalf("got %+v, want %+v", got, req)
|
||||
}
|
||||
plain, err := ParseFetchRequest(map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40),
|
||||
"base": strings.Repeat("c", 40), "beside": []any{}})
|
||||
if err != nil || plain.Paths != nil || plain.Base != strings.Repeat("c", 40) {
|
||||
t.Fatalf("a request without paths: %+v, %v", plain, err)
|
||||
}
|
||||
|
||||
ok := map[string]any{"repository": "novox/x", "commit": strings.Repeat("b", 40)}
|
||||
for _, c := range []struct {
|
||||
set map[string]any
|
||||
says string
|
||||
}{
|
||||
{map[string]any{"repository": "novox"}, "is not a repository"},
|
||||
{map[string]any{"repository": "novox/../x"}, "is not a repository"},
|
||||
{map[string]any{"commit": "abc"}, "is not a full commit"},
|
||||
{map[string]any{"base": "abc"}, "is not a trunk commit"},
|
||||
{map[string]any{"paths": []any{"src"}}, "names no module directory"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{"../b"}}, "leaves the module's directory"},
|
||||
{map[string]any{"dir": "../a", "paths": []any{"src"}}, "leaves the repository"},
|
||||
{map[string]any{"dir": "modules/a", "paths": []any{3}}, "paths is a list of strings"},
|
||||
{map[string]any{"dir": "modules/a"}, "names a module directory and no path"},
|
||||
} {
|
||||
args := map[string]any{}
|
||||
for k, v := range ok {
|
||||
args[k] = v
|
||||
}
|
||||
for k, v := range c.set {
|
||||
args[k] = v
|
||||
}
|
||||
if _, err := ParseFetchRequest(args); err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%v: %v, wanted a refusal saying %q", c.set, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// SeatTools are a build seat's verbs on one node, as its holder serves them (novox/hq ADR 0525 §10; mesh-sdk catalogue
|
||||
// buildSeatVerbs): `<seat>.current` (what the node runs, its capacity, whether it is paused), `<seat>.pause` and
|
||||
// `<seat>.resume`, the pause kept on the node's pause key by the puller. what names the asks in the descriptions ("Go").
|
||||
func SeatTools(seat, what string, p *Puller) []stdio.Tool {
|
||||
verb := func(name, description string, run func() (any, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: seat + "." + name, Description: description,
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return run() }}
|
||||
}
|
||||
current := func() (any, error) {
|
||||
capacity := 0
|
||||
if p.Capacity != nil {
|
||||
capacity = p.Capacity()
|
||||
}
|
||||
return map[string]any{"node": p.Node, "capacity": capacity, "running": p.Running(), "paused": p.Paused()}, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
verb("current", "The "+what+" asks this node runs now (each one's id, kind and when it started), its build capacity, "+
|
||||
"and whether it is paused.", current),
|
||||
verb("pause", "Take no new "+what+" ask on this node until resumed; an ask running here ends in its place. Kept "+
|
||||
"across a restart (the node's pause key).", func() (any, error) {
|
||||
if err := p.Pause("the seat's pause verb"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return current()
|
||||
}),
|
||||
verb("resume", "Take "+what+" asks again on this node.", func() (any, error) {
|
||||
if err := p.Resume("the seat's resume verb"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return current()
|
||||
}),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1887/A9a, ADR 0525 §10: a build seat's verbs are <seat>.current, .pause and .resume; current names the ask running,
|
||||
// pause stops the puller taking asks, resume lets it take them again.
|
||||
func TestSeatToolsCurrentPauseResume(t *testing.T) {
|
||||
src, st := NewFakeAsks(), newSteps()
|
||||
release := make(chan struct{})
|
||||
p := &Puller{Source: src, Language: delivery.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 },
|
||||
PullWait: 10 * time.Millisecond, Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { <-release; return nil, nil }}
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tool := range SeatTools("node-build-go", "Go", p) {
|
||||
tools[tool.Name] = tool.Run
|
||||
}
|
||||
for _, name := range []string{"node-build-go.current", "node-build-go.pause", "node-build-go.resume"} {
|
||||
if tools[name] == nil {
|
||||
t.Fatalf("no %s", name)
|
||||
}
|
||||
}
|
||||
a := testAsk(delivery.AskDelivery)
|
||||
_ = src.PublishAsk(a)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go p.Pull(ctx)
|
||||
waitFor(t, "the ask running", func() bool { return p.InHand() == 1 })
|
||||
out, _ := tools["node-build-go.current"](nil)
|
||||
cur := out.(map[string]any)
|
||||
running := cur["running"].([]RunningAsk)
|
||||
if len(running) != 1 || running[0].ID != a.ID || running[0].Module != "m" || cur["capacity"] != 1 || cur["paused"] != false {
|
||||
t.Fatalf("current = %+v", cur)
|
||||
}
|
||||
if out, err := tools["node-build-go.pause"](nil); err != nil || out.(map[string]any)["paused"] != true {
|
||||
t.Fatalf("pause = %v, %v", out, err)
|
||||
}
|
||||
close(release)
|
||||
_ = src.PublishAsk(testAsk(delivery.AskDelivery))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if src.Waiting(delivery.AskDelivery) != 1 {
|
||||
t.Fatal("a paused node took an ask")
|
||||
}
|
||||
if _, err := tools["node-build-go.resume"](nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, "the ask taken after the resume", func() bool { return src.Waiting(delivery.AskDelivery) == 0 })
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// Package buildstep is what both build modules share: the runner of a long step that answers a tool call at once
|
||||
// and ends on its key in the module's `steps` state (novox/hq ADR 0469 §2; issue 1471), the toolchain guard
|
||||
// (issue 1473) and the fetch of one commit over SSH (issue 1474).
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Put writes one key of a state: stdio.State(delivery.StepsState).Put's shape (go/state.go).
|
||||
type Put func(key string, value any) (uint64, error)
|
||||
|
||||
// EndAttempts and EndRetry bound how a step's end is written: a write that fails is tried again, since the end
|
||||
// is the only way the step's reader learns it is over.
|
||||
var (
|
||||
EndAttempts = 5
|
||||
EndRetry = 2 * time.Second
|
||||
)
|
||||
|
||||
var now = time.Now
|
||||
|
||||
// NewID is a step id: the time in milliseconds in base 36 and six random hex digits, lower-case letters and
|
||||
// digits only, so it is one key token and differs between two steps of one node even in one millisecond.
|
||||
func NewID() string {
|
||||
var b [3]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return strconv.FormatInt(now().UnixMilli(), 36) + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// Start begins a step of tool on node and answers its id at once: the key <node>.<id> holds state running when
|
||||
// Start returns. run is then called in a goroutine of its own, under ctx (the process's, not the tool call's: the
|
||||
// call is cut at 30 s, the step is not). When it returns, the key is written passed with run's result, failed with
|
||||
// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and
|
||||
// the process goes on. If the running write fails, nothing was started and Start answers the error.
|
||||
func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) {
|
||||
step, key, err := begin(put, node, tool, NewID())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
go end(ctx, put, key, step, run)
|
||||
return step.ID, nil
|
||||
}
|
||||
|
||||
// begin writes the step running under id.
|
||||
func begin(put Put, node, tool, id string) (delivery.Step, string, error) {
|
||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
|
||||
key, err := delivery.StepKey(node, step.ID)
|
||||
if err != nil {
|
||||
return delivery.Step{}, "", err
|
||||
}
|
||||
if err := write(put, key, step); err != nil {
|
||||
return delivery.Step{}, "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err)
|
||||
}
|
||||
return step, key, nil
|
||||
}
|
||||
|
||||
// end runs run and writes the step's end.
|
||||
func end(ctx context.Context, put Put, key string, step delivery.Step, run func(ctx context.Context) (any, error)) {
|
||||
result, err := protect(ctx, run)
|
||||
_ = finish(put, key, step, result, err, log.Printf) // finish says a refused end itself
|
||||
}
|
||||
|
||||
// finish writes the step's end from run's result or error: passed, failed or refused (a *Refusal). A write that fails is
|
||||
// tried EndAttempts times. When the whole end is refused every time (a result the state will not take: issue 1930), the
|
||||
// refusal is logged and a small failed end without the result is written instead, naming the write's error, so the
|
||||
// step's reader learns it is over and why. finish answers an error only when no end could be written at all, and then
|
||||
// the step's ask is not acknowledged as done.
|
||||
func finish(put Put, key string, step delivery.Step, result any, err error, logf func(format string, args ...any)) error {
|
||||
step.Ended = now().UTC()
|
||||
switch ref := (*Refusal)(nil); {
|
||||
case err == nil:
|
||||
step.State = delivery.StepPassed
|
||||
if result != nil {
|
||||
raw, merr := json.Marshal(result)
|
||||
if merr != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error()
|
||||
} else {
|
||||
step.Result = raw
|
||||
}
|
||||
}
|
||||
case errors.As(err, &ref):
|
||||
step.State, step.Reason = delivery.StepRefused, err.Error()
|
||||
default:
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
}
|
||||
werr := writeEnd(put, key, step)
|
||||
if werr == nil {
|
||||
return nil
|
||||
}
|
||||
if logf != nil {
|
||||
logf("the end of the step %s (%s) could not be written in %d tries: %v", key, step.State, EndAttempts, werr)
|
||||
}
|
||||
ended := step.State
|
||||
why := fmt.Sprintf("the step ended %s, and its end could not be written: %v", ended, werr)
|
||||
if step.Reason != "" {
|
||||
why = fmt.Sprintf("the step ended %s (%s), and its end could not be written: %v", ended, step.Reason, werr)
|
||||
}
|
||||
step.State, step.Reason, step.Result = delivery.StepFailed, why, nil
|
||||
if ferr := writeEnd(put, key, step); ferr != nil {
|
||||
if logf != nil {
|
||||
logf("the failed end of the step %s could not be written either: %v", key, ferr)
|
||||
}
|
||||
return fmt.Errorf("buildstep: no end of the step %s could be written: %w", key, ferr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeEnd writes the step's end, trying EndAttempts times.
|
||||
func writeEnd(put Put, key string, step delivery.Step) error {
|
||||
for attempt := 1; ; attempt++ {
|
||||
werr := write(put, key, step)
|
||||
if werr == nil || attempt >= EndAttempts {
|
||||
return werr
|
||||
}
|
||||
time.Sleep(EndRetry)
|
||||
}
|
||||
}
|
||||
|
||||
// protect is run with a panic turned into the error that ends the step.
|
||||
func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
result, err = nil, fmt.Errorf("the step panicked: %v", p)
|
||||
}
|
||||
}()
|
||||
return run(ctx)
|
||||
}
|
||||
|
||||
func write(put Put, key string, step delivery.Step) error {
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = put(key, json.RawMessage(raw))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// keys is a steps state in memory: every value put, in order, per key, and a signal for each end.
|
||||
type keys struct {
|
||||
mu sync.Mutex
|
||||
puts map[string][]delivery.Step
|
||||
ended chan string
|
||||
}
|
||||
|
||||
func newKeys() *keys { return &keys{puts: map[string][]delivery.Step{}, ended: make(chan string, 16)} }
|
||||
|
||||
func (k *keys) put(key string, value any) (uint64, error) {
|
||||
step, err := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
k.mu.Lock()
|
||||
k.puts[key] = append(k.puts[key], step)
|
||||
n := uint64(len(k.puts[key]))
|
||||
k.mu.Unlock()
|
||||
if step.State != delivery.StepRunning {
|
||||
k.ended <- key
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (k *keys) last(key string) delivery.Step {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
s := k.puts[key]
|
||||
return s[len(s)-1]
|
||||
}
|
||||
|
||||
func (k *keys) waitEnd(t *testing.T) string {
|
||||
t.Helper()
|
||||
select {
|
||||
case key := <-k.ended:
|
||||
return key
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the step never ended on its key")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// 1471/AC1: a run that blocks: Start answers an id within 100 ms and the key holds running.
|
||||
func TestStartAnswersAtOnceWithTheStepRunning(t *testing.T) {
|
||||
k := newKeys()
|
||||
release := make(chan struct{})
|
||||
defer close(release)
|
||||
began := time.Now()
|
||||
id, err := Start(context.Background(), k.put, "ace", "fetch", func(ctx context.Context) (any, error) {
|
||||
<-release
|
||||
return nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d := time.Since(began); d > 100*time.Millisecond {
|
||||
t.Errorf("Start took %v", d)
|
||||
}
|
||||
s := k.last("ace." + id)
|
||||
if s.State != delivery.StepRunning || s.Tool != "fetch" || s.Node != "ace" || s.ID != id || s.Started.IsZero() {
|
||||
t.Errorf("step = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1471/AC2 and 1473/AC4: a result, an error and a Refusal end the step passed, failed and refused.
|
||||
func TestStepEndsPassedFailedOrRefusedOnItsKey(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
run func(context.Context) (any, error)
|
||||
state string
|
||||
reason string
|
||||
result string
|
||||
}{
|
||||
{"result", func(context.Context) (any, error) { return map[string]string{"sha": "abc"}, nil }, delivery.StepPassed, "", `{"sha":"abc"}`},
|
||||
{"error", func(context.Context) (any, error) { return nil, errors.New("exit 2") }, delivery.StepFailed, "exit 2", ""},
|
||||
{"refusal", func(context.Context) (any, error) {
|
||||
return nil, &Refusal{Text: "go 1.27.1 installed, the recipe builds with go 1.27.0", Want: "1.27.0", Have: "1.27.1"}
|
||||
}, delivery.StepRefused, "go 1.27.1 installed, the recipe builds with go 1.27.0", ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
k := newKeys()
|
||||
id, err := Start(context.Background(), k.put, "ace", "test", c.run)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := k.waitEnd(t); got != "ace."+id {
|
||||
t.Fatalf("ended key %q", got)
|
||||
}
|
||||
s := k.last("ace." + id)
|
||||
if s.State != c.state || s.Reason != c.reason || string(s.Result) != c.result || s.Ended.IsZero() {
|
||||
t.Errorf("step = %+v", s)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1471/AC3: a panic in run ends the step failed, naming it, and the test process goes on.
|
||||
func TestAPanicEndsTheStepFailed(t *testing.T) {
|
||||
k := newKeys()
|
||||
id, _ := Start(context.Background(), k.put, "ace", "build", func(context.Context) (any, error) { panic("boom") })
|
||||
k.waitEnd(t)
|
||||
s := k.last("ace." + id)
|
||||
if s.State != delivery.StepFailed || s.Reason != "the step panicked: boom" {
|
||||
t.Errorf("step = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1471/AC5: ids made in one and the same millisecond differ, and each is one key token. The clock is fixed
|
||||
// while no step runs, so no goroutine reads it meanwhile.
|
||||
func TestTwoIDsInOneMillisecondDiffer(t *testing.T) {
|
||||
was := now
|
||||
defer func() { now = was }()
|
||||
at := time.Date(2026, 10, 20, 12, 0, 0, 0, time.UTC)
|
||||
now = func() time.Time { return at }
|
||||
seen := map[string]bool{}
|
||||
for i := 0; i < 50; i++ {
|
||||
id := NewID()
|
||||
if seen[id] {
|
||||
t.Fatalf("id %q handed out twice", id)
|
||||
}
|
||||
seen[id] = true
|
||||
if _, err := delivery.StepKey("ace", id); err != nil {
|
||||
t.Errorf("id %q: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A first write that fails starts nothing.
|
||||
func TestStartAnswersTheErrorWhenTheRunningWriteFails(t *testing.T) {
|
||||
ran := make(chan struct{}, 1)
|
||||
_, err := Start(context.Background(), func(string, any) (uint64, error) { return 0, errors.New("no grant") }, "ace", "fetch",
|
||||
func(context.Context) (any, error) { ran <- struct{}{}; return nil, nil })
|
||||
if err == nil {
|
||||
t.Fatal("Start answered no error")
|
||||
}
|
||||
select {
|
||||
case <-ran:
|
||||
t.Error("run was called")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
// The probe a run step's isolation is proved with (novox/hq ADR 0448; issue 1309, S19): run inside the step, it says
|
||||
// on stdout every way it reached what a step must not, and "shut off" when it reached none. Its arguments: the files
|
||||
// it must not read (the forge key, a bus login), a TCP address and a unix socket of the machine it must not reach, a
|
||||
// name it must not resolve, and a cache file it writes, which must be unchanged after the step.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 7 {
|
||||
fmt.Println("usage: probe <key> <login> <tcp address> <unix socket> <name> <cache file>")
|
||||
os.Exit(2)
|
||||
}
|
||||
key, login, tcp, socket, name, cache := os.Args[1], os.Args[2], os.Args[3], os.Args[4], os.Args[5], os.Args[6]
|
||||
reached := 0
|
||||
say := func(f string, a ...any) { reached++; fmt.Printf(f+"\n", a...) }
|
||||
for _, f := range []string{key, login} {
|
||||
if _, err := os.ReadFile(f); err == nil {
|
||||
say("read %s", f)
|
||||
}
|
||||
}
|
||||
if c, err := net.DialTimeout("tcp", tcp, 2*time.Second); err == nil {
|
||||
c.Close()
|
||||
say("reached %s", tcp)
|
||||
}
|
||||
if c, err := net.DialTimeout("unix", socket, 2*time.Second); err == nil {
|
||||
c.Close()
|
||||
say("reached %s", socket)
|
||||
}
|
||||
if addrs, err := net.LookupHost(name); err == nil {
|
||||
say("resolved %s to %v", name, addrs)
|
||||
}
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
if entries, err := os.ReadDir(home); err == nil && len(entries) > 0 {
|
||||
say("the home %s holds %d entries", home, len(entries))
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(cache, []byte("written by the step\n"), 0o644); err != nil {
|
||||
say("could not write the cache under its throwaway layer: %v", err)
|
||||
}
|
||||
if reached == 0 {
|
||||
fmt.Println("shut off")
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,123 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// ModuleTree is what a merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256
|
||||
// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs
|
||||
// (novox/hq ADR 0484 clause 3).
|
||||
type ModuleTree struct {
|
||||
Tree string `json:"tree"`
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
// Paths are, of a module whose every artifact names its paths, each named path's git object id at the commit; Tree
|
||||
// is then their delivery.NamedTree with Manifest (novox/hq ADR 0499 clause 2). Absent otherwise.
|
||||
Paths map[string]string `json:"paths,omitempty"`
|
||||
}
|
||||
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory
|
||||
// (the repository's root is "."); for a module whose every artifact names its paths, the named tree of those paths and
|
||||
// their objects (novox/hq ADR 0499 clause 2). The workspace is at commit, so the manifests and the lock files are read from it. env is
|
||||
// the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge.
|
||||
func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) {
|
||||
r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
trees := map[string]ModuleTree{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
mdir := filepath.ToSlash(filepath.Dir(l))
|
||||
spec := commit + ":" + mdir
|
||||
if mdir == "." {
|
||||
spec = commit + "^{tree}"
|
||||
}
|
||||
t := run(ctx, env, "git", "-C", dir, "rev-parse", spec)
|
||||
if t.Status != 0 || t.Err != "" {
|
||||
return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr))
|
||||
}
|
||||
mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mt.Manifest = sha256Hex(raw)
|
||||
if err := narrow(ctx, run, env, dir, commit, mdir, raw, &mt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, lock := range lockFiles {
|
||||
if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil {
|
||||
if mt.Locks == nil {
|
||||
mt.Locks = map[string]string{}
|
||||
}
|
||||
mt.Locks[lock] = sha256Hex(raw)
|
||||
}
|
||||
}
|
||||
trees[mdir] = mt
|
||||
}
|
||||
return trees, nil
|
||||
}
|
||||
|
||||
// narrow sets mt's Paths and its named Tree when the module's manifest names paths on every artifact (novox/hq ADR 0499
|
||||
// clause 2): one `git ls-tree -z` gives each named path's object at commit. The merge preview reads the manifest here, as
|
||||
// the indexer does, with catalogue.ParseManifestForInputs (issue 1751), and no other build step reads paths from one (ADR 0495 clause 1). A manifest that does not parse
|
||||
// is not narrowed: the indexer gives it none. A named path absent at commit fails, naming the module and the path.
|
||||
func narrow(ctx context.Context, run forgekey.Runner, env []string, dir, commit, mdir string, raw []byte, mt *ModuleTree) error {
|
||||
m, err := catalogue.ParseManifestForInputs(raw)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
paths, every := m.NamedPaths()
|
||||
if !every {
|
||||
return nil
|
||||
}
|
||||
full := make([]string, len(paths))
|
||||
for i, p := range paths {
|
||||
full[i] = path.Join(mdir, p)
|
||||
}
|
||||
r := run(ctx, env, "git", append([]string{"-C", dir, "ls-tree", "-z", commit, "--"}, full...)...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("git ls-tree %s of the module %s: %s", commit, mdir, firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
objects := map[string]string{}
|
||||
for _, l := range strings.Split(r.Stdout, "\x00") {
|
||||
meta, name, ok := strings.Cut(l, "\t")
|
||||
fields := strings.Fields(meta)
|
||||
if !ok || len(fields) != 3 {
|
||||
continue
|
||||
}
|
||||
objects[name] = fields[2]
|
||||
}
|
||||
mt.Paths = map[string]string{}
|
||||
for i, p := range paths {
|
||||
object := objects[full[i]]
|
||||
if object == "" {
|
||||
return fmt.Errorf("the module %s names the path %q, which is not in %s", mdir, p, commit)
|
||||
}
|
||||
mt.Paths[p] = object
|
||||
}
|
||||
named, err := delivery.NamedTree(mt.Manifest, mt.Paths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the module %s: %w", mdir, err)
|
||||
}
|
||||
mt.Tree = named
|
||||
return nil
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
@@ -0,0 +1,63 @@
|
||||
//go:build integration
|
||||
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// On a real repository shaped like mesh-sdk (novox/hq ADR 0499 clauses 2 and 6), a commit touching only the Go package
|
||||
// leaves the TypeScript package's tree value equal, and a commit touching its source changes it; a module that names no
|
||||
// path keeps its directory's tree id.
|
||||
func TestAGoOnlyCommitLeavesTheNamedTreeEqualOnRealGit(t *testing.T) {
|
||||
repo := t.TempDir()
|
||||
realGit(t, repo, "init", "-q", "-b", "main")
|
||||
writeFile(t, repo, "module.json", `{"module":"mesh-sdk","version":"1","build":{"artifacts":[`+
|
||||
`{"name":"lib","kind":"package","language":"typescript","paths":["src","test","package.json","package-lock.json","tsconfig.json"]}]}}`)
|
||||
writeFile(t, repo, "src/index.ts", "export const a = 1\n")
|
||||
writeFile(t, repo, "test/a.test.ts", "// a\n")
|
||||
writeFile(t, repo, "package.json", "{}\n")
|
||||
writeFile(t, repo, "package-lock.json", "{}\n")
|
||||
writeFile(t, repo, "tsconfig.json", "{}\n")
|
||||
writeFile(t, repo, "go/delivery/x.go", "package delivery\n")
|
||||
writeFile(t, repo, "other/module.json", `{"module":"other","version":"1"}`)
|
||||
writeFile(t, repo, "other/a.txt", "a\n")
|
||||
commit := func(msg string) string {
|
||||
realGit(t, repo, "add", "-A")
|
||||
realGit(t, repo, "commit", "-q", "-m", msg)
|
||||
return realGit(t, repo, "rev-parse", "HEAD")
|
||||
}
|
||||
trees := func(c string) map[string]ModuleTree {
|
||||
got, err := ModuleTrees(context.Background(), forgekey.ExecRunner, nil, repo, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
first := trees(commit("first"))
|
||||
if !strings.HasPrefix(first["."].Tree, "named:") || len(first["."].Paths) != 5 {
|
||||
t.Fatalf("the root module was not narrowed: %+v", first["."])
|
||||
}
|
||||
if want := realGit(t, repo, "rev-parse", "HEAD:other"); first["other"].Tree != want || first["other"].Paths != nil {
|
||||
t.Fatalf("a module naming no path: %+v, want its directory's tree %s", first["other"], want)
|
||||
}
|
||||
|
||||
writeFile(t, repo, "go/delivery/x.go", "package delivery\n\nconst B = 2\n")
|
||||
goOnly := trees(commit("go only"))
|
||||
if goOnly["."].Tree != first["."].Tree {
|
||||
t.Fatalf("a Go-only commit changed the TypeScript package's tree: %s, was %s", goOnly["."].Tree, first["."].Tree)
|
||||
}
|
||||
|
||||
writeFile(t, repo, "src/index.ts", "export const a = 2\n")
|
||||
src := trees(commit("src"))
|
||||
if src["."].Tree == first["."].Tree {
|
||||
t.Fatal("a commit to src/ left the TypeScript package's tree equal")
|
||||
}
|
||||
if src["."].Paths["src"] != realGit(t, repo, "rev-parse", "HEAD:src") {
|
||||
t.Fatalf("the object of src is %s, git says %s", src["."].Paths["src"], realGit(t, repo, "rev-parse", "HEAD:src"))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// A module whose every artifact names its paths takes its tree from them (novox/hq ADR 0499 clause 2): ModuleTrees asks
|
||||
// git for each named path's object at the commit, keeps them in Paths and answers their delivery.NamedTree as Tree; a
|
||||
// module naming none keeps its directory's tree, as before.
|
||||
func TestModuleTreesNarrowsAModuleWhoseArtifactsNameTheirPaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
sdk := `{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src","package.json"]}]}}`
|
||||
writeTreeFile(t, root, "module.json", sdk)
|
||||
writeTreeFile(t, root, "modules/b/module.json", `{"module":"b","version":"1"}`)
|
||||
merge := strings.Repeat("c", 40)
|
||||
var lsTree []string
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\nmodules/b/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "tree-of-" + args[3] + "\n"}
|
||||
case "ls-tree":
|
||||
lsTree = append(lsTree, strings.Join(args[4:], " "))
|
||||
return forgekey.Ran{Stdout: "100644 blob " + strings.Repeat("b", 40) + "\tpackage.json\x00" +
|
||||
"040000 tree " + strings.Repeat("a", 40) + "\tsrc\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected " + args[2]}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, nil, root, merge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := map[string]string{"src": strings.Repeat("a", 40), "package.json": strings.Repeat("b", 40)}
|
||||
want, _ := delivery.NamedTree(hexOf(sdk), paths)
|
||||
if g := got["."]; g.Tree != want || len(g.Paths) != 2 || g.Paths["src"] != paths["src"] || g.Paths["package.json"] != paths["package.json"] {
|
||||
t.Fatalf("the root module = %+v, want tree %s and paths %v", g, want, paths)
|
||||
}
|
||||
if len(lsTree) != 1 || lsTree[0] != merge+" -- package.json src" {
|
||||
t.Fatalf("git ls-tree ran as %v", lsTree)
|
||||
}
|
||||
if g := got["modules/b"]; g.Tree != "tree-of-"+merge+":modules/b" || g.Paths != nil {
|
||||
t.Fatalf("a module naming no path was narrowed: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A named path absent at the commit fails the answer, naming the module and the path: a tree computed without it would
|
||||
// pin a build source that does not exist.
|
||||
func TestModuleTreesFailsNamingANamedPathAbsentAtTheCommit(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTreeFile(t, root, "modules/a/module.json", `{"module":"a","version":"1","build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src","gone"]}]}}`)
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "modules/a/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "t\n"}
|
||||
case "ls-tree":
|
||||
if strings.Join(args[4:], " ") != strings.Repeat("c", 40)+" -- modules/a/gone modules/a/src" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "wrong arguments " + strings.Join(args[3:], " ")}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "040000 tree " + strings.Repeat("a", 40) + "\tmodules/a/src\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1}
|
||||
}
|
||||
_, err := ModuleTrees(context.Background(), run, nil, root, strings.Repeat("c", 40))
|
||||
if err == nil || !strings.Contains(err.Error(), `"gone"`) || !strings.Contains(err.Error(), "modules/a") {
|
||||
t.Fatalf("an absent named path gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest whose config value falls back to a predefined key is narrowed here as the indexer narrows it: both read it
|
||||
// with catalogue.ParseManifestForInputs (issue 1751), so a build module holding no published list computes the same
|
||||
// tree (ADR 0499 clause 2).
|
||||
func TestModuleTreesNarrowsAManifestFallingBackToAPredefinedKey(t *testing.T) {
|
||||
was := catalogue.ReplacePublished(nil) // the build modules' case: no published list
|
||||
defer catalogue.ReplacePublished(was)
|
||||
root := t.TempDir()
|
||||
raw := `{"module":"xterm","version":"1","config":{"size":{"type":"number","description":"The text size.","predefined":"font-size","default":12,"runtime":true}},` +
|
||||
`"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript","paths":["src"]}]}}`
|
||||
if _, err := catalogue.ParseManifest([]byte(raw)); err == nil {
|
||||
t.Fatal("ParseManifest reads the manifest with no list; the test proves nothing")
|
||||
}
|
||||
if _, err := catalogue.ParseManifestForInputs([]byte(raw)); err != nil {
|
||||
t.Fatalf("the indexer's read refuses the manifest: %v", err)
|
||||
}
|
||||
writeTreeFile(t, root, "module.json", raw)
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "whole\n"}
|
||||
case "ls-tree":
|
||||
return forgekey.Ran{Stdout: "040000 tree " + strings.Repeat("a", 40) + "\tsrc\x00"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, nil, root, strings.Repeat("c", 40))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want, _ := delivery.NamedTree(hexOf(raw), map[string]string{"src": strings.Repeat("a", 40)})
|
||||
if g := got["."]; g.Tree != want || g.Paths["src"] == "" {
|
||||
t.Fatalf("the module = %+v, want the named tree %s", g, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
func hexOf(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) }
|
||||
|
||||
func writeTreeFile(t *testing.T, root, name, content string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(root, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ModuleTrees answers, for each directory holding a module.json (the root as "."), the tree id git gives, the sha256 of
|
||||
// module.json and of each lock file present, and reads them from the workspace.
|
||||
func TestModuleTreesAnswersTreeManifestAndLocksPerModuleDirectory(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTreeFile(t, root, "module.json", `{"root":1}`)
|
||||
writeTreeFile(t, root, "go.sum", "rootsum")
|
||||
writeTreeFile(t, root, "modules/a/module.json", `{"a":1}`)
|
||||
writeTreeFile(t, root, "modules/a/go.sum", "asum")
|
||||
writeTreeFile(t, root, "modules/a/package-lock.json", "alock")
|
||||
writeTreeFile(t, root, "modules/b/module.json", `{"b":1}`)
|
||||
merge := strings.Repeat("c", 40)
|
||||
var calls []string
|
||||
var envs [][]string
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
calls = append(calls, name+" "+strings.Join(args[2:], " "))
|
||||
envs = append(envs, env)
|
||||
switch args[2] {
|
||||
case "ls-files":
|
||||
return forgekey.Ran{Stdout: "module.json\nmodules/a/module.json\nmodules/b/module.json\n"}
|
||||
case "rev-parse":
|
||||
return forgekey.Ran{Stdout: "tree-of-" + args[3] + "\n"}
|
||||
}
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected"}
|
||||
}
|
||||
got, err := ModuleTrees(context.Background(), run, []string{"X=1"}, root, merge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]ModuleTree{
|
||||
".": {Tree: "tree-of-" + merge + "^{tree}", Manifest: hexOf(`{"root":1}`), Locks: map[string]string{"go.sum": hexOf("rootsum")}},
|
||||
"modules/a": {Tree: "tree-of-" + merge + ":modules/a", Manifest: hexOf(`{"a":1}`), Locks: map[string]string{"go.sum": hexOf("asum"), "package-lock.json": hexOf("alock")}},
|
||||
"modules/b": {Tree: "tree-of-" + merge + ":modules/b", Manifest: hexOf(`{"b":1}`)},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("trees = %+v\nwant %+v", got, want)
|
||||
}
|
||||
if got["modules/b"].Locks != nil {
|
||||
t.Error("a module with no lock file carries a lock map")
|
||||
}
|
||||
for _, e := range envs {
|
||||
if len(e) != 1 || e[0] != "X=1" {
|
||||
t.Errorf("git ran with env %v, want the one given", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleTreesFailsNamingTheGitCommandOrTheUnreadableManifest(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
fail := func(on string) forgekey.Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if args[2] == on {
|
||||
return forgekey.Ran{Status: 128, Stderr: "fatal: no\n"}
|
||||
}
|
||||
return forgekey.Ran{Stdout: "modules/a/module.json\n"}
|
||||
}
|
||||
}
|
||||
for _, on := range []string{"ls-files", "rev-parse"} {
|
||||
if _, err := ModuleTrees(context.Background(), fail(on), nil, root, sha); err == nil || !strings.Contains(err.Error(), on) || !strings.Contains(err.Error(), "fatal: no") {
|
||||
t.Errorf("%s failing: %v", on, err)
|
||||
}
|
||||
}
|
||||
if _, err := ModuleTrees(context.Background(), fail("none"), nil, root, sha); err == nil {
|
||||
t.Error("a module.json that is not in the workspace was read")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package buildstep
|
||||
|
||||
// A build module's room to work and its workspaces (novox/hq issue 1839, ADR 0509): a step is refused while the
|
||||
// filesystem it writes to has less free than the floor, and a workspace is removed when its caller releases it, or,
|
||||
// unheld, once its last use is past a bound (a lost caller's), never while a running step holds it.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// FreeSpaceFloor is the free space a build module needs on its work and cache filesystems to take a step: the floor
|
||||
// build-agent keeps (mesh-controller internal/builder/floor.go, issue 1834).
|
||||
const FreeSpaceFloor uint64 = 20 << 30
|
||||
|
||||
// UnheldBound is how long a workspace no step holds is kept after its last use, when its caller never released it
|
||||
// (ADR 0509: a lost caller's workspace goes after two hours).
|
||||
const UnheldBound = 2 * time.Hour
|
||||
|
||||
// FreeBytes reads the bytes an unprivileged writer can still use on the filesystem holding path.
|
||||
func FreeBytes(path string) (uint64, error) {
|
||||
var s syscall.Statfs_t
|
||||
if err := syscall.Statfs(path, &s); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.Bavail * uint64(s.Bsize), nil
|
||||
}
|
||||
|
||||
// Floor refuses a step while a filesystem it writes to has less free than Bytes. A zero Bytes checks nothing.
|
||||
type Floor struct {
|
||||
// Node is named in the refusal.
|
||||
Node string
|
||||
Bytes uint64
|
||||
// Free reads a path's free bytes; FreeBytes when nil. A test answers for a disk it does not have.
|
||||
Free func(path string) (uint64, error)
|
||||
}
|
||||
|
||||
// Check answers a *Refusal, "refused: <node> has N GiB free under the floor of F GiB (on <path>)", when any path's
|
||||
// filesystem has less free than the floor, or its free space cannot be read; nil otherwise. A path not made yet is
|
||||
// read at its nearest folder that is; an empty path is skipped.
|
||||
func (f Floor) Check(paths ...string) error {
|
||||
if f.Bytes == 0 {
|
||||
return nil
|
||||
}
|
||||
free := f.Free
|
||||
if free == nil {
|
||||
free = FreeBytes
|
||||
}
|
||||
for _, p := range paths {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
at := p
|
||||
for {
|
||||
if _, err := os.Stat(at); err == nil || filepath.Dir(at) == at {
|
||||
break
|
||||
}
|
||||
at = filepath.Dir(at)
|
||||
}
|
||||
n, err := free(at)
|
||||
if err != nil {
|
||||
return &Refusal{Text: fmt.Sprintf("refused: the free space of %s on %s cannot be read: %v", p, f.Node, err)}
|
||||
}
|
||||
if n < f.Bytes {
|
||||
return &Refusal{Text: fmt.Sprintf("refused: %s has %d GiB free under the floor of %d GiB (on %s)", f.Node, n>>30, f.Bytes>>30, p),
|
||||
Want: fmt.Sprintf("%d GiB", f.Bytes>>30), Have: fmt.Sprintf("%d GiB", n>>30)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var workspaceID = regexp.MustCompile(`^[a-z0-9]+$`)
|
||||
|
||||
// Workspaces are a build module's workspaces, each <Dir>/<id>, and which of them running steps hold.
|
||||
type Workspaces struct {
|
||||
Dir string
|
||||
|
||||
mu sync.Mutex
|
||||
held map[string]int
|
||||
}
|
||||
|
||||
// Hold marks the workspace held by a running step until the answered func is called; both mark it used now, so the
|
||||
// sweep counts its bound from the step's end. An id that is not a workspace's holds nothing.
|
||||
func (w *Workspaces) Hold(id string) func() {
|
||||
if !workspaceID.MatchString(id) {
|
||||
return func() {}
|
||||
}
|
||||
w.mu.Lock()
|
||||
if w.held == nil {
|
||||
w.held = map[string]int{}
|
||||
}
|
||||
w.held[id]++
|
||||
w.mu.Unlock()
|
||||
w.touch(id)
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
w.touch(id)
|
||||
w.mu.Lock()
|
||||
if w.held[id]--; w.held[id] <= 0 {
|
||||
delete(w.held, id)
|
||||
}
|
||||
w.mu.Unlock()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Workspaces) touch(id string) {
|
||||
now := time.Now()
|
||||
_ = os.Chtimes(filepath.Join(w.Dir, id), now, now) // a workspace not made yet has nothing to mark
|
||||
}
|
||||
|
||||
// Release removes the workspace at once: its caller's chain of steps has ended. One already gone is no error; one a
|
||||
// running step holds is refused and kept; an id that is not a workspace's is refused.
|
||||
func (w *Workspaces) Release(id string) error {
|
||||
if !workspaceID.MatchString(id) {
|
||||
return fmt.Errorf("%q is not a workspace id", id)
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.held[id] > 0 {
|
||||
return fmt.Errorf("the workspace %s is held by a running step", id)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(w.Dir, id)); err != nil {
|
||||
return fmt.Errorf("the workspace %s could not be removed: %w", id, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sweep removes every workspace no step holds whose last use is older than bound, and answers the ids it removed,
|
||||
// sorted. It is the safety net for a caller that never released its workspace.
|
||||
func (w *Workspaces) Sweep(now time.Time, bound time.Duration) []string {
|
||||
entries, err := os.ReadDir(w.Dir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
removed := []string{}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() || !workspaceID.MatchString(e.Name()) || w.held[e.Name()] > 0 {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil || now.Sub(info.ModTime()) < bound {
|
||||
continue
|
||||
}
|
||||
if os.RemoveAll(filepath.Join(w.Dir, e.Name())) == nil {
|
||||
removed = append(removed, e.Name())
|
||||
}
|
||||
}
|
||||
sort.Strings(removed)
|
||||
return removed
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 1839/AC1: a filesystem under the floor refuses the step, naming the node, what is free and the floor.
|
||||
func TestAFilesystemUnderTheFloorRefusesNamingNodeFreeAndFloor(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
f := Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 4<<30 + 5, nil }}
|
||||
err := f.Check(dir)
|
||||
var ref *Refusal
|
||||
if !errors.As(err, &ref) {
|
||||
t.Fatalf("Check = %v, want a *Refusal", err)
|
||||
}
|
||||
if want := "refused: ace has 4 GiB free under the floor of 20 GiB (on " + dir + ")"; err.Error() != want {
|
||||
t.Errorf("Check = %q, want %q", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: at or above the floor, and with no floor set, nothing is refused.
|
||||
func TestAtTheFloorOrWithNoFloorNothingIsRefused(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := (Floor{Node: "ace", Bytes: 20 << 30, Free: func(string) (uint64, error) { return 20 << 30, nil }}).Check(dir); err != nil {
|
||||
t.Errorf("at the floor: %v", err)
|
||||
}
|
||||
if err := (Floor{Node: "ace", Free: func(string) (uint64, error) { return 0, nil }}).Check(dir); err != nil {
|
||||
t.Errorf("no floor: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC1: a path not made yet is read at its nearest folder that is; a free space that cannot be read is a refusal.
|
||||
func TestAPathNotMadeIsReadAtItsParentAndAnUnreadableOneRefuses(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
var read string
|
||||
f := Floor{Node: "ace", Bytes: 1, Free: func(p string) (uint64, error) { read = p; return 2, nil }}
|
||||
if err := f.Check(filepath.Join(dir, "not", "made")); err != nil || read != dir {
|
||||
t.Errorf("Check = %v, read %q, want nil and %q", err, read, dir)
|
||||
}
|
||||
f.Free = func(string) (uint64, error) { return 0, errors.New("statfs: boom") }
|
||||
var ref *Refusal
|
||||
if err := f.Check(dir); !errors.As(err, &ref) {
|
||||
t.Errorf("Check = %v, want a refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
func makeWorkspace(t *testing.T, w *Workspaces, id string, age time.Duration) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(w.Dir, id)
|
||||
if err := os.MkdirAll(filepath.Join(p, "tree"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Now().Add(-age)
|
||||
if err := os.Chtimes(p, at, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func gone(p string) bool { _, err := os.Stat(p); return os.IsNotExist(err) }
|
||||
|
||||
// 1839/AC2: a released workspace is gone at once; one already gone is no error; one a running step holds is refused
|
||||
// and kept; an id that is not a workspace's is refused.
|
||||
func TestReleaseRemovesAWorkspaceAtOnce(t *testing.T) {
|
||||
w := &Workspaces{Dir: t.TempDir()}
|
||||
p := makeWorkspace(t, w, "abc1", 0)
|
||||
if err := w.Release("abc1"); err != nil || !gone(p) {
|
||||
t.Fatalf("Release = %v, gone %v", err, gone(p))
|
||||
}
|
||||
if err := w.Release("abc1"); err != nil {
|
||||
t.Errorf("a second release = %v, want nil", err)
|
||||
}
|
||||
held := makeWorkspace(t, w, "abc2", 0)
|
||||
done := w.Hold("abc2")
|
||||
if err := w.Release("abc2"); err == nil || gone(held) {
|
||||
t.Errorf("Release of a held workspace = %v, gone %v; want refused and kept", err, gone(held))
|
||||
}
|
||||
done()
|
||||
if err := w.Release("abc2"); err != nil || !gone(held) {
|
||||
t.Errorf("Release after the hold ended = %v, gone %v", err, gone(held))
|
||||
}
|
||||
if err := w.Release("../etc"); err == nil {
|
||||
t.Errorf("Release of ../etc answered nil")
|
||||
}
|
||||
}
|
||||
|
||||
// 1839/AC3: the sweep removes a workspace no step holds whose last use is past the bound, and keeps a held one and a
|
||||
// recent one; a hold ending counts as a use.
|
||||
func TestSweepRemovesUnheldWorkspacesPastTheBound(t *testing.T) {
|
||||
w := &Workspaces{Dir: t.TempDir()}
|
||||
old := makeWorkspace(t, w, "old1", 3*time.Hour)
|
||||
heldOld := makeWorkspace(t, w, "held1", 3*time.Hour)
|
||||
recent := makeWorkspace(t, w, "new1", time.Minute)
|
||||
if err := os.WriteFile(filepath.Join(w.Dir, "not-a-workspace.txt"), nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
done := w.Hold("held1")
|
||||
removed := w.Sweep(time.Now(), 2*time.Hour)
|
||||
if !gone(old) || gone(heldOld) || gone(recent) {
|
||||
t.Errorf("after the sweep: old gone %v, held kept %v, recent kept %v", gone(old), !gone(heldOld), !gone(recent))
|
||||
}
|
||||
if len(removed) != 1 || removed[0] != "old1" {
|
||||
t.Errorf("Sweep answered %v, want [old1]", removed)
|
||||
}
|
||||
done()
|
||||
if w.Sweep(time.Now(), 2*time.Hour); gone(heldOld) {
|
||||
t.Errorf("a workspace whose hold just ended was swept: its end is a use")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package stdio
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"sync"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Unproven is the caller of a call the runtime named no caller for: one that arrived on the subject that names
|
||||
// none, or from a runtime that names none (novox/hq issue 365). Said as a word, never left empty, so a record
|
||||
// shows it was not proven.
|
||||
const Unproven = "unproven"
|
||||
|
||||
// MetaCaller is where in a `tools/call`'s `_meta` the runtime names the caller: the bus user that published the
|
||||
// call, which the runtime takes from the subject the call arrived on — that subject's last token, which the
|
||||
// bus lets each user publish with its own name alone (novox/hq issue 365, by ADR 0259 §3's precedent). The
|
||||
// runtime writes `_meta`; a caller writes only `arguments`, so no caller can name another.
|
||||
const MetaCaller = "mesh/caller"
|
||||
|
||||
// busUser is the shape of a bus user's name: `<node>.<module>`, `person.<name>`, `controller` — names whose
|
||||
// parts are safe in a subject. Anything else the runtime named is not a caller and reads as Unproven.
|
||||
var busUser = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`)
|
||||
|
||||
// calls holds the caller of every call while its tool runs, by the arguments map that call was handed.
|
||||
var calls sync.Map // unsafe.Pointer → string
|
||||
|
||||
func argsKey(args map[string]any) unsafe.Pointer {
|
||||
return reflect.ValueOf(args).UnsafePointer()
|
||||
}
|
||||
|
||||
// CallerOf is who called the tool that was handed args, as the runtime proved it: the bus user that
|
||||
// published the call, or Unproven. Read from the runtime, never from the arguments' contents — a `caller`
|
||||
// argument names nobody. Valid while the tool runs; arguments no call was handed have no caller.
|
||||
func CallerOf(args map[string]any) string {
|
||||
if args == nil {
|
||||
return Unproven
|
||||
}
|
||||
if c, ok := calls.Load(argsKey(args)); ok {
|
||||
return c.(string)
|
||||
}
|
||||
return Unproven
|
||||
}
|
||||
|
||||
// callerIn is the caller a `tools/call`'s `_meta` names, or Unproven.
|
||||
func callerIn(meta map[string]any) string {
|
||||
if s, ok := meta[MetaCaller].(string); ok && busUser.MatchString(s) {
|
||||
return s
|
||||
}
|
||||
return Unproven
|
||||
}
|
||||
|
||||
// runAs runs a tool with the caller held for its arguments for as long as it runs.
|
||||
func runAs(t Tool, caller string, args map[string]any) (any, error) {
|
||||
key := argsKey(args)
|
||||
calls.Store(key, caller)
|
||||
defer calls.Delete(key)
|
||||
return t.Run(args)
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package stdio
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// callWith serves one tool that answers the caller CallerOf reads from its arguments, sends it one tools/call
|
||||
// with the params given, and returns the caller it answered.
|
||||
func callWith(t *testing.T, params map[string]any) string {
|
||||
t.Helper()
|
||||
inR, inW := io.Pipe()
|
||||
outR, outW := io.Pipe()
|
||||
go func() {
|
||||
_ = ServeOn("tracker", []Tool{{Name: "who", Run: func(args map[string]any) (any, error) {
|
||||
return map[string]any{"caller": CallerOf(args)}, nil
|
||||
}}}, inR, outW)
|
||||
}()
|
||||
defer inW.Close()
|
||||
b, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": params})
|
||||
_, _ = inW.Write(append(b, '\n'))
|
||||
lines := bufio.NewScanner(outR)
|
||||
done := make(chan string, 1)
|
||||
go func() {
|
||||
if lines.Scan() {
|
||||
var m struct {
|
||||
Result struct {
|
||||
Content []struct {
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
} `json:"result"`
|
||||
}
|
||||
_ = json.Unmarshal(lines.Bytes(), &m)
|
||||
var answer struct {
|
||||
Caller string `json:"caller"`
|
||||
}
|
||||
if len(m.Result.Content) > 0 {
|
||||
_ = json.Unmarshal([]byte(m.Result.Content[0].Text), &answer)
|
||||
}
|
||||
done <- answer.Caller
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case got := <-done:
|
||||
return got
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the bundle did not answer the call")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// **A tool reads its caller from the runtime, never from the call's arguments** (novox/hq issue 365): the
|
||||
// runtime names the caller it took from the subject the call arrived on in the call's `_meta`, and CallerOf
|
||||
// hands it to the tool by the arguments the tool was given.
|
||||
func TestAToolReadsTheCallerTheRuntimeNamed(t *testing.T) {
|
||||
got := callWith(t, map[string]any{"name": "who", "arguments": map[string]any{"title": "x"},
|
||||
"_meta": map[string]any{MetaCaller: "shanks.node-tools"}})
|
||||
if got != "shanks.node-tools" {
|
||||
t.Fatalf("the tool read caller %q, want the one the runtime named", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A caller named in the arguments is not the caller: an argument is the caller's to write.
|
||||
func TestACallerInTheArgumentsIsNeverTheCaller(t *testing.T) {
|
||||
got := callWith(t, map[string]any{"name": "who", "arguments": map[string]any{"caller": "novox.node-tools",
|
||||
"_meta": map[string]any{MetaCaller: "novox.node-tools"}, MetaCaller: "novox.node-tools"}})
|
||||
if got != Unproven {
|
||||
t.Fatalf("a caller the arguments claimed was read as %q, want %q", got, Unproven)
|
||||
}
|
||||
got = callWith(t, map[string]any{"name": "who", "arguments": map[string]any{"caller": "novox.node-tools"},
|
||||
"_meta": map[string]any{MetaCaller: "shanks.node-tools"}})
|
||||
if got != "shanks.node-tools" {
|
||||
t.Fatalf("with a caller in the arguments the tool read %q, want the runtime's", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A runtime that names no caller — one from before issue 365, or a call on the subject that names none — is
|
||||
// said as Unproven, explicitly; a caller that is not a bus user's name is Unproven too.
|
||||
func TestACallTheRuntimeNamedNoCallerForIsUnproven(t *testing.T) {
|
||||
for _, params := range []map[string]any{
|
||||
{"name": "who", "arguments": map[string]any{}},
|
||||
{"name": "who"},
|
||||
{"name": "who", "arguments": map[string]any{}, "_meta": map[string]any{}},
|
||||
{"name": "who", "arguments": map[string]any{}, "_meta": map[string]any{MetaCaller: ""}},
|
||||
{"name": "who", "arguments": map[string]any{}, "_meta": map[string]any{MetaCaller: "a b"}},
|
||||
{"name": "who", "arguments": map[string]any{}, "_meta": map[string]any{MetaCaller: 7}},
|
||||
} {
|
||||
if got := callWith(t, params); got != Unproven {
|
||||
t.Errorf("%v: the tool read caller %q, want %q", params, got, Unproven)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Arguments the runtime never handed a tool have no caller.
|
||||
func TestArgumentsOfNoCallHaveNoCaller(t *testing.T) {
|
||||
if got := CallerOf(map[string]any{"x": 1}); got != Unproven {
|
||||
t.Fatalf("CallerOf of a map no call was given answered %q", got)
|
||||
}
|
||||
if got := CallerOf(nil); got != Unproven {
|
||||
t.Fatalf("CallerOf(nil) answered %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq issue 213 (ADR 0188 §1, §3): a module's own Go service is a bundle the host runs as a
|
||||
// process, not an image. Each test holds one thing that had to change in the composer for the
|
||||
// controller to be declared that way.
|
||||
|
||||
const aServiceDigest = "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||
|
||||
// aServiceModule is the controller's shape in miniature: it answers tools of its own, its code is a
|
||||
// Go bundle a process runs as an account it declares, its secrets belong to that account, it
|
||||
// prepares its state, and its process replaces the container it used to run as.
|
||||
func aServiceModule(t *testing.T) Manifest {
|
||||
t.Helper()
|
||||
raw := `{
|
||||
"module": "svc", "version": "1", "tools": ["status"], "prepares": true,
|
||||
"own-secrets": {"store": "${dir:state}/store"},
|
||||
"secrets-owner": "svc",
|
||||
"resources": [
|
||||
{"id": "state", "type": "directory", "mode": "0700", "place": "mesh", "owner": "svc"},
|
||||
{"id": "service", "type": "process", "name": "svc", "artifact": "code",
|
||||
"run": ["./svc", "serve"], "user": "svc", "replaces": ["server"],
|
||||
"env": {"SVC_STORE_FILE": "${dir:state}/store", "SVC_STORE_PORT": "${seat:mesh-store:5432}"}},
|
||||
{"id": "account", "type": "user", "name": "svc", "shell": "/usr/bin/nologin", "home": "/var/lib/svc"}
|
||||
],
|
||||
"build": {"artifacts": [{"name": "code", "kind": "bundle", "language": "go", "system": "arch",
|
||||
"from": "cmd/svc", "binary": "svc"}]}
|
||||
}`
|
||||
m, err := ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatalf("the service's manifest is refused: %v", err)
|
||||
}
|
||||
resolved, err := m.Resolve([]Built{{Name: "code", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "svc/code@" + aServiceDigest, Digest: aServiceDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func composeTheService(t *testing.T, with Rendering) []map[string]any {
|
||||
t.Helper()
|
||||
with.Needed = map[string]map[string]string{"svc": {"store": "sealed-store"}}
|
||||
with.ArtifactStore = "anchor.internal:5100"
|
||||
out, err := Resolution{Node: "anchor", Modules: []Manifest{aServiceModule(t)}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the service does not compose: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func indexOf(out []map[string]any, id string) int {
|
||||
for i, r := range out {
|
||||
if r["id"] == id {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// A module that declares tools has every bundle served by the node's runtime unless it says
|
||||
// otherwise — and the controller declares the verbs it answers as tools. Its service bundle is run
|
||||
// by its own process; launched a second time by the runtime it would be a second controller
|
||||
// pretending to be an MCP server.
|
||||
func TestABundleItsOwnProcessRunsIsNotServedByTheRuntime(t *testing.T) {
|
||||
m := aServiceModule(t)
|
||||
if len(m.Bundles) != 1 {
|
||||
t.Fatalf("the service's bundle was not kept: %+v", m.Bundles)
|
||||
}
|
||||
if loads := m.Bundles[0].Loads; len(loads) != 0 {
|
||||
t.Fatalf("the runtime would launch the service's own bundle as tools: %v", loads)
|
||||
}
|
||||
// And a bundle no resource runs still is served, as a module declaring tools always had it.
|
||||
tools := Manifest{Module: "t", Version: "1", Tools: []string{"x"},
|
||||
Build: &Build{Artifacts: []Artifact{{Name: "tools", Kind: ArtifactBundle, Language: "go",
|
||||
System: "arch", From: "cmd/t"}}}}
|
||||
resolved, err := tools.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle,
|
||||
Reference: ArtifactStoreScheme + "t/tools@" + aServiceDigest, Digest: aServiceDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loads := resolved.Bundles[0].Loads; len(loads) != 1 || loads[0] != "t" {
|
||||
t.Fatalf("a tools bundle nothing runs is no longer served: %v", loads)
|
||||
}
|
||||
}
|
||||
|
||||
// The account is created before anything is given to it. Its secrets are mesh-computed and so
|
||||
// placed before the module's own resources; given to a user the machine did not have yet, they were
|
||||
// refused on the first apply and the process started without them.
|
||||
func TestAModulesAccountComesBeforeWhatBelongsToIt(t *testing.T) {
|
||||
out := composeTheService(t, Rendering{})
|
||||
account, secret := indexOf(out, "svc.account"), indexOf(out, "svc."+NeedID("store"))
|
||||
if account < 0 || secret < 0 {
|
||||
t.Fatalf("the account or the secret is missing: %v", out)
|
||||
}
|
||||
if account > secret {
|
||||
t.Fatalf("the secret owned by svc is written before svc exists: account at %d, secret at %d",
|
||||
account, secret)
|
||||
}
|
||||
if owner := out[secret]["owner"]; owner != "svc" {
|
||||
t.Errorf("the secret belongs to %v, not the account its process runs as", owner)
|
||||
}
|
||||
}
|
||||
|
||||
// The process is the module's program; its preparation is the same program asked to prepare, as a
|
||||
// step before it — with the same account and environment, and handing nothing over.
|
||||
func TestAProcessIsPreparedByItsOwnProgram(t *testing.T) {
|
||||
out := composeTheService(t, Rendering{})
|
||||
step, process := indexOf(out, "svc.service-prepare"), indexOf(out, "svc.service")
|
||||
if step < 0 || process < 0 || step > process {
|
||||
t.Fatalf("the preparation is not a step before the process (%d, %d): %v", step, process, out)
|
||||
}
|
||||
s := out[step]
|
||||
if s["type"] != "process" || s["run-once"] != true || s["name"] != "svc-prepare" {
|
||||
t.Errorf("the preparation is not a run-once process: %v", s)
|
||||
}
|
||||
if run, _ := json.Marshal(s["run"]); string(run) != `["./svc","prepare"]` {
|
||||
t.Errorf("the preparation runs %s", run)
|
||||
}
|
||||
if s["user"] != "svc" || s["source"] != out[process]["source"] {
|
||||
t.Errorf("the preparation does not run the same bundle as the same account: %v", s)
|
||||
}
|
||||
if env, _ := s["env"].(map[string]any); env["SVC_STORE_FILE"] == nil {
|
||||
t.Errorf("the preparation is not given the process's environment: %v", s["env"])
|
||||
}
|
||||
if _, has := s["replaces"]; has {
|
||||
t.Errorf("the preparation would hand over what the process replaces: %v", s)
|
||||
}
|
||||
if _, has := s["args"]; has {
|
||||
t.Errorf("the preparation carries a container's args: %v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// What the process replaces is named as the host recorded it, `<module>.<id>`; unprefixed, the host
|
||||
// matches nothing and removes the container first, as before.
|
||||
func TestWhatAProcessReplacesIsNamedAsTheHostRecordedIt(t *testing.T) {
|
||||
out := composeTheService(t, Rendering{})
|
||||
p := out[indexOf(out, "svc.service")]
|
||||
if got, _ := json.Marshal(p["replaces"]); string(got) != `["svc.server"]` {
|
||||
t.Fatalf("the process replaces %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatReplacesMayNameIsRefusedNearItsAuthor(t *testing.T) {
|
||||
for what, resource := range map[string]string{
|
||||
"a container": `{"id":"c","type":"container","name":"c","image":"x@` + aServiceDigest + `","replaces":["old"]}`,
|
||||
"a step": `{"id":"p","type":"process","name":"p","run":["./p"],"run-once":true,"replaces":["old"]}`,
|
||||
"something declared": `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["p"]}`,
|
||||
"another module's": `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["other.old"]}`,
|
||||
"not a list": `{"id":"p","type":"process","name":"p","run":["./p"],"replaces":"old"}`,
|
||||
} {
|
||||
raw := `{"module":"m","version":"1","resources":[` + resource + `]}`
|
||||
if _, err := ParseManifest([]byte(raw)); err == nil || !strings.Contains(err.Error(), "replace") {
|
||||
t.Errorf("replaces on %s was accepted: %v", what, err)
|
||||
}
|
||||
}
|
||||
ok := `{"module":"m","version":"1","resources":[{"id":"p","type":"process","name":"p","run":["./p"],"replaces":["old"]}]}`
|
||||
if _, err := ParseManifest([]byte(ok)); err != nil {
|
||||
t.Errorf("a process replacing what its module no longer declares was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The fault 04-ISSUES/036 records: a media stack is several modules — a library server, the
|
||||
// acquisition managers, a download client — that must share directories on one machine. Written
|
||||
// as owned `directory` resources, two of them on one node are refused as rivalrous owners, which
|
||||
// is right in general and wrong here: sharing those directories is the whole point of the stack.
|
||||
//
|
||||
// Declared as accesses to an operator-owned path (novox/hq ADR 0051), they co-resolve. An access
|
||||
// is not a resource and never enters the duplicate-owner map, so this is the exact case the
|
||||
// resolver refuses above when the same path is owned — and does not, when it is merely accessed.
|
||||
func TestTwoModulesAccessingOnePathCoResolve(t *testing.T) {
|
||||
a := mod("a", nil, nil, nil)
|
||||
a.Accesses = []Access{{Path: "/services/media/downloads", Mode: AccessReadWrite}}
|
||||
b := mod("b", nil, nil, nil)
|
||||
b.Accesses = []Access{{Path: "/services/media/downloads", Mode: AccessRead}}
|
||||
|
||||
got, err := Resolve(shelf(a, b), []string{"a", "b"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatalf("two modules accessing one operator-owned path were refused: %v", err)
|
||||
}
|
||||
|
||||
// And each accessing module's grant reaches the machine as its own `access` resource, so the
|
||||
// host can find the path present or refuse — both naming the one operator-owned path.
|
||||
var accesses int
|
||||
for _, r := range mustDeclare(t, got) {
|
||||
if r["type"] == "access" && r["path"] == "/services/media/downloads" {
|
||||
accesses++
|
||||
}
|
||||
}
|
||||
if accesses != 2 {
|
||||
t.Errorf("expected both modules' accesses in the declaration, got %d", accesses)
|
||||
}
|
||||
}
|
||||
|
||||
// The mirror of the case above, so the sharing vocabulary does not weaken the rule it sits beside:
|
||||
// two modules OWNING one path is still refused, exactly as before accesses existed.
|
||||
func TestTwoModulesOwningOnePathAreStillRefused(t *testing.T) {
|
||||
a := mod("a", nil, nil, nil)
|
||||
a.Resources = []map[string]any{
|
||||
{"id": "lib", "type": "directory", "path": "/services/media/movies", "mode": "0755"}}
|
||||
b := mod("b", nil, nil, nil)
|
||||
b.Resources = []map[string]any{
|
||||
{"id": "lib", "type": "directory", "path": "/services/media/movies", "mode": "0755"}}
|
||||
|
||||
_, err := Resolve(shelf(a, b), []string{"a", "b"}, workstation(), World{})
|
||||
if err == nil {
|
||||
t.Fatal("two modules owning the same directory were both assigned")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "/services/media/movies") {
|
||||
t.Errorf("the refusal does not name the path: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Shared data is the operator's, owned by no module (novox/hq ADR 0051). A module owning the path
|
||||
// another module was told to expect the operator to provide would create and chown it — and the
|
||||
// two intentions cannot both hold, so it is refused by name. This is what keeps the distinction
|
||||
// machine-checkable rather than a convention nobody enforces.
|
||||
func TestAModuleMayNotOwnWhatAnotherAccesses(t *testing.T) {
|
||||
owns := mod("owns", nil, nil, nil)
|
||||
owns.Resources = []map[string]any{
|
||||
{"id": "lib", "type": "directory", "path": "/services/media/movies", "mode": "0755"}}
|
||||
uses := mod("uses", nil, nil, nil)
|
||||
uses.Accesses = []Access{{Path: "/services/media/movies", Mode: AccessRead}}
|
||||
|
||||
_, err := Resolve(shelf(owns, uses), []string{"owns", "uses"}, workstation(), World{})
|
||||
if err == nil {
|
||||
t.Fatal("a module owning a path another accesses was allowed")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "/services/media/movies") ||
|
||||
!strings.Contains(err.Error(), "the operator's") {
|
||||
t.Errorf("the refusal does not explain the ownership conflict: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest states an access's extent, the way a listening port states its source. An absolute
|
||||
// path and a mode of read or read-write; anything else is refused rather than acted on wrongly.
|
||||
func TestAnAccessIsValidated(t *testing.T) {
|
||||
good := []byte(`{"module":"m","accesses":[{"path":"/services/media","mode":"read-write"}]}`)
|
||||
if _, err := ParseManifest(good); err != nil {
|
||||
t.Fatalf("a valid access was refused: %v", err)
|
||||
}
|
||||
|
||||
// Mode is optional and narrows to read — the safe default, because the danger with an access
|
||||
// is being given more than was meant, not less.
|
||||
bare := []byte(`{"module":"m","accesses":[{"path":"/services/media"}]}`)
|
||||
m, err := ParseManifest(bare)
|
||||
if err != nil {
|
||||
t.Fatalf("an access with no mode was refused: %v", err)
|
||||
}
|
||||
if m.Accesses[0].At() != AccessRead {
|
||||
t.Errorf("an access with no mode is %q, want %q", m.Accesses[0].At(), AccessRead)
|
||||
}
|
||||
|
||||
relative := []byte(`{"module":"m","accesses":[{"path":"services/media","mode":"read"}]}`)
|
||||
if _, err := ParseManifest(relative); err == nil {
|
||||
t.Error("a relative access path was accepted")
|
||||
}
|
||||
|
||||
wrong := []byte(`{"module":"m","accesses":[{"path":"/services/media","mode":"append"}]}`)
|
||||
if _, err := ParseManifest(wrong); err == nil {
|
||||
t.Error("an access with an unknown mode was accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What an adopted node is declared in place of a filter (novox/hq ADR 0100).
|
||||
//
|
||||
// On an adopted node the firewall found on the machine stays in force: the mesh loads no table
|
||||
// that drops by default or holds an accept. What the mesh needs reachable is declared as
|
||||
// openings, which the host converges through the found firewall in its own terms; and the mesh
|
||||
// guards its own foundation ports itself, in a table that only refuses.
|
||||
|
||||
// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is
|
||||
// never a module's, so none of it is ever held as found.
|
||||
const AdoptionPrefix = "adoption."
|
||||
|
||||
// Where an opening admits from, on the wire.
|
||||
const (
|
||||
OpeningFromEverywhere = "everywhere"
|
||||
OpeningFromMesh = "mesh"
|
||||
)
|
||||
|
||||
// The two paths a packet reaches a port by: received by the machine, or forwarded to a
|
||||
// container that publishes it.
|
||||
const (
|
||||
PathIncoming = "incoming"
|
||||
PathForwarded = "forwarded"
|
||||
)
|
||||
|
||||
// Guard resources: the refusal-only table, the unit that loads it, and that unit running.
|
||||
const (
|
||||
GuardPath = "/etc/mesh/guard.nft"
|
||||
GuardUnit = "mesh-guard.service"
|
||||
// GuardUnitPath is where the unit is written.
|
||||
GuardUnitPath = "/etc/systemd/system/" + GuardUnit
|
||||
)
|
||||
|
||||
// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer
|
||||
// raises the same three on an adopted genesis, so the first push finds them already there.
|
||||
func GuardID() string { return AdoptionPrefix + "guard" }
|
||||
func GuardUnitID() string { return AdoptionPrefix + "guard-unit" }
|
||||
func GuardRunningID() string { return AdoptionPrefix + "guard-running" }
|
||||
|
||||
// GuardPackageID is the tool that loads the guard, declared first: a node joining adopted has
|
||||
// no filter module and may have no nft at all, and a table nothing can load guards nothing.
|
||||
func GuardPackageID() string { return AdoptionPrefix + "guard-package" }
|
||||
|
||||
// GuardPackage is the package that carries nft.
|
||||
const GuardPackage = "nftables"
|
||||
|
||||
// OpeningID is an opening's resource identity: its protocol, port and path say what it is.
|
||||
func OpeningID(protocol string, port int, path string) string {
|
||||
return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path)
|
||||
}
|
||||
|
||||
// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the
|
||||
// filter it would load were the node converged, each from where that filter would admit it.
|
||||
//
|
||||
// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure —
|
||||
// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine
|
||||
// only opens nothing. `published` maps a machine port a container publishes to the container's
|
||||
// port: a published port is forwarded, not received, so its opening names the forwarded path and
|
||||
// the port the packet is forwarded to.
|
||||
func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any {
|
||||
type key struct {
|
||||
protocol string
|
||||
port int
|
||||
}
|
||||
from := map[key]string{}
|
||||
var order []key
|
||||
widen := func(k key, f string) {
|
||||
was, seen := from[k]
|
||||
if !seen {
|
||||
order = append(order, k)
|
||||
}
|
||||
if !seen || was != OpeningFromEverywhere {
|
||||
from[k] = f
|
||||
}
|
||||
}
|
||||
for _, rule := range rules {
|
||||
switch rule.From {
|
||||
case FromEverywhere:
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere)
|
||||
case FromMesh:
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromMesh)
|
||||
case FromLocalNetwork:
|
||||
// The private network's part only (novox/hq ADR 0340). An adopted node's found firewall
|
||||
// stays, and it is what admits the machine's local network; an opening has no word for a
|
||||
// local network, and widening it to everywhere would open the port to the internet.
|
||||
widen(key{rule.Protocol, rule.Port}, OpeningFromMesh)
|
||||
}
|
||||
}
|
||||
for _, port := range foundation {
|
||||
widen(key{"tcp", port}, OpeningFromEverywhere)
|
||||
}
|
||||
sort.Slice(order, func(a, b int) bool {
|
||||
if order[a].port != order[b].port {
|
||||
return order[a].port < order[b].port
|
||||
}
|
||||
return order[a].protocol < order[b].protocol
|
||||
})
|
||||
out := make([]map[string]any, 0, len(order))
|
||||
for _, k := range order {
|
||||
opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol,
|
||||
"from": from[k]}
|
||||
if to, forwarded := published[k.protocol][k.port]; forwarded {
|
||||
opening["id"] = OpeningID(k.protocol, k.port, PathForwarded)
|
||||
opening["path"] = PathForwarded
|
||||
opening["to"] = to
|
||||
} else {
|
||||
opening["id"] = OpeningID(k.protocol, k.port, PathIncoming)
|
||||
opening["path"] = PathIncoming
|
||||
}
|
||||
out = append(out, opening)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Published is every port the given containers publish on the machine, by protocol and machine
|
||||
// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off
|
||||
// the machine reaches it, forwarded or not.
|
||||
func Published(resources []map[string]any) map[string]map[int]int {
|
||||
out := map[string]map[int]int{}
|
||||
for _, r := range resources {
|
||||
if fmt.Sprint(r["type"]) != "container" {
|
||||
continue
|
||||
}
|
||||
listed, _ := r["ports"].([]any)
|
||||
for _, entry := range listed {
|
||||
written := strings.TrimSpace(fmt.Sprint(entry))
|
||||
protocol := "tcp"
|
||||
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||
protocol = written[cut+1:]
|
||||
}
|
||||
// Indexed from the end, so an IPv6 address's own colons never shift the ports.
|
||||
outer, inner, address, ok := mapping(written)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch strings.Trim(address, "[]") {
|
||||
case "127.0.0.1", "localhost", "::1":
|
||||
continue
|
||||
}
|
||||
if out[protocol] == nil {
|
||||
out[protocol] = map[int]int{}
|
||||
}
|
||||
out[protocol][outer] = inner
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AsGuard renders the mesh's refusal-only table for the given machine ports.
|
||||
//
|
||||
// It passes everything by default and holds nothing but a refusal, so it cannot close anything
|
||||
// the machine serves; and it is the mesh's own table, so the found firewall reloading does not
|
||||
// touch it. It refuses only packets addressed to this machine, and the ports except from the
|
||||
// machine itself — its loopback and the container runtime's own networks — and from the private
|
||||
// network, known by the interface a packet arrives on and never by its source address. At
|
||||
// prerouting, ahead of the runtime's destination translation, so it matches the port the packet
|
||||
// was sent to; in the inet family, so both address families.
|
||||
//
|
||||
// **The machine's own interfaces are named, where the derived filter names address ranges.** The
|
||||
// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo,
|
||||
// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is
|
||||
// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name)
|
||||
// would have its containers' traffic to a guarded port refused, which reads as the port being
|
||||
// down. Names rather than addresses is deliberate: a source address can be claimed by whoever
|
||||
// sends the packet, and this table exists to refuse what the found firewall never sees. Widening
|
||||
// it means adding names here and in the installer's copy together, which the golden test holds to
|
||||
// one text.
|
||||
//
|
||||
// The same text the installer raises on an adopted genesis; a test holds both to it.
|
||||
func AsGuard(ports []int) string {
|
||||
sorted := append([]int{}, ports...)
|
||||
sort.Ints(sorted)
|
||||
listed := make([]string, len(sorted))
|
||||
for i, p := range sorted {
|
||||
listed[i] = strconv.Itoa(p)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("table inet mesh_guard {}\n")
|
||||
b.WriteString("delete table inet mesh_guard\n")
|
||||
b.WriteString("table inet mesh_guard {\n")
|
||||
b.WriteString("\tchain prerouting {\n")
|
||||
b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n")
|
||||
// Only packets addressed to this machine: traffic it routes for others — a predecessor's hub,
|
||||
// say — is never the guard's business (novox/hq ADR 0103).
|
||||
fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" "+
|
||||
"iifname != \"br-*\" iifname != \"mesh0\" tcp dport { %s } drop\n",
|
||||
strings.Join(listed, ", "))
|
||||
b.WriteString("\t}\n")
|
||||
b.WriteString("}\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never
|
||||
// a flush, which would take the container runtime's rules and the found firewall with it.
|
||||
func GuardUnitText() string {
|
||||
return "[Unit]\n" +
|
||||
"Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" +
|
||||
// Early, before the network is up, and without the default dependencies that would
|
||||
// order it after the network; stopped at shutdown like any unit.
|
||||
"DefaultDependencies=no\n" +
|
||||
"Wants=network-pre.target\n" +
|
||||
"Before=network-pre.target shutdown.target\n" +
|
||||
"Conflicts=shutdown.target\n" +
|
||||
"\n" +
|
||||
"[Service]\n" +
|
||||
"Type=oneshot\n" +
|
||||
"RemainAfterExit=yes\n" +
|
||||
"ExecStart=nft -f " + GuardPath + "\n" +
|
||||
"ExecReload=nft -f " + GuardPath + "\n" +
|
||||
"ExecStop=nft delete table inet mesh_guard\n" +
|
||||
"\n" +
|
||||
"[Install]\n" +
|
||||
"WantedBy=multi-user.target\n"
|
||||
}
|
||||
|
||||
// GuardResources are the guard as four resources of the existing kinds: the tool that loads it,
|
||||
// the table, the unit, and the unit running — reloaded when the table changes, so the new table
|
||||
// replaces the old in one `nft -f` through the unit's ExecReload with no moment unguarded, and
|
||||
// restarted only when the unit itself changes. Nothing when there is nothing to guard: an empty
|
||||
// set is not a table nft loads.
|
||||
func GuardResources(ports []int) []map[string]any {
|
||||
if len(ports) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []map[string]any{
|
||||
{"id": GuardPackageID(), "type": "package", "package": GuardPackage},
|
||||
{"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports),
|
||||
"mode": "0644"},
|
||||
{"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(),
|
||||
"mode": "0644"},
|
||||
{"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running",
|
||||
"boot": "enabled", "restart-on": []any{GuardUnitID()}, "reload-on": []any{GuardID()}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,683 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0100: on an adopted node the found firewall stays in force. The mesh declares
|
||||
// openings where it would have loaded a filter, and guards its own ports in a table that only
|
||||
// refuses.
|
||||
|
||||
// hub is the private network's generator on the hub: it opens the hub's port from anywhere.
|
||||
type hub struct{}
|
||||
|
||||
func (hub) Resources(string) ([]map[string]any, bool, error) {
|
||||
return []map[string]any{{"id": "config", "type": "file", "path": "/etc/wireguard/mesh0.conf",
|
||||
"content": "[Interface]\n"}}, true, nil
|
||||
}
|
||||
func (hub) Listens(string) ([]Listening, error) {
|
||||
return []Listening{{Port: 51820, Protocol: "udp", From: FromEverywhere}}, nil
|
||||
}
|
||||
|
||||
// anAdoptedAnchor is the control-node's set: the store, the bus, the registry, the private network,
|
||||
// a served module and the filter module.
|
||||
func anAdoptedAnchor() Resolution {
|
||||
return Resolution{Node: "anchor", Modules: []Manifest{
|
||||
{Module: "network", Computed: "overlay"},
|
||||
{Module: "postgres", Guards: []int{5432},
|
||||
Listens: []Listening{{Port: 5432, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}}}},
|
||||
{Module: "lavinmq", Guards: []int{15672},
|
||||
Listens: []Listening{{Port: 5671, From: FromMesh}, {Port: 5672, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}}}},
|
||||
{Module: "distribution",
|
||||
Listens: []Listening{{Port: 5000, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "store", "type": "container", "name": "registry",
|
||||
"ports": []any{"5000"}}}},
|
||||
{Module: "hello-web",
|
||||
Listens: []Listening{{Port: 8080, From: FromEverywhere}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "hello-web",
|
||||
"ports": []any{"8080"}}}},
|
||||
{Module: "helper", Listens: []Listening{{Port: 9000, From: FromMachine}}},
|
||||
{Module: "nftables", Filtering: &Filtering{Into: "/etc/nftables.conf"},
|
||||
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
||||
"state": "running", "restart-on": []any{"filtering"}}}},
|
||||
}}
|
||||
}
|
||||
|
||||
func anchorRendering(adopted bool) Rendering {
|
||||
return Rendering{
|
||||
Generators: map[string]Generator{"overlay": hub{}},
|
||||
Ports: map[string]map[int]int{"distribution": {5000: 5000}, "hello-web": {8080: 20001}},
|
||||
Settings: SettingsBy{"distribution": {{From: "node anchor",
|
||||
Placement: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}},
|
||||
Mesh: []string{"10.42.0.1"},
|
||||
Foundation: []int{5671},
|
||||
// What the machine reported faces outside, which every rule in the filter is written
|
||||
// around (novox/hq ADR 0140).
|
||||
OutwardLinks: []string{"eth0"},
|
||||
TunnelInterface: "mesh0",
|
||||
Adopted: adopted,
|
||||
// Genesis takes the foundation's modules.
|
||||
Taken: map[string]bool{"postgres": true, "lavinmq": true},
|
||||
}
|
||||
}
|
||||
|
||||
func byID(resources []map[string]any) map[string]map[string]any {
|
||||
out := map[string]map[string]any{}
|
||||
for _, r := range resources {
|
||||
out[r["id"].(string)] = r
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnAdoptedNodeIsDeclaredOpeningsFromTheSameInputsAsTheFilter(t *testing.T) {
|
||||
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
want := map[string]map[string]any{
|
||||
// The hub's port, from anywhere, received.
|
||||
"adoption.opening-udp-51820-incoming": {"port": 51820, "protocol": "udp",
|
||||
"from": "everywhere", "path": "incoming"},
|
||||
// The store's port from the private network only, and forwarded: a container publishes it.
|
||||
"adoption.opening-tcp-5432-forwarded": {"port": 5432, "protocol": "tcp", "from": "mesh",
|
||||
"path": "forwarded", "to": 5432},
|
||||
// The bus from anywhere: a node enrols over it before it has a private address.
|
||||
"adoption.opening-tcp-5671-forwarded": {"port": 5671, "protocol": "tcp",
|
||||
"from": "everywhere", "path": "forwarded", "to": 5671},
|
||||
"adoption.opening-tcp-5672-forwarded": {"port": 5672, "protocol": "tcp", "from": "mesh",
|
||||
"path": "forwarded", "to": 5672},
|
||||
// The registry from anywhere, by its node's exposure setting.
|
||||
"adoption.opening-tcp-5000-forwarded": {"port": 5000, "protocol": "tcp",
|
||||
"from": "everywhere", "path": "forwarded", "to": 5000},
|
||||
// A published port names the machine port and the container port it is forwarded to.
|
||||
"adoption.opening-tcp-20001-forwarded": {"port": 20001, "protocol": "tcp",
|
||||
"from": "everywhere", "path": "forwarded", "to": 8080},
|
||||
}
|
||||
for id, fields := range want {
|
||||
opening, ok := got[id]
|
||||
if !ok {
|
||||
t.Errorf("no %s among %v", id, keys(got))
|
||||
continue
|
||||
}
|
||||
if opening["type"] != "opening" {
|
||||
t.Errorf("%s is a %v", id, opening["type"])
|
||||
}
|
||||
for k, v := range fields {
|
||||
if opening[k] != v {
|
||||
t.Errorf("%s: %s is %v, want %v", id, k, opening[k], v)
|
||||
}
|
||||
}
|
||||
}
|
||||
for id := range got {
|
||||
if strings.HasPrefix(id, "adoption.opening-") && want[id] == nil {
|
||||
t.Errorf("an opening nothing asked for: %s", id)
|
||||
}
|
||||
}
|
||||
// A port for this machine only opens nothing, and the management port is not opened at all.
|
||||
for id := range got {
|
||||
if strings.Contains(id, "-9000-") || strings.Contains(id, "-15672-") {
|
||||
t.Errorf("%s is opened", id)
|
||||
}
|
||||
}
|
||||
|
||||
// And openings come first, in the order the machine applies them.
|
||||
if !strings.HasPrefix(composed.Resources[0]["id"].(string), "adoption.opening-") {
|
||||
t.Errorf("openings are not first: %v", composed.Resources[0]["id"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) {
|
||||
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range composed.Resources {
|
||||
if r["path"] == "/etc/nftables.conf" || strings.HasPrefix(r["id"].(string), "nftables.") {
|
||||
t.Fatalf("an adopted node is declared the filter module's %v", r["id"])
|
||||
}
|
||||
if content, _ := r["content"].(string); strings.Contains(content, "policy drop") {
|
||||
t.Fatalf("an adopted node is declared a table that drops by default: %v", r["id"])
|
||||
}
|
||||
// Nothing but refusals: the only accept in the guard is its policy.
|
||||
if content, _ := r["content"].(string); r["id"] == GuardID() &&
|
||||
strings.Count(content, "accept") != 1 {
|
||||
t.Fatalf("the guard holds an accept:\n%s", content)
|
||||
}
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
guard := got[GuardID()]
|
||||
if guard == nil || got[GuardUnitID()] == nil || got[GuardRunningID()] == nil {
|
||||
t.Fatalf("no guard: %v", keys(got))
|
||||
}
|
||||
if guard["content"] != AsGuard([]int{5432, 5672, 15672}) {
|
||||
t.Fatalf("the guard does not guard the store, the broker and its management port:\n%s",
|
||||
guard["content"])
|
||||
}
|
||||
// The tool that loads it comes first, and the table after it: a node joining adopted has no
|
||||
// filter module and may have no nft.
|
||||
pkg, table := -1, -1
|
||||
for i, r := range composed.Resources {
|
||||
switch r["id"] {
|
||||
case GuardPackageID():
|
||||
pkg = i
|
||||
if r["type"] != "package" || r["package"] != "nftables" {
|
||||
t.Fatalf("the guard's package is %v", r)
|
||||
}
|
||||
case GuardID():
|
||||
table = i
|
||||
}
|
||||
}
|
||||
if pkg < 0 || pkg > table {
|
||||
t.Fatalf("nftables is not declared before the guard's table (%d, %d)", pkg, table)
|
||||
}
|
||||
// A changed table is reloaded — one `nft -f`, atomic — never restarted, which would delete the
|
||||
// table and leave the ports unguarded until it is loaded again. Only a changed unit restarts.
|
||||
if !reflect.DeepEqual(got[GuardRunningID()]["reload-on"], []any{GuardID()}) ||
|
||||
!reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardUnitID()}) {
|
||||
t.Fatalf("the guard is not reloaded on its table and restarted on its unit: %v",
|
||||
got[GuardRunningID()])
|
||||
}
|
||||
// Nothing of the mesh's own is anybody's to hold.
|
||||
for id, module := range composed.Owner {
|
||||
if strings.HasPrefix(id, AdoptionPrefix) {
|
||||
t.Fatalf("%s is owned by %s", id, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConvergedNodeIsDeclaredItsFilterAndNoOpenings(t *testing.T) {
|
||||
composed, err := anAdoptedAnchor().Compose(anchorRendering(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if got["nftables.filtering"] == nil || got["nftables.load"] == nil {
|
||||
t.Fatalf("a converged node lost its filter: %v", keys(got))
|
||||
}
|
||||
for id := range got {
|
||||
if strings.HasPrefix(id, AdoptionPrefix) {
|
||||
t.Fatalf("a converged node is declared %s", id)
|
||||
}
|
||||
}
|
||||
plain, err := anAdoptedAnchor().Declaration(anchorRendering(false))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := json.Marshal(plain)
|
||||
b, _ := json.Marshal(composed.Resources)
|
||||
if string(a) != string(b) {
|
||||
t.Fatal("Compose and Declaration disagree on a converged node")
|
||||
}
|
||||
}
|
||||
|
||||
// The table the installer raises and the controller declares, character for character.
|
||||
func TestTheGuardIsExactlyThisTable(t *testing.T) {
|
||||
const golden = `table inet mesh_guard {}
|
||||
delete table inet mesh_guard
|
||||
table inet mesh_guard {
|
||||
chain prerouting {
|
||||
type filter hook prerouting priority raw; policy accept;
|
||||
fib daddr type local iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop
|
||||
}
|
||||
}
|
||||
`
|
||||
if got := AsGuard([]int{15672, 5432}); got != golden {
|
||||
t.Fatalf("the guard changed:\n%s", got)
|
||||
}
|
||||
const unit = `[Unit]
|
||||
Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)
|
||||
DefaultDependencies=no
|
||||
Wants=network-pre.target
|
||||
Before=network-pre.target shutdown.target
|
||||
Conflicts=shutdown.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=nft -f /etc/mesh/guard.nft
|
||||
ExecReload=nft -f /etc/mesh/guard.nft
|
||||
ExecStop=nft delete table inet mesh_guard
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`
|
||||
if got := GuardUnitText(); got != unit {
|
||||
t.Fatalf("the guard's unit changed:\n%s", got)
|
||||
}
|
||||
if GuardResources(nil) != nil {
|
||||
t.Fatal("a guard with nothing to guard is an empty set nft refuses to load")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGuardedPortMustBeAPort(t *testing.T) {
|
||||
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[5432]}`)); err != nil {
|
||||
t.Fatalf("guards is refused: %v", err)
|
||||
}
|
||||
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[0]}`)); err == nil {
|
||||
t.Fatal("guarding port 0 was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func keys[V any](m map[string]V) []string {
|
||||
return sortedKeys(m)
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the foundation's ports are the node's. Given 5433 for the store, every place
|
||||
// that uses the port reads it from there: the container, the filter, the openings, the guard.
|
||||
func TestAGivenPortIsUsedEverywhereThePortIs(t *testing.T) {
|
||||
given := map[string]map[int]int{"postgres": {5432: 5433}, "lavinmq": {15672: 15673}}
|
||||
for _, adopted := range []bool{true, false} {
|
||||
with := anchorRendering(adopted)
|
||||
with.Given = given
|
||||
with.Ports["postgres"] = map[int]int{5432: 5433}
|
||||
composed, err := anAdoptedAnchor().Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if ports := got["postgres.server"]["ports"]; !reflect.DeepEqual(ports, []any{"5433:5432"}) {
|
||||
t.Fatalf("the store's container publishes %v", ports)
|
||||
}
|
||||
if ports := got["lavinmq.server"]["ports"]; !reflect.DeepEqual(ports,
|
||||
[]any{"5671:5671", "5672:5672", "127.0.0.1:15673:15672"}) {
|
||||
t.Fatalf("the broker's container publishes %v", ports)
|
||||
}
|
||||
if !adopted {
|
||||
filter, _ := got["nftables.filtering"]["content"].(string)
|
||||
if !strings.Contains(filter, "tcp dport 5433 accept") || strings.Contains(filter, "5432") {
|
||||
t.Fatalf("the filter does not use the given port:\n%s", filter)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if o := got["adoption.opening-tcp-5433-forwarded"]; o == nil || o["to"] != 5432 {
|
||||
t.Fatalf("no opening for the given port: %v", keys(got))
|
||||
}
|
||||
if guard := got[GuardID()]["content"]; guard != AsGuard([]int{5433, 5672, 15673}) {
|
||||
t.Fatalf("the guard does not guard the given ports:\n%s", guard)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) {
|
||||
store := anAdoptedAnchor().Modules[1]
|
||||
node := func(v any) []Layer {
|
||||
return []Layer{{From: "anchor", Placement: map[string]any{PortsSetting: v}}}
|
||||
}
|
||||
if got, err := GivenPorts(store, node(map[string]any{"5432": float64(5433)})); err != nil ||
|
||||
got[5432] != 5433 {
|
||||
t.Fatalf("a node's given port was not read: %v %v", got, err)
|
||||
}
|
||||
if _, err := GivenPorts(store, []Layer{{From: MeshWideLayer,
|
||||
Placement: map[string]any{PortsSetting: map[string]any{"5432": float64(5433)}}}}); err == nil {
|
||||
t.Fatal("a port given for the whole mesh was accepted")
|
||||
}
|
||||
if _, err := GivenPorts(store, node(map[string]any{"6000": float64(6001)})); err == nil {
|
||||
t.Fatal("a port the module neither listens on, publishes nor guards was given")
|
||||
}
|
||||
if _, err := GivenPorts(store, node(map[string]any{"5432": float64(70000)})); err == nil {
|
||||
t.Fatal("a machine port that is not a port was given")
|
||||
}
|
||||
if _, err := GivenPorts(store, node(map[string]any{"5432": float64(22)})); err == nil {
|
||||
t.Fatal("ssh's port was given")
|
||||
}
|
||||
broker := anAdoptedAnchor().Modules[2]
|
||||
if _, err := GivenPorts(broker, node(map[string]any{"5671": float64(5700),
|
||||
"5672": float64(5700)})); err == nil {
|
||||
t.Fatal("one machine port was given for two of the module's ports")
|
||||
}
|
||||
if stray := UnusedSettings(store, node(map[string]any{"5432": float64(5433)})); len(stray) != 0 {
|
||||
t.Fatalf("a given port is called stray: %v", stray)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the guard is derived, and from taken modules only — every machine port a taken
|
||||
// module publishes that the filter admits from the private network only, and the ports its
|
||||
// manifest guards. A module assigned but not taken is not guarded: its port may still be the
|
||||
// predecessor's.
|
||||
func TestTheGuardIsDerivedFromTakenModulesOnly(t *testing.T) {
|
||||
guardOf := func(with Rendering) string {
|
||||
t.Helper()
|
||||
composed, err := anAdoptedAnchor().Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, _ := byID(composed.Resources)[GuardID()]["content"].(string)
|
||||
return content
|
||||
}
|
||||
|
||||
// The broker taken, the store not: the broker's plain port follows from its listens (from
|
||||
// the mesh, published), its management port from its manifest; the store is not guarded, and
|
||||
// neither is the bus, which the mesh needs from everywhere.
|
||||
with := anchorRendering(true)
|
||||
with.Taken = map[string]bool{"lavinmq": true}
|
||||
if got := guardOf(with); got != AsGuard([]int{5672, 15672}) {
|
||||
t.Fatalf("the guard is not the taken broker's ports:\n%s", got)
|
||||
}
|
||||
|
||||
// A taken module publishing a port admitted from everywhere is not guarded; one admitted from
|
||||
// the mesh is. The registry is exposed everywhere on this node, and hello-web listens from
|
||||
// everywhere.
|
||||
with.Taken = map[string]bool{"distribution": true, "hello-web": true}
|
||||
if got := guardOf(with); got != "" {
|
||||
t.Fatalf("a port admitted from everywhere is guarded:\n%s", got)
|
||||
}
|
||||
with.Settings = nil
|
||||
if got := guardOf(with); got != AsGuard([]int{5000}) {
|
||||
t.Fatalf("the registry, from the mesh only, is not guarded:\n%s", got)
|
||||
}
|
||||
|
||||
// Nothing taken, nothing guarded — and no guard at all rather than an empty set.
|
||||
with = anchorRendering(true)
|
||||
with.Taken = nil
|
||||
if got := guardOf(with); got != "" {
|
||||
t.Fatalf("an untaken store is guarded:\n%s", got)
|
||||
}
|
||||
|
||||
// A given port is followed: where the machine put it is what is refused.
|
||||
with = anchorRendering(true)
|
||||
with.Given = map[string]map[int]int{"lavinmq": {5672: 5682, 15672: 15673}}
|
||||
with.Ports["lavinmq"] = map[int]int{5671: 5671, 5672: 5682}
|
||||
if got := guardOf(with); got != AsGuard([]int{5432, 5682, 15673}) {
|
||||
t.Fatalf("the guard does not follow the given ports:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A mapping bound to loopback is not published to anything off the machine — in either address
|
||||
// family — and an address's own colons never shift the ports.
|
||||
func TestPublishedLeavesOutLoopbackInBothFamilies(t *testing.T) {
|
||||
got := Published([]map[string]any{{"type": "container", "ports": []any{
|
||||
"127.0.0.1:15672:15672", "[::1]:8080:80", "localhost:9090:90",
|
||||
"[::]:8443:443", "0.0.0.0:5000:5000", "5353:53/udp"}}})
|
||||
want := map[string]map[int]int{"tcp": {8443: 443, 5000: 5000}, "udp": {5353: 53}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("published is %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0038: only a published port is the mesh's to move. A module that binds the machine
|
||||
// itself listens where its software was told to, so giving it a machine port is refused.
|
||||
func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) {
|
||||
onTheMachine := Manifest{Module: "daemon",
|
||||
Listens: []Listening{{Port: 9000, From: FromMesh}}, Guards: []int{9000}}
|
||||
layers := []Layer{{From: "node anchor",
|
||||
Placement: map[string]any{PortsSetting: map[string]any{"9000": float64(9100)}}}}
|
||||
_, err := GivenPorts(onTheMachine, layers)
|
||||
if err == nil || !strings.Contains(err.Error(), "does not publish") {
|
||||
t.Fatalf("a port no container publishes was given: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: a guarded port this node is told to open to everyone is opened, not guarded.
|
||||
// An opening from everywhere beside a guard dropping it is one statement refusing the other.
|
||||
func TestAGuardedPortOpenedToEveryoneIsNotGuarded(t *testing.T) {
|
||||
with := anchorRendering(true)
|
||||
with.Settings["postgres"] = []Layer{{From: "node anchor",
|
||||
Placement: map[string]any{ExposeSetting: map[string]any{"5432": FromEverywhere}}}}
|
||||
composed, err := anAdoptedAnchor().Compose(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if o := got["adoption.opening-tcp-5432-forwarded"]; o == nil || o["from"] != OpeningFromEverywhere {
|
||||
t.Fatalf("the store's port is not opened to everyone: %v", o)
|
||||
}
|
||||
if guard, _ := got[GuardID()]["content"].(string); guard != AsGuard([]int{5672, 15672}) {
|
||||
t.Fatalf("a port opened to everyone is still guarded:\n%s", guard)
|
||||
}
|
||||
}
|
||||
|
||||
// A module that publishes its ssh port the long way — `2222:22`, because the machine's own daemon
|
||||
// holds 22 — and says it listens on the machine side of that mapping, which is what anything
|
||||
// reaching it dials.
|
||||
func aForge() Manifest {
|
||||
return Manifest{Module: "forge",
|
||||
Provides: []Offer{{Name: "git-over-ssh", Scope: ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}},
|
||||
Listens: []Listening{{Port: 3000, From: FromMesh}, {Port: 2222, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge",
|
||||
"ports": []any{"3000", "2222:22"}}}}
|
||||
}
|
||||
|
||||
// portsAsThePlanWould is where this machine puts each of a module's ports, derived the way
|
||||
// cmd/mesh-controller/plan.go derives it: a given port first, looked up by the port the module
|
||||
// says it listens on, and otherwise wherever the manifest's own mapping already put it. Written
|
||||
// here because everything below — the filter, the openings, the guard, what a consumer is told —
|
||||
// reads that map, and a given port that the lookup does not find moves the container's mapping
|
||||
// and nothing else.
|
||||
//
|
||||
// It stands in for the plan only where the plan does not allocate: a port the manifest already
|
||||
// placed, or one a node was given. For a short form with no given port the real plan asks the
|
||||
// inventory for a machine port and may come back with one from the pool, which needs a store and
|
||||
// is what cmd/mesh-controller's own tests exercise. So an assertion here about such a port asserts
|
||||
// this helper, not the mesh; keep the assertions to the ports under test.
|
||||
func portsAsThePlanWould(m Manifest, given map[int]int) map[int]int {
|
||||
out := map[int]int{}
|
||||
for _, l := range m.Listens {
|
||||
if at, is := given[l.Port]; is {
|
||||
out[l.Port] = at
|
||||
continue
|
||||
}
|
||||
at, _ := m.MachineSide(l.Port)
|
||||
out[l.Port] = at
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100 and 0038: the machine side of a long-form mapping is a port the module
|
||||
// publishes, so a node may move it — and a module may name a mapping by either end.
|
||||
func TestAGivenPortNamesEitherEndOfWhatTheModulePublishes(t *testing.T) {
|
||||
forge := aForge()
|
||||
node := func(v any) []Layer {
|
||||
return []Layer{{From: "node anchor", Placement: map[string]any{PortsSetting: v}}}
|
||||
}
|
||||
|
||||
// The machine side — the number this module says it listens on, and the one the predecessor
|
||||
// had somewhere else. Answered under 2222, the end the module itself names, which is what
|
||||
// every reader of this map asks for. One entry, not two: a second key for the same answer is
|
||||
// an entry another mapping's reader could find instead.
|
||||
given, err := GivenPorts(forge, node(map[string]any{"2222": float64(222)}))
|
||||
if err != nil {
|
||||
t.Fatalf("the machine side of a mapping cannot be given a port: %v", err)
|
||||
}
|
||||
if want := map[int]int{2222: 222}; !reflect.DeepEqual(given, want) {
|
||||
t.Fatalf("the forge was given %v; the mapping it names is filed under %v", given, want)
|
||||
}
|
||||
|
||||
// The container's own port names the same mapping and means the same thing — and is filed
|
||||
// under the same name, because the module's name for it has not changed.
|
||||
if inside, err := GivenPorts(forge, node(map[string]any{"22": float64(222)})); err != nil ||
|
||||
!reflect.DeepEqual(inside, map[int]int{2222: 222}) {
|
||||
t.Fatalf("the container's end of the mapping was given %v: %v", inside, err)
|
||||
}
|
||||
|
||||
// A short form is published on the number it names, and is unchanged by any of this.
|
||||
if short, err := GivenPorts(forge, node(map[string]any{"3000": float64(2999)})); err != nil ||
|
||||
short[3000] != 2999 {
|
||||
t.Fatalf("the short form was given %v: %v", short, err)
|
||||
}
|
||||
|
||||
// A port no container publishes is still refused, in the same words.
|
||||
if _, err := GivenPorts(forge, node(map[string]any{"9000": float64(9100)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "does not publish") {
|
||||
t.Fatalf("a port the forge does not publish was given: %v", err)
|
||||
}
|
||||
|
||||
// And the two ends of one mapping given two different numbers is one setting contradicting
|
||||
// the other: the machine publishes it once.
|
||||
if _, err := GivenPorts(forge, node(map[string]any{
|
||||
"2222": float64(222), "22": float64(300)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "one mapping") {
|
||||
t.Fatalf("the two ends of one mapping were given different ports: %v", err)
|
||||
}
|
||||
// Said at both ends with the same number, it is still said twice, and refused where every
|
||||
// other repeated machine port is — as the inventory refuses it when the setting is stored,
|
||||
// which is the layer that sees it first.
|
||||
if _, err := GivenPorts(forge, node(map[string]any{
|
||||
"2222": float64(222), "22": float64(222)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "to both its 22 and its 2222") {
|
||||
t.Fatalf("one mapping given one machine port at both ends: %v", err)
|
||||
}
|
||||
// A number that names two different mappings names neither: which one to move is not said.
|
||||
twice := aForge()
|
||||
twice.Resources[0]["ports"] = []any{"22", "2222:22"}
|
||||
if _, err := GivenPorts(twice, node(map[string]any{"22": float64(222)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "twice") {
|
||||
t.Fatalf("a number naming two of the module's mappings was accepted: %v", err)
|
||||
}
|
||||
|
||||
// And the same refusal when the number naming two mappings is not the one the setting used
|
||||
// but the one the answer would be filed under. Here `80` is the module's own name for a
|
||||
// mapping, and two mappings wear it; filing an answer there is one container's port standing
|
||||
// where the other's is read, and both containers then publish it.
|
||||
shared := Manifest{Module: "gallery",
|
||||
Listens: []Listening{{Port: 80, From: FromMesh}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "a", "type": "container", "name": "a", "ports": []any{"4001:80"}},
|
||||
{"id": "b", "type": "container", "name": "b", "ports": []any{"4002:80"}}}}
|
||||
if _, err := GivenPorts(shared, node(map[string]any{"4001": float64(1234)})); err == nil ||
|
||||
!strings.Contains(err.Error(), "twice") {
|
||||
t.Fatalf("two containers were put on one machine port: %v", err)
|
||||
}
|
||||
|
||||
// A module whose mappings chain — one's machine side is another's container port — keeps them
|
||||
// apart, because each is filed under the port the module names it by and neither name is
|
||||
// shared. Given both, each moves on its own and neither overwrites the other.
|
||||
chained := Manifest{Module: "chain",
|
||||
Listens: []Listening{{Port: 80, From: FromMesh}, {Port: 9090, From: FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "chain",
|
||||
"ports": []any{"8080:80", "9090:8080"}}}}
|
||||
both, err := GivenPorts(chained, node(map[string]any{"80": float64(1234), "9090": float64(5678)}))
|
||||
if err != nil || !reflect.DeepEqual(both, map[int]int{80: 1234, 9090: 5678}) {
|
||||
t.Fatalf("chained mappings were given %v: %v", both, err)
|
||||
}
|
||||
if moved := givenOuter("8080:80", both); moved != "1234:80" {
|
||||
t.Fatalf("the first mapping moved to %q, and it was given 1234", moved)
|
||||
}
|
||||
if moved := givenOuter("9090:8080", both); moved != "5678:8080" {
|
||||
t.Fatalf("the second mapping moved to %q, and it was given 5678", moved)
|
||||
}
|
||||
}
|
||||
|
||||
// And the number reaches everything derived from it. The fault this is written against moved the
|
||||
// container's mapping alone: the filter opened the port the software had left, the adopted node's
|
||||
// opening named it too, the guard refused it, and a consumer was sent to it.
|
||||
func TestAGivenMachineSideReachesTheFilterTheOpeningAndTheConsumer(t *testing.T) {
|
||||
forge := aForge()
|
||||
given, err := GivenPorts(forge, []Layer{{From: "node anchor",
|
||||
Placement: map[string]any{PortsSetting: map[string]any{"2222": float64(222)}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("the machine side of a mapping cannot be given a port: %v", err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{forge}}
|
||||
with := Rendering{
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, given)},
|
||||
Given: map[string]map[int]int{"forge": given},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
OutwardLinks: []string{"eth0"},
|
||||
TunnelInterface: "mesh0",
|
||||
Taken: map[string]bool{"forge": true},
|
||||
}
|
||||
|
||||
// What the runtime is handed: the machine's own port on the outside, the container's within.
|
||||
composed, err := r.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the forge does not compose: %v", err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
if ports := got["forge.server"]["ports"]; !reflect.DeepEqual(ports, []any{"3000:3000", "222:22"}) {
|
||||
t.Fatalf("the forge's container publishes %v", ports)
|
||||
}
|
||||
|
||||
// What the filter would open, were the node converged.
|
||||
rules, err := r.Rules(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var opened []int
|
||||
for _, rule := range rules {
|
||||
opened = append(opened, rule.Port)
|
||||
}
|
||||
// Only the moved port is asserted: the forge's other port is a short form the real plan would
|
||||
// allocate rather than read off the manifest, so its number here is portsAsThePlanWould's and
|
||||
// not the mesh's.
|
||||
if !slices.Contains(opened, 222) || slices.Contains(opened, 2222) {
|
||||
t.Fatalf("the filter opens %v, not where the machine puts the forge", opened)
|
||||
}
|
||||
|
||||
// And what it is declared instead, adopted: an opening on the machine's port, naming the
|
||||
// container's port on the forwarded path — a published port is forwarded, never received.
|
||||
opening := got[OpeningID("tcp", 222, PathForwarded)]
|
||||
if opening == nil || opening["to"] != 22 || opening["from"] != OpeningFromMesh {
|
||||
t.Fatalf("no opening for the port this node gave the forge: %v", keys(got))
|
||||
}
|
||||
for id := range got {
|
||||
if strings.HasPrefix(id, "adoption.opening-tcp-2222-") {
|
||||
t.Errorf("an opening for the port the forge was moved off: %s", id)
|
||||
}
|
||||
}
|
||||
|
||||
// The guard refuses it where the machine put it, and nothing where it used to be.
|
||||
guard, _ := got[GuardID()]["content"].(string)
|
||||
if !strings.Contains(guard, "222") || strings.Contains(guard, "2222") {
|
||||
t.Fatalf("the guard does not follow the given port:\n%s", guard)
|
||||
}
|
||||
|
||||
// And a consumer is sent to the same number, which is read from what the module serves.
|
||||
if told := ServedOn(forge, "git-over-ssh", with.Ports["forge"])["port"]; told != 222 {
|
||||
t.Fatalf("a consumer is told the forge answers on %v", told)
|
||||
}
|
||||
}
|
||||
|
||||
// And the mapping itself moves under either name, because Rendering.Given is a map anybody
|
||||
// composing a declaration hands in: keyed by the machine side, which is what a module declaring
|
||||
// 2222 calls its port, only the outside moves and the container's own port stays as written.
|
||||
func TestAMappingIsMovedUnderEitherOfItsNames(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
written string
|
||||
given map[int]int
|
||||
want string
|
||||
}{
|
||||
{"2222:22", map[int]int{2222: 222}, "222:22"},
|
||||
{"2222:22", map[int]int{22: 222}, "222:22"},
|
||||
{"127.0.0.1:15672:15672/tcp", map[int]int{15672: 15673}, "127.0.0.1:15673:15672/tcp"},
|
||||
{"2222:22", map[int]int{3000: 2999}, "2222:22"},
|
||||
{"2222:22", nil, "2222:22"},
|
||||
} {
|
||||
if got := givenOuter(c.written, c.given); got != c.want {
|
||||
t.Errorf("%s given %v is published as %s, want %s", c.written, c.given, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The same on a node that was given nothing, which is where the derivation was never at fault:
|
||||
// a long-form mapping is published where the manifest put it, and an adopted node opens that port
|
||||
// on the forwarded path like any other. What broke it was the number reaching only the mapping.
|
||||
func TestALongFormPortIsOpenedWhereTheManifestPublishesIt(t *testing.T) {
|
||||
forge := aForge()
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{forge}}
|
||||
composed, err := r.Compose(Rendering{
|
||||
Ports: map[string]map[int]int{"forge": portsAsThePlanWould(forge, nil)},
|
||||
Mesh: []string{"10.77.0.1"},
|
||||
Adopted: true,
|
||||
OutwardLinks: []string{"eth0"},
|
||||
TunnelInterface: "mesh0",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := byID(composed.Resources)
|
||||
opening := got[OpeningID("tcp", 2222, PathForwarded)]
|
||||
if opening == nil || opening["to"] != 22 {
|
||||
t.Fatalf("no opening for the forge's published ssh port: %v", keys(got))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The account agents run as (novox/hq ADR 0266): a module names it as a machine fact — the agent account
|
||||
// where the node names one, the operator's otherwise — and asks the node-engine to judge it never to become
|
||||
// root only where it is the agents' own.
|
||||
|
||||
func TestTheAgentAccountFactFallsBackToTheOperatorAndIsNeverRootOnlyWhenItsOwn(t *testing.T) {
|
||||
facts := machineFacts(Resolution{Node: "anchor", Account: "ops"}, nil, "")
|
||||
if facts["agent-account"] != "ops" || facts["agent-home"] != "/home/ops" || facts["agent-root"] != "" {
|
||||
t.Errorf("with no agent account named, agents run as the operator: %v", facts)
|
||||
}
|
||||
facts = machineFacts(Resolution{Node: "anchor", Account: "ops", AccountHome: "/srv/ops"}, nil, "")
|
||||
if facts["agent-home"] != "/srv/ops" {
|
||||
t.Errorf("the operator's stated home is the agent's home when they are one account: %v", facts)
|
||||
}
|
||||
facts = machineFacts(Resolution{Node: "anchor", Account: "ops", AgentAccount: "agent"}, nil, "")
|
||||
if facts["agent-account"] != "agent" || facts["agent-home"] != "/home/agent" || facts["agent-root"] != RootNever {
|
||||
t.Errorf("a named agent account is the agents', never root: %v", facts)
|
||||
}
|
||||
if facts["account"] != "ops" {
|
||||
t.Errorf("the operator account is still the operator's: %v", facts)
|
||||
}
|
||||
facts = machineFacts(Resolution{Node: "anchor", AgentAccount: "agent", AgentAccountHome: "/var/lib/agent"}, nil, "")
|
||||
if facts["agent-home"] != "/var/lib/agent" || facts["agent-root"] != RootNever {
|
||||
t.Errorf("an agent account with a stated home on a machine with no operator: %v", facts)
|
||||
}
|
||||
if _, has := machineFacts(Resolution{Node: "anchor"}, nil, "")["agent-account"]; has {
|
||||
t.Error("a machine with no account at all names an agent account")
|
||||
}
|
||||
}
|
||||
|
||||
// The agent's module, in the shape the catalogue's declares it: the account, never root where it is its
|
||||
// own; its directory under that home, owned by it.
|
||||
const agentModule = `{"module": "agent", "version": "1", "resources": [
|
||||
{"id": "account", "type": "user", "name": "${machine:agent-account}", "home": "${machine:agent-home}",
|
||||
"root": "${machine:agent-root}"},
|
||||
{"id": "home", "type": "directory", "path": "${machine:agent-home}/.agent", "mode": "0700",
|
||||
"owner": "${machine:agent-account}"}
|
||||
]}`
|
||||
|
||||
func TestTheAgentAccountIsDeclaredNeverRootOnlyToAnEngineThatJudgesIt(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(agentModule))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := func(r Resolution, with Rendering) (user, home map[string]any) {
|
||||
t.Helper()
|
||||
r.Node, r.Modules = "anchor", []Manifest{m}
|
||||
out, err := r.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fileNamed(out, "agent.account"), fileNamed(out, "agent.home")
|
||||
}
|
||||
|
||||
user, home := compose(Resolution{Account: "ops", AgentAccount: "agent"}, Rendering{JudgesRoot: true})
|
||||
if user["name"] != "agent" || user[RootField] != RootNever || user["home"] != "/home/agent" {
|
||||
t.Errorf("an engine that judges root is sent the agent account never to become root: %v", user)
|
||||
}
|
||||
if home["path"] != "/home/agent/.agent" || home["owner"] != "agent" {
|
||||
t.Errorf("the agent's directory is under its own home, its own: %v", home)
|
||||
}
|
||||
|
||||
user, _ = compose(Resolution{Account: "ops", AgentAccount: "agent", AgentAccountHome: "/srv/agent"}, Rendering{JudgesRoot: true})
|
||||
if user["home"] != "/srv/agent" {
|
||||
t.Errorf("an agent account named with a home of its own is made there: %v", user)
|
||||
}
|
||||
|
||||
user, _ = compose(Resolution{Account: "ops", AgentAccount: "agent"}, Rendering{})
|
||||
if _, sent := user[RootField]; sent || user["name"] != "agent" {
|
||||
t.Errorf("an older engine, which parses strictly, is sent root: %v", user)
|
||||
}
|
||||
|
||||
user, home = compose(Resolution{Account: "ops"}, Rendering{JudgesRoot: true})
|
||||
if _, sent := user[RootField]; sent || user["name"] != "ops" {
|
||||
t.Errorf("where agents run as the operator, root asserts nothing and is not sent: %v", user)
|
||||
}
|
||||
if home["path"] != "/home/ops/.agent" || home["owner"] != "ops" {
|
||||
t.Errorf("with no agent account, the agent's directory is the operator's: %v", home)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRuntimeIsToldTheAgentAccount(t *testing.T) {
|
||||
with := Rendering{ArtifactStore: "anchor.internal:5101",
|
||||
Needed: map[string]map[string]string{RuntimeModule: {"broker": "sealed-credential"}}}
|
||||
envOf := func(r Resolution) map[string]string {
|
||||
t.Helper()
|
||||
r.Node, r.Modules = "anchor", []Manifest{aToolsModule(t, "nftables", "tools/index.js"), theRuntime(t)}
|
||||
out, err := r.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
process := fileNamed(out, RuntimeModule+"."+RuntimeProcessID())
|
||||
if process == nil {
|
||||
t.Fatal("no runtime process was composed")
|
||||
}
|
||||
return process["env"].(map[string]string)
|
||||
}
|
||||
env := envOf(Resolution{Account: "ops", AgentAccount: "agent"})
|
||||
if env[RuntimeAgentAccount] != "agent" || env[RuntimeAgentHome] != "/home/agent" || env[RuntimeOperatorAccount] != "ops" {
|
||||
t.Errorf("the runtime is not told whom agents run as: %v", env)
|
||||
}
|
||||
env = envOf(Resolution{Account: "ops"})
|
||||
if env[RuntimeAgentAccount] != "ops" || env[RuntimeAgentHome] != "/home/ops" {
|
||||
t.Errorf("with no agent account, agents run as the operator: %v", env)
|
||||
}
|
||||
env = envOf(Resolution{})
|
||||
if _, set := env[RuntimeAgentAccount]; set {
|
||||
t.Errorf("a machine with no account names an agent account: %v", env)
|
||||
}
|
||||
if problems := bundleEnvProblems("x", Artifact{Name: "b", Kind: ArtifactBundle, Loads: []string{"x"},
|
||||
Env: map[string]string{RuntimeAgentAccount: "me"}}); len(problems) == 0 {
|
||||
t.Error("a bundle may tell the runtime whom agents run as")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/internal/beside"
|
||||
)
|
||||
|
||||
// **The word does not come back through a manifest** (novox/hq ADR 0131). A module that wants
|
||||
// messaging wants the mesh's bus, reached through the sdk and named by the `mesh-broker` seat. Naming
|
||||
// the old wire protocol asks for the one server being retired, so both directions are refused at the
|
||||
// parser — this is judged from the manifest alone, no store needed.
|
||||
|
||||
func TestAManifestProvidingAmqpIsRefused(t *testing.T) {
|
||||
raw := []byte(`{"module":"old-broker","version":"1","provides":[{"name":"amqp","scope":"mesh"}]}`)
|
||||
_, err := ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), `provides "amqp", which is not a provision`) {
|
||||
t.Fatalf("a module providing amqp was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManifestRequiringAmqpIsRefused(t *testing.T) {
|
||||
raw := []byte(`{"module":"forwarder","version":"1","requires":["amqp"]}`)
|
||||
_, err := ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), `requires "amqp", which is not a provision`) {
|
||||
t.Fatalf("a module requiring amqp was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// And the catalogue (internal/beside) names it nowhere — the three modules that did are removed under
|
||||
// design 28 task 5.4, not converted.
|
||||
func TestNoCatalogueManifestNamesAmqp(t *testing.T) {
|
||||
modules, err := filepath.Glob(filepath.Join(beside.Catalogue(t), "*", "module.json"))
|
||||
if err != nil || len(modules) == 0 {
|
||||
t.Fatalf("no manifests in the catalogue, so this proved nothing: %v", err)
|
||||
}
|
||||
for _, path := range modules {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(raw), `"amqp"`) {
|
||||
t.Errorf("%s names amqp, which is not a provision (novox/hq ADR 0131)",
|
||||
filepath.Base(filepath.Dir(path)))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package catalogue
|
||||
|
||||
import "testing"
|
||||
|
||||
// A claim written before the rename still holds (novox/hq ADR 0122, ADR 0156): with the store's
|
||||
// aliases loaded, the former name resolves to the seat.
|
||||
func TestTheArtifactStoresFormerNameResolvesToIt(t *testing.T) {
|
||||
was := aliases
|
||||
t.Cleanup(func() { aliases = was })
|
||||
UseAliases(map[string]string{"the-artifact-store": "mesh-artifact-store"})
|
||||
seat, known := SeatNamed("the-artifact-store")
|
||||
if !known || seat.Name != "mesh-artifact-store" || seat.Delivers != "artifact-store" {
|
||||
t.Fatalf("the former name did not resolve: %+v %v", seat, known)
|
||||
}
|
||||
if _, known := SeatNamed("mesh-artifact-store"); !known {
|
||||
t.Fatal("the seat is not in the set under its name")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The artifact store's seat is one per mesh, read from the catalogue beside this checkout.
|
||||
//
|
||||
// **A second store anywhere is refused by name, not discovered as a consumer failure.** The seat
|
||||
// was node-scoped, so a second `distribution` on another machine resolved cleanly there — and a
|
||||
// node-scoped requirement with one candidate installs that candidate on the node, so anything that
|
||||
// required the store's presence beside it would have raised a fresh, empty store on the wrong
|
||||
// machine. Only afterwards did the mesh notice: `artifact-store` offered by two nodes, and every
|
||||
// consumer elsewhere refusing to choose. The claim says it first, where the second store is
|
||||
// assigned.
|
||||
func TestASecondArtifactStoreAnywhereIsRefusedByName(t *testing.T) {
|
||||
store := catalogueManifest(t, "distribution")
|
||||
|
||||
// The first store resolves as it always has.
|
||||
if _, err := Resolve(shelf(store), []string{"distribution"}, workstation(), World{}); err != nil {
|
||||
t.Fatalf("the store alone does not resolve: %v", err)
|
||||
}
|
||||
|
||||
// A second one, on any other machine, is refused — and the refusal names the seat.
|
||||
elsewhere := World{Held: []Held{{Claim: "mesh-artifact-store", Scope: ScopeMesh,
|
||||
Node: "anchor", Module: "distribution"}}}
|
||||
other := workstation()
|
||||
other.Name = "laptop"
|
||||
_, err := Resolve(shelf(store), []string{"distribution"}, other, elsewhere)
|
||||
if err == nil {
|
||||
t.Fatal("a second store was accepted on another machine; it would offer artifact-store a " +
|
||||
"second time and every consumer elsewhere would refuse to choose")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mesh-artifact-store") || !strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Fatalf("refused without naming the seat: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What the mesh built, named by what it is rather than by where it was pushed.
|
||||
//
|
||||
// **An image is recorded by its digest and its path; the registry's address is a route to it**
|
||||
// (novox/hq 04-ISSUES/102). A build used to be recorded as `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||
// — the reference the builder pushed to, kept whole — and every declaration carried that literal.
|
||||
// Move the registry's port, or the registry, and every fresh pull of a mesh image fails: a new
|
||||
// node, a recreate after eviction. The digest is the identity; the address is the node's setting
|
||||
// for the module that serves the artifact store, and it is read from there when a reference is
|
||||
// composed, never written into a record.
|
||||
//
|
||||
// So a kept reference has a scheme of the mesh's own, named after the provision that answers it:
|
||||
//
|
||||
// artifact-store://<module>/<artifact>@sha256:<hex> an image
|
||||
// artifact-store://<module>/<artifact>/blobs/sha256:<hex> an archive
|
||||
//
|
||||
// No runtime knows the scheme. That is the point of it being one: a reference that leaks to a
|
||||
// machine uncomposed is refused by the runtime as malformed, in front of whoever sent it, rather
|
||||
// than pulled from a public registry that happens to have a repository by that name — which is
|
||||
// what an address-less `<module>/<artifact>@sha256:…` would be.
|
||||
|
||||
// ArtifactStoreScheme marks a reference to something in the mesh's artifact store, kept without
|
||||
// the store's address.
|
||||
const ArtifactStoreScheme = ArtifactStoreProvision + "://"
|
||||
|
||||
// Recorded is a reference as the mesh records it: the artifact store's address, if the builder wrote
|
||||
// one, taken off.
|
||||
//
|
||||
// For a reference the builder announced — one it pushed to the store — which is the only kind
|
||||
// this is called on. An image the builder named `<host>/<path>@sha256:…` is kept as its path; an
|
||||
// archive it named `http://<host>/v2/<path>/blobs/<digest>` likewise. A reference with no address
|
||||
// in it — an image id from a genesis build that had nowhere to publish, a package version — is
|
||||
// what it was.
|
||||
func Recorded(reference string) string {
|
||||
if strings.HasPrefix(reference, ArtifactStoreScheme) {
|
||||
return reference
|
||||
}
|
||||
if rest, isURL := strings.CutPrefix(reference, "http://"); isURL {
|
||||
if _, path, ok := strings.Cut(rest, "/v2/"); ok && strings.Contains(path, "/blobs/") {
|
||||
return ArtifactStoreScheme + path
|
||||
}
|
||||
return reference
|
||||
}
|
||||
host, path, ok := strings.Cut(reference, "/")
|
||||
if !ok || !isRegistryHost(host) || !strings.Contains(path, "@sha256:") {
|
||||
return reference
|
||||
}
|
||||
return ArtifactStoreScheme + path
|
||||
}
|
||||
|
||||
// isRegistryHost is the runtime's own rule for reading the first component of a reference as a
|
||||
// registry rather than as a namespace: it has a dot or a port in it, or it is localhost.
|
||||
func isRegistryHost(component string) bool {
|
||||
return component == "localhost" || strings.ContainsAny(component, ".:")
|
||||
}
|
||||
|
||||
// InArtifactStore reports whether a reference is a kept one, and what it names there.
|
||||
func InArtifactStore(reference string) (path string, kept bool) {
|
||||
return strings.CutPrefix(reference, ArtifactStoreScheme)
|
||||
}
|
||||
|
||||
// Routed is a kept reference as a machine fetches it, through the artifact store at `address`
|
||||
// (host:port). A reference that is not a kept one is what it was.
|
||||
func Routed(reference, address string) string {
|
||||
path, kept := InArtifactStore(reference)
|
||||
if !kept {
|
||||
return reference
|
||||
}
|
||||
if strings.Contains(path, "/blobs/") {
|
||||
return "http://" + address + "/v2/" + path
|
||||
}
|
||||
return address + "/" + path
|
||||
}
|
||||
|
||||
// Rerouted is a reference the mesh recorded, whichever way it was recorded, as a machine fetches
|
||||
// it now: a kept one composed with the store's address, and one recorded before references were
|
||||
// kept without their address — the builder's own `<host>/<path>@sha256:…` — re-routed to where
|
||||
// the store is now. Only for references that are the mesh's own: everything a build record
|
||||
// holds is, by construction.
|
||||
func Rerouted(reference, address string) string {
|
||||
return Routed(Recorded(reference), address)
|
||||
}
|
||||
|
||||
// artifactsInto composes the artifact store's address into a resource's `image` and `source`.
|
||||
//
|
||||
// **Composed here, at the last moment before a machine, and stored nowhere.** A kept reference is
|
||||
// routed through the store as this network reaches it now. A reference recorded with an address
|
||||
// before references were kept without one is re-routed the same way — but only when the mesh
|
||||
// built it (`with.Built` names every `<module>/<artifact>` it has), because a module may run an
|
||||
// image from a public registry under its own name and that one is exactly where it says.
|
||||
//
|
||||
// A kept reference with no store to route it through is refused: sent as it is, the runtime would
|
||||
// refuse the scheme on the machine, one push away from the reason.
|
||||
//
|
||||
// **What this does not reach: the images genesis pinned.** The installer builds the control plane
|
||||
// and the builder before the mesh exists, pushes them itself and pins their manifests to
|
||||
// `<registry>:<port>/mesh-controller@…` and `<registry>:<port>/mesh-builder@…` — single-segment
|
||||
// repositories with no build record, so `with.Built` does not name them and they are left as
|
||||
// written until each is rebuilt through the mesh, which records it by digest and path. Until then
|
||||
// a registry that moves strands exactly those two on a recreate, and the control plane's is the
|
||||
// one that cannot be repaired through the mesh. Rebuild both through `build` before moving the
|
||||
// store (novox/hq 04-ISSUES/102, finding F4).
|
||||
func artifactsInto(resource map[string]any, module string, with Rendering) error {
|
||||
for _, key := range []string{"image", "source"} {
|
||||
written, ok := resource[key].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, kept := InArtifactStore(written); kept {
|
||||
if with.ArtifactStore == "" {
|
||||
return fmt.Errorf(
|
||||
"%s's %v names %s, which is in the mesh's artifact store, and this mesh has no "+
|
||||
"artifact store on its network to fetch it from — nothing assigned offers "+
|
||||
"%s, or the machine offering it is not on the private network",
|
||||
module, resource["id"], written, ArtifactStoreProvision)
|
||||
}
|
||||
resource[key] = Routed(written, with.ArtifactStore)
|
||||
continue
|
||||
}
|
||||
if with.ArtifactStore == "" {
|
||||
continue
|
||||
}
|
||||
recorded := Recorded(written)
|
||||
if recorded == written {
|
||||
continue
|
||||
}
|
||||
path, _ := InArtifactStore(recorded)
|
||||
repository := path
|
||||
if at := strings.IndexAny(path, "@"); at >= 0 {
|
||||
repository = path[:at]
|
||||
} else if blobs := strings.Index(path, "/blobs/"); blobs >= 0 {
|
||||
repository = path[:blobs]
|
||||
}
|
||||
if with.Built[repository] {
|
||||
resource[key] = Routed(recorded, with.ArtifactStore)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A build is recorded by what it is; where it is pushed is composed where it is used (novox/hq
|
||||
// 04-ISSUES/102).
|
||||
|
||||
var digest = "sha256:" + strings.Repeat("d", 64)
|
||||
|
||||
func TestAReferenceIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"anchor.internal:5100/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
"localhost:5000/gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
"http://anchor.internal:5100/v2/gitea/config/blobs/" + digest: ArtifactStoreScheme + "gitea/config/blobs/" + digest,
|
||||
ArtifactStoreScheme + "gitea/server@" + digest: ArtifactStoreScheme + "gitea/server@" + digest,
|
||||
digest: digest,
|
||||
"@novox/sdk@1.2.3": "@novox/sdk@1.2.3",
|
||||
"gitea/gitea@" + digest: "gitea/gitea@" + digest,
|
||||
"https://registry.example/v2/gitea/config/blobs/" + digest: "https://registry.example/v2/gitea/config/blobs/" + digest,
|
||||
}
|
||||
for announced, want := range cases {
|
||||
if got := Recorded(announced); got != want {
|
||||
t.Errorf("Recorded(%q) = %q, want %q", announced, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecordedReferenceIsRoutedThroughTheStoreAsItIsNow(t *testing.T) {
|
||||
if got := Routed(ArtifactStoreScheme+"gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("an image is fetched as %q", got)
|
||||
}
|
||||
if got := Routed(ArtifactStoreScheme+"gitea/config/blobs/"+digest, "anchor.internal:5101"); got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
|
||||
t.Errorf("an archive is fetched as %q", got)
|
||||
}
|
||||
if got := Routed("gitea/gitea@"+digest, "anchor.internal:5101"); got != "gitea/gitea@"+digest {
|
||||
t.Errorf("a reference that is not the store's was routed: %q", got)
|
||||
}
|
||||
// One recorded before references were kept without their address follows the store too.
|
||||
if got := Rerouted("anchor.internal:5100/gitea/server@"+digest, "anchor.internal:5101"); got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("a reference recorded with the old address stays there: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **The address is composed into a declaration, and the record never carries it.**
|
||||
func TestAnImageTheMeshBuiltIsRoutedThroughTheStoreWhenDeclared(t *testing.T) {
|
||||
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea", "artifact": "server"},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "artifact": "config"},
|
||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache", "image": "valkey/valkey@" + digest},
|
||||
}, Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "server", Kind: ArtifactImage, From: "Dockerfile"},
|
||||
{Name: "config", Kind: ArtifactArchive, From: "config"},
|
||||
}}}
|
||||
resolved, err := m.Resolve([]Built{
|
||||
{Name: "server", Kind: ArtifactImage, Reference: ArtifactStoreScheme + "gitea/server@" + digest},
|
||||
{Name: "config", Kind: ArtifactArchive, Reference: ArtifactStoreScheme + "gitea/config/blobs/" + digest, Digest: digest},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{resolved}}
|
||||
|
||||
out, err := r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.server")["image"]; got != "anchor.internal:5101/gitea/server@"+digest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.config")["source"]; got != "http://anchor.internal:5101/v2/gitea/config/blobs/"+digest {
|
||||
t.Errorf("the archive the mesh built is fetched from %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.cache")["image"]; got != "valkey/valkey@"+digest {
|
||||
t.Errorf("an image from a public registry was routed through the store: %v", got)
|
||||
}
|
||||
// The manifest the mesh holds still says what it is, not where it was fetched from.
|
||||
if got := resolved.Resources[0]["image"]; got != ArtifactStoreScheme+"gitea/server@"+digest {
|
||||
t.Errorf("composing wrote the address into the catalogue's copy: %v", got)
|
||||
}
|
||||
|
||||
// And the store moves: the same record, another address, without a rebuild.
|
||||
out, err = r.Declaration(Rendering{ArtifactStore: "laptop.internal:5000"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "gitea.server")["image"]; got != "laptop.internal:5000/gitea/server@"+digest {
|
||||
t.Errorf("after the store moved, the image is still fetched as %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageInTheStoreWithNoStoreToFetchItFromIsRefused(t *testing.T) {
|
||||
m := Manifest{Module: "gitea", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||
"image": ArtifactStoreScheme + "gitea/server@" + digest},
|
||||
}}
|
||||
_, err := Resolution{Node: "anchor", Modules: []Manifest{m}}.Declaration(Rendering{})
|
||||
if err == nil || !strings.Contains(err.Error(), "no artifact store") {
|
||||
t.Fatalf("a reference nothing can fetch was sent to a machine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A reference recorded with an address before this follows the store too** — when the mesh
|
||||
// built it. A module running an image straight from a public registry under its own name is left
|
||||
// exactly where it says: `quay.io/keycloak/keycloak` is not the mesh's, whatever it is called.
|
||||
func TestAReferenceRecordedWithAnAddressFollowsTheStoreWhenTheMeshBuiltIt(t *testing.T) {
|
||||
m := Manifest{Module: "keycloak", Version: "1", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-keycloak",
|
||||
"image": "anchor.internal:5100/keycloak/server@" + digest},
|
||||
{"id": "upstream", "type": "container", "name": "mesh-keycloak-upstream",
|
||||
"image": "quay.io/keycloak/keycloak@" + digest},
|
||||
}}
|
||||
r := Resolution{Node: "anchor", Modules: []Manifest{m}}
|
||||
out, err := r.Declaration(Rendering{
|
||||
ArtifactStore: "anchor.internal:5101",
|
||||
Built: map[string]bool{"keycloak/server": true},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5101/keycloak/server@"+digest {
|
||||
t.Errorf("an image the mesh built, recorded with the old address, is fetched as %v", got)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.upstream")["image"]; got != "quay.io/keycloak/keycloak@"+digest {
|
||||
t.Errorf("a public image was re-routed through the store: %v", got)
|
||||
}
|
||||
// Nothing known to be built: nothing re-routed, nothing refused.
|
||||
out, err = r.Declaration(Rendering{ArtifactStore: "anchor.internal:5101"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "keycloak.server")["image"]; got != "anchor.internal:5100/keycloak/server@"+digest {
|
||||
t.Errorf("with no build record, a reference was rewritten: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// What the machine's modules back up is placement (design 57 L3, issue 794). The holder of node-backup names it as
|
||||
// `${backup:lines}` (what to run and which paths to keep) and `${backup:data}` (every item with its class and
|
||||
// protection): the text `${contribution:node-backup:backup}` and `${contribution:node-backup:data}` render, from the
|
||||
// same function, derived from each placed module's declared data (novox/hq ADR 0233). A handed holder has the text in
|
||||
// its plan's placement for its client to fill; one not handed has the controller fill it where it stands.
|
||||
|
||||
// BackupNamespace is the placeholder kind of the machine's backup lines and data.
|
||||
const BackupNamespace = "backup"
|
||||
|
||||
// ofBackup is where a holder places them: ${backup:<key>}. Loose inside the braces, so a misspelt key is found and
|
||||
// refused rather than written out as text.
|
||||
var ofBackup = regexp.MustCompile(`\$\{backup:([^}]*)\}`)
|
||||
|
||||
// backupKinds is the contribution kind each key of the backup placement renders.
|
||||
var backupKinds = map[string]string{"lines": BackupKindBackup, "data": BackupKindData}
|
||||
|
||||
// backupPlacer is the machine's modules the backup placement is composed over, as the old contributions are.
|
||||
type backupPlacer struct {
|
||||
modules []Manifest
|
||||
with Rendering
|
||||
facts map[string]string
|
||||
caps map[string]bool
|
||||
unplaced *[]string
|
||||
}
|
||||
|
||||
// placed is the text of one key for a holder, or what is wrong with naming it.
|
||||
func (b backupPlacer) placed(holder Manifest, key string, item bool) (string, string) {
|
||||
kind, known := backupKinds[key]
|
||||
switch {
|
||||
case item:
|
||||
return "", "an item is kept per item; the machine's backup lines are placed only in a file"
|
||||
case !known:
|
||||
return "", fmt.Sprintf("the backup placement has `lines` and `data`, not %q", key)
|
||||
case !holder.ClaimsSeat(BackupSeat):
|
||||
return "", fmt.Sprintf("%s does not claim %s; the machine's backup lines are placed by its holder alone "+
|
||||
"(novox/hq ADR 0212)", holder.Module, BackupSeat)
|
||||
}
|
||||
text, left, err := seatContributions(b.modules, holder, BackupSeat+":"+kind, b.with, b.facts, b.caps)
|
||||
if b.unplaced != nil {
|
||||
*b.unplaced = append(*b.unplaced, left...)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err.Error()
|
||||
}
|
||||
return text, ""
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A machine's backup lines are placement (design 57 L3, issue 794): the holder of node-backup names them as
|
||||
// `${backup:lines}` and `${backup:data}`, which a handed holder has its client fill from the plan and an unhanded
|
||||
// one has the controller fill as it fills `${contribution:node-backup:backup}` and `${...:data}`: the same text.
|
||||
|
||||
const (
|
||||
oldBackupText = "# lines\n${contribution:node-backup:backup}"
|
||||
oldDataText = "# data\n${contribution:node-backup:data}"
|
||||
placedBackupTxt = "# lines\n${backup:lines}"
|
||||
placedDataTxt = "# data\n${backup:data}"
|
||||
backupHolder = `{"module":"restic","version":"1","moved":true,"claims":[{"name":"node-backup","scope":"node"}],
|
||||
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
|
||||
"resources":[{"id":"declared","type":"file","path":"/etc/backups.conf","template":"files/backups"},
|
||||
{"id":"data-declared","type":"file","path":"/etc/data.conf","template":"files/data"}]}`
|
||||
)
|
||||
|
||||
func dataModule(name string) string {
|
||||
return `{"module":"` + name + `","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}],
|
||||
"data":{"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable","why":"the ` + name + ` data"}]}}`
|
||||
}
|
||||
|
||||
// backupsOn is a machine of a holder whose files say these texts and the modules that declare data.
|
||||
func backupsOn(t *testing.T, lines, data string, modules ...string) Resolution {
|
||||
t.Helper()
|
||||
all := []Manifest{held(t, backupHolder, builtWith("restic", map[string]string{"backups": lines, "data": data}))}
|
||||
names := []string{"restic"}
|
||||
for _, name := range modules {
|
||||
all = append(all, mustParse(t, dataModule(name)))
|
||||
names = append(names, name)
|
||||
}
|
||||
r, err := Resolve(shelf(all...), names, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Account = "op"
|
||||
return r
|
||||
}
|
||||
|
||||
var backupModules = []string{"alpha", "beta", "gamma"}
|
||||
|
||||
// gammaLeftOut is a machine whose gamma does not place: its lines are kept, best effort.
|
||||
func gammaLeftOut(with Rendering) Rendering {
|
||||
with.NotPlaced = map[string]string{"gamma": "its placement setting does not read"}
|
||||
return with
|
||||
}
|
||||
|
||||
func TestTheBackupPlacementIsTheTextTheOldPlaceholdersRender(t *testing.T) {
|
||||
// The old form is an unmoved holder: its placeholders are the controller's alone.
|
||||
oldHolder := strings.Replace(backupHolder, `"moved":true,`, "", 1)
|
||||
all := []Manifest{held(t, oldHolder, builtWith("restic", map[string]string{"backups": oldBackupText, "data": oldDataText}))}
|
||||
for _, name := range backupModules {
|
||||
all = append(all, mustParse(t, dataModule(name)))
|
||||
}
|
||||
old, err := Resolve(shelf(all...), append([]string{"restic"}, backupModules...), workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old.Account = "op"
|
||||
wantLines, wantData := fileOf(t, old, gammaLeftOut(Rendering{}), "restic.declared"), fileOf(t, old, gammaLeftOut(Rendering{}), "restic.data-declared")
|
||||
if !strings.Contains(wantLines, "# alpha\npath ") || !strings.Contains(wantLines, "# gamma\n") || !strings.Contains(wantData, "item d") {
|
||||
t.Fatalf("the old form renders\n%s\n%s", wantLines, wantData)
|
||||
}
|
||||
moved := backupsOn(t, placedBackupTxt, placedDataTxt, backupModules...)
|
||||
// Not handed: the controller fills the new placeholders with the old text.
|
||||
if got := fileOf(t, moved, gammaLeftOut(Rendering{}), "restic.declared"); got != wantLines {
|
||||
t.Errorf("not handed, the lines are\n%s\nwant\n%s", got, wantLines)
|
||||
}
|
||||
if got := fileOf(t, moved, gammaLeftOut(Rendering{}), "restic.data-declared"); got != wantData {
|
||||
t.Errorf("not handed, the data is\n%s\nwant\n%s", got, wantData)
|
||||
}
|
||||
// Handed: the plan carries the same text as placement, and the files are the client's.
|
||||
c, _ := composeWith(t, moved, gammaLeftOut(handingTo("store:5000")))
|
||||
p := c.Plans["restic"].Placement
|
||||
if got := "# lines\n" + p["backup:lines"]; got != wantLines {
|
||||
t.Errorf("handed, the plan's lines are\n%s\nwant\n%s", got, wantLines)
|
||||
}
|
||||
if got := "# data\n" + p["backup:data"]; got != wantData {
|
||||
t.Errorf("handed, the plan's data is\n%s\nwant\n%s", got, wantData)
|
||||
}
|
||||
if len(p) != 2 {
|
||||
t.Errorf("the plan's placement is %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleAddedOrTakenAwayChangesTheBackupPlan(t *testing.T) {
|
||||
plan := func(modules ...string) map[string]string {
|
||||
c, _ := composeWith(t, backupsOn(t, placedBackupTxt, placedDataTxt, modules...), handingTo("store:5000"))
|
||||
return c.Plans["restic"].Placement
|
||||
}
|
||||
two, three := plan("alpha", "beta"), plan("alpha", "beta", "gamma")
|
||||
if reflect.DeepEqual(two, three) || strings.Contains(two["backup:lines"], "gamma") || !strings.Contains(three["backup:lines"], "# gamma\n") {
|
||||
t.Errorf("two modules give %v, three give %v", two, three)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownBackupKeyIsRefused(t *testing.T) {
|
||||
r := backupsOn(t, "${backup:rows}", placedDataTxt, backupModules...)
|
||||
if _, err := r.Compose(handingTo("store:5000")); err == nil || !strings.Contains(err.Error(), "backup:rows") {
|
||||
t.Errorf("handed, an unknown key is %v", err)
|
||||
}
|
||||
if _, err := r.Declaration(Rendering{}); err == nil || !strings.Contains(err.Error(), "backup:rows") {
|
||||
t.Errorf("not handed, an unknown key is %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func fileOf(t *testing.T, r Resolution, with Rendering, id string) string {
|
||||
t.Helper()
|
||||
out, err := r.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, res := range out {
|
||||
if res["id"] == id {
|
||||
return res["content"].(string)
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s was not composed", id)
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A backup contribution written by hand still names its module's own directories; one it does not
|
||||
// declare is refused where it is written rather than reaching the holder as the literal text. (The
|
||||
// catalogue check refuses a hand-written backup line at all since ADR 0233; parsing still reads one,
|
||||
// because a module on the shelf was written before.)
|
||||
func TestABackupNamingAnUndeclaredDirectoryIsRefused(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:nowhere}"}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "nowhere") {
|
||||
t.Fatalf("a backup of an undeclared directory was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A module on the shelf from before ADR 0233, which writes its backup lines by hand and declares no
|
||||
// data, is still backed up: its lines are placed as written, each module's under a comment naming it.
|
||||
func TestHandWrittenBackupLinesOfAnOlderModuleArePlaced(t *testing.T) {
|
||||
pg, err := ParseManifest([]byte(`{"module":"pg","version":"1",
|
||||
"resources":[{"id":"dumps","type":"directory","path":"/srv/pg/dumps","mode":"0700"}],
|
||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"run pg-dump-all\npath ${dir:dumps}"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mail, err := ParseManifest([]byte(`{"module":"mail","version":"1",
|
||||
"resources":[{"id":"spool","type":"directory","mode":"0700"}],
|
||||
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:spool}"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
placed, _, err := seatContributions([]Manifest{pg, mail}, Manifest{}, BackupSeat+":"+"backup", Rendering{}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"# pg\nrun pg-dump-all\npath /srv/pg/dumps\n", "# mail\npath /var/lib/mail/spool\n"} {
|
||||
if !strings.Contains(placed, want) {
|
||||
t.Errorf("placed contributions lack %q:\n%s", want, placed)
|
||||
}
|
||||
}
|
||||
if strings.Contains(placed, "${dir:") {
|
||||
t.Errorf("a directory reached the holder unfilled:\n%s", placed)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The module that provides the artifact store may not be delivered through it.
|
||||
//
|
||||
// Building publishes to the store and the builder will not start without one, so a module that
|
||||
// provides the store and also builds something asks the mesh to put an artifact into the thing
|
||||
// that artifact is needed to create. On a mesh new enough to have no registry, that is a build
|
||||
// that never returns (novox/hq 04-ISSUES/029).
|
||||
func TestTheArtifactStoreCannotBeDeliveredThroughItself(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"registry","version":"1",` +
|
||||
`"provides":[{"name":"artifact-store","scope":"mesh"}],` +
|
||||
`"build":{"artifacts":[{"name":"registry","kind":"upstream","from":"registry:2"}]},` +
|
||||
`"resources":[{"id":"store","type":"container","name":"mesh-registry",` +
|
||||
`"artifact":"registry","ports":["5000:5000"]}]}`))
|
||||
if err == nil {
|
||||
t.Fatal("a registry module that builds its own image was accepted; the build has " +
|
||||
"nowhere to publish until the module it belongs to is already running")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "artifact-store") {
|
||||
t.Fatalf("refused without naming the provision the cycle turns on: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Naming the image directly is the way out, and must stay accepted.
|
||||
func TestAnArtifactStoreThatNamesItsImageIsAccepted(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"registry","version":"1",` +
|
||||
`"provides":[{"name":"artifact-store","scope":"mesh"}],` +
|
||||
`"resources":[{"id":"store","type":"container","name":"mesh-registry",` +
|
||||
`"image":"registry@sha256:` +
|
||||
`266f282fabd7cd3df053ee7c658c77b42380d1a2f0d8e5a1c0d7a6d5b5c4a3b2",` +
|
||||
`"ports":["5000:5000"]}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("the one way an artifact store can be delivered was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// And an ordinary module still builds whatever it likes.
|
||||
func TestAModuleThatDoesNotProvideTheStoreStillBuilds(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"forge","version":"1",` +
|
||||
`"build":{"artifacts":[{"name":"forge","kind":"upstream","from":"gitea/gitea:1.22"}]},` +
|
||||
`"resources":[{"id":"run","type":"container","name":"forge","artifact":"forge"}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("an ordinary module was caught by a rule about the artifact store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A mapping that names an address still names the port, and the machine side is still the middle.
|
||||
//
|
||||
// The foundation bundle writes "127.0.0.1:5432:5432" today, so the shape is not hypothetical.
|
||||
// Found in review: the first cut split on the first colon, read "127.0.0.1" as the machine port,
|
||||
// failed to parse it, and silently skipped the mapping — which put the filter back on the
|
||||
// declared port, the exact fault MachineSide was written to end.
|
||||
func TestAnAddressedMappingStillNamesThePort(t *testing.T) {
|
||||
m := Manifest{Module: "store", Resources: []map[string]any{
|
||||
{"type": "container", "id": "server", "ports": []any{"127.0.0.1:8080:80"}},
|
||||
}}
|
||||
for _, named := range []int{8080, 80} {
|
||||
at, mayAssign := m.MachineSide(named)
|
||||
if mayAssign {
|
||||
t.Fatalf("%d was reassigned though the manifest published it explicitly", named)
|
||||
}
|
||||
if at != 8080 {
|
||||
t.Fatalf("naming %d gave %d; the machine side of 127.0.0.1:8080:80 is 8080", named, at)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A consumer of a provision that keeps its data stays bound where its data is (novox/hq ADR 0232).
|
||||
//
|
||||
// **What a resolution chooses is not where a consumer's data is.** Resolving answers "which provider
|
||||
// would I pick now", from the seats' holders, the pins and what is assigned where, and every one of
|
||||
// those can change under a consumer without anybody meaning to move it. For a resolver that is the
|
||||
// point: any provider answers alike. For a database it is the consumer's whole state: on 2026-10-05
|
||||
// one change to how a seat's holder answers (issue 258) re-bound five database consumers on one
|
||||
// machine to the store on another, each was given a fresh, empty database there, and nothing warned
|
||||
// for twenty hours (issue 273). Nothing was lost only because the old provider kept everything.
|
||||
//
|
||||
// So the mesh records where each such consumer was last sent (the store's `binding` table), and a
|
||||
// resolution that would answer it from anywhere else keeps the recorded provider instead and says so.
|
||||
// **Only a pin moves it**, because a pin is a person: the one act that says "answer this machine's
|
||||
// consumers from there", taken knowing the data must go first.
|
||||
|
||||
// KeptBinding is one consumer this resolution would have moved, and did not.
|
||||
type KeptBinding struct {
|
||||
// Machine is where the consumer runs, and Consumer the module there that is bound.
|
||||
Machine string
|
||||
Consumer string
|
||||
// Provision is what it is bound for.
|
||||
Provision string
|
||||
// Bound is the provider it was recorded at, and still is; Would is the one the resolution chose.
|
||||
Bound Chosen
|
||||
Would Chosen
|
||||
}
|
||||
|
||||
// String is the condition's sentence: the move, where the data is, and the act that confirms it.
|
||||
func (k KeptBinding) String() string {
|
||||
return fmt.Sprintf("would move %s's %s from %s to %s — its data is on %s; kept there. "+
|
||||
"`pin %s %s %s %s` to confirm a move (and move the data first)",
|
||||
k.Consumer, k.Provision, k.Bound, k.Would, k.Bound, k.Machine, k.Provision, k.Would.Node, k.Would.Module)
|
||||
}
|
||||
|
||||
// keepBound holds every need for a provision that keeps its consumers' data at the provider its
|
||||
// consumer was bound to, where the resolution chose another.
|
||||
//
|
||||
// A need with no record is a binding being made, and is left as resolved: it is recorded when it is
|
||||
// first sent. A pin naming the provider the resolution chose is a person moving it, and is left too.
|
||||
// Otherwise the recorded provider answers, if it still provides the provision — beside the consumer,
|
||||
// or offered from elsewhere — and the move is returned as kept. **One that no longer does is refused,
|
||||
// never answered by the provider chosen**: answering it there is exactly the silent move this exists
|
||||
// to stop, and the consumer's data is still wherever it was.
|
||||
func keepBound(needs []Needed, catalogue map[string]Manifest, node Node, world World,
|
||||
keeps map[string]bool, here func(string) bool) ([]Needed, []KeptBinding, []string) {
|
||||
var kept []KeptBinding
|
||||
var problems []string
|
||||
refused := map[[2]string]bool{}
|
||||
out := make([]Needed, 0, len(needs))
|
||||
for _, n := range needs {
|
||||
if n.ByRecord || !keeps[n.Name] {
|
||||
out = append(out, n)
|
||||
continue
|
||||
}
|
||||
n.KeepsData = true
|
||||
bound, recorded := world.Bound[n.For][n.Name]
|
||||
chose := Chosen{Node: n.From, Module: n.Module}
|
||||
if !recorded || sameProvider(bound, chose) {
|
||||
out = append(out, n)
|
||||
continue
|
||||
}
|
||||
if pin, pinned := world.Pinned[n.Name]; pinned && pin.matches(Provider{Node: chose.Node, Module: chose.Module}) {
|
||||
out = append(out, n)
|
||||
continue
|
||||
}
|
||||
held, ok, why := boundNeed(n, bound, catalogue, node, world, here)
|
||||
if !ok {
|
||||
if key := [2]string{n.For, n.Name}; !refused[key] {
|
||||
refused[key] = true
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s on %s is bound to %s for %q, which keeps its data, and %s — it would move to %s, "+
|
||||
"which holds none of it. Move its data and say so with `pin %s %s %s %s`, or give "+
|
||||
"%s back what it provided",
|
||||
n.For, node.Name, bound, n.Name, why, chose, node.Name, n.Name, chose.Node, chose.Module,
|
||||
bound.Node))
|
||||
}
|
||||
continue
|
||||
}
|
||||
out = append(out, held)
|
||||
kept = append(kept, KeptBinding{Machine: node.Name, Consumer: n.For, Provision: n.Name, Bound: bound, Would: chose})
|
||||
}
|
||||
return out, kept, problems
|
||||
}
|
||||
|
||||
// sameProvider is whether a recorded provider is the one chosen. A record naming no module (none
|
||||
// is written without one, but a person's hand might) matches any module on its node.
|
||||
func sameProvider(bound, chose Chosen) bool {
|
||||
return bound.Node == chose.Node && (bound.Module == "" || bound.Module == chose.Module)
|
||||
}
|
||||
|
||||
// boundNeed is the need answered by the recorded provider, or why it cannot be.
|
||||
func boundNeed(n Needed, bound Chosen, catalogue map[string]Manifest, node Node, world World,
|
||||
here func(string) bool) (Needed, bool, string) {
|
||||
held := Needed{Name: n.Name, For: n.For, Local: n.Local, KeepsData: true}
|
||||
if bound.Node == node.Name {
|
||||
m, known := catalogue[bound.Module]
|
||||
if !known || !here(bound.Module) || !providesAt(m, n.Name, ScopeMesh) {
|
||||
return Needed{}, false, fmt.Sprintf("%s no longer runs on %s", bound.Module, node.Name)
|
||||
}
|
||||
at := node.At
|
||||
if at == "" {
|
||||
at = "127.0.0.1"
|
||||
}
|
||||
held.From, held.At, held.Module = node.Name, at, m.Module
|
||||
held.Serves, held.SharedOwn, held.Identity = servedByOne(m, n.Name), sharedByOne(m, n.Name), m.IdentityBoundOf(n.Name)
|
||||
held.CredentialForm, held.CredentialTool = m.CredentialFormOf(n.Name), m.CredentialToolOf(n.Name)
|
||||
return held, true, ""
|
||||
}
|
||||
matching := bound.among(world.Offered[n.Name])
|
||||
if len(matching) != 1 {
|
||||
return Needed{}, false, fmt.Sprintf("%s no longer provides it", bound)
|
||||
}
|
||||
p := matching[0]
|
||||
if node.At == "" || p.At == "" {
|
||||
return Needed{}, false, fmt.Sprintf("%s and %s are not both on the private network", node.Name, p.Node)
|
||||
}
|
||||
identity := DefaultIdentityBound
|
||||
if pm, known := catalogue[p.Module]; known {
|
||||
held.SharedOwn, _ = pm.SharedCredentialOf(n.Name)
|
||||
held.CredentialForm, held.CredentialTool = pm.CredentialFormOf(n.Name), pm.CredentialToolOf(n.Name)
|
||||
identity = pm.IdentityBoundOf(n.Name)
|
||||
}
|
||||
held.From, held.At, held.Module, held.Serves, held.Identity = p.Node, p.At, p.Module, p.Serves, identity
|
||||
held.LeftOut = p.LeftOut
|
||||
return held, true, ""
|
||||
}
|
||||
|
||||
// Unbound is one consumer that still asks for a provision and whose own resolution binds it to
|
||||
// another provider than the one a pair credential on record was made with (novox/hq issue 274).
|
||||
//
|
||||
// **A provider is granted exactly the consumers bound to it.** The credential from the old provider
|
||||
// stays on record — it is the key to a login that provider keeps, disabled, with the consumer's data,
|
||||
// until a person deletes it with `cleanup delete` (ADR 0230), and a pin back must find it — but it is
|
||||
// no longer granted, so the provider stops being asked for it and retires it. Said on every plan and
|
||||
// push of the provider, so a credential the mesh keeps and does not use is never kept silently.
|
||||
type Unbound struct {
|
||||
// Provision is what was required, Provider the machine the credential on record is from.
|
||||
Provision string `json:"provision"`
|
||||
Provider string `json:"provider"`
|
||||
// Consumer is the machine, Module the module on it that requires it, Local the credential's
|
||||
// name inside it where it keeps several (ADR 0094).
|
||||
Consumer string `json:"consumer"`
|
||||
Module string `json:"module"`
|
||||
Local string `json:"local,omitempty"`
|
||||
// BoundTo is every provider the consumer's resolution binds this credential to now; empty when
|
||||
// it binds it nowhere — the module asks for the provision under other local names.
|
||||
BoundTo []string `json:"bound_to,omitempty"`
|
||||
}
|
||||
|
||||
func (u Unbound) String() string {
|
||||
who := u.Module
|
||||
if u.Local != "" {
|
||||
who += " (as " + u.Local + ")"
|
||||
}
|
||||
now := "is bound to no provider under that name"
|
||||
if len(u.BoundTo) > 0 {
|
||||
now = "is bound to " + strings.Join(u.BoundTo, ", ")
|
||||
}
|
||||
return fmt.Sprintf("%s on %s %s for %s, not to %s — %s no longer grants it, so it retires that "+
|
||||
"login and keeps its data until `cleanup delete` (ADR 0230); the credential from %s stays on "+
|
||||
"record while that login does", who, u.Consumer, now, u.Provision, u.Provider, u.Provider, u.Provider)
|
||||
}
|
||||
|
||||
// BindsFrom is the providers this resolution binds a pair credential's need to — the provision, the
|
||||
// module that requires it and the credential's local name — answered by a machine rather than by a
|
||||
// record. None means this machine states no such binding.
|
||||
func (r Resolution) BindsFrom(provision, module, local string) []string {
|
||||
var from []string
|
||||
for _, n := range r.Needs {
|
||||
if n.ByRecord || n.Name != provision || n.For != module || n.Local != local {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(from, n.From) {
|
||||
from = append(from, n.From)
|
||||
}
|
||||
}
|
||||
return from
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The half of a connection that is not secret, put where the program reading it can find it.
|
||||
//
|
||||
// **The asymmetry this removes was backwards** (novox/hq 04-ISSUES/023). A sealed credential can
|
||||
// be placed inside any configuration file a module writes: the module leaves a hole, the mesh
|
||||
// delivers the value sealed beside it, and the host — the only thing that sees both — fills it in.
|
||||
// The host and the port and the name to present are ordinary facts the mesh holds in the clear,
|
||||
// and they were the ones stuck: readable only inside a JSON binding, which a program reading
|
||||
// `KEY=value` cannot use.
|
||||
//
|
||||
// So the same shape, and simpler. These values are not secret, so **the control plane substitutes
|
||||
// them itself** before the declaration is sent. Nothing new reaches the host, which learns no
|
||||
// formats and gains no fields.
|
||||
//
|
||||
// **It stays name-agnostic** ([ADR 0027]). The mesh does not learn what a `postgres-database` is:
|
||||
// `at`, `as`, `from` and `address` (the providing machine's private address, novox/hq ADR 0194) are
|
||||
// facts about any provision at all, and everything else comes from what the provider said it
|
||||
// serves — whose keys are agreed by the requirement's name, not by this file.
|
||||
|
||||
// bound is where a module says a value from one of its bindings belongs:
|
||||
// ${bound:<provision>:<key>}, and for an optional requirement the value it takes while absent,
|
||||
// ${bound:<provision>:<key>|default:<value>} (novox/hq ADR 0330 §4). A default is never empty: an
|
||||
// absent requirement is never rendered as an empty string.
|
||||
var bound = regexp.MustCompile(`\$\{bound:([a-z0-9][a-z0-9.-]*[a-z0-9]):([a-z0-9][a-z0-9_-]*)(?:\|default:([^}\n]+))?\}`)
|
||||
|
||||
// knownFor is everything a module may name from one of its bindings.
|
||||
//
|
||||
// Three facts the mesh states about any provision, plus whatever the provider said it serves. A
|
||||
// module may not reach a binding it does not have — the same boundary as a secret, for the same
|
||||
// reason.
|
||||
func knownFor(m Manifest, needs []Needed, node string) (map[string]map[string]string, error) {
|
||||
out := map[string]map[string]string{}
|
||||
for _, want := range m.Wants() {
|
||||
for i := range needs {
|
||||
n := needs[i]
|
||||
if n.Name != want || n.For != m.Module {
|
||||
continue
|
||||
}
|
||||
as := ConsumerIdentity(node, IdentitySource(m.Slug, m.Module))
|
||||
values := map[string]string{
|
||||
"at": n.At,
|
||||
"from": n.From,
|
||||
"as": as,
|
||||
}
|
||||
// What the provider derives for this consumer rather than for all of them
|
||||
// (novox/hq ADR 0202). Filled here, the one place a provision and the module
|
||||
// requiring it are both in hand.
|
||||
served, err := ServedTo(n.Serves, as)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s requires %s: %w", m.Module, want, err)
|
||||
}
|
||||
for key, value := range served {
|
||||
// The provider's own vocabulary. Rendered plainly: a port is 5432, not 5432.000000,
|
||||
// which is what a float would write and what a connection string would refuse.
|
||||
values[key] = plainly(value)
|
||||
}
|
||||
out[want] = values
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// withOwnNames adds a module's own composed names to what it may name from one binding:
|
||||
// `${bound:<provision>:name}` and `:internal-name`, and for several contributions to one requirement
|
||||
// `:name-<local>` / `:internal-name-<local>`. Set over anything the provider serves under those keys:
|
||||
// what the module is called is the mesh's statement, not the provider's.
|
||||
func withOwnNames(values map[string]string, own map[string]any) {
|
||||
for _, key := range []string{"name", "internal-name"} {
|
||||
if v, ok := own[key].(string); ok {
|
||||
values[key] = v
|
||||
}
|
||||
}
|
||||
many, _ := own["names"].(map[string]any)
|
||||
for local, raw := range many {
|
||||
names, _ := raw.(map[string]any)
|
||||
for _, key := range []string{"name", "internal-name"} {
|
||||
if v, ok := names[key].(string); ok {
|
||||
values[key+"-"+local] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// plainly renders a served value as a program would expect to read it.
|
||||
func plainly(value any) string {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return v
|
||||
case float64:
|
||||
if v == float64(int64(v)) {
|
||||
return fmt.Sprintf("%d", int64(v))
|
||||
}
|
||||
return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%f", v), "0"), ".")
|
||||
case bool:
|
||||
return fmt.Sprintf("%t", v)
|
||||
case nil:
|
||||
return ""
|
||||
default:
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
}
|
||||
|
||||
// boundInto replaces a file's ${bound:…} placeholders with what the mesh knows.
|
||||
//
|
||||
// A placeholder naming something the module does not require, or a key the provider does not
|
||||
// serve, is refused. Left as it was, the literal `${bound:x:y}` would be written into a
|
||||
// configuration file and read as a value — a connection to a host called `${bound:x:y}`, failing
|
||||
// somewhere that names neither the module nor the mesh.
|
||||
//
|
||||
// **An optional requirement left absent takes the default its placeholder names** (novox/hq ADR
|
||||
// 0330 §4), and one that names none is refused rather than rendered empty. The module check refuses
|
||||
// that already; this is the same rule where the file is made, for a manifest registered before it.
|
||||
func boundInto(resource map[string]any, known map[string]map[string]string, module string, absent map[string]bool) error {
|
||||
if fmt.Sprint(resource["type"]) != "file" {
|
||||
return nil
|
||||
}
|
||||
content, ok := resource["content"].(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var refused error
|
||||
filled := bound.ReplaceAllStringFunc(content, func(token string) string {
|
||||
if refused != nil {
|
||||
return token
|
||||
}
|
||||
m := bound.FindStringSubmatch(token)
|
||||
provision, key, fallback := m[1], m[2], m[3]
|
||||
values, has := known[provision]
|
||||
if !has {
|
||||
if absent[provision] {
|
||||
if fallback != "" {
|
||||
return fallback
|
||||
}
|
||||
refused = fmt.Errorf(
|
||||
"%s has a file that says ${bound:%s:%s}, and %q is optional and absent here: it names no "+
|
||||
"value for then (${bound:%s:%s|default:<value>}), so it would be written empty "+
|
||||
"(novox/hq ADR 0330 §4)", module, provision, key, provision, provision, key)
|
||||
return token
|
||||
}
|
||||
refused = fmt.Errorf(
|
||||
"%s has a file that says ${bound:%s:%s}, and %s does not require %q. It may name %s",
|
||||
module, provision, key, module, provision, orNothing(namesOfBindings(known)))
|
||||
return token
|
||||
}
|
||||
value, said := values[key]
|
||||
if !said {
|
||||
refused = fmt.Errorf(
|
||||
"%s asks its %s binding for %q, and what answers it says %s",
|
||||
module, provision, key, orNothing(namesOfKeys(values)))
|
||||
return token
|
||||
}
|
||||
return value
|
||||
})
|
||||
if refused != nil {
|
||||
return refused
|
||||
}
|
||||
resource["content"] = filled
|
||||
return nil
|
||||
}
|
||||
|
||||
func namesOfBindings(known map[string]map[string]string) []string {
|
||||
var names []string
|
||||
for name := range known {
|
||||
names = append(names, fmt.Sprintf("%q", name))
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func namesOfKeys(values map[string]string) []string {
|
||||
var names []string
|
||||
for key := range values {
|
||||
names = append(names, fmt.Sprintf("%q", key))
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func orNothing(names []string) string {
|
||||
if len(names) == 0 {
|
||||
return "nothing"
|
||||
}
|
||||
return join(names)
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A consumer writes its own connection string (novox/hq 04-ISSUES/023).
|
||||
//
|
||||
// Everything needed for one now reaches the file that needs it: the address and port from what the
|
||||
// provider serves, the name to present from what the mesh decided, the database name from what the
|
||||
// module itself contributed, and the password through the sealed hole the host fills.
|
||||
func TestAConsumerCanWriteAConnectionString(t *testing.T) {
|
||||
r := Resolution{
|
||||
Node: "workstation",
|
||||
Modules: []Manifest{{
|
||||
Module: "keycloak",
|
||||
Requires: []string{"postgres-database"},
|
||||
Secrets: map[string]string{"postgres-database": "/var/lib/keycloak/db.secret"},
|
||||
Resources: []map[string]any{{
|
||||
"id": "dbenv", "type": "file", "path": "/var/lib/keycloak/database.env",
|
||||
"mode": "0600",
|
||||
"content": "KC_DB_URL=jdbc:postgresql://${bound:postgres-database:at}:" +
|
||||
"${bound:postgres-database:port}/keycloak\n" +
|
||||
"KC_DB_USERNAME=${bound:postgres-database:as}\n" +
|
||||
"KC_DB_PASSWORD=${secret:postgres-database}\n",
|
||||
}},
|
||||
}},
|
||||
Needs: []Needed{{
|
||||
Name: "postgres-database", From: "anchor", At: "anchor.internal", For: "keycloak",
|
||||
Sealed: "sealed-db", Serves: map[string]any{"port": float64(5432)},
|
||||
}},
|
||||
}
|
||||
out, err := r.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
file := fileNamed(out, "keycloak.dbenv")
|
||||
if file == nil {
|
||||
t.Fatalf("no file in %v", out)
|
||||
}
|
||||
got := file["content"].(string)
|
||||
|
||||
want := "KC_DB_URL=jdbc:postgresql://anchor.internal:5432/keycloak\n" +
|
||||
"KC_DB_USERNAME=mesh_workstation_keycloak\n" +
|
||||
"KC_DB_PASSWORD=${secret:postgres-database}\n"
|
||||
if got != want {
|
||||
t.Errorf("the consumer cannot connect with what it was given:\n got %q\nwant %q", got, want)
|
||||
}
|
||||
// The password is still a hole. Only the host may fill that one, and only on the machine.
|
||||
if sealed, _ := file["secrets"].(map[string]any); sealed["postgres-database"] != "sealed-db" {
|
||||
t.Errorf("the credential did not travel with the file: %v", file)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider answered on its consumer's own machine is announced at the port the machine PUBLISHES
|
||||
// it on, not the port the module declared (novox/hq 04-ISSUES/038).
|
||||
//
|
||||
// Issue 018 made the same-node provider announced at all; this is the promise after it — that the
|
||||
// port a co-located consumer is told is the one actually listening at the address it is handed. The
|
||||
// mesh assigns a bare `ports` mapping a host port (here the container's 5432 becomes 15432 on this
|
||||
// machine), and it must announce 15432, or the consumer dials workstation.internal:5432 where the
|
||||
// provider is not published. Cross-node consumers already got this (the plan re-derives the served
|
||||
// facts with the provider's assignment); the same-node need was settled before the assignment and
|
||||
// carried the declared port until now.
|
||||
func TestASameNodeProviderIsAnnouncedAtThePortItIsPublishedOn(t *testing.T) {
|
||||
provider := Manifest{
|
||||
Module: "postgres", Version: "1",
|
||||
Provides: FromAnywhere("postgres-database"),
|
||||
Listens: []Listening{{Port: 5432, Protocol: "tcp", From: "mesh"}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Resources: []map[string]any{{
|
||||
"id": "server", "type": "container", "name": "postgres",
|
||||
"ports": []any{"5432"},
|
||||
}},
|
||||
}
|
||||
consumer := Manifest{
|
||||
Module: "keycloak", Version: "1", Requires: []string{"postgres-database"},
|
||||
Resources: []map[string]any{{
|
||||
"id": "dbenv", "type": "file", "path": "/var/lib/keycloak/database.env", "mode": "0600",
|
||||
"content": "KC_DB_URL=jdbc:postgresql://${bound:postgres-database:at}:" +
|
||||
"${bound:postgres-database:port}/keycloak\n",
|
||||
}},
|
||||
}
|
||||
r, err := Resolve(shelf(provider, consumer),
|
||||
[]string{"postgres", "keycloak"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The mesh assigned the container's 5432 to 15432 on this machine.
|
||||
out, err := r.Declaration(Rendering{Ports: map[string]map[int]int{"postgres": {5432: 15432}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Where the provider is actually published: the assigned host port, on the machine at large.
|
||||
server := fileNamed(out, "postgres.server")
|
||||
if server == nil {
|
||||
t.Fatalf("the provider container is not in the declaration: %v", out)
|
||||
}
|
||||
if published := fmt.Sprint(server["ports"]); !strings.Contains(published, "15432:5432") {
|
||||
t.Fatalf("the provider was not published on the assigned port: %v", server["ports"])
|
||||
}
|
||||
|
||||
// What the consumer was told to dial: the SAME port, at the announced address — not the 5432 the
|
||||
// module declared, which is closed on that address.
|
||||
dbenv := fileNamed(out, "keycloak.dbenv")
|
||||
if dbenv == nil {
|
||||
t.Fatalf("the consumer's binding file is not in the declaration: %v", out)
|
||||
}
|
||||
got := dbenv["content"].(string)
|
||||
want := "KC_DB_URL=jdbc:postgresql://workstation.internal:15432/keycloak\n"
|
||||
if got != want {
|
||||
t.Errorf("the consumer dials a port the provider does not listen on:\n got %q\nwant %q",
|
||||
got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A port is 5432, not 5432.000000 — which is what a number decoded from JSON would write, and what
|
||||
// every connection string in the world would refuse.
|
||||
func TestAPortIsWrittenAsAPort(t *testing.T) {
|
||||
if got := plainly(float64(5432)); got != "5432" {
|
||||
t.Errorf("a port was rendered as %q", got)
|
||||
}
|
||||
if got := plainly("postgres"); got != "postgres" {
|
||||
t.Errorf("a string was rendered as %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The name the provider will create is the name the consumer is told to present. One derivation,
|
||||
// so the two agree by construction rather than by two conventions written on the same day.
|
||||
func TestTheProviderAndTheConsumerAgreeOnTheName(t *testing.T) {
|
||||
provider, err := Resolve(shelf(Manifest{
|
||||
Module: "postgres", Version: "1", Provides: FromAnywhere("postgres-database"),
|
||||
Grants: map[string]string{"postgres-database": "/var/lib/postgres/grants"},
|
||||
Receives: map[string]string{"postgres-database": "/var/lib/postgres/grants/mesh.json"},
|
||||
}), []string{"postgres"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := provider.Declaration(Rendering{Grants: []Grant{{
|
||||
Provision: "postgres-database", Consumer: "workstation", From: "keycloak",
|
||||
Values: map[string]any{"name": "keycloak"}, Sealed: "sealed",
|
||||
}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var told string
|
||||
for _, res := range out {
|
||||
if res["path"] == "/var/lib/postgres/grants/mesh.json" {
|
||||
told = res["content"].(string)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(told, `"as": "mesh_workstation_keycloak"`) {
|
||||
t.Errorf("the provider was not told what to call the login: %s", told)
|
||||
}
|
||||
if told == "" {
|
||||
t.Fatal("the provider was told nothing at all")
|
||||
}
|
||||
// Which is exactly what the consumer, on its own machine, is told to present.
|
||||
if ConsumerIdentity("workstation", "keycloak") != "mesh_workstation_keycloak" {
|
||||
t.Errorf("got %q", ConsumerIdentity("workstation", "keycloak"))
|
||||
}
|
||||
}
|
||||
|
||||
// A binding the module does not have is a typo, refused where it was written.
|
||||
func TestAFileCannotNameABindingTheModuleDoesNotHave(t *testing.T) {
|
||||
r := Resolution{
|
||||
Node: "workstation",
|
||||
Modules: []Manifest{{
|
||||
Module: "keycloak", Requires: []string{"postgres-database"},
|
||||
Resources: []map[string]any{{
|
||||
"id": "conf", "type": "file", "path": "/etc/keycloak.conf",
|
||||
"content": "host=${bound:s3-bucket:at}\n",
|
||||
}},
|
||||
}},
|
||||
Needs: []Needed{{Name: "postgres-database", From: "anchor", At: "a", For: "keycloak"}},
|
||||
}
|
||||
_, err := r.Declaration(Rendering{})
|
||||
if err == nil {
|
||||
t.Fatal("a module read a binding it does not have")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "postgres-database") {
|
||||
t.Errorf("the refusal does not say what it could have named: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A key the provider does not serve is refused too. Written through, the literal placeholder
|
||||
// becomes a hostname, and the failure names neither the module nor the mesh.
|
||||
func TestAKeyTheProviderDoesNotServeIsRefused(t *testing.T) {
|
||||
r := Resolution{
|
||||
Node: "workstation",
|
||||
Modules: []Manifest{{
|
||||
Module: "keycloak", Requires: []string{"postgres-database"},
|
||||
Resources: []map[string]any{{
|
||||
"id": "conf", "type": "file", "path": "/etc/keycloak.conf",
|
||||
"content": "sock=${bound:postgres-database:socket}\n",
|
||||
}},
|
||||
}},
|
||||
Needs: []Needed{{
|
||||
Name: "postgres-database", From: "anchor", At: "a", For: "keycloak",
|
||||
Serves: map[string]any{"port": float64(5432)},
|
||||
}},
|
||||
}
|
||||
_, err := r.Declaration(Rendering{})
|
||||
if err == nil {
|
||||
t.Fatal("a module read a key nothing serves, and the placeholder became a value")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `"port"`) {
|
||||
t.Errorf("the refusal does not say what is served: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A name too long to stay distinct is refused before anything is created.
|
||||
func TestAnIdentityTooLongToBeDistinctIsRefused(t *testing.T) {
|
||||
if err := CheckIdentity("anchor", "keycloak"); err != nil {
|
||||
t.Fatalf("an ordinary consumer was refused: %v", err)
|
||||
}
|
||||
if err := CheckIdentity(strings.Repeat("n", 40), strings.Repeat("m", 40)); err == nil {
|
||||
t.Fatal("an identity that providers would shorten was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// Two modules on one node get two identities, which is the whole point of both issues.
|
||||
func TestTwoModulesOnOneNodeAreTwoIdentities(t *testing.T) {
|
||||
if ConsumerIdentity("anchor", "gitea") == ConsumerIdentity("anchor", "keycloak") {
|
||||
t.Fatal("two consumers on one machine share an identity")
|
||||
}
|
||||
// And one module on two nodes, likewise.
|
||||
if ConsumerIdentity("anchor", "gitea") == ConsumerIdentity("laptop", "gitea") {
|
||||
t.Fatal("one module on two machines shares an identity")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's resolver configuration must name its resolver by address — it cannot resolve the name
|
||||
// of the thing it resolves names with (novox/hq ADR 0194). So a binding offers the providing
|
||||
// machine's private address beside its name, and only when the machine has one.
|
||||
func TestABindingOffersTheProvidersAddress(t *testing.T) {
|
||||
consumer := func() Resolution {
|
||||
return Resolution{
|
||||
Node: "workstation",
|
||||
Modules: []Manifest{{
|
||||
Module: "resolv-conf",
|
||||
Requires: []string{"wildcard-resolution"},
|
||||
Resources: []map[string]any{{
|
||||
"id": "resolv", "type": "file", "path": "/etc/resolv.conf", "mode": "0644",
|
||||
"content": "nameserver ${bound:wildcard-resolution:address}\n",
|
||||
}},
|
||||
}},
|
||||
Needs: []Needed{{Name: "wildcard-resolution", From: "anchor", At: "anchor.internal", For: "resolv-conf"}},
|
||||
}
|
||||
}
|
||||
out, err := consumer().Declaration(Rendering{Machines: map[string]string{"anchor.internal": "10.77.0.1"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fileNamed(out, "resolv-conf.resolv")["content"]; got != "nameserver 10.77.0.1\n" {
|
||||
t.Fatalf("the resolver is not named by its address: %q", got)
|
||||
}
|
||||
// A machine with no address yet: refused, never written with a blank where the address belongs.
|
||||
if _, err := consumer().Declaration(Rendering{}); err == nil {
|
||||
t.Fatal("a file naming an address the mesh does not have was composed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A consumer of a provision that keeps its data stays bound where its data is (novox/hq ADR 0232,
|
||||
// issue 273).
|
||||
//
|
||||
// The incident, exactly: a machine runs its own store and five consumers of it; the mesh's store seat
|
||||
// is held by the store on another machine. Issue 258's rule — the seat's holder elsewhere answers
|
||||
// before this machine's own provider — re-bound all five to the holder, each was made a fresh, empty
|
||||
// database there, and nothing said so.
|
||||
|
||||
var incidentConsumers = []string{"board", "listings", "workflows", "agents", "game"}
|
||||
|
||||
func storeMesh() map[string]Manifest {
|
||||
shelf := map[string]Manifest{
|
||||
"store": {Module: "store", Version: "1",
|
||||
Provides: []Offer{{Name: "postgres-database", Scope: ScopeMesh}},
|
||||
Claims: []Claim{{Name: "mesh-store", Scope: ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Grants: map[string]string{"postgres-database": "/grants"}},
|
||||
"resolver": {Module: "resolver", Version: "1",
|
||||
Provides: []Offer{{Name: "wildcard-resolution", Scope: ScopeMesh}},
|
||||
Claims: []Claim{{Name: "mesh-dns-resolver", Scope: ScopeMesh}}},
|
||||
"network": {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}},
|
||||
}
|
||||
for _, c := range incidentConsumers {
|
||||
shelf[c] = Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}}
|
||||
}
|
||||
return shelf
|
||||
}
|
||||
|
||||
// storeWorld is the rest of the mesh as the home server's plan sees it: the anchor runs a store and a
|
||||
// resolver and holds both seats; the home server runs its own of each.
|
||||
func storeWorld() World {
|
||||
w := World{Offered: map[string][]Provider{
|
||||
"postgres-database": {
|
||||
{Node: "anchor", At: "anchor.internal", Module: "store", Serves: map[string]any{"port": 5432}},
|
||||
{Node: "home", At: "home.internal", Module: "store", Serves: map[string]any{"port": 5434}},
|
||||
},
|
||||
"wildcard-resolution": {
|
||||
{Node: "anchor", At: "anchor.internal", Module: "resolver"},
|
||||
{Node: "home", At: "home.internal", Module: "resolver"},
|
||||
},
|
||||
}}
|
||||
w.Held = []Held{
|
||||
{Claim: "mesh-store", Scope: ScopeMesh, Node: "anchor", Module: "store"},
|
||||
{Claim: "mesh-dns-resolver", Scope: ScopeMesh, Node: "anchor", Module: "resolver"},
|
||||
}
|
||||
w.Holdings = w.Held
|
||||
return w
|
||||
}
|
||||
|
||||
var home = Node{Name: "home", At: "home.internal"}
|
||||
|
||||
func homeAssigned() []string {
|
||||
return append([]string{"store", "resolver", "network"}, incidentConsumers...)
|
||||
}
|
||||
|
||||
func boundFrom(t *testing.T, r Resolution, consumer, provision string) Needed {
|
||||
t.Helper()
|
||||
for _, n := range r.Needs {
|
||||
if n.For == consumer && n.Name == provision {
|
||||
return n
|
||||
}
|
||||
}
|
||||
t.Fatalf("no binding of %s for %s: %+v", consumer, provision, r.Needs)
|
||||
return Needed{}
|
||||
}
|
||||
|
||||
func TestTheIncidentAMachinesOwnStoreKeepsItsConsumersWhenTheSeatIsHeldElsewhere(t *testing.T) {
|
||||
got, err := Resolve(storeMesh(), homeAssigned(), home, storeWorld())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range incidentConsumers {
|
||||
if n := boundFrom(t, got, c, "postgres-database"); n.From != "home" || n.Module != "store" {
|
||||
t.Errorf("%s bound to %s/%s; its data is on the store beside it (issue 273)", c, n.From, n.Module)
|
||||
}
|
||||
}
|
||||
// And the resolver, which keeps nothing of anybody's, still answers from the seat's holder (258).
|
||||
if n := boundFrom(t, got, "network", "wildcard-resolution"); n.From != "anchor" {
|
||||
t.Errorf("the resolver bound to %s; the seat is held on anchor (issue 258)", n.From)
|
||||
}
|
||||
if len(got.Kept) != 0 {
|
||||
t.Errorf("nothing was moved, and %d kept: %+v", len(got.Kept), got.Kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheIncidentWithEveryConsumerOnRecordStillMovesNothing(t *testing.T) {
|
||||
w := storeWorld()
|
||||
w.Bound = map[string]map[string]Chosen{}
|
||||
for _, c := range incidentConsumers {
|
||||
w.Bound[c] = map[string]Chosen{"postgres-database": {Node: "home", Module: "store"}}
|
||||
}
|
||||
got, err := Resolve(storeMesh(), homeAssigned(), home, w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range incidentConsumers {
|
||||
if n := boundFrom(t, got, c, "postgres-database"); n.From != "home" {
|
||||
t.Errorf("%s bound to %s", c, n.From)
|
||||
}
|
||||
}
|
||||
if len(got.Kept) != 0 {
|
||||
t.Errorf("kept %+v", got.Kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPinElsewhereStillMovesAStoresConsumers(t *testing.T) {
|
||||
w := storeWorld()
|
||||
w.Pinned = map[string]Chosen{"postgres-database": {Node: "anchor", Module: "store"}}
|
||||
w.Bound = map[string]map[string]Chosen{"board": {"postgres-database": {Node: "home", Module: "store"}}}
|
||||
got, err := Resolve(storeMesh(), homeAssigned(), home, w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := boundFrom(t, got, "board", "postgres-database"); n.From != "anchor" || n.Module != "store" {
|
||||
t.Errorf("bound to %s/%s; a person pinned it to anchor", n.From, n.Module)
|
||||
}
|
||||
if len(got.Kept) != 0 {
|
||||
t.Errorf("a pin is a person's move, not one to keep: %+v", got.Kept)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer on a machine with no store of its own, bound to one store, when the seat moves to
|
||||
// another: the holder would answer, and the binding stays.
|
||||
func laptopWorld(holder string) World {
|
||||
w := storeWorld()
|
||||
w.Held = []Held{{Claim: "mesh-store", Scope: ScopeMesh, Node: holder, Module: "store"}}
|
||||
w.Holdings = w.Held
|
||||
return w
|
||||
}
|
||||
|
||||
var laptop = Node{Name: "laptop", At: "laptop.internal"}
|
||||
|
||||
func TestABoundConsumerStaysWhenTheSeatsHolderChanges(t *testing.T) {
|
||||
w := laptopWorld("home")
|
||||
w.Bound = map[string]map[string]Chosen{"board": {"postgres-database": {Node: "anchor", Module: "store"}}}
|
||||
got, err := Resolve(storeMesh(), []string{"board"}, laptop, w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n := boundFrom(t, got, "board", "postgres-database")
|
||||
if n.From != "anchor" || n.At != "anchor.internal" || n.Serves["port"] != 5432 {
|
||||
t.Fatalf("bound to %s at %s %v; its data is on anchor", n.From, n.At, n.Serves)
|
||||
}
|
||||
if len(got.Kept) != 1 {
|
||||
t.Fatalf("the move was not said: %+v", got.Kept)
|
||||
}
|
||||
k := got.Kept[0]
|
||||
if k.Bound != (Chosen{"anchor", "store"}) || k.Would != (Chosen{"home", "store"}) || k.Consumer != "board" {
|
||||
t.Fatalf("kept %+v", k)
|
||||
}
|
||||
for _, want := range []string{"would move board's postgres-database from anchor/store to home/store",
|
||||
"its data is on anchor/store", "pin laptop postgres-database home store", "move the data first"} {
|
||||
if !strings.Contains(k.String(), want) {
|
||||
t.Errorf("%q does not say %q", k.String(), want)
|
||||
}
|
||||
}
|
||||
// Without a record it is a binding being made, and the holder answers it as before.
|
||||
w.Bound = nil
|
||||
got, err = Resolve(storeMesh(), []string{"board"}, laptop, w)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := boundFrom(t, got, "board", "postgres-database"); n.From != "home" {
|
||||
t.Errorf("a new consumer bound to %s; the seat is held on home", n.From)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABoundConsumerWhoseProviderIsGoneIsRefusedNotMoved(t *testing.T) {
|
||||
w := laptopWorld("anchor")
|
||||
w.Offered["postgres-database"] = w.Offered["postgres-database"][:1] // only anchor's store is left
|
||||
w.Bound = map[string]map[string]Chosen{"board": {"postgres-database": {Node: "home", Module: "store"}}}
|
||||
_, err := Resolve(storeMesh(), []string{"board"}, laptop, w)
|
||||
if err == nil {
|
||||
t.Fatal("bound to home's store, which is gone, and answered by anchor's — the silent move")
|
||||
}
|
||||
for _, want := range []string{"board on laptop is bound to home/store", "home/store no longer provides it",
|
||||
"pin laptop postgres-database anchor store"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("%q does not say %q", err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachinesOwnStoreUnassignedRefusesItsBoundConsumers(t *testing.T) {
|
||||
w := storeWorld()
|
||||
w.Bound = map[string]map[string]Chosen{"board": {"postgres-database": {Node: "home", Module: "store"}}}
|
||||
_, err := Resolve(storeMesh(), []string{"board"}, home, w)
|
||||
if err == nil || !strings.Contains(err.Error(), "store no longer runs on home") {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The first pass asks only what a machine offers, and is never refused by a binding.
|
||||
func TestTheFirstPassKeepsNoBinding(t *testing.T) {
|
||||
w := storeWorld()
|
||||
w.Unchecked = true
|
||||
w.Bound = map[string]map[string]Chosen{"board": {"postgres-database": {Node: "gone", Module: "store"}}}
|
||||
if _, err := Resolve(storeMesh(), []string{"board"}, laptop, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOfferSaysWhetherItKeepsConsumerData(t *testing.T) {
|
||||
var o Offer
|
||||
if err := json.Unmarshal([]byte(`{"name":"wildcard-resolution","scope":"mesh","keeps-consumer-data":false}`), &o); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o.KeepsConsumerData == nil || *o.KeepsConsumerData {
|
||||
t.Fatalf("read %+v", o)
|
||||
}
|
||||
out, err := json.Marshal(o)
|
||||
if err != nil || !strings.Contains(string(out), `"keeps-consumer-data":false`) {
|
||||
t.Fatalf("wrote %s, %v", out, err)
|
||||
}
|
||||
yes, no := true, false
|
||||
granting := Manifest{Module: "g", Provides: []Offer{{Name: "p", Scope: ScopeMesh}}, Grants: map[string]string{"p": "/g"}}
|
||||
if !granting.KeepsConsumerData("p") {
|
||||
t.Error("a provider granting each consumer a credential keeps what it writes, unsaid")
|
||||
}
|
||||
if (Manifest{Module: "r", Provides: []Offer{{Name: "p", Scope: ScopeMesh}}}).KeepsConsumerData("p") {
|
||||
t.Error("a provider granting nothing keeps nothing, unsaid")
|
||||
}
|
||||
granting.Provides[0].KeepsConsumerData = &no
|
||||
if granting.KeepsConsumerData("p") {
|
||||
t.Error("what an offer says, it gets")
|
||||
}
|
||||
said := Manifest{Module: "s", Provides: []Offer{{Name: "p", Scope: ScopeMesh, KeepsConsumerData: &yes}}}
|
||||
if !said.KeepsConsumerData("p") || !KeepsConsumerData(map[string]Manifest{"s": said, "r": {Module: "r",
|
||||
Provides: []Offer{{Name: "p", Scope: ScopeMesh, KeepsConsumerData: &no}}}}, "p") {
|
||||
t.Error("a name any provider says keeps data keeps data")
|
||||
}
|
||||
}
|
||||
|
||||
// An offer saying it keeps nothing is answered by the seat's holder as the resolver is, even where it
|
||||
// grants a credential.
|
||||
func TestAStoreSayingItKeepsNothingFollowsTheSeat(t *testing.T) {
|
||||
shelf := storeMesh()
|
||||
no := false
|
||||
s := shelf["store"]
|
||||
s.Provides = []Offer{{Name: "postgres-database", Scope: ScopeMesh, KeepsConsumerData: &no}}
|
||||
shelf["store"] = s
|
||||
got, err := Resolve(shelf, homeAssigned(), home, storeWorld())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := boundFrom(t, got, "board", "postgres-database"); n.From != "anchor" {
|
||||
t.Errorf("bound to %s; it keeps nothing, and the seat is held on anchor", n.From)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The mesh's bus is one per mesh, read from the catalogue beside this checkout.
|
||||
//
|
||||
// **This is step 2's claim, and it is checked here rather than in a bed** (novox/hq ADR 0116):
|
||||
// adoption puts the NATS server into the `mesh-broker` seat on a mesh that is already running,
|
||||
// and the property that matters is that a second one anywhere is refused *when it is assigned*,
|
||||
// not discovered later as two servers holding different halves of the mesh's traffic. A second
|
||||
// bus is not a degraded mesh; it is two meshes that both believe they are the one.
|
||||
func TestASecondMeshBusAnywhereIsRefusedByName(t *testing.T) {
|
||||
nats := catalogueManifest(t, "nats")
|
||||
|
||||
if _, err := Resolve(shelf(nats), []string{"nats"}, workstation(), World{}); err != nil {
|
||||
t.Fatalf("the bus alone does not resolve: %v", err)
|
||||
}
|
||||
|
||||
elsewhere := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
|
||||
Node: "anchor", Module: "nats"}}}
|
||||
other := workstation()
|
||||
other.Name = "laptop"
|
||||
_, err := Resolve(shelf(nats), []string{"nats"}, other, elsewhere)
|
||||
if err == nil {
|
||||
t.Fatal("a second bus was accepted on another machine")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "one per mesh") {
|
||||
t.Fatalf("refused without naming the seat: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The seat is the server's role, not the product's name (novox/hq ADR 0079). A different
|
||||
// implementation of the bus claims the same seat, and the mesh refuses it for the same reason —
|
||||
// which is the property that lets the bus be replaced at all.
|
||||
func TestTheSeatRefusesADifferentBusToo(t *testing.T) {
|
||||
nats := catalogueManifest(t, "nats")
|
||||
held := World{Held: []Held{{Claim: "mesh-broker", Scope: ScopeMesh,
|
||||
Node: "anchor", Module: "some-other-broker"}}}
|
||||
other := workstation()
|
||||
other.Name = "laptop"
|
||||
if _, err := Resolve(shelf(nats), []string{"nats"}, other, held); err == nil {
|
||||
t.Fatal("the seat admitted a second holder because the module's name differed")
|
||||
}
|
||||
}
|
||||
|
||||
// The old broker is gone from the catalogue (novox/hq ADR 0131, design 28 task 5.4), so it is no
|
||||
// longer a fixture here. That two eligible holders stand beside each other with one on record is
|
||||
// pinned in holdings_test.go against manifests this package owns.
|
||||
|
||||
// **A seat and the interface it delivers are different names, and renaming one must not rename
|
||||
// the other** (novox/hq ADR 0118). This nearly went wrong: the seats were renamed to the `mesh-*`
|
||||
// prefix, and a blanket search-and-replace also renamed `npm-package-registry` and `git` where
|
||||
// they are *provisions* — which a consumer requires and a provider offers. The tests failed with
|
||||
// "the package registry is served on <nil>", which does not say "you renamed an interface".
|
||||
func TestRenamingASeatDidNotRenameTheInterfaceItDelivers(t *testing.T) {
|
||||
for _, pair := range []struct{ seat, delivers string }{
|
||||
{"git", "git"},
|
||||
{"npm-package-registry", "npm-package-registry"},
|
||||
{"mesh-artifact-store", "artifact-store"},
|
||||
{"mesh-store", "postgres-database"},
|
||||
{"mesh-broker", "mesh-bus"},
|
||||
} {
|
||||
s, known := SeatNamed(pair.seat)
|
||||
if !known {
|
||||
t.Fatalf("%q is not a seat", pair.seat)
|
||||
}
|
||||
if s.Delivers != pair.delivers {
|
||||
t.Errorf("the %s seat delivers %q, expected %q — renaming the seat moved the "+
|
||||
"interface with it, and every consumer requiring it would stop resolving",
|
||||
pair.seat, s.Delivers, pair.delivers)
|
||||
}
|
||||
// **Three of these deliberately share a name with what they deliver**, and that is not an
|
||||
// incomplete rename. Renaming a seat that delivers a provision cascades to every consumer
|
||||
// requiring it, with a mesh-wide window where a holder stops resolving mid-flight — so the
|
||||
// trunk deferred exactly those three (novox/hq ADR 0121) while renaming the node-scoped ones.
|
||||
// What this test is for is the other direction: that renaming a seat never moves the
|
||||
// interface, which once produced "the package registry is served on <nil>".
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest written against an old seat name is told what it became rather than refused as
|
||||
// unknown. **That map is the controller's store now, not this package** (novox/hq ADR 0122): a
|
||||
// rename is a row, so the courtesy survives a rename nobody recompiled for. Checked where the
|
||||
// table is read, not here, where there is no longer a hardcoded list to check against.
|
||||
@@ -0,0 +1,332 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Where the answer to a requirement is allowed to live.
|
||||
//
|
||||
// A shell, a display server and a private network have to be on the machine that needs them. A
|
||||
// database does not — it runs somewhere and is reached over the network. Both were written
|
||||
// `requires`, so both were answered the same way, and the second answer was to install PostgreSQL
|
||||
// on every machine that runs a web application.
|
||||
|
||||
// onNetwork is a set of providers, all of them reachable. Written as a helper because a machine
|
||||
// that cannot reach the one answering its requirement is its own case, tested separately.
|
||||
// reachable is this machine, on the private network.
|
||||
func reachable() Node {
|
||||
n := workstation()
|
||||
n.At = "workstation.internal"
|
||||
return n
|
||||
}
|
||||
|
||||
func onNetwork(nodes ...string) map[string][]Provider {
|
||||
out := make([]Provider, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
out = append(out, Provider{Node: n, At: n + ".internal", Module: "postgres"})
|
||||
}
|
||||
return map[string][]Provider{"postgres-database": out}
|
||||
}
|
||||
|
||||
func brokeredShelf() map[string]Manifest {
|
||||
return shelf(
|
||||
Manifest{Module: "postgres", Version: "1", Provides: FromAnywhere("postgres-database")},
|
||||
Manifest{Module: "meshboard", Version: "1", Requires: []string{"postgres-database"}},
|
||||
)
|
||||
}
|
||||
|
||||
func TestADatabaseIsNotInstalledOnEveryMachineThatUsesOne(t *testing.T) {
|
||||
// The fault this whole distinction exists for.
|
||||
got, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
|
||||
World{Offered: onNetwork("anchor")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have := strings.Join(names(got), " "); strings.Contains(have, "postgres") {
|
||||
t.Fatalf("using a database installed one here: %s", have)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAMachineTakesFromElsewhereIsRecorded(t *testing.T) {
|
||||
// It is the only part of a node's set that stops working when a *different* machine goes
|
||||
// away, and it is where a credential will have to be handed back.
|
||||
got, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
|
||||
World{Offered: onNetwork("anchor")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Needs) != 1 {
|
||||
t.Fatalf("got %v", got.Needs)
|
||||
}
|
||||
if got.Needs[0].Name != "postgres-database" || got.Needs[0].From != "anchor" {
|
||||
t.Fatalf("got %v", got.Needs[0])
|
||||
}
|
||||
if got.Needs[0].For != "meshboard" {
|
||||
t.Fatalf("it does not say what wanted it: %v", got.Needs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingInTheMeshProvidingItIsRefusedWithSomewhereToPutIt(t *testing.T) {
|
||||
// Refused rather than installed here. Choosing a machine to put a database on is a decision
|
||||
// with consequences, and nothing resolving a web application should make it silently.
|
||||
_, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(), World{})
|
||||
if err == nil {
|
||||
t.Fatal("a database was found in a mesh that has none")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "assign") || !strings.Contains(err.Error(), "postgres") {
|
||||
t.Fatalf("the refusal does not say what to do: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoNodesProvidingItIsRefusedRatherThanPicked(t *testing.T) {
|
||||
// Same rule as everywhere else. Picking one would be a guess about which database a person
|
||||
// meant, and the wrong guess is somebody's data in the wrong place.
|
||||
_, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
|
||||
World{Offered: onNetwork("anchor", "archive")})
|
||||
if err == nil {
|
||||
t.Fatal("one of two databases was picked silently")
|
||||
}
|
||||
for _, want := range []string{"anchor", "archive", "pin"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not name %s: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSayingWhichOneSettlesIt(t *testing.T) {
|
||||
got, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
|
||||
World{
|
||||
Offered: onNetwork("anchor", "archive"),
|
||||
Pinned: map[string]Chosen{"postgres-database": {Node: "archive", Module: "postgres"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Needs) != 1 || got.Needs[0].From != "archive" {
|
||||
t.Fatalf("the choice was not taken: %v", got.Needs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBeingPointedAtAMachineThatDoesNotProvideItIsRefused(t *testing.T) {
|
||||
// Rather than falling back to one that does. A fallback would quietly move somebody's data to
|
||||
// a machine they did not choose, which is the whole reason the question is asked.
|
||||
_, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
|
||||
World{
|
||||
Offered: onNetwork("anchor", "archive"),
|
||||
Pinned: map[string]Chosen{"postgres-database": {Node: "somewhere-else", Module: "postgres"}},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("a machine was silently given a different database from the one chosen")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "somewhere-else") {
|
||||
t.Fatalf("the refusal does not say what was chosen: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneProviderDoesNotOverruleAChoice(t *testing.T) {
|
||||
// A single answer is normally taken silently. Not when somebody said they wanted a different
|
||||
// one -- that is the mesh overruling a person, which it does nowhere else.
|
||||
_, err := Resolve(brokeredShelf(), []string{"meshboard"}, reachable(),
|
||||
World{
|
||||
Offered: onNetwork("anchor"),
|
||||
Pinned: map[string]Chosen{"postgres-database": {Node: "archive", Module: "postgres"}},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("the only database was used although another was chosen")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "only anchor/postgres provides it") {
|
||||
t.Fatalf("the refusal does not say what is available: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACatalogueThatDisagreesAboutScopeIsRefused(t *testing.T) {
|
||||
// If one module says a database is local and another says it is anywhere, the same
|
||||
// requirement means two things depending on which one happens to answer it.
|
||||
_, err := Resolve(shelf(
|
||||
Manifest{Module: "postgres", Version: "1", Provides: FromAnywhere("postgres-database")},
|
||||
Manifest{Module: "sqlite", Version: "1", Provides: Offers("postgres-database")},
|
||||
Manifest{Module: "meshboard", Version: "1", Requires: []string{"postgres-database"}},
|
||||
), []string{"meshboard"}, workstation(), World{})
|
||||
if err == nil {
|
||||
t.Fatal("a catalogue that disagrees about where a database lives was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "two things") {
|
||||
t.Fatalf("unhelpful refusal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALocalRequirementIsStillAnsweredLocally(t *testing.T) {
|
||||
// The change must not have made everything brokered. A shell is still installed here.
|
||||
got, err := Resolve(shelf(
|
||||
Manifest{Module: "zsh", Version: "1", Provides: Offers("shell")},
|
||||
Manifest{Module: "tools", Version: "1", Requires: []string{"shell"}},
|
||||
), []string{"tools"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if have := strings.Join(names(got), " "); !strings.Contains(have, "zsh") {
|
||||
t.Fatalf("a shell was not installed on the machine that needs one: %s", have)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheShortFormStillMeansHere(t *testing.T) {
|
||||
// `"provides": ["shell"]` must keep meaning what it meant, or every existing manifest
|
||||
// silently changes meaning.
|
||||
m, err := ParseManifest([]byte(`{"module":"zsh","version":"1","provides":["shell"]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.Provides) != 1 || m.Provides[0].At() != ScopeNode {
|
||||
t.Fatalf("a plain name is no longer node-scoped: %v", m.Provides)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASiteScopedProvisionIsRefused(t *testing.T) {
|
||||
// Meaningful for a claim — one DHCP server per segment — and not yet meaningful for a
|
||||
// provision, because nothing knows how to reach "the one at my site".
|
||||
_, err := ParseManifest([]byte(
|
||||
`{"module":"dns","version":"1","provides":[{"name":"resolver","scope":"site"}]}`))
|
||||
if err == nil {
|
||||
t.Fatal("a site-scoped provision was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "site") {
|
||||
t.Fatalf("unhelpful refusal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Being told, which is the difference between knowing and being able to.
|
||||
|
||||
func boundShelf() map[string]Manifest {
|
||||
return shelf(
|
||||
Manifest{Module: "postgres", Version: "1", Provides: FromAnywhere("postgres-database"),
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432, "driver": "postgres"}}},
|
||||
Manifest{Module: "meshboard", Version: "1", Requires: []string{"postgres-database"},
|
||||
Binds: map[string]string{"postgres-database": "/etc/meshboard/database.json"}},
|
||||
)
|
||||
}
|
||||
|
||||
func binding(t *testing.T, out []map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range out {
|
||||
if r["path"] != "/etc/meshboard/database.json" {
|
||||
continue
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil {
|
||||
t.Fatalf("what the app is told is not readable: %v", err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
t.Fatalf("the app was told nothing: %v", out)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAnAppIsToldWhereItsDatabaseIs(t *testing.T) {
|
||||
// Knowing it needs the anchor's database is useless to the program that needs it unless the
|
||||
// program is told. This is the whole point of the field.
|
||||
got, err := Resolve(boundShelf(), []string{"meshboard"}, reachable(),
|
||||
World{Offered: map[string][]Provider{"postgres-database": {{Node: "anchor", At: "anchor.internal",
|
||||
Serves: map[string]any{"port": 5432, "driver": "postgres"}}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
told := binding(t, mustDeclare(t, got))
|
||||
if told["from"] != "anchor" || told["at"] != "anchor.internal" {
|
||||
t.Fatalf("it was not told where: %v", told)
|
||||
}
|
||||
serves, _ := told["serves"].(map[string]any)
|
||||
if serves["port"] != float64(5432) || serves["driver"] != "postgres" {
|
||||
t.Fatalf("it was not told how: %v", serves)
|
||||
}
|
||||
}
|
||||
|
||||
func TestItSaysWhereTheCredentialIsInstead(t *testing.T) {
|
||||
// A missing field looks like a bug; a stated absence looks like a boundary. Somebody wiring
|
||||
// this up must not spend an afternoon looking for the password field.
|
||||
//
|
||||
// It used to say only that the mesh had no way to issue one, which stopped being true when
|
||||
// 021 was fixed. A stated absence is only useful while it is accurate — once it is not, it
|
||||
// sends the reader somewhere there is nothing to find.
|
||||
got, _ := Resolve(boundShelf(), []string{"meshboard"}, reachable(),
|
||||
World{Offered: map[string][]Provider{"postgres-database": {{Node: "anchor", At: "anchor.internal"}}}})
|
||||
told := binding(t, mustDeclare(t, got))
|
||||
note, _ := told["generated"].(string)
|
||||
if !strings.Contains(note, "not here") || !strings.Contains(note, "secrets") {
|
||||
t.Fatalf("the file does not say where the credential is instead: %v", note)
|
||||
}
|
||||
for key := range told {
|
||||
if strings.Contains(key, "password") || strings.Contains(key, "secret") {
|
||||
t.Fatalf("something that looks like a credential appeared: %q", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoMachinesThatCannotReachEachOtherAreRefused(t *testing.T) {
|
||||
// An app on one machine and a database on another that share no private network is a mesh
|
||||
// that reports itself configured and does not work. Said here rather than discovered as a
|
||||
// connection timing out.
|
||||
_, err := Resolve(boundShelf(), []string{"meshboard"}, workstation(), // not on the network
|
||||
World{Offered: map[string][]Provider{"postgres-database": {{Node: "anchor", At: "anchor.internal"}}}})
|
||||
if err == nil {
|
||||
t.Fatal("an app was pointed at a database it has no path to")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "private network") {
|
||||
t.Fatalf("the refusal does not say what is wrong: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), meshNetwork) {
|
||||
t.Fatalf("the refusal does not say what to assign: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheProviderBeingOffTheNetworkIsAlsoRefused(t *testing.T) {
|
||||
// Both directions, because the failure is identical from either end and the remedy differs.
|
||||
_, err := Resolve(boundShelf(), []string{"meshboard"}, reachable(),
|
||||
World{Offered: map[string][]Provider{"postgres-database": {{Node: "anchor"}}}})
|
||||
if err == nil {
|
||||
t.Fatal("an app was pointed at a database that is not on the private network")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "anchor") {
|
||||
t.Fatalf("the refusal does not name the unreachable end: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindingSomethingAnsweredHereWritesNothing(t *testing.T) {
|
||||
// A file saying "it is on this node" is a fact nobody needs and one more thing to keep true.
|
||||
got, err := Resolve(shelf(
|
||||
Manifest{Module: "zsh", Version: "1", Provides: Offers("shell")},
|
||||
Manifest{Module: "tools", Version: "1", Requires: []string{"shell"},
|
||||
Binds: map[string]string{"shell": "/etc/tools/shell.json"}},
|
||||
), []string{"tools"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range mustDeclare(t, got) {
|
||||
if r["path"] == "/etc/tools/shell.json" {
|
||||
t.Fatalf("a binding was written for something on this machine: %v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBindingWhatYouDoNotRequireIsRefused(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"app","version":"1",
|
||||
"binds":{"postgres-database":"/etc/app/db.json"}}`))
|
||||
if err == nil {
|
||||
t.Fatal("a module was told about something it never asked for")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does not require") {
|
||||
t.Fatalf("unhelpful refusal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServingWhatYouDoNotProvideIsRefused(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"app","version":"1",
|
||||
"serves":{"postgres-database":{"port":5432}}}`))
|
||||
if err == nil {
|
||||
t.Fatal("a module served something it does not provide")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does not provide") {
|
||||
t.Fatalf("unhelpful refusal: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// BucketShare is one entry of `shares` on a module's requirement of `s3-bucket` (novox/hq ADR 0468 §8): who else may
|
||||
// put or get in the consumer's bucket, and with which methods. Not the NFS shares of shares.go (ADR 0263).
|
||||
//
|
||||
// "requires": [{"name": "s3-bucket", "shares": [{"to": "seat:node-files", "methods": ["put"]},
|
||||
// {"to": "print-and-scan", "methods": ["get"]}]}]
|
||||
//
|
||||
// `to` is a module name, or `seat:<seat name>`. `methods` is a non-empty subset of get and put, without a repeat.
|
||||
// Shares are allowed on `s3-bucket` only, and no two entries name the same `to`.
|
||||
type BucketShare struct {
|
||||
To string `json:"to"`
|
||||
Methods []string `json:"methods"`
|
||||
}
|
||||
|
||||
// BucketRequirement is the requirement `shares` may be written on.
|
||||
const BucketRequirement = "s3-bucket"
|
||||
|
||||
// RequireSharesKey is where a stored manifest keeps the shares of its requirements: beside `requires`, which stays a
|
||||
// plain list of names, for the reason OptionalRequiresKey is beside it. An older controller sets the unknown key
|
||||
// aside instead of failing the catalogue.
|
||||
const RequireSharesKey = "requires-shares"
|
||||
|
||||
// SharesValue is the key of the value a consumer gives the provider of s3-bucket in its grant: the list of
|
||||
// {to, methods} sorted by `to`. A consumer's own `contributes` may not name it.
|
||||
const SharesValue = "shares"
|
||||
|
||||
var shareTarget = regexp.MustCompile(`^(seat:)?[a-z0-9][a-z0-9-]*$`)
|
||||
|
||||
// checkBucketShares refuses a malformed `shares` of one requirement, naming the entry and ADR 0468 §8.
|
||||
func checkBucketShares(requirement string, shares []BucketShare) error {
|
||||
if len(shares) == 0 {
|
||||
return nil
|
||||
}
|
||||
if requirement != BucketRequirement {
|
||||
return fmt.Errorf("requires %s: `shares` is allowed on %s only (novox/hq ADR 0468 §8)", requirement, BucketRequirement)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for i, s := range shares {
|
||||
at := fmt.Sprintf("requires %s: shares[%d] (to %q)", requirement, i, s.To)
|
||||
if !shareTarget.MatchString(s.To) {
|
||||
return fmt.Errorf("%s: `to` is a module name or seat:<seat name> (novox/hq ADR 0468 §8)", at)
|
||||
}
|
||||
if seen[s.To] {
|
||||
return fmt.Errorf("%s: a second entry for the same `to` (novox/hq ADR 0468 §8)", at)
|
||||
}
|
||||
seen[s.To] = true
|
||||
if len(s.Methods) == 0 {
|
||||
return fmt.Errorf("%s: `methods` is empty; name get, put or both (novox/hq ADR 0468 §8)", at)
|
||||
}
|
||||
used := map[string]bool{}
|
||||
for _, method := range s.Methods {
|
||||
if method != "get" && method != "put" {
|
||||
return fmt.Errorf("%s: method %q is not get or put (novox/hq ADR 0468 §8)", at, method)
|
||||
}
|
||||
if used[method] {
|
||||
return fmt.Errorf("%s: method %q twice (novox/hq ADR 0468 §8)", at, method)
|
||||
}
|
||||
used[method] = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readRequireShares reads the stored form's side key.
|
||||
func readRequireShares(raw json.RawMessage) (map[string][]BucketShare, error) {
|
||||
var out map[string][]BucketShare
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("%s: an object of requirement to a list of {to, methods}: %w", RequireSharesKey, err)
|
||||
}
|
||||
for requirement, shares := range out {
|
||||
if err := checkBucketShares(requirement, shares); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sortedShares is a requirement's shares as they are handed to the provider: sorted by `to` (a file whose lines move
|
||||
// looks changed), each entry's methods sorted, and a copy so the manifest is not reordered.
|
||||
func sortedShares(shares []BucketShare) []BucketShare {
|
||||
out := make([]BucketShare, 0, len(shares))
|
||||
for _, s := range shares {
|
||||
methods := append([]string(nil), s.Methods...)
|
||||
sort.Strings(methods)
|
||||
out = append(out, BucketShare{To: s.To, Methods: methods})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].To < out[j].To })
|
||||
return out
|
||||
}
|
||||
|
||||
// withShares adds this module's shares of the requirement to the values it gives the provider. A module whose own
|
||||
// `contributes` already names `shares` is refused: the field is the only way to say it (ADR 0468 §8).
|
||||
func withShares(m Manifest, requirement string, values map[string]any) (map[string]any, error) {
|
||||
shares := m.RequireShares[requirement]
|
||||
if _, named := values[SharesValue]; named && requirement == BucketRequirement {
|
||||
return nil, fmt.Errorf("%s contributing to %s names `shares`; write it on its requirement instead (novox/hq ADR 0468 §8)",
|
||||
m.Module, requirement)
|
||||
}
|
||||
if len(shares) == 0 {
|
||||
return values, nil
|
||||
}
|
||||
out := make(map[string]any, len(values)+1)
|
||||
for k, v := range values {
|
||||
out[k] = v
|
||||
}
|
||||
listed := []any{}
|
||||
for _, s := range sortedShares(shares) {
|
||||
methods := []any{}
|
||||
for _, method := range s.Methods {
|
||||
methods = append(methods, method)
|
||||
}
|
||||
listed = append(listed, map[string]any{"to": s.To, "methods": methods})
|
||||
}
|
||||
out[SharesValue] = listed
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issue 1591: `shares` on a requirement of s3-bucket (novox/hq ADR 0468 §8).
|
||||
|
||||
const filesWithShares = `{"module":"files","version":"1","requires":["shell",
|
||||
{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["get","put"]}]}]}`
|
||||
|
||||
func minioLike() Manifest {
|
||||
return Manifest{Module: "minio", Version: "1", Provides: Offers("s3-bucket"),
|
||||
Receives: map[string]string{"s3-bucket": "/grants/mesh.json"}}
|
||||
}
|
||||
|
||||
func givenToMinio(t *testing.T, out []map[string]any) []Contribution {
|
||||
t.Helper()
|
||||
for _, r := range out {
|
||||
if r["path"] != "/grants/mesh.json" {
|
||||
continue
|
||||
}
|
||||
var parsed struct {
|
||||
Given []Contribution `json:"given"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(r["content"].(string)), &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return parsed.Given
|
||||
}
|
||||
t.Fatalf("minio was given no file: %v", out)
|
||||
return nil
|
||||
}
|
||||
|
||||
func sharesOf(t *testing.T, values map[string]any) string {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(values["shares"])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
const sortedSharesJSON = `[{"methods":["get","put"],"to":"print-and-scan"},{"methods":["put"],"to":"seat:node-files"}]`
|
||||
|
||||
// 1591/AC1: parsed, the shares are on the manifest; stored, `requires` is plain names and the side key holds them.
|
||||
func TestSharesParseAndAreStoredBesideRequires(t *testing.T) {
|
||||
m := mustParse(t, filesWithShares)
|
||||
want := []BucketShare{{To: "seat:node-files", Methods: []string{"put"}}, {To: "print-and-scan", Methods: []string{"get", "put"}}}
|
||||
if !reflect.DeepEqual(m.RequireShares["s3-bucket"], want) {
|
||||
t.Fatalf("the shares read as %+v", m.RequireShares)
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &keys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(keys["requires"]) != `["shell","s3-bucket"]` {
|
||||
t.Errorf("the stored requires is not plain names: %s", keys["requires"])
|
||||
}
|
||||
if !strings.Contains(string(keys[RequireSharesKey]), `"seat:node-files"`) {
|
||||
t.Errorf("the side key does not hold the shares: %s", raw)
|
||||
}
|
||||
var again Manifest
|
||||
if err := json.Unmarshal(raw, &again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(again, m) {
|
||||
t.Fatalf("not lossless:\n%+v\n%+v", again, m)
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC2: a malformed share is refused, naming the entry and ADR 0468 §8.
|
||||
func TestAMalformedShareIsRefused(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"unknown method": `{"name":"s3-bucket","shares":[{"to":"a","methods":["delete"]}]}`,
|
||||
"empty to": `{"name":"s3-bucket","shares":[{"to":"","methods":["get"]}]}`,
|
||||
"malformed to": `{"name":"s3-bucket","shares":[{"to":"Seat:X y","methods":["get"]}]}`,
|
||||
"empty methods": `{"name":"s3-bucket","shares":[{"to":"a","methods":[]}]}`,
|
||||
"duplicate to": `{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]},{"to":"a","methods":["put"]}]}`,
|
||||
"duplicate verb": `{"name":"s3-bucket","shares":[{"to":"a","methods":["get","get"]}]}`,
|
||||
"other require": `{"name":"route","shares":[{"to":"a","methods":["get"]}]}`,
|
||||
}
|
||||
for name, entry := range cases {
|
||||
_, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":[` + entry + `]}`))
|
||||
if err == nil {
|
||||
t.Errorf("%s: accepted", name)
|
||||
continue
|
||||
}
|
||||
if !strings.Contains(err.Error(), "ADR 0468 §8") || !strings.Contains(err.Error(), "requires") {
|
||||
t.Errorf("%s: the error does not name the entry and the ADR: %v", name, err)
|
||||
}
|
||||
}
|
||||
// The stored form is held to the same rules.
|
||||
_, err := ParseManifest([]byte(`{"module":"files","version":"1","requires":["s3-bucket"],
|
||||
"requires-shares":{"s3-bucket":[{"to":"a","methods":[]}]}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") {
|
||||
t.Errorf("the stored form was accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC3: the values a consumer gives carry the shares sorted by `to`, remote (grant) and co-located alike.
|
||||
func TestTheGrantCarriesTheSharesSorted(t *testing.T) {
|
||||
m := mustParse(t, filesWithShares)
|
||||
remote := Resolution{Node: "laptop", Modules: []Manifest{m}}
|
||||
values, asks, err := remote.ContributionsFrom("s3-bucket", "files", SettingsBy{})
|
||||
if err != nil || !asks {
|
||||
t.Fatalf("asks %v, err %v", asks, err)
|
||||
}
|
||||
if got := sharesOf(t, values); got != sortedSharesJSON {
|
||||
t.Fatalf("the grant values carry %s", got)
|
||||
}
|
||||
// As the provider's file shows it, through a grant.
|
||||
provider, err := Resolve(shelf(minioLike()), []string{"minio"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := provider.Declaration(Rendering{Grants: []Grant{{Provision: "s3-bucket", Consumer: "laptop", From: "files", Values: values}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given := givenToMinio(t, out)
|
||||
if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON {
|
||||
t.Fatalf("the remote consumer's shares did not reach the file: %+v", given)
|
||||
}
|
||||
|
||||
// Co-located: the consumer contributes to s3-bucket too, and no grant is passed.
|
||||
co := mustParse(t, `{"module":"files","version":"1","contributes":{"s3-bucket":{"name":"files"}},
|
||||
"requires":[{"name":"s3-bucket","shares":[{"to":"seat:node-files","methods":["put"]},{"to":"print-and-scan","methods":["put","get"]}]}]}`)
|
||||
both, err := Resolve(shelf(minioLike(), co), []string{"files"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err = both.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
given = givenToMinio(t, out)
|
||||
if len(given) != 1 || sharesOf(t, given[0].Values) != sortedSharesJSON || given[0].Values["name"] != "files" {
|
||||
t.Fatalf("the co-located consumer's shares did not reach the file: %+v", given)
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC4: only the shares change, so the module is not recreated and held for no one; the provider's file changes.
|
||||
func TestChangingOnlyTheSharesDoesNotRecreateTheConsumer(t *testing.T) {
|
||||
was := mustParse(t, filesWithShares)
|
||||
now := mustParse(t, `{"module":"files","version":"1","requires":["shell",
|
||||
{"name":"s3-bucket","shares":[{"to":"print-and-scan","methods":["get"]}]}]}`)
|
||||
if said := Recreates(was, now).Say("files"); said != "" {
|
||||
t.Errorf("a change of shares recreates the module: %s", said)
|
||||
}
|
||||
if said := Holds(was, now); said != "" {
|
||||
t.Errorf("a change of shares holds the module: %s", said)
|
||||
}
|
||||
before := Resolution{Node: "laptop", Modules: []Manifest{was}}
|
||||
after := Resolution{Node: "laptop", Modules: []Manifest{now}}
|
||||
a, _, _ := before.ContributionsFrom("s3-bucket", "files", SettingsBy{})
|
||||
b, _, _ := after.ContributionsFrom("s3-bucket", "files", SettingsBy{})
|
||||
if sharesOf(t, a) == sharesOf(t, b) {
|
||||
t.Fatalf("the provider's values did not change: %s", sharesOf(t, a))
|
||||
}
|
||||
}
|
||||
|
||||
// 1591/AC5: a consumer whose contributes to s3-bucket names `shares` itself is refused.
|
||||
func TestAConsumerCannotContributeSharesItself(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"files","version":"1",
|
||||
"contributes":{"s3-bucket":{"shares":"x"}},
|
||||
"requires":[{"name":"s3-bucket","shares":[{"to":"a","methods":["get"]}]}]}`))
|
||||
if err == nil || !strings.Contains(err.Error(), "ADR 0468 §8") {
|
||||
t.Errorf("both written, accepted: %v", err)
|
||||
}
|
||||
// Without the field the grant is still refused, so the provider is never told something the manifest did not say.
|
||||
m := Manifest{Module: "files", Requires: []string{"s3-bucket"},
|
||||
Contributes: map[string]map[string]any{"s3-bucket": {"shares": "x"}}}
|
||||
r := Resolution{Node: "laptop", Modules: []Manifest{m}}
|
||||
if _, _, err := r.ContributionsFrom("s3-bucket", "files", SettingsBy{}); err == nil {
|
||||
t.Error("a contributed `shares` reached the grant")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,717 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Turning a manifest that names artifacts into one that names digests.
|
||||
//
|
||||
// **Two documents, deliberately.** The manifest in a repository says *this resource uses the
|
||||
// archive called `config`*; the manifest the mesh holds says *this resource is sha256:…*. A digest
|
||||
// is not knowable until something is built, so a repository carrying one would be a repository
|
||||
// whose file is wrong the moment anybody edits anything — and the mesh would be pinning a value
|
||||
// nobody could have checked.
|
||||
//
|
||||
// So the built manifest is **derived**, and the record of which commit it was derived from is what
|
||||
// makes "is this current?" answerable without building (novox/hq ADR 0009).
|
||||
|
||||
// Built is one artifact after it exists: where it is and what it hashes to.
|
||||
type Built struct {
|
||||
// Name is what the manifest called it.
|
||||
Name string
|
||||
// Kind is "image" or "archive".
|
||||
Kind string
|
||||
// Reference is what a machine uses to get it — an image reference for an image, a URL for an
|
||||
// archive. Both already carry the digest for an image; an archive carries it separately.
|
||||
Reference string
|
||||
// Digest is "sha256:<hex>", for an archive. An image reference already ends in one.
|
||||
Digest string
|
||||
// Launchers are, for a bundle in an interpreted language, the executable the build wrote beside
|
||||
// each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it.
|
||||
Launchers map[string]string
|
||||
// Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the
|
||||
// build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on
|
||||
// the wire or kept: Resolve reads them into the files' content, which is what the mesh holds.
|
||||
Texts map[string]string `json:"-"`
|
||||
}
|
||||
|
||||
// Resolve fills a manifest's resources in from what was built.
|
||||
//
|
||||
// Every resource naming an artifact is rewritten to name the thing itself, and the `artifact` key
|
||||
// is removed — because it is a build-time word and the host has never heard of it. A resource
|
||||
// naming an artifact nothing produced is refused: it would otherwise reach a machine with an
|
||||
// empty image or an unpinned archive, which is the shape of failure that looks like success.
|
||||
func (m Manifest) Resolve(built []Built) (Manifest, error) {
|
||||
if m.Build == nil && len(built) == 0 {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
by := map[string]Built{}
|
||||
for _, b := range built {
|
||||
by[b.Name] = b
|
||||
}
|
||||
// Declared and not produced is a build that did not do what the manifest asked, and saying so
|
||||
// here beats a machine reporting it later.
|
||||
var missing []string
|
||||
if m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if _, ok := by[a.Name]; !ok {
|
||||
missing = append(missing, a.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
sort.Strings(missing)
|
||||
return Manifest{}, fmt.Errorf(
|
||||
"%s says it builds %s and the build did not produce %s — the build did not do what "+
|
||||
"the manifest asked, which is a different fault from a resource asking for the "+
|
||||
"wrong thing",
|
||||
m.Module, strings.Join(missing, " and "), oneOrOther(len(missing)))
|
||||
}
|
||||
|
||||
// **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into
|
||||
// the file's content, so what the mesh holds and composes is what the same text inline would be.
|
||||
var handed *Handed
|
||||
if m.Templated() {
|
||||
name := TemplatesArtifact(m)
|
||||
texts := by[name].Texts
|
||||
if name == "" || texts == nil {
|
||||
return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+
|
||||
"archive it packs %s/ into (%q): the builder that built it does not read templates",
|
||||
m.Module, m.TemplatesDir(), name)
|
||||
}
|
||||
dir := m.TemplatesDir()
|
||||
filled, err := fillTemplates(m.Module, dir, m.Resources, func(template string) (string, bool) {
|
||||
t, ok := texts[strings.TrimPrefix(template, dir+"/")]
|
||||
return t, ok
|
||||
}, fmt.Sprintf("the archive %q", name))
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
items, err := fillItemTemplates(m, m.Items, func(template string) (string, bool) {
|
||||
t, ok := texts[strings.TrimPrefix(template, dir+"/")]
|
||||
return t, ok
|
||||
}, fmt.Sprintf("the archive %q", name))
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
// And, for a moved module, where its templates are and which one each file and item is filled from
|
||||
// (handed.go): read while its resources and items still name their templates.
|
||||
handed = handedBy(m, by[name])
|
||||
m.Resources = filled
|
||||
m.Items = items
|
||||
}
|
||||
|
||||
out := m
|
||||
out.Handed = handed
|
||||
out.Build = nil
|
||||
// Read with the build, like Build: the manifest the mesh holds carries each file's text (issue 731).
|
||||
out.Templates = ""
|
||||
out.Resources = nil
|
||||
// What the build compiled, kept on the resolved manifest (novox/hq ADR 0175): a tools bundle is
|
||||
// named by no resource of the module's own — the node's runtime loads it — so this is the only
|
||||
// place the mesh would otherwise not have it. In artifact order, so two resolutions of one
|
||||
// build compare equal.
|
||||
out.Bundles = nil
|
||||
if m.Build != nil {
|
||||
run := runByAResource(m)
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != ArtifactBundle {
|
||||
continue
|
||||
}
|
||||
made := by[a.Name]
|
||||
// What the runtime loads: what the artifact said, else every entrypoint of a module
|
||||
// that declares tools, else nothing (the field's own rule; see Artifact.Loads).
|
||||
//
|
||||
// **Never, unasked, a bundle one of the module's own resources runs** (novox/hq issue 213).
|
||||
// A process the host runs is the module's service, not its tools: the controller declares
|
||||
// the verbs it answers as `tools` and serves them itself, and its bundle would otherwise
|
||||
// have been launched a second time by the node's runtime, as an MCP child it is not.
|
||||
loads := append([]string(nil), a.Loads...)
|
||||
if a.Loads == nil && len(m.Tools) > 0 && !run[a.Name] {
|
||||
loads = append([]string(nil), a.Entrypoints...)
|
||||
// A bundle compiled to a binary has no entrypoints: the binary is what it is, and what
|
||||
// the runtime starts to serve it (novox/hq ADR 0193). So a Go tools bundle is served
|
||||
// as Go — the runtime execs it — exactly as a TypeScript one is through its launcher.
|
||||
if bin := BinaryOf(a); bin != "" {
|
||||
loads = []string{bin}
|
||||
}
|
||||
}
|
||||
// **Kept, never routed** (ADR 0155): the builder publishes to the store at the address
|
||||
// it reached it by, and a manifest carrying that address names an installation —
|
||||
// registration refused node-tools for exactly this on 2026-10-02. The build record
|
||||
// already keeps the store-relative form; the resolved manifest keeps the same, and
|
||||
// composition routes it through the store a machine reaches (Routed).
|
||||
out.Bundles = append(out.Bundles, Bundle{
|
||||
Name: a.Name, Source: Recorded(made.Reference), Digest: made.Digest,
|
||||
Language: a.Language, Entrypoints: append([]string(nil), a.Entrypoints...),
|
||||
Loads: loads, Env: copyWords(a.Env), Launchers: copyWords(made.Launchers),
|
||||
Binary: BinaryOf(a),
|
||||
})
|
||||
}
|
||||
sort.Slice(out.Bundles, func(i, j int) bool { return out.Bundles[i].Name < out.Bundles[j].Name })
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
named, _ := r["artifact"].(string)
|
||||
if named == "" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
artifact, ok := by[named]
|
||||
if !ok {
|
||||
return Manifest{}, fmt.Errorf(
|
||||
"%s: %v uses the artifact %q, and this module builds no such thing",
|
||||
m.Module, r["id"], named)
|
||||
}
|
||||
|
||||
filled := map[string]any{}
|
||||
for k, v := range r {
|
||||
filled[k] = v
|
||||
}
|
||||
delete(filled, "artifact")
|
||||
switch artifact.Kind {
|
||||
case ArtifactPackage:
|
||||
// A package is not a resource on any machine; it is consumed by other builds. A
|
||||
// resource that names one is a manifest error, named here rather than shipped.
|
||||
return Manifest{}, fmt.Errorf(
|
||||
"%s: %v uses %q, which is a package — a build input, not a resource a machine runs",
|
||||
m.Module, r["id"], named)
|
||||
case ArtifactImage, ArtifactUpstream:
|
||||
filled["image"] = artifact.Reference
|
||||
// An image is not unpacked anywhere, so it has no directory to be named for its
|
||||
// version and `${version}` has nothing to mean. Refused rather than left as literal
|
||||
// text in a path, which is how it would reach a machine and be created as a directory
|
||||
// called `${version}`.
|
||||
for key, value := range filled {
|
||||
if text, isText := value.(string); isText && strings.Contains(text, versionRef) {
|
||||
return Manifest{}, fmt.Errorf(
|
||||
"%s: %v says %s in %q, and %q is an image — an image is not unpacked, so "+
|
||||
"it has no versioned place. %s is for an archive or a bundle",
|
||||
m.Module, r["id"], versionRef, key, named, versionRef)
|
||||
}
|
||||
}
|
||||
case ArtifactArchive, ArtifactBundle:
|
||||
// The same on the wire: both are bytes fetched by digest and unpacked. They differ in
|
||||
// how they were made — one packed as it stood, the other compiled first — and a
|
||||
// machine has no reason to care which.
|
||||
// Kept, not routed, for the reason the bundles above are (ADR 0155).
|
||||
filled["source"] = Recorded(artifact.Reference)
|
||||
filled["digest"] = artifact.Digest
|
||||
// **And `${version}`, so a resource can name a place that is this build's alone**
|
||||
// (novox/hq ADR 0141, 04-ISSUES/142). A component is unpacked into a directory named
|
||||
// for its version so it can read its own version from its path — and until this,
|
||||
// nothing could compose that path: an archive named a fixed one in the manifest and
|
||||
// nothing interpolated the build into it, so nothing could ask for
|
||||
// `…/versions/<version>/` and every machine took a hand-placed fallback.
|
||||
//
|
||||
// The version is the artifact's own digest, short. Not the commit: two builds of one
|
||||
// commit are meant to be the same bytes (the toolchains are `-trimpath` for this), and
|
||||
// a content-addressed version means an unchanged build resolves to the path it already
|
||||
// had — so re-composing a declaration moves nothing, where a commit would move the
|
||||
// path of an identical binary and recreate everything that reads it.
|
||||
for key, value := range filled {
|
||||
filled[key] = withVersion(value, versionOf(artifact.Digest))
|
||||
}
|
||||
default:
|
||||
return Manifest{}, fmt.Errorf("%s: %q is a %q, and an artifact is %q, %q, %q or %q",
|
||||
m.Module, named, artifact.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream,
|
||||
ArtifactBundle)
|
||||
}
|
||||
out.Resources = append(out.Resources, filled)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// withVersion is a resource's value with `${version}` filled: in a string, and in each string of a map —
|
||||
// a process's env, where the controller is told which build it is (novox/hq issue 352). Anything else is
|
||||
// left as it is.
|
||||
func withVersion(value any, version string) any {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
if strings.Contains(v, versionRef) {
|
||||
return strings.ReplaceAll(v, versionRef, version)
|
||||
}
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(v))
|
||||
for k, x := range v {
|
||||
out[k] = withVersion(x, version)
|
||||
}
|
||||
return out
|
||||
case map[string]string:
|
||||
out := make(map[string]string, len(v))
|
||||
for k, x := range v {
|
||||
out[k] = strings.ReplaceAll(x, versionRef, version)
|
||||
}
|
||||
return out
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// checkBuild is the manifest's own account of what it builds.
|
||||
func (b *Build) problems(module string) []string {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
seen := map[string]bool{}
|
||||
for _, a := range b.Artifacts {
|
||||
if a.Name == "" {
|
||||
problems = append(problems, module+" builds an artifact with no name")
|
||||
continue
|
||||
}
|
||||
if seen[a.Name] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s builds two artifacts called %q, and a resource naming it could mean either",
|
||||
module, a.Name))
|
||||
}
|
||||
seen[a.Name] = true
|
||||
switch a.Kind {
|
||||
case ArtifactImage, ArtifactArchive, ArtifactUpstream, ArtifactBundle, ArtifactPackage:
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a %q, and an artifact is %q, %q, %q or %q",
|
||||
module, a.Name, a.Kind, ArtifactImage, ArtifactArchive, ArtifactUpstream,
|
||||
ArtifactBundle+", "+ArtifactPackage))
|
||||
}
|
||||
// **A bundle is built from the module itself, so it says a language instead.** Everything
|
||||
// else names what it is built from: a Dockerfile, a directory, somebody else's reference.
|
||||
// A bundle's source is the module's own directory by definition, and what it needs to say
|
||||
// is which compiler — because the mesh chooses that, and cannot choose for a module that
|
||||
// has not said.
|
||||
if len(a.External) > 0 && (a.Kind != ArtifactBundle || a.Language != "typescript") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q names packages it keeps external, and only a TypeScript bundle is bundled into "+
|
||||
"one file with some kept out (novox/hq ADR 0193)", module, a.Name))
|
||||
}
|
||||
if len(a.Env) > 0 && a.Kind != ArtifactBundle {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a %q and says what it is given (env). Only a bundle the node's runtime "+
|
||||
"serves is given words (novox/hq ADR 0192); a container says its own environment",
|
||||
module, a.Name, a.Kind))
|
||||
}
|
||||
if a.Kind == ArtifactBundle || a.Kind == ArtifactPackage {
|
||||
// **Except for a language that compiles to a binary, where it names which one**
|
||||
// (novox/hq 04-ISSUES/142). A bundle in an interpreted language is the module's own
|
||||
// directory compiled whole, and naming a source would be describing its own build. A
|
||||
// repository written in a compiled language holds several commands — the host and its
|
||||
// bootstrap live in one, and the mesh needs the host — and "the module's own directory"
|
||||
// is then not a package at all. So the compiled case may say which package, and says
|
||||
// the module root by saying nothing.
|
||||
if a.From != "" && !compilesToABinary(a.Language) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a bundle and names what it is built from (%q). A bundle is built "+
|
||||
"from the module's own directory; what it says is the language",
|
||||
module, a.Name, a.From))
|
||||
}
|
||||
if strings.TrimSpace(a.Language) == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a bundle and says no language, so nothing can choose a compiler "+
|
||||
"for it", module, a.Name))
|
||||
}
|
||||
problems = append(problems, bundleEnvProblems(module, a)...)
|
||||
// What the runtime loads is among what was compiled (ADR 0175): a name here that is
|
||||
// not an entrypoint is a file the bundle does not contain, and the runtime would
|
||||
// fail to import it on every machine rather than here.
|
||||
for _, load := range a.Loads {
|
||||
found := false
|
||||
for _, e := range a.Entrypoints {
|
||||
found = found || e == load
|
||||
}
|
||||
// A bundle compiled to a binary is one executable: the runtime loads that or nothing
|
||||
// (novox/hq ADR 0193).
|
||||
if bin := BinaryOf(a); bin != "" {
|
||||
found = load == bin
|
||||
}
|
||||
if !found {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q says the runtime loads %q, which is not among its entrypoints — "+
|
||||
"what is loaded is compiled, so it is named there too", module, a.Name, load))
|
||||
}
|
||||
}
|
||||
// **A system, for a language that compiles to a binary** (novox/hq ADR 0142). A binary
|
||||
// is pinned to one operating system at link time so a host refuses to touch a machine
|
||||
// it was not built for (novox/hq ADR 0005); an artifact that says nothing would be
|
||||
// compiled for whatever the build machine happened to be, which reads as portable and
|
||||
// is not.
|
||||
if compiled := compilesToABinary(a.Language); compiled && strings.TrimSpace(a.System) == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is compiled to a binary and says no system, so it would be built for "+
|
||||
"whatever the build machine happens to be. Declare one artifact per "+
|
||||
"system: %s", module, a.Name, spokenSystems()))
|
||||
} else if !compiled && strings.TrimSpace(a.System) != "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q names the system %q and is written in %q, which compiles to code that "+
|
||||
"runs anywhere — a system that decides nothing reads as though it did",
|
||||
module, a.Name, a.System, a.Language))
|
||||
} else if compiled && !knownSystem(a.System) {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is built for %q, and a system is %s",
|
||||
module, a.Name, a.System, spokenSystems()))
|
||||
}
|
||||
} else {
|
||||
if a.From == "" && a.URL == "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q says nothing about what it is built from", module, a.Name))
|
||||
}
|
||||
if a.Language != "" {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a %q and names a language. Only a bundle is compiled by the mesh; "+
|
||||
"everything else brings its own recipe", module, a.Name, a.Kind))
|
||||
}
|
||||
}
|
||||
if a.Compiles != "" {
|
||||
if a.Kind != ArtifactImage {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is a %q and names a Go package it compiles. Only an image's recipe is "+
|
||||
"told which; a bundle names its package in from (novox/hq ADR 0267)", module, a.Name, a.Kind))
|
||||
} else if c := strings.TrimSpace(a.Compiles); strings.HasPrefix(c, "/") || c == ".." ||
|
||||
strings.HasPrefix(c, "../") || strings.Contains(c, "/../") || strings.HasSuffix(c, "/..") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q compiles %q, which leaves the tree it is built in", module, a.Name, a.Compiles))
|
||||
}
|
||||
}
|
||||
// An upstream image is named, not read from the repository, so the path rule does not
|
||||
// apply to it — and applying it anyway would refuse every reference with a registry host
|
||||
// in it.
|
||||
if a.Kind != ArtifactUpstream &&
|
||||
(strings.HasPrefix(a.From, "/") || strings.Contains(a.From, "..")) {
|
||||
// A build reads its own repository and nothing else. A path leaving it would make
|
||||
// what gets built depend on whatever happens to be on the machine building it.
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q is built from %q, which is outside its own repository",
|
||||
module, a.Name, a.From))
|
||||
}
|
||||
if a.Kind == ArtifactUpstream && !strings.Contains(a.From, ":") {
|
||||
// Without a tag or digest, what gets mirrored is whatever `latest` means today, and
|
||||
// a module pinned to that is not pinned.
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From))
|
||||
}
|
||||
problems = append(problems, upstreamFileProblems(module, a)...)
|
||||
problems = append(problems, pathsProblems(module, a)...)
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// NamedPaths are the paths a module's fingerprint takes its tree from (novox/hq ADR 0499 clause 2): the union of the
|
||||
// paths its artifacts name, cleaned and sorted, and every true only when the module builds something and every artifact
|
||||
// names paths; otherwise none, and the module keeps its directory's tree.
|
||||
//
|
||||
// **Read by the core alone** (ADR 0495 clause 1, the lead's decision L8 of R4e): the indexer and its merge preview
|
||||
// (buildstep.ModuleTrees) call it. A build module's fetch step is handed the paths by the builder, from the indexer's
|
||||
// kept inputs (delivery.KeptInputs2.Paths), and reads no manifest for them.
|
||||
func (m Manifest) NamedPaths() (paths []string, every bool) {
|
||||
if m.Build == nil || len(m.Build.Artifacts) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if len(a.Paths) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
for _, p := range a.Paths {
|
||||
seen[path.Clean(p)] = true
|
||||
}
|
||||
}
|
||||
for p := range seen {
|
||||
paths = append(paths, p)
|
||||
}
|
||||
sort.Strings(paths)
|
||||
return paths, true
|
||||
}
|
||||
|
||||
// pathsProblems is what is wrong with an artifact's `paths` (novox/hq ADR 0499 clause 1): named on a kind whose build
|
||||
// source is derived or named otherwise, or a path that leaves the module's directory, is its whole directory, or is a
|
||||
// pattern.
|
||||
func pathsProblems(module string, a Artifact) []string {
|
||||
if a.Paths == nil {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case a.Kind == ArtifactBundle && a.Language == "go":
|
||||
return []string{fmt.Sprintf("%s: %q is a Go program and names paths: its build source is derived from the "+
|
||||
"package it compiles (novox/hq ADR 0267 clause 1, ADR 0499 clause 1)", module, a.Name)}
|
||||
case a.Kind != ArtifactBundle && a.Kind != ArtifactPackage:
|
||||
return []string{fmt.Sprintf("%s: %q is a %q and names paths; only a package or a bundle in a language other "+
|
||||
"than Go names the paths it is built from (novox/hq ADR 0499 clause 1)", module, a.Name, a.Kind)}
|
||||
case len(a.Paths) == 0:
|
||||
return []string{fmt.Sprintf("%s: %q names no path in paths; leave the word out to build from the module's "+
|
||||
"directory", module, a.Name)}
|
||||
}
|
||||
var problems []string
|
||||
for _, p := range a.Paths {
|
||||
clean := path.Clean(p)
|
||||
switch {
|
||||
case strings.TrimSpace(p) == "":
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names an empty path", module, a.Name))
|
||||
case strings.HasPrefix(p, "/") || clean == ".." || strings.HasPrefix(clean, "../"):
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names the path %q, which leaves the module's directory",
|
||||
module, a.Name, p))
|
||||
case clean == ".":
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names the module's whole directory (%q) as a path; leave "+
|
||||
"the word out for that", module, a.Name, p))
|
||||
case strings.ContainsAny(p, "*?[]{}\\"):
|
||||
problems = append(problems, fmt.Sprintf("%s: %q names %q, which is a pattern; paths names files and "+
|
||||
"directories", module, a.Name, p))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// reservedSuffixes are names no public vendor publishes under: an installation's own and documentation's.
|
||||
var reservedSuffixes = []string{".internal", ".local", ".lan", ".home", ".corp", ".localhost", ".home.arpa",
|
||||
".test", ".invalid", ".example", ".intranet", ".private"}
|
||||
|
||||
// notAPublicHost says why a host is not a public vendor's name, or "".
|
||||
func notAPublicHost(host string) string {
|
||||
h := strings.ToLower(strings.TrimSuffix(host, "."))
|
||||
if net.ParseIP(h) != nil {
|
||||
return "an address rather than a name"
|
||||
}
|
||||
if !strings.Contains(h, ".") {
|
||||
return "a name with no domain"
|
||||
}
|
||||
for _, s := range reservedSuffixes {
|
||||
if strings.HasSuffix(h, s) {
|
||||
return "a name under " + s + ", which is nobody's public domain"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// sha256Hex is a pinned sha256 as a manifest writes it: 64 lower-case hexadecimal characters.
|
||||
var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
// upstreamFileProblems is what is wrong with an archive pinned by URL (novox/hq ADR 0316 §2, §6): it is an
|
||||
// archive, fetched over https, pinned by a sha256 and a version, and it names no directory of the repository
|
||||
// besides; a file name or a directory within is a plain path that stays where it is put.
|
||||
func upstreamFileProblems(module string, a Artifact) []string {
|
||||
pinned := a.URL != "" || a.SHA256 != "" || a.Version != "" || a.File != "" || a.Within != ""
|
||||
if !pinned {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
say := func(format string, args ...any) {
|
||||
problems = append(problems, fmt.Sprintf("%s: %q ", module, a.Name)+fmt.Sprintf(format, args...))
|
||||
}
|
||||
if a.Kind != ArtifactArchive {
|
||||
say("is a %q and names an upstream file (url, sha256, version, file, within): only an archive is "+
|
||||
"mirrored from one (novox/hq ADR 0316)", a.Kind)
|
||||
return problems
|
||||
}
|
||||
if a.URL == "" {
|
||||
say("names a sha256, a version, a file or a directory within, and no url to fetch it from")
|
||||
return problems
|
||||
}
|
||||
if a.From != "" {
|
||||
say("is built from %q and fetched from %s: an archive is one or the other", a.From, a.URL)
|
||||
}
|
||||
if u, err := url.Parse(a.URL); err != nil || u.Scheme != "https" || u.Host == "" {
|
||||
say("is fetched from %q, and an upstream file is fetched over https from a named host", a.URL)
|
||||
} else if why := notAPublicHost(u.Hostname()); why != "" {
|
||||
say("is fetched from %s, %s: an upstream file is somebody else's release, published under a public name", a.URL, why)
|
||||
}
|
||||
if !sha256Hex.MatchString(a.SHA256) {
|
||||
say("is fetched from %s without a pinned sha256 (64 lower-case hexadecimal characters): what that "+
|
||||
"address serves changes under the mesh, and only the digest says it is what was reviewed", a.URL)
|
||||
}
|
||||
if strings.TrimSpace(a.Version) == "" {
|
||||
say("pins %s and says no version, so a person reading the pin cannot tell which release it is", a.URL)
|
||||
}
|
||||
if a.File != "" && a.Within != "" {
|
||||
say("names a file (%q) and a directory within (%q): a file is kept whole, a directory is taken out "+
|
||||
"of an archive", a.File, a.Within)
|
||||
}
|
||||
for field, p := range map[string]string{"file": a.File, "within": a.Within} {
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
clean := path.Clean(p)
|
||||
if clean != p || strings.HasPrefix(p, "/") || clean == "." || clean == ".." || strings.HasPrefix(clean, "../") ||
|
||||
(field == "file" && strings.Contains(p, "/")) {
|
||||
say("names %s %q, which is not a plain relative %s", field, p, map[string]string{
|
||||
"file": "file name", "within": "directory"}[field])
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// oneOrOther keeps the message readable for one artifact and for several, because a message that
|
||||
// says "neither" about one thing reads as a bug in the message.
|
||||
func oneOrOther(n int) string {
|
||||
if n == 1 {
|
||||
return "it"
|
||||
}
|
||||
return "them"
|
||||
}
|
||||
|
||||
// Systems the mesh builds binaries for, which is the set a host may be pinned to (novox/hq ADR 0005).
|
||||
//
|
||||
// **A closed list, and the host's own, not the compiler's.** These are not the values a Go toolchain
|
||||
// would call an operating system — the difference between two of them is a C library, not a kernel.
|
||||
// They are what a machine reports itself to be and what a host is linked to refuse, so the list that
|
||||
// matters is the one the host understands.
|
||||
var systems = []string{"alpine", "android", "arch"}
|
||||
|
||||
// knownSystem is whether the mesh builds for it.
|
||||
func knownSystem(system string) bool {
|
||||
want := strings.ToLower(strings.TrimSpace(system))
|
||||
for _, s := range systems {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// spokenSystems is the list as a refusal says it, so a reader is one edit from right.
|
||||
func spokenSystems() string {
|
||||
return strings.Join(systems, ", ")
|
||||
}
|
||||
|
||||
// compilesToABinary is whether this language's bundle is a binary for one operating system rather
|
||||
// than code that runs wherever its interpreter does.
|
||||
//
|
||||
// **Asked of the language, not of the artifact.** A module says what it is written in; what that
|
||||
// implies is the mesh's to know, exactly as the compiler is (novox/hq ADR 0142). Asking the artifact
|
||||
// would let two artifacts in one language disagree about whether they are portable.
|
||||
func compilesToABinary(language string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(language)) {
|
||||
case "go":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// versionRef is how a resource names the version of the artifact it uses: ${version}.
|
||||
//
|
||||
// No artifact name in it, because the resource already says which artifact it is for — a second
|
||||
// name would be a second thing to keep in step with the first.
|
||||
const versionRef = "${version}"
|
||||
|
||||
// versionOf is an artifact's version as a path names it: its digest, short.
|
||||
//
|
||||
// **Content-addressed on purpose.** The alternative is the commit a build came from, and two builds
|
||||
// of one commit are meant to produce the same bytes — every toolchain here is `-trimpath` for that
|
||||
// reason. A commit-named path would move for an identical binary, and everything reading that path
|
||||
// would be recreated for a change that is not one. A digest-named path moves exactly when the bytes
|
||||
// do.
|
||||
//
|
||||
// Twelve hex characters: enough that two of this mesh's builds will not collide, short enough to
|
||||
// read in a path and in a journal line. The `sha256:` prefix goes, because a directory name carrying
|
||||
// a colon is a directory name people quote wrong.
|
||||
func versionOf(digest string) string {
|
||||
hex := strings.TrimPrefix(strings.TrimSpace(digest), "sha256:")
|
||||
if len(hex) > 12 {
|
||||
return hex[:12]
|
||||
}
|
||||
return hex
|
||||
}
|
||||
|
||||
// bundleEnvWords are the words the runtime sets for itself; a bundle that named one would be
|
||||
// telling the runtime what it is, which is the mesh's to say (novox/hq ADR 0192).
|
||||
var bundleEnvWords = map[string]bool{
|
||||
RuntimeToolModules: true, RuntimeBrokerFile: true, RuntimeOperatorAccount: true,
|
||||
RuntimeOperatorHome: true, RuntimeToolEnv: true, RuntimeAgentAccount: true, RuntimeAgentHome: true,
|
||||
}
|
||||
|
||||
// bundleEnvProblems says what is wrong with what a bundle says it is given (novox/hq ADR 0192):
|
||||
// a value is a path or a constant written with the references a container's environment may use
|
||||
// for a place or a port, and never a secret's content or another module's binding — a secret
|
||||
// reaches a tool as a file whose path is named.
|
||||
func bundleEnvProblems(module string, a Artifact) []string {
|
||||
if len(a.Env) == 0 {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
for _, word := range sortedKeys(a.Env) {
|
||||
value := a.Env[word]
|
||||
if bundleEnvWords[word] {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q gives itself %s, which the node's runtime sets for itself; a bundle is "+
|
||||
"given its own words beside the runtime's, never in place of them (novox/hq ADR 0192)",
|
||||
module, a.Name, word))
|
||||
}
|
||||
rest := ofPort.ReplaceAllString(dirRef.ReplaceAllString(value, ""), "")
|
||||
if strings.Contains(rest, "${") {
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: %q gives %s the value %q. A bundle's word is a path or a constant, written with "+
|
||||
"${dir:…} and ${port:…} only; a secret reaches a tool as a file the mesh places, "+
|
||||
"named by its path, never as its content (novox/hq ADR 0192)",
|
||||
module, a.Name, word, value))
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
func copyWords(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BinaryOf is what a bundle compiled to a binary is called once built: what the artifact names, else
|
||||
// the package it is built from, else the artifact's own name (novox/hq 04-ISSUES/142). Empty for a
|
||||
// language that does not compile to one. The builder writes the binary under this name, and the
|
||||
// composer runs it by it, so both ask here.
|
||||
func BinaryOf(a Artifact) string {
|
||||
if !compilesToABinary(a.Language) {
|
||||
return ""
|
||||
}
|
||||
if name := strings.TrimSpace(a.Binary); name != "" {
|
||||
return name
|
||||
}
|
||||
if from := strings.Trim(a.From, "./"); from != "" {
|
||||
return path.Base(from)
|
||||
}
|
||||
return a.Name
|
||||
}
|
||||
|
||||
// runByAResource is the artifacts one of a module's own resources names — a process that runs it,
|
||||
// a step, an archive that unpacks it — by name.
|
||||
func runByAResource(m Manifest) map[string]bool {
|
||||
named := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if a, ok := r["artifact"].(string); ok && a != "" {
|
||||
named[a] = true
|
||||
}
|
||||
}
|
||||
return named
|
||||
}
|
||||
|
||||
// undeliveredBundles says which of a module's bundles nothing would ever put on a machine (novox/hq
|
||||
// 04-ISSUES/216). A bundle reaches a machine three ways: the node's runtime serves it (it says
|
||||
// `loads`, or its module declares `tools`), a resource names it (a process, a step, an archive), or
|
||||
// it is the runtime itself. One reached by none of them was built, recorded and pushed as success,
|
||||
// and was simply absent — seven modules' tools went missing that way on 2026-10-03. Refused here,
|
||||
// naming the field that would deliver it.
|
||||
func undeliveredBundles(m Manifest) []string {
|
||||
if m.Build == nil || m.Module == RuntimeModule {
|
||||
return nil
|
||||
}
|
||||
named := runByAResource(m)
|
||||
var problems []string
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != ArtifactBundle || named[a.Name] || len(a.Loads) > 0 || len(m.Tools) > 0 {
|
||||
continue
|
||||
}
|
||||
problems = append(problems, fmt.Sprintf(
|
||||
"%s: the bundle %q would be built and never reach a machine: nothing loads it, runs it or "+
|
||||
"unpacks it. A tools bundle says `loads` (the entrypoints the node's runtime serves) or its "+
|
||||
"module lists its `tools`; a daemon or a step is a resource naming it (novox/hq 04-ISSUES/216)",
|
||||
m.Module, a.Name))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// bundleFor is a manifest whose one artifact is a bundle in the given language and system.
|
||||
func bundleFor(language, system string) Manifest {
|
||||
return Manifest{Module: "a-component", Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "binary", Kind: ArtifactBundle, Language: language, System: system},
|
||||
}}}
|
||||
}
|
||||
|
||||
func problemsOf(t *testing.T, m Manifest) string {
|
||||
t.Helper()
|
||||
return strings.Join(m.Build.problems(m.Module), "\n")
|
||||
}
|
||||
|
||||
// **A language that compiles to a binary must say which system.**
|
||||
//
|
||||
// A binary is pinned to one operating system at link time, so a host refuses to touch a machine it
|
||||
// was not built for. An artifact that says nothing would be compiled for whatever the build machine
|
||||
// happened to be — which reads as portable and is not, and is the fault this check exists for.
|
||||
func TestABinaryMustSayWhichSystemItIsFor(t *testing.T) {
|
||||
got := problemsOf(t, bundleFor("go", ""))
|
||||
if !strings.Contains(got, "says no system") {
|
||||
t.Fatalf("a compiled bundle with no system was accepted:\n%s", got)
|
||||
}
|
||||
// And the refusal names what it could have said, so a reader is one edit from right.
|
||||
for _, system := range []string{"alpine", "android", "arch"} {
|
||||
if !strings.Contains(got, system) {
|
||||
t.Fatalf("the refusal does not name %q as a choice:\n%s", system, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABinaryThatNamesASystemIsAccepted(t *testing.T) {
|
||||
if got := problemsOf(t, bundleFor("go", "arch")); got != "" {
|
||||
t.Fatalf("a compiled bundle naming a system was refused:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A system the mesh does not build for is refused where it is written. These are the host's own
|
||||
// names, not a compiler's: the difference between two of them is a C library rather than a kernel,
|
||||
// so a value that looks like an operating system to a toolchain is still wrong here.
|
||||
func TestASystemTheMeshDoesNotBuildForIsRefused(t *testing.T) {
|
||||
for _, wrong := range []string{"linux", "debian", "darwin"} {
|
||||
got := problemsOf(t, bundleFor("go", wrong))
|
||||
if !strings.Contains(got, "and a system is") {
|
||||
t.Fatalf("%q was accepted as a system:\n%s", wrong, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **And a language that runs anywhere must not name one.** A system that decides nothing reads as
|
||||
// though it did, which is the same fault as a restriction that restricts nothing (novox/hq ADR 0045).
|
||||
func TestAPortableBundleMayNotNameASystem(t *testing.T) {
|
||||
got := problemsOf(t, bundleFor("typescript", "arch"))
|
||||
if !strings.Contains(got, "runs anywhere") {
|
||||
t.Fatalf("a portable bundle was allowed to name a system:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPortableBundleNamingNoSystemIsAccepted(t *testing.T) {
|
||||
if got := problemsOf(t, bundleFor("typescript", "")); got != "" {
|
||||
t.Fatalf("an ordinary bundle was refused:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// One component, one artifact per system: the shape the mesh's own binaries are declared in, and the
|
||||
// reason the target is the artifact's rather than the recipe's.
|
||||
func TestOneArtifactPerSystemIsAccepted(t *testing.T) {
|
||||
m := Manifest{Module: "the-host", Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "arch", Kind: ArtifactBundle, Language: "go", System: "arch"},
|
||||
{Name: "alpine", Kind: ArtifactBundle, Language: "go", System: "alpine"},
|
||||
{Name: "android", Kind: ArtifactBundle, Language: "go", System: "android"},
|
||||
}}}
|
||||
if got := problemsOf(t, m); got != "" {
|
||||
t.Fatalf("one artifact per system was refused:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The manifest in a repository and the manifest the mesh holds are two documents.
|
||||
//
|
||||
// A digest is not knowable until something is built, so a repository carrying one would be a
|
||||
// repository whose file is wrong the moment anybody edits anything — and the mesh would be pinning
|
||||
// a value nobody could have checked.
|
||||
|
||||
func buildable() Manifest {
|
||||
return Manifest{
|
||||
Module: "meshboard", Version: "1",
|
||||
Build: &Build{Artifacts: []Artifact{
|
||||
{Name: "server", Kind: ArtifactImage, From: "Dockerfile"},
|
||||
{Name: "theme", Kind: ArtifactArchive, From: "files"},
|
||||
}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "svc", "type": "container", "name": "meshboard", "artifact": "server"},
|
||||
{"id": "look", "type": "archive", "path": "/opt/meshboard", "artifact": "theme"},
|
||||
{"id": "dir", "type": "directory", "path": "/opt/meshboard"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func wasBuilt() []Built {
|
||||
return []Built{
|
||||
{Name: "server", Kind: ArtifactImage,
|
||||
Reference: "registry.invalid/meshboard@sha256:" + strings.Repeat("a", 64)},
|
||||
{Name: "theme", Kind: ArtifactArchive,
|
||||
Reference: "https://store.invalid/theme.tar.gz",
|
||||
Digest: "sha256:" + strings.Repeat("b", 64)},
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResourceNamingAnArtifactBecomesOneNamingTheThing(t *testing.T) {
|
||||
got, err := buildable().Resolve(wasBuilt())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range got.Resources {
|
||||
if r["artifact"] != nil {
|
||||
// A build-time word the host has never heard of. Leaving it would be a field the
|
||||
// strict decoder refuses on the machine, at the worst moment.
|
||||
t.Fatalf("%v still names an artifact: %v", r["id"], r)
|
||||
}
|
||||
}
|
||||
if got.Resources[0]["image"] != "registry.invalid/meshboard@sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("the image was not filled in: %v", got.Resources[0])
|
||||
}
|
||||
if got.Resources[1]["source"] != "https://store.invalid/theme.tar.gz" ||
|
||||
got.Resources[1]["digest"] != "sha256:"+strings.Repeat("b", 64) {
|
||||
t.Fatalf("the archive was not filled in: %v", got.Resources[1])
|
||||
}
|
||||
// And a resource that names nothing is untouched.
|
||||
if got.Resources[2]["path"] != "/opt/meshboard" || len(got.Resources[2]) != 3 {
|
||||
t.Fatalf("an ordinary resource was changed: %v", got.Resources[2])
|
||||
}
|
||||
// The built manifest carries no build section: it is the derived document, and something
|
||||
// holding both would invite somebody to build from it again.
|
||||
if got.Build != nil {
|
||||
t.Fatal("the built manifest still says how to build itself")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtifactDeclaredAndNotBuiltIsRefused(t *testing.T) {
|
||||
// A build that did not do what the manifest asked. Said here rather than by a machine later
|
||||
// reporting an empty image.
|
||||
_, err := buildable().Resolve(wasBuilt()[:1])
|
||||
if err == nil {
|
||||
t.Fatal("a manifest resolved with an artifact missing")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "theme") {
|
||||
t.Fatalf("the refusal does not name what is missing: %v", err)
|
||||
}
|
||||
// And it must blame the build rather than the resource. A resource asking for something that
|
||||
// does not exist is a manifest fault; a build not producing what the manifest declared is a
|
||||
// build fault, and telling somebody to fix the wrong one costs an afternoon.
|
||||
if !strings.Contains(err.Error(), "the build did not produce") {
|
||||
t.Fatalf("the refusal blames the wrong thing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResourceNamingSomethingTheModuleDoesNotBuildIsRefused(t *testing.T) {
|
||||
m := buildable()
|
||||
m.Resources = append(m.Resources,
|
||||
map[string]any{"id": "other", "type": "archive", "path": "/x", "artifact": "nothing"})
|
||||
if _, err := m.Resolve(wasBuilt()); err == nil {
|
||||
t.Fatal("a resource naming an artifact nobody builds was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleThatBuildsNothingIsOrdinary(t *testing.T) {
|
||||
// Most of what a person installs is configuration. Requiring an empty build section would be
|
||||
// a field that exists to be left blank.
|
||||
m := Manifest{Module: "shell", Version: "1", Resources: []map[string]any{
|
||||
{"id": "rc", "type": "file", "path": "/etc/zsh/zshrc", "content": "setopt"},
|
||||
}}
|
||||
got, err := m.Resolve(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Resources) != 1 {
|
||||
t.Fatalf("got %v", got.Resources)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoArtifactsWithOneNameAreRefused(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"x","kind":"image","from":"Dockerfile"},
|
||||
{"name":"x","kind":"archive","from":"files"}]}}`))
|
||||
if err == nil {
|
||||
t.Fatal("two artifacts with one name were accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "could mean either") {
|
||||
t.Fatalf("unhelpful refusal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtifactBuiltFromOutsideItsRepositoryIsRefused(t *testing.T) {
|
||||
// A build reads its own repository and nothing else. A path leaving it would make what gets
|
||||
// built depend on whatever happens to be on the machine doing the building.
|
||||
for _, from := range []string{"/etc/passwd", "../elsewhere"} {
|
||||
_, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"x","kind":"archive","from":"` + from + `"}]}}`))
|
||||
if err == nil {
|
||||
t.Fatalf("%q was accepted as a build input", from)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "outside its own repository") {
|
||||
t.Fatalf("refused for the wrong reason: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtifactOfAnUnknownKindIsRefused(t *testing.T) {
|
||||
_, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"x","kind":"binary","from":"main.go"}]}}`))
|
||||
if err == nil {
|
||||
t.Fatal("an artifact of an unknown kind was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPackageParsesLikeABundleAndNeedsALanguage(t *testing.T) {
|
||||
// The SDK's shape: a package built from the module's own directory, naming a language.
|
||||
m, err := ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","slug":"sdk",
|
||||
"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]},
|
||||
"resources":[]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("the SDK's package manifest did not parse: %v", err)
|
||||
}
|
||||
if m.Build.Artifacts[0].Kind != ArtifactPackage {
|
||||
t.Fatalf("expected a package artifact, got %q", m.Build.Artifacts[0].Kind)
|
||||
}
|
||||
|
||||
// A package that names what it is built from is refused, exactly as a bundle is: it is built
|
||||
// from the module's own directory.
|
||||
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"lib","kind":"package","language":"typescript","from":"Dockerfile"}]}}`)); err == nil {
|
||||
t.Fatal("a package naming a source was accepted")
|
||||
}
|
||||
// A package with no language cannot choose a toolchain.
|
||||
if _, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"lib","kind":"package"}]}}`)); err == nil {
|
||||
t.Fatal("a package with no language was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAResourceNamingAPackageIsRefused(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"a","version":"1","slug":"a",
|
||||
"build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]},
|
||||
"resources":[{"id":"svc","type":"container","name":"a","artifact":"lib"}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
_, err = m.Resolve([]Built{{Name: "lib", Kind: ArtifactPackage, Reference: "@novox/a@1.0.0"}})
|
||||
if err == nil {
|
||||
t.Fatal("a resource backed by a package was accepted; a package is not a resource")
|
||||
}
|
||||
}
|
||||
|
||||
// An image names the Go package its recipe compiles (novox/hq ADR 0267): within the tree it is built in,
|
||||
// and only an image says one.
|
||||
func TestOnlyAnImageNamesThePackageItCompilesWithinItsTree(t *testing.T) {
|
||||
m, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[
|
||||
{"name":"x","kind":"image","from":"Dockerfile","compiles":"cmd/x"}]}}`))
|
||||
if err != nil || m.Build.Artifacts[0].Compiles != "cmd/x" {
|
||||
t.Fatalf("an image naming its package was refused or lost it: %v", err)
|
||||
}
|
||||
for _, c := range []struct{ artifact, says string }{
|
||||
{`{"name":"x","kind":"archive","from":"files","compiles":"cmd/x"}`, "Only an image"},
|
||||
{`{"name":"x","kind":"image","from":"Dockerfile","compiles":"../x"}`, "leaves the tree"},
|
||||
{`{"name":"x","kind":"image","from":"Dockerfile","compiles":"/x"}`, "leaves the tree"},
|
||||
{`{"name":"x","kind":"image","from":"Dockerfile","compiles":"a/../../x"}`, "leaves the tree"},
|
||||
} {
|
||||
_, err := ParseManifest([]byte(`{"module":"a","version":"1","build":{"artifacts":[` + c.artifact + `]}}`))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted a refusal saying %q", c.artifact, err, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/internal/beside"
|
||||
)
|
||||
|
||||
// The module holding mesh-broker is the bus, and its account is granted the bus's snapshot API and
|
||||
// nothing else (novox/hq ADR 0235). Anything it declared to say or hear on the bus would be granted
|
||||
// nothing, so it is refused at the parser rather than silently dropped.
|
||||
func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) {
|
||||
raw := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"emits":["something.happened"]}`)
|
||||
_, err := ParseManifest(raw)
|
||||
if err == nil || !strings.Contains(err.Error(), "granted the bus's snapshot API and nothing else") {
|
||||
t.Fatalf("a bus module declaring what it emits was not refused, or not for the reason: %v", err)
|
||||
}
|
||||
|
||||
// Its tools bundle says what it emits (novox/hq ADR 0357): the node's runtime carries the bundle and is
|
||||
// granted the module's events, so a bus module that ships one may declare them.
|
||||
bundled := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"emits":["bus.slow-consumer"],"tools":["bus_streams"],
|
||||
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/t","binary":"t"}]}}`)
|
||||
if _, err := ParseManifest(bundled); err != nil {
|
||||
t.Fatalf("a bus module whose tools bundle declares what it emits was refused: %v", err)
|
||||
}
|
||||
|
||||
quiet := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],
|
||||
"claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"},
|
||||
"tools":["bus_streams"]}`)
|
||||
m, err := ParseManifest(quiet)
|
||||
if err != nil {
|
||||
t.Fatalf("a bus module with an account and tools was refused: %v", err)
|
||||
}
|
||||
if !m.ClaimsSeat(BrokerSeat) {
|
||||
t.Fatal("it does not read as holding the broker seat")
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's bus protects its streams by the snapshot, not as live files: its streams' item is a
|
||||
// dump into its snapshots, it has the account the dump runs as, and the dump runs the snapshot
|
||||
// program in the bus's own container.
|
||||
func TestTheCataloguesBusIsBackedUpBySnapshot(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join(beside.Catalogue(t), "nats", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, account := m.OwnSecrets["broker"]; !account {
|
||||
t.Fatal("the bus declares no account, so its snapshot could not reach it")
|
||||
}
|
||||
var found bool
|
||||
if m.Data == nil {
|
||||
t.Fatal("the bus declares no data")
|
||||
}
|
||||
for _, it := range m.Data.Own {
|
||||
if it.ID != "jetstream" {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if !it.Backup.IsDump() || it.Backup.Into != "snapshots" {
|
||||
t.Fatalf("the bus's streams are protected by %+v, not a snapshot into its snapshots", it.Backup)
|
||||
}
|
||||
if !strings.Contains(it.Backup.Dump, "mesh-nats-snapshot snapshot") {
|
||||
t.Fatalf("the dump does not run the snapshot program: %s", it.Backup.Dump)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("the bus declares no item for its streams")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1).
|
||||
//
|
||||
// **Beside its health, on the container**: a command run inside it, by its shell, as an exec health check
|
||||
// is, that exits 0 while the container holds work a replacement would lose and non-zero when it is idle;
|
||||
// and how long one ask may take. Being asked is also its notice that it is about to be replaced: from the
|
||||
// first ask it takes no new work. The node-engine asks before it replaces the container, leaves it running
|
||||
// while it says busy, and replaces it on the first pass it says idle, or at 15 minutes — the engine's bound,
|
||||
// never the module's.
|
||||
//
|
||||
// Sent only to a node-engine that reads it (link.BusyContract): an older, strict engine refuses a field it
|
||||
// does not know, whole, and to it the container is replaced at once, as before.
|
||||
|
||||
// BusyField is the resource field.
|
||||
const BusyField = "busy"
|
||||
|
||||
// The bounds of one ask (ADR 0335), the node-engine's own, held here near the author.
|
||||
const (
|
||||
BusyTimeoutDefault = 10 * time.Second
|
||||
BusyTimeoutCeiling = time.Minute
|
||||
)
|
||||
|
||||
// Busy is one container's busy, read.
|
||||
type Busy struct {
|
||||
Command string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// ReadBusy reads a resource's `busy` field, its default applied. False when it carries none.
|
||||
func ReadBusy(r map[string]any) (Busy, bool, []string) {
|
||||
raw, present := r[BusyField]
|
||||
if !present {
|
||||
return Busy{}, false, nil
|
||||
}
|
||||
id := fmt.Sprint(r["id"])
|
||||
fields, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return Busy{}, true, []string{fmt.Sprintf("%s: busy is %T; it is an object with a command", id, raw)}
|
||||
}
|
||||
var problems []string
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if k != "command" && k != "timeout" {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy says %q; busy has a command and a timeout", id, k))
|
||||
}
|
||||
}
|
||||
b := Busy{Timeout: BusyTimeoutDefault}
|
||||
if v, has := fields["command"]; has {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's command is %T; it is text", id, v))
|
||||
}
|
||||
b.Command = strings.TrimSpace(s)
|
||||
}
|
||||
if v, has := fields["timeout"]; has {
|
||||
s, _ := v.(string)
|
||||
d, err := time.ParseDuration(strings.TrimSpace(s))
|
||||
switch {
|
||||
case err != nil:
|
||||
problems = append(problems, fmt.Sprintf("%s: its busy's timeout is %v; it is a duration such as \"10s\"", id, v))
|
||||
case d <= 0 || d > BusyTimeoutCeiling:
|
||||
problems = append(problems, fmt.Sprintf("%s: an ask of its busy takes more than nothing and at most %s, not %s",
|
||||
id, BusyTimeoutCeiling, d))
|
||||
default:
|
||||
b.Timeout = d
|
||||
}
|
||||
}
|
||||
return b, true, problems
|
||||
}
|
||||
|
||||
// busyProblems is everything wrong with a manifest's `busy` fields, in the manifest's words: a command on
|
||||
// one line, a timeout within its bounds, and only on a container that stays up.
|
||||
func busyProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
for _, r := range m.Resources {
|
||||
b, has, read := ReadBusy(r)
|
||||
if !has {
|
||||
continue
|
||||
}
|
||||
where := m.Module + ": " + fmt.Sprint(r["id"])
|
||||
for _, p := range read {
|
||||
problems = append(problems, m.Module+": "+p)
|
||||
}
|
||||
if fmt.Sprint(r["type"]) != "container" || !LongRunning(r) {
|
||||
problems = append(problems, fmt.Sprintf("%s declares busy and is not a container that stays up: busy is "+
|
||||
"asked of a running container before it is replaced (ADR 0335)", where))
|
||||
}
|
||||
if b.Command == "" {
|
||||
problems = append(problems, where+"'s busy names no command: one run inside the container, exiting 0 "+
|
||||
"while it is busy")
|
||||
}
|
||||
if strings.ContainsAny(b.Command, "\n\r") {
|
||||
problems = append(problems, where+"'s busy command is one line")
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// busyInto composes a container's `busy` into the node-engine's words, or takes it away: for an engine that
|
||||
// reads it, the command and its timeout written out; for one that does not, the field is not sent, and the
|
||||
// container is replaced at once, as before.
|
||||
func busyInto(resource map[string]any, with Rendering) {
|
||||
if _, has := resource[BusyField]; !has {
|
||||
return
|
||||
}
|
||||
if !with.ReadsBusy {
|
||||
delete(resource, BusyField)
|
||||
return
|
||||
}
|
||||
b, _, _ := ReadBusy(resource)
|
||||
resource[BusyField] = map[string]any{"command": b.Command, "timeout": b.Timeout.String()}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/validate"
|
||||
)
|
||||
|
||||
// A long-running container may say it is busy (novox/hq ADR 0335 §1): `module check` holds the field to its
|
||||
// shape near its author, and it is sent only to a node-engine that reads it, in words the engine's own
|
||||
// validator accepts.
|
||||
|
||||
func busyManifest(t *testing.T, on string, busy any) []byte {
|
||||
t.Helper()
|
||||
extra := map[string]any{"id": "agent", "type": "container", "name": "app-agent",
|
||||
"image": "registry.example/app" + healthDigest}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(healthManifest(t, "", nil, extra), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range m["resources"].([]any) {
|
||||
if r := r.(map[string]any); r["id"] == on {
|
||||
r[BusyField] = busy
|
||||
}
|
||||
}
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestABusyIsHeldToItsShape(t *testing.T) {
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"})); err != nil {
|
||||
t.Fatalf("a well-formed busy was refused: %v", err)
|
||||
}
|
||||
if _, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "x", "timeout": "30s"})); err != nil {
|
||||
t.Fatalf("a busy with its timeout was refused: %v", err)
|
||||
}
|
||||
for name, c := range map[string]struct {
|
||||
on string
|
||||
busy any
|
||||
says string
|
||||
}{
|
||||
"text": {"agent", "mesh-builder busy", "it is an object"},
|
||||
"no command": {"agent", map[string]any{"timeout": "5s"}, "names no command"},
|
||||
"two lines": {"agent", map[string]any{"command": "a\nb"}, "one line"},
|
||||
"a long timeout": {"agent", map[string]any{"command": "x", "timeout": "5m"}, "at most 1m0s"},
|
||||
"an unknown key": {"agent", map[string]any{"command": "x", "every": "1m"}, `says "every"`},
|
||||
"on a step": {"seed", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
"on a service": {"daemon", map[string]any{"command": "x"}, "not a container that stays up"},
|
||||
} {
|
||||
_, err := ParseManifest(busyManifest(t, c.on, c.busy))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: want a refusal saying %q, got %v", name, c.says, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestABusyIsSentOnlyToAnEngineThatReadsIt(t *testing.T) {
|
||||
m, err := ParseManifest(busyManifest(t, "agent", map[string]any{"command": "mesh-builder busy"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := func(with Rendering) []map[string]any {
|
||||
out, err := Resolution{Node: "laptop", Modules: []Manifest{m}}.Declaration(with)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
older := compose(Rendering{})
|
||||
if b, sent := older[indexOfID(older, "app.agent")][BusyField]; sent {
|
||||
t.Fatalf("busy was sent to an engine that does not read it: %v", b)
|
||||
}
|
||||
newer := compose(Rendering{ReadsBusy: true})
|
||||
got, _ := newer[indexOfID(newer, "app.agent")][BusyField].(map[string]any)
|
||||
if got["command"] != "mesh-builder busy" || got["timeout"] != "10s" {
|
||||
t.Fatalf("busy composed as %v", got)
|
||||
}
|
||||
// And the engine's own validator takes the container with it.
|
||||
body, err := json.Marshal(map[string]any{"declaration": validate.Version, "resources": newer})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := validate.Declaration(body); len(problems) > 0 {
|
||||
t.Fatalf("the node-engine's validator refuses the composed busy: %v", problems)
|
||||
}
|
||||
if _, has := m.Resources[len(m.Resources)-1][BusyField].(map[string]any)["timeout"]; has {
|
||||
t.Error("composing wrote the timeout into the catalogue's own manifest")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/internal/beside"
|
||||
)
|
||||
|
||||
// TestEveryCatalogueManifestParses runs the real catalogue through the real gate.
|
||||
//
|
||||
// Not a fixture: the point is whether the manifests as written are accepted by the control plane that
|
||||
// will read them, and a copy of one manifest proves nothing about the other seventy-one.
|
||||
// catalogueRoot is the catalogue these checks run over: in a merge check the clone the build seat put
|
||||
// beside this one, elsewhere the copy captured in testdata/beside (internal/beside). A check that only
|
||||
// ran when somebody remembered a variable was a check nobody ran (novox/hq issue 134), and one that read
|
||||
// whatever checkout sat beside judged the machine, not the change (novox/hq issue 432): it never skips.
|
||||
func catalogueRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
return beside.Dir(t, "mesh-catalog")
|
||||
}
|
||||
|
||||
func TestEveryCatalogueManifestParses(t *testing.T) {
|
||||
_, found := catalogueManifests(t)
|
||||
named, routed := 0, 0
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", filepath.Base(filepath.Dir(r.path)), err)
|
||||
continue
|
||||
}
|
||||
for _, l := range m.Listens {
|
||||
if l.Name != "" {
|
||||
named++
|
||||
}
|
||||
}
|
||||
for port := range RoutedPorts(m) {
|
||||
_ = port
|
||||
routed++
|
||||
}
|
||||
}
|
||||
t.Logf("%d manifests, %d named endpoints, %d routed endpoints resolved", len(found), named, routed)
|
||||
if named == 0 {
|
||||
t.Fatal("no endpoint in the catalogue is named, so this proved nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoCatalogueManifestNamesAnInstallation is ADR 0112's check, run over the real catalogue: no
|
||||
// definition names a domain or a public address the mesh acts on, and every value that must for now
|
||||
// carries its reason (novox/hq ADR 0155, issue 134). The list it prints is the one that shrinks.
|
||||
func TestNoCatalogueManifestNamesAnInstallation(t *testing.T) {
|
||||
_, found := catalogueManifests(t)
|
||||
var named []string
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", r.path, err)
|
||||
continue
|
||||
}
|
||||
named = append(named, InstallationProblems(m)...)
|
||||
}
|
||||
if len(named) > 0 {
|
||||
t.Fatalf("%d value(s) name an installation:\n %s", len(named), strings.Join(named, "\n "))
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryCatalogueModuleDeclaresItsData is ADR 0233's check over the real catalogue (it replaced ADR
|
||||
// 0214's store list): every provider that grants says what it keeps for its consumers, nothing writes a
|
||||
// backup line by hand, every directory a container writes is declared, and every irreplaceable item is
|
||||
// backed up — so a store added is a store backed up, without a list of stores to keep in step.
|
||||
func TestEveryCatalogueModuleDeclaresItsData(t *testing.T) {
|
||||
_, found := catalogueManifests(t)
|
||||
shelf := Shelf{}
|
||||
granting := 0
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
continue // TestEveryCatalogueManifestParses says why
|
||||
}
|
||||
if len(m.Grants) > 0 {
|
||||
granting++
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, problem := range DataProblems(shelf) {
|
||||
t.Error(problem)
|
||||
}
|
||||
if granting == 0 {
|
||||
t.Fatal("no module in the catalogue grants a provision, so this proved nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryCatalogueIdentityFitsWhatItRequires is ADR 0225's check over the real catalogue: every
|
||||
// module's identity, on the longest machine name, fits the bound of every provision it wants. An
|
||||
// overflow fails here, in the pull request that introduces it, rather than on the provider's machine
|
||||
// the first time a real machine's name meets the module's (issue 263).
|
||||
func TestEveryCatalogueIdentityFitsWhatItRequires(t *testing.T) {
|
||||
_, found := catalogueManifests(t)
|
||||
shelf := Shelf{}
|
||||
for _, r := range found {
|
||||
m, err := r.manifest, r.err
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", r.path, err)
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
for _, p := range IdentityProblems(shelf, DefaultLongestMachine) {
|
||||
t.Error(p)
|
||||
}
|
||||
// The night it was found: the resolver provision bounds nothing, and the object store still 20.
|
||||
if dns, ok := shelf["dnsmasq"]; ok {
|
||||
if b := dns.IdentityBoundOf("wildcard-resolution"); b.Bounded() {
|
||||
t.Errorf("the resolver provision bounds its consumers' identities: %+v", b)
|
||||
}
|
||||
}
|
||||
if store, ok := shelf["minio"]; ok {
|
||||
if b := store.IdentityBoundOf("s3-bucket"); b.Max != 20 {
|
||||
t.Errorf("the object store's access key is not bounded at 20: %+v", b)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// readManifest is one manifest of the catalogue as ReadTreeManifest read it.
|
||||
type readManifest struct {
|
||||
path string // the module.json
|
||||
manifest Manifest
|
||||
err error
|
||||
}
|
||||
|
||||
var (
|
||||
readOnce sync.Once
|
||||
readAll []readManifest
|
||||
readRoot string
|
||||
)
|
||||
|
||||
// catalogueManifests is every manifest of the catalogue (catalogueRoot), read through ReadTreeManifest once
|
||||
// per test binary and not once per test: the checks over the real catalogue each read and parsed every
|
||||
// manifest again, and together took 1.5 s of mesh-sdk's unit bound of 2 s (issue 1756). What it returns is
|
||||
// shared and read-only: a test that changes a manifest copies it first.
|
||||
func catalogueManifests(t *testing.T) (string, []readManifest) {
|
||||
t.Helper()
|
||||
root := catalogueRoot(t)
|
||||
readOnce.Do(func() {
|
||||
readRoot = root
|
||||
found, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, p := range found {
|
||||
m, err := ReadTreeManifest(root, filepath.Dir(p))
|
||||
readAll = append(readAll, readManifest{path: p, manifest: m, err: err})
|
||||
}
|
||||
})
|
||||
if len(readAll) == 0 {
|
||||
t.Fatalf("no manifests under %s", root)
|
||||
}
|
||||
return readRoot, readAll
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/internal/beside"
|
||||
)
|
||||
|
||||
// **A machine trusts the mesh's authority because a module put its root there** (novox/hq ADR
|
||||
// 0147, issue 129). The module carries a shell script and a unit, and both are worthless unless
|
||||
// the mesh fills in where the authority is — which is the one thing about it the manifest cannot
|
||||
// state, because the authority's address is a fact about the mesh and not about the module.
|
||||
//
|
||||
// So what is checked here is the rendering, not the parsing: the script the machine will run
|
||||
// names the authority it was bound to, and the unit runs that script both ways. The verification
|
||||
// itself — a plain client trusting an internal name on a machine holding this, and failing on one
|
||||
// that does not — is the lab's, and cannot be had here.
|
||||
func TestCaTrustRendersTheAuthorityItWasBoundTo(t *testing.T) {
|
||||
m, err := ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Join(beside.Catalogue(t), "ca-trust"))
|
||||
if err != nil {
|
||||
t.Fatalf("the trust module does not parse:\n%v", err)
|
||||
}
|
||||
|
||||
r := Resolution{
|
||||
Node: "workstation",
|
||||
Modules: []Manifest{m},
|
||||
Needs: []Needed{{
|
||||
Name: "internal-acme-ca", From: "anchor", At: "anchor.internal", For: "ca-trust",
|
||||
Serves: map[string]any{
|
||||
"port": float64(9000), "path": "/acme/acme/directory", "roots": "/roots.pem",
|
||||
},
|
||||
}},
|
||||
}
|
||||
out, err := r.Declaration(Rendering{})
|
||||
if err != nil {
|
||||
t.Fatalf("the trust module could not be composed for a machine: %v", err)
|
||||
}
|
||||
|
||||
script := fileNamed(out, "ca-trust.anchor")
|
||||
if script == nil {
|
||||
t.Fatalf("nothing writes the script the unit runs: %v", out)
|
||||
}
|
||||
body, _ := script["content"].(string)
|
||||
if !strings.Contains(body, "https://anchor.internal:9000/roots.pem") {
|
||||
t.Errorf("the script does not fetch from the authority it was bound to:\n%s", body)
|
||||
}
|
||||
if script["mode"] != "0755" {
|
||||
t.Errorf("the script is written %v, which systemd cannot execute", script["mode"])
|
||||
}
|
||||
|
||||
unit := fileNamed(out, "ca-trust.unit")
|
||||
if unit == nil {
|
||||
t.Fatalf("no unit: %v", out)
|
||||
}
|
||||
text, _ := unit["content"].(string)
|
||||
// Both halves. A unit that only installs the anchor leaves a machine trusting an authority
|
||||
// nobody assigned it to any more, which is the half issue 129 asked for by name.
|
||||
for _, want := range []string{
|
||||
"ExecStart=" + script["path"].(string) + " install",
|
||||
"ExecStop=" + script["path"].(string) + " remove",
|
||||
"RemainAfterExit=yes",
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Errorf("the unit does not say %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Chosen is the provider somebody named for a provision: the module, and the node it runs on. Both,
|
||||
// always (novox/hq #258) — a provision comes from a module, and the same module on two machines is
|
||||
// two answers, so neither half alone says which. Module is empty only on a record made before this
|
||||
// was asked, and such a record is honoured exactly as long as it is unambiguous.
|
||||
type Chosen struct {
|
||||
Node string
|
||||
Module string
|
||||
}
|
||||
|
||||
func (c Chosen) String() string {
|
||||
if c.Module == "" {
|
||||
return c.Node
|
||||
}
|
||||
return c.Node + "/" + c.Module
|
||||
}
|
||||
|
||||
// matches is whether this provider is the one chosen.
|
||||
func (c Chosen) matches(p Provider) bool {
|
||||
return p.Node == c.Node && (c.Module == "" || p.Module == c.Module)
|
||||
}
|
||||
|
||||
// among is every offered provider the choice names — one, when the choice is whole.
|
||||
func (c Chosen) among(where []Provider) []Provider {
|
||||
var out []Provider
|
||||
for _, p := range where {
|
||||
if c.matches(p) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// nameOf is how a refusal names a provider: the node and the module on it.
|
||||
func nameOf(p Provider) string {
|
||||
return Chosen{Node: p.Node, Module: p.Module}.String()
|
||||
}
|
||||
|
||||
// providerNames is every provider named, sorted, for a refusal to list.
|
||||
func providerNames(where []Provider) []string {
|
||||
out := make([]string, 0, len(where))
|
||||
for _, p := range where {
|
||||
out = append(out, nameOf(p))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// providersHere is which modules in this node's own set offer a provision, sorted.
|
||||
func providersHere(catalogue map[string]Manifest, here func(string) bool, want string) []string {
|
||||
var out []string
|
||||
for name, m := range catalogue {
|
||||
if !here(name) {
|
||||
continue
|
||||
}
|
||||
for _, o := range m.Offers() {
|
||||
if o == want {
|
||||
out = append(out, name)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// servedByOne is what one provider beside the consumer says a consumer needs to know, or nothing.
|
||||
//
|
||||
// Serving is *whether* a need is created at all when the provider is on this same machine (novox/hq
|
||||
// 04-ISSUES/038's sibling): a need never created is a binding the consumer never gets. The manifest
|
||||
// alone answers that; the values are settled later, with the node's settings.
|
||||
func servedByOne(m Manifest, want string) map[string]any {
|
||||
if _, ok := m.Serves[want]; ok {
|
||||
return ServedOn(m, want, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sharedByOne is the own secret that provider names as its credential (ADR 0158), or "" when it
|
||||
// gives each consumer its own.
|
||||
func sharedByOne(m Manifest, want string) string {
|
||||
if own, shared := m.SharedCredentialOf(want); shared {
|
||||
return own
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func oneOf(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A provider and its consumer on ONE machine, which is what ADR 0066's anchor is.
|
||||
//
|
||||
// **Every fault in this file is the same shape: the same-node path diverging from the cross-node
|
||||
// one.** A provider on another machine is walked by the control plane — its served facts are
|
||||
// re-derived from its manifest with that machine's port assignments and settled with that node's
|
||||
// settings layers — and only then handed to the consumer. A provider on the consumer's OWN machine
|
||||
// never passes through that walk, so every step of it had to be repeated here, and each step that
|
||||
// was not is a promise the co-located arrangement quietly breaks.
|
||||
//
|
||||
// 04-ISSUES/038 was the first of them (the port). These are the rest.
|
||||
|
||||
// A root certificate, as a certificate authority serves one: a real, self-signed one made for these tests (its key
|
||||
// thrown away), because the one setting that may hold lines must parse as a certificate (novox/hq issue 339).
|
||||
const servedRoot = `-----BEGIN CERTIFICATE-----
|
||||
MIIBPjCB8aADAgECAhRZG3p93hUUB5bz00uxhYo/nJnTQjAFBgMrZXAwFDESMBAG
|
||||
A1UEAwwJdGVzdCByb290MCAXDTI2MTAwODIyMjE0NloYDzIxMjYwOTE0MjIyMTQ2
|
||||
WjAUMRIwEAYDVQQDDAl0ZXN0IHJvb3QwKjAFBgMrZXADIQA0pt/ld+W0MXwBhPfO
|
||||
cuAt56kIW6Qcn+4vqWpuvHTiqaNTMFEwHQYDVR0OBBYEFIjgaLk4OVGIOeZQiqnN
|
||||
p9vhciFjMB8GA1UdIwQYMBaAFIjgaLk4OVGIOeZQiqnNp9vhciFjMA8GA1UdEwEB
|
||||
/wQFMAMBAf8wBQYDK2VwA0EAl9uWeSM2XAV8u0reyV3BLRxNVik+4FCRO1QKPs2k
|
||||
IlB1rK9oAOYManH+VFuBMI/JJ31ajSti81q4E0CSw8r5DQ==
|
||||
-----END CERTIFICATE-----
|
||||
`
|
||||
|
||||
// stepCA is an internal ACME authority: it answers `acme-ca` from anywhere in the mesh, and what a
|
||||
// consumer must know is the directory URL and the root to trust. **The root is not knowable when
|
||||
// the module is written** — it exists only once the CA has been initialised — so the manifest
|
||||
// declares the key and the operator supplies the value as a setting, which is exactly the shape the
|
||||
// cross-node path settles and the same-node path did not.
|
||||
func stepCA() Manifest {
|
||||
return Manifest{
|
||||
Module: "step-ca", Version: "1",
|
||||
Provides: FromAnywhere("acme-ca"),
|
||||
Listens: []Listening{{Port: 9000, Protocol: "tcp", From: FromMesh}},
|
||||
Serves: map[string]map[string]any{"acme-ca": {
|
||||
"directory": "https://anchor.internal/acme/acme/directory",
|
||||
// Declared empty: a manifest is the same on every mesh, and this mesh's root is not.
|
||||
"root": "",
|
||||
}},
|
||||
Resources: []map[string]any{{
|
||||
"id": "server", "type": "container", "name": "step-ca", "ports": []any{"9000"},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// stepCAWithItsRoot is the authority whose root the operator supplies: the served key asks for it as
|
||||
// `${setting:root}` (ADR 0155), the only way a setting reaches what a provider serves since a layer
|
||||
// stopped replacing a served key (novox/hq issue 696; before it, the empty `root` was replaced whole).
|
||||
func stepCAWithItsRoot() Manifest {
|
||||
m := stepCA()
|
||||
m.Serves["acme-ca"]["root"] = "${setting:root}"
|
||||
return m
|
||||
}
|
||||
|
||||
// routeProxy issues from that authority, so it must be told the root to verify it with.
|
||||
func routeProxy() Manifest {
|
||||
return Manifest{
|
||||
Module: "route-proxy", Version: "1",
|
||||
Requires: []string{"acme-ca"},
|
||||
Provides: FromAnywhere("route"),
|
||||
Binds: map[string]string{"acme-ca": "/var/lib/route-proxy/acme-ca.json"},
|
||||
Receives: map[string]string{"route": "/var/lib/route-proxy/routes.json"},
|
||||
Resources: []map[string]any{{
|
||||
"id": "bundle", "type": "file", "path": "/var/lib/route-proxy/ca.pem", "mode": "0644",
|
||||
"content": "${bound:acme-ca:root}",
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// A co-located provider's served VALUES reach its consumer, not just its served keys.
|
||||
//
|
||||
// The mesh walks a provider on ANOTHER machine and settles what it serves with that node's settings
|
||||
// layers before offering it (cmd/mesh-controller plan.go, theRestOfTheMesh). A provider on the
|
||||
// consumer's own machine was never settled at all: resolve.go's servedHere and declaration.go's
|
||||
// here() both read the manifest and stop there. So a served value the operator supplied — the one
|
||||
// kind of value a manifest cannot carry, because it is different on every mesh — arrived as the
|
||||
// manifest's empty default.
|
||||
//
|
||||
// The consequence is silent and total: route-proxy wrote an empty CA bundle, fell back to the
|
||||
// system trust store, could not verify the internal authority, and no certificate was ever issued.
|
||||
func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) {
|
||||
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
|
||||
[]string{"step-ca", "route-proxy"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// What the operator set on the provider, on this node — the CA's root, which only exists once
|
||||
// the CA has been initialised.
|
||||
out, err := r.Declaration(Rendering{Settings: SettingsBy{
|
||||
"step-ca": {{From: "the operator", Values: map[string]any{"root": servedRoot}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bundle := fileNamed(out, "route-proxy.bundle")
|
||||
if bundle == nil {
|
||||
t.Fatalf("the consumer was given no bundle at all: %v", out)
|
||||
}
|
||||
if got := bundle["content"]; got != servedRoot {
|
||||
t.Errorf("the co-located consumer was not told the root it must trust:\n got %q\nwant %q",
|
||||
got, servedRoot)
|
||||
}
|
||||
}
|
||||
|
||||
// And the binding file says the same thing, for a consumer that reads the binding rather than a
|
||||
// substituted placeholder. The two are one fact and must not be able to disagree.
|
||||
func TestACoLocatedConsumersBindingCarriesTheSettledValues(t *testing.T) {
|
||||
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()),
|
||||
[]string{"step-ca", "route-proxy"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := r.Declaration(Rendering{
|
||||
Settings: SettingsBy{
|
||||
"step-ca": {{From: "the operator", Values: map[string]any{"root": servedRoot}}},
|
||||
},
|
||||
// And the machine moved the provider's port while it was at it, so both halves of the
|
||||
// cross-node derivation are exercised at once.
|
||||
Ports: map[string]map[int]int{"step-ca": {9000: 19000}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
binding := fileNamed(out, "route-proxy.bound-acme-ca")
|
||||
if binding == nil {
|
||||
t.Fatalf("the consumer was given no binding at all: %v", out)
|
||||
}
|
||||
var said struct {
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(binding["content"].(string)), &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said.Serves["root"] != servedRoot {
|
||||
t.Errorf("the binding does not carry the settled root: %q", said.Serves["root"])
|
||||
}
|
||||
if port, _ := asPort(said.Serves["port"]); port != 19000 {
|
||||
t.Errorf("the binding does not carry the port the machine publishes: %v", said.Serves["port"])
|
||||
}
|
||||
}
|
||||
|
||||
// A provider PULLED IN rather than assigned is settled the same way.
|
||||
//
|
||||
// The resolver's same-node need is built during the walk, from whichever modules had been chosen by
|
||||
// the time the requirement came up — so which path a co-located binding took depended on the order
|
||||
// a person happened to assign things in. Settling after the closure is known removes that: both
|
||||
// orders now produce the same file.
|
||||
func TestACoLocatedProviderIsSettledWhicheverWayItWasPulledIn(t *testing.T) {
|
||||
for _, assigned := range [][]string{
|
||||
{"step-ca", "route-proxy"},
|
||||
{"route-proxy", "step-ca"},
|
||||
} {
|
||||
r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), assigned, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := r.Declaration(Rendering{Settings: SettingsBy{
|
||||
"step-ca": {{From: "the operator", Values: map[string]any{"root": servedRoot}}},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundle := fileNamed(out, "route-proxy.bundle")
|
||||
if bundle == nil || bundle["content"] != servedRoot {
|
||||
t.Errorf("assigned %v: the consumer was not told the root", assigned)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A same-node contribution names the port the MACHINE publishes, not the one the module declared.
|
||||
//
|
||||
// 04-ISSUES/038 fixed this for what a consumer is TOLD about a provider. This is the other
|
||||
// direction: what a workload TELLS the provider about itself. gitea declares a bare container port
|
||||
// 3000, the mesh publishes it as 20000:3000 — and gitea's route contribution still said 3000, so
|
||||
// the proxy beside it dialled a port nothing listens on and answered 502 for every request.
|
||||
//
|
||||
// The redirect uses the CONTRIBUTING module's assignment: the port belongs to the workload, and
|
||||
// using the provider's map would move it to wherever the proxy happens to be published.
|
||||
func TestASameNodeContributionNamesThePortTheMachinePublishes(t *testing.T) {
|
||||
gitea := Manifest{
|
||||
Module: "gitea", Version: "1",
|
||||
Listens: []Listening{{Port: 3000, Protocol: "tcp", From: FromMesh}},
|
||||
Contributes: map[string]map[string]any{"route": {"label": "git", "port": 3000}},
|
||||
Resources: []map[string]any{{
|
||||
"id": "server", "type": "container", "name": "gitea", "ports": []any{"3000"},
|
||||
}},
|
||||
}
|
||||
r, err := Resolve(shelf(gitea, routeProxy(), stepCA()),
|
||||
[]string{"gitea", "route-proxy", "step-ca"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := r.Declaration(Rendering{
|
||||
// The machine put gitea's 3000 on 20000, and published it that way.
|
||||
Ports: map[string]map[int]int{"gitea": {3000: 20000}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Where the workload is actually published.
|
||||
server := fileNamed(out, "gitea.server")
|
||||
if published := strings.Join(asStrings(server["ports"]), ","); published != "20000:3000" {
|
||||
t.Fatalf("the workload was not published on the assigned port: %v", server["ports"])
|
||||
}
|
||||
|
||||
// What the proxy beside it was told to dial: the SAME port.
|
||||
routes := fileNamed(out, "route-proxy.received-route")
|
||||
if routes == nil {
|
||||
t.Fatalf("the proxy was given no routes file at all: %v", out)
|
||||
}
|
||||
var given struct {
|
||||
Given []Contribution `json:"given"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(routes["content"].(string)), &given); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(given.Given) != 1 {
|
||||
t.Fatalf("expected one route, got %v", given.Given)
|
||||
}
|
||||
port, ok := asPort(given.Given[0].Values["port"])
|
||||
if !ok || port != 20000 {
|
||||
t.Errorf("the proxy was told to dial a port nothing listens on: %v",
|
||||
given.Given[0].Values["port"])
|
||||
}
|
||||
}
|
||||
|
||||
// A contribution from ANOTHER machine is left exactly as it was.
|
||||
//
|
||||
// Its port belongs to that machine's assignment, which this node's map knows nothing about —
|
||||
// applying this node's map to it would move a remote workload's port to wherever a local module of
|
||||
// the same name happens to be published, which is worse than the fault being fixed.
|
||||
func TestAContributionFromAnotherMachineKeepsItsOwnPort(t *testing.T) {
|
||||
r, err := Resolve(shelf(routeProxy(), stepCA()),
|
||||
[]string{"route-proxy", "step-ca"}, reachable(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := r.Declaration(Rendering{
|
||||
// A same-named module on this machine, moved. The grant below is a laptop's, and must not
|
||||
// be dragged along with it.
|
||||
Ports: map[string]map[int]int{"gitea": {3000: 20000}},
|
||||
Grants: []Grant{{
|
||||
Provision: "route", Consumer: "laptop", From: "gitea", At: "laptop.internal",
|
||||
Values: map[string]any{"name": "git.example", "port": 3000},
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
routes := fileNamed(out, "route-proxy.received-route")
|
||||
var given struct {
|
||||
Given []Contribution `json:"given"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(routes["content"].(string)), &given); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(given.Given) != 1 {
|
||||
t.Fatalf("expected one route, got %v", given.Given)
|
||||
}
|
||||
if port, _ := asPort(given.Given[0].Values["port"]); port != 3000 {
|
||||
t.Errorf("another machine's contribution was rewritten with this machine's ports: %v",
|
||||
given.Given[0].Values["port"])
|
||||
}
|
||||
}
|
||||
|
||||
// The cross-node half of the same fault: a grant assembled on the providing machine carries the
|
||||
// consumer's declared port until the CONSUMER's machine's assignments are applied to it. The
|
||||
// declaration layer cannot do it — those assignments are not this machine's, which is the line the
|
||||
// test above holds — so the grant layer is handed them and does it there.
|
||||
func TestAContributionIsRedirectedToWhereItsOwnMachinePublishedIt(t *testing.T) {
|
||||
published := map[int]int{3000: 20000}
|
||||
|
||||
got := AtPublishedPort(map[string]any{"name": "git.example.tld", "port": 3000}, "forge", published)
|
||||
if port, _ := asPort(got["port"]); port != 20000 {
|
||||
t.Errorf("the proxy would dial a port that machine never published: %v", got["port"])
|
||||
}
|
||||
if got["name"] != "git.example.tld" {
|
||||
t.Errorf("redirecting the port disturbed the rest of the contribution: %v", got)
|
||||
}
|
||||
|
||||
// Idempotent, and silent about a port nobody moved: a module that pinned its own mapping has no
|
||||
// assignment, and must come back exactly as it was written.
|
||||
again := AtPublishedPort(got, "forge", published)
|
||||
if port, _ := asPort(again["port"]); port != 20000 {
|
||||
t.Errorf("applying it twice moved the port again: %v", again["port"])
|
||||
}
|
||||
pinned := AtPublishedPort(map[string]any{"port": 9070}, "office", published)
|
||||
if port, _ := asPort(pinned["port"]); port != 9070 {
|
||||
t.Errorf("a port the mesh never assigned was rewritten: %v", pinned["port"])
|
||||
}
|
||||
}
|
||||
|
||||
func asStrings(v any) []string {
|
||||
listed, ok := v.([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(listed))
|
||||
for _, one := range listed {
|
||||
out = append(out, strings.TrimSpace(plainly(one)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,635 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Collections (novox/hq ADR 0349 clause 13, design 57 L1, issue 684): a provision with many providers.
|
||||
//
|
||||
// **A collector requires a collection; a contributor provides an item from a template.** A module taking additions
|
||||
// from others names a collection in a file of its own, `${collected:<collection>}`, and declares it under
|
||||
// `collections`, with the comment its tool writes a line in. Each contributor ships its item as a template of its
|
||||
// own repository and declares it under `items`: `{"collection": …, "template": "files/<name>"}`. Its build packs the
|
||||
// template with the module's others, and the mesh holds its text as it holds a file's (template.go).
|
||||
//
|
||||
// **The node's provisioner client fills both.** An item is filled on the node by the client of the node its
|
||||
// contributor is placed on, from the same owners as a file, and recorded in mesh-provisioner_collect; a collector's
|
||||
// file is filled by the same client from every item of its collection recorded for that node, so an item goes when
|
||||
// its contributor leaves. So a module contributing an item is a moved module (handed.go), and the controller renders
|
||||
// no item: the plan of a contributor names its items and their templates, and the plan of a collector names each
|
||||
// collection it reads and its comment. Neither carries a contributor's text.
|
||||
//
|
||||
// **The three old namespaces read as collections.** `${shell:<shell>:<slot>}`, `${environment:<format>}` and
|
||||
// `${contribution:<seat>:<kind>}` name, in a handed collector's file, the collection of the same name —
|
||||
// `shell:zsh:normal` — with the comment the controller wrote before each module's piece. An item is contributed to
|
||||
// one by that name, by a module the old form would have let contribute it. Each file is filled one way, never
|
||||
// both (ADR 0349 clause 17): a handed collector is filled from items alone and an unhanded one from the old forms
|
||||
// alone, and a piece of the other form on the same machine is said as not placed, never dropped quietly.
|
||||
|
||||
// Item is one item a module contributes to a collection: its template in the module's repository, or, in the
|
||||
// manifest the mesh holds, the template's text, read back from the build's templates archive like a file's.
|
||||
//
|
||||
// **Attributes, a capability and a name** (design 57 L3, issue 777): an item may carry `attributes` (string keys and
|
||||
// values; `order`, a whole number 0–99 defaulting to 50, orders the items of one placeholder) and `if-capability`,
|
||||
// the capability a machine must report for the item to be placed on it. A module may give several items to one
|
||||
// collection, each then with a `name` unique within the module and the collection. A collector takes only the items
|
||||
// whose attributes match, `${collected:<collection>:<k>=<v>,…}`, and all of them without the selector.
|
||||
type Item struct {
|
||||
Collection string `json:"collection"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Template string `json:"template,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Attributes map[string]string `json:"attributes,omitempty"`
|
||||
IfCapability string `json:"if-capability,omitempty"`
|
||||
}
|
||||
|
||||
// ItemKey is an item's key among its module's: the collection, and `/<name>` for a named item.
|
||||
func ItemKey(collection, name string) string {
|
||||
if name == "" {
|
||||
return collection
|
||||
}
|
||||
return collection + "/" + name
|
||||
}
|
||||
|
||||
func (it Item) key() string { return ItemKey(it.Collection, it.Name) }
|
||||
|
||||
// order is the item's place among the items of one placeholder: its `order` attribute, or OrderDefault.
|
||||
func (it Item) order() int {
|
||||
if n, err := strconv.Atoi(it.Attributes[OrderAttribute]); err == nil && n >= 0 && n <= 99 {
|
||||
return n
|
||||
}
|
||||
return OrderDefault
|
||||
}
|
||||
|
||||
// OrderAttribute is the attribute that orders items.
|
||||
const OrderAttribute = "order"
|
||||
|
||||
// matches is whether the item's attributes carry every pair of the selector.
|
||||
func (it Item) matches(where map[string]string) bool {
|
||||
for k, v := range where {
|
||||
if got, has := it.Attributes[k]; !has || got != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// itemsInOrder is the module's items by order, then name; the order the old form placed a module's pieces in.
|
||||
func itemsInOrder(items []Item) []Item {
|
||||
out := append([]Item(nil), items...)
|
||||
sort.SliceStable(out, func(a, b int) bool {
|
||||
if oa, ob := out[a].order(), out[b].order(); oa != ob {
|
||||
return oa < ob
|
||||
}
|
||||
return out[a].Name < out[b].Name
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// Collection is one collection a module collects, and the comment its tool writes a line in: each contributor's item
|
||||
// is introduced by `<comment> <contributor>`, and by nothing without one.
|
||||
type Collection struct {
|
||||
Name string `json:"name"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
}
|
||||
|
||||
// CollectedNamespace is a collection's placeholder namespace, `${collected:<collection>}`.
|
||||
const CollectedNamespace = "collected"
|
||||
|
||||
// EnvironmentCollection is the one collection of the session environment (design 57 L3, issue 795): its items are
|
||||
// typed by attributes, a variable (`kind=variable`, `name`) or a PATH entry (`kind=path`, `at=start|end`), and the
|
||||
// template's text is the value or the entry. A collector names the rendering, `${collected:environment:posix}` or
|
||||
// `${collected:environment:systemd}`, which is not a selector.
|
||||
const EnvironmentCollection = "environment"
|
||||
|
||||
// oldCollectionNamespaces are the namespaces read as collections of their own name.
|
||||
var oldCollectionNamespaces = []string{"shell", "environment", "contribution"}
|
||||
|
||||
var (
|
||||
// ofCollected is where a collector places a collection. Loose inside the braces, so a misspelt name is refused.
|
||||
ofCollected = regexp.MustCompile(`\$\{collected:([^}]*)\}`)
|
||||
// collectionName is words joined by colons, so an old namespace's placeholder names its collection as it reads.
|
||||
collectionName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)*$`)
|
||||
// itemName, attributeKey and attributeValue are the words an item's name and attributes are made of; none holds
|
||||
// a colon, a comma or an equals sign, which tell a selector from a collection's name.
|
||||
itemName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
attributeKey = regexp.MustCompile(`^[a-z][a-z0-9-]*$`)
|
||||
attributeValue = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`)
|
||||
)
|
||||
|
||||
// splitSelector is a placeholder's name without its selector, and the selector: `widgets:bar=top,place=left` is
|
||||
// `widgets` and `bar=top,place=left`. A collection's name holds no `=`, so the selector begins at the last colon
|
||||
// before the first one.
|
||||
func splitSelector(name string) (collection, selector string) {
|
||||
eq := strings.Index(name, "=")
|
||||
if eq < 0 {
|
||||
return name, ""
|
||||
}
|
||||
colon := strings.LastIndex(name[:eq], ":")
|
||||
if colon < 0 {
|
||||
return name, ""
|
||||
}
|
||||
return name[:colon], name[colon+1:]
|
||||
}
|
||||
|
||||
// selectorOf is a selector's pairs, or why it is none.
|
||||
func selectorOf(selector string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
if selector == "" {
|
||||
return out, nil
|
||||
}
|
||||
for _, pair := range strings.Split(selector, ",") {
|
||||
k, v, ok := strings.Cut(pair, "=")
|
||||
switch {
|
||||
case !ok || !attributeKey.MatchString(k) || !attributeValue.MatchString(v):
|
||||
return nil, fmt.Errorf("%q in %q is not <attribute>=<value>", pair, selector)
|
||||
case out[k] != "":
|
||||
return nil, fmt.Errorf("%q names %s twice", selector, k)
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// collectionOfPlaceholder is the collection a placeholder's namespace and key name, and whether they name one.
|
||||
func collectionOfPlaceholder(namespace, key string) (string, bool) {
|
||||
key, _ = splitSelector(key)
|
||||
switch {
|
||||
case namespace == CollectedNamespace && strings.HasPrefix(key, EnvironmentCollection+":"):
|
||||
// The environment is one collection, and the rest of the name picks the rendering (issue 795).
|
||||
return EnvironmentCollection, true
|
||||
case namespace == CollectedNamespace:
|
||||
return key, true
|
||||
case namespace == "environment":
|
||||
return EnvironmentCollection, true
|
||||
case slices.Contains(oldCollectionNamespaces, namespace):
|
||||
return namespace + ":" + key, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// oldCollection is an old namespace's collection by its name: its comment, or why no item may be contributed to it.
|
||||
// ok is false for a name that is not an old namespace's.
|
||||
//
|
||||
// forItem is whether an item is to be contributed to it: an item may be contributed to a kind received as data,
|
||||
// which a collector may place as a collection only where its shape says its pieces are text (design 57 L3).
|
||||
func oldCollection(name string, forItem bool) (comment string, old bool, wrong error) {
|
||||
namespace, key, _ := strings.Cut(name, ":")
|
||||
if !slices.Contains(oldCollectionNamespaces, namespace) {
|
||||
return "", false, nil
|
||||
}
|
||||
switch namespace {
|
||||
case "shell":
|
||||
target, slot, two := strings.Cut(key, ":")
|
||||
if !two || !oneOf(contributionTargets(), target) || !oneOf(knownSlots, slot) {
|
||||
return "", true, fmt.Errorf("%s is no slot: shell code is shell:<shell>:<slot>, the shell one of %s, "+
|
||||
"the session's file one of %s or the power moment one of %s, and the slot one of %s", name,
|
||||
strings.Join(knownShells, ", "), strings.Join(sessionFiles, ", "), strings.Join(powerMoments, ", "),
|
||||
strings.Join(knownSlots, ", "))
|
||||
}
|
||||
// shellCode's line before each module's code.
|
||||
return "#", true, nil
|
||||
case "environment":
|
||||
if key == "" {
|
||||
// The collection itself, which an item is contributed to; a placeholder names a rendering.
|
||||
return "#", true, nil
|
||||
}
|
||||
if key != EnvironmentPOSIX && key != EnvironmentSystemd {
|
||||
return "", true, fmt.Errorf("%s is no format: the environment is environment:%s or environment:%s",
|
||||
name, EnvironmentPOSIX, EnvironmentSystemd)
|
||||
}
|
||||
return "#", true, nil
|
||||
}
|
||||
seat, kind, _ := strings.Cut(key, ":")
|
||||
s, r, ok := receivable(seat, kind)
|
||||
switch {
|
||||
case !ok && s.Name == "":
|
||||
return "", true, fmt.Errorf("%s names the seat %q, which the mesh does not define", name, seat)
|
||||
case !ok:
|
||||
return "", true, fmt.Errorf("%s: %s; %s", name, s.Name, kindsOf(s))
|
||||
case r.Shape != nil && (forItem || collectableAsText(r)):
|
||||
return r.Comment, true, nil
|
||||
case r.Shape != nil:
|
||||
return "", true, fmt.Errorf("%s is received by %s as data in its own shape, which no item of text is "+
|
||||
"(novox/hq ADR 0255)", name, s.Name)
|
||||
case r.Dirs:
|
||||
return "", true, fmt.Errorf("%s names its contributors' directories, which the holder reads as paths "+
|
||||
"the controller fills; it is not yet a collection (design 57 L1)", name)
|
||||
case s.Name != canonicalSeat(seat):
|
||||
return "", true, fmt.Errorf("%s names the seat by another name; a collection names it %s", name, s.Name)
|
||||
}
|
||||
return r.Comment, true, nil
|
||||
}
|
||||
|
||||
// placerOfOld is the seat whose holder alone may collect an old namespace's collection (ADR 0203, 0204, 0208,
|
||||
// 0211, 0212): the same seat that places it the old way.
|
||||
func placerOfOld(name string) string {
|
||||
namespace, key, _ := strings.Cut(name, ":")
|
||||
switch namespace {
|
||||
case "shell":
|
||||
target, _, _ := strings.Cut(key, ":")
|
||||
return placerOf(target)
|
||||
case "environment":
|
||||
return EnvironmentSeat
|
||||
}
|
||||
seat, _, _ := strings.Cut(key, ":")
|
||||
return canonicalSeat(seat)
|
||||
}
|
||||
|
||||
// collectionProblems is what is wrong with a module's collections and items, from the manifest alone.
|
||||
func collectionProblems(m Manifest) []string {
|
||||
var problems []string
|
||||
declared := map[string]bool{}
|
||||
for i, c := range m.Collections {
|
||||
switch {
|
||||
case !collectionName.MatchString(c.Name):
|
||||
problems = append(problems, fmt.Sprintf("%s's collection %d is %q, which is not a collection's name: "+
|
||||
"words of lower-case letters, digits and hyphens, joined by colons", m.Module, i+1, c.Name))
|
||||
case declared[c.Name]:
|
||||
problems = append(problems, fmt.Sprintf("%s collects %s twice", m.Module, c.Name))
|
||||
}
|
||||
if _, old, _ := oldCollection(c.Name, false); old {
|
||||
problems = append(problems, fmt.Sprintf("%s collects %s, which is read from its placeholder: a module "+
|
||||
"places it as ${%s} where it holds the seat that places it, and declares nothing", m.Module, c.Name,
|
||||
c.Name))
|
||||
}
|
||||
if strings.ContainsAny(c.Comment, "\n\r") {
|
||||
problems = append(problems, fmt.Sprintf("%s's collection %s has a comment holding a line break",
|
||||
m.Module, c.Name))
|
||||
}
|
||||
declared[c.Name] = true
|
||||
}
|
||||
// A collection declared and placed nowhere is a word read by nothing; judged where every file's text is here.
|
||||
if !m.Templated() {
|
||||
named := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if text, ok := r["content"].(string); ok {
|
||||
for _, f := range ofCollected.FindAllStringSubmatch(text, -1) {
|
||||
base, _ := splitSelector(f[1])
|
||||
named[base] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range m.Collections {
|
||||
if !named[c.Name] && collectionName.MatchString(c.Name) {
|
||||
problems = append(problems, fmt.Sprintf("%s collects %s and no file of its places it as "+
|
||||
"${collected:%s}", m.Module, c.Name, c.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
contributed := map[string]bool{}
|
||||
for i, it := range m.Items {
|
||||
where := fmt.Sprintf("%s's item %d", m.Module, i+1)
|
||||
if !collectionName.MatchString(it.Collection) {
|
||||
problems = append(problems, fmt.Sprintf("%s is to %q, which is not a collection's name", where,
|
||||
it.Collection))
|
||||
} else if _, old, wrong := oldCollection(it.Collection, true); old && wrong != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s: %v", where, wrong))
|
||||
}
|
||||
switch {
|
||||
case it.Name != "" && !itemName.MatchString(it.Name):
|
||||
problems = append(problems, fmt.Sprintf("%s: the name %q is not a name: lower-case letters, digits and "+
|
||||
"hyphens", where, it.Name))
|
||||
case contributed[it.key()]:
|
||||
problems = append(problems, fmt.Sprintf("%s contributes to %s twice; a module contributes one item to "+
|
||||
"a collection, or several each with a name of its own", m.Module, it.key()))
|
||||
case it.Name == "" && slices.ContainsFunc(m.Items, func(o Item) bool {
|
||||
return o.Collection == it.Collection && o.Name != ""
|
||||
}):
|
||||
problems = append(problems, fmt.Sprintf("%s contributes several items to %s and %s has no name; each "+
|
||||
"of several items has a name of its own", m.Module, it.Collection, where))
|
||||
}
|
||||
contributed[it.key()] = true
|
||||
problems = append(problems, itemAttributeProblems(m.Module, where, it)...)
|
||||
if it.Collection == EnvironmentCollection {
|
||||
problems = append(problems, environmentItemProblems(where, it)...)
|
||||
}
|
||||
switch {
|
||||
case it.Template == "" && it.Content == "":
|
||||
problems = append(problems, fmt.Sprintf("%s names no template: an item is a template of the module's "+
|
||||
"own, \"template\": \"%s/<name>\" (novox/hq ADR 0349 §13)", where, m.TemplatesDir()))
|
||||
case it.Template != "" && it.Content != "":
|
||||
problems = append(problems, fmt.Sprintf("%s names its template and says its content too", where))
|
||||
case it.Template != "":
|
||||
if wrong := itemTemplateProblem(m, it.Template); wrong != "" {
|
||||
problems = append(problems, fmt.Sprintf("%s: %s", where, wrong))
|
||||
}
|
||||
// **Filled by the provisioner's client alone**: the controller renders no item.
|
||||
if !m.Moved {
|
||||
problems = append(problems, fmt.Sprintf("%s: an item is filled by the provisioner's client of the "+
|
||||
"node its contributor is placed on, so a module contributing one is marked moved "+
|
||||
"(\"moved\": true; novox/hq ADR 0349 §13, §17)", where))
|
||||
}
|
||||
}
|
||||
if it.Content != "" {
|
||||
problems = append(problems, itemTextProblems(m.Module, it)...)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// itemAttributeProblems is what is wrong with an item's attributes and capability. The attributes of an item of a kind
|
||||
// received as data are the fields its shape places by (and `order`), each with a value the shape takes.
|
||||
func itemAttributeProblems(module, where string, it Item) []string {
|
||||
var problems []string
|
||||
for _, k := range sortedKeys(it.Attributes) {
|
||||
v := it.Attributes[k]
|
||||
switch {
|
||||
case !attributeKey.MatchString(k):
|
||||
problems = append(problems, fmt.Sprintf("%s: the attribute %q is not a name: lower-case letters, digits "+
|
||||
"and hyphens", where, k))
|
||||
continue
|
||||
case !attributeFits(it.Collection, k, v):
|
||||
problems = append(problems, fmt.Sprintf("%s: %s=%q is not a value: lower-case letters, digits, hyphens, "+
|
||||
"dots and underscores (an environment variable's name, letters, digits and underscores)", where, k, v))
|
||||
continue
|
||||
}
|
||||
if n, err := strconv.Atoi(v); k == OrderAttribute && (err != nil || n < 0 || n > 99) {
|
||||
problems = append(problems, fmt.Sprintf("%s: order=%q is not a whole number from 0 to 99", where, v))
|
||||
}
|
||||
}
|
||||
if namespace, key, _ := strings.Cut(it.Collection, ":"); namespace == "contribution" {
|
||||
seat, kind, _ := strings.Cut(key, ":")
|
||||
if _, r, ok := receivable(seat, kind); ok && r.Shape != nil {
|
||||
for _, k := range sortedKeys(it.Attributes) {
|
||||
var place *PlaceField
|
||||
for i := range r.Shape.Places {
|
||||
if r.Shape.Places[i].Field == k {
|
||||
place = &r.Shape.Places[i]
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case k == OrderAttribute:
|
||||
case place == nil:
|
||||
problems = append(problems, fmt.Sprintf("%s: %s is placed by %s, and %q is none of them", where,
|
||||
it.Collection, placeFields(r), k))
|
||||
case !oneOf(place.Values, it.Attributes[k]):
|
||||
problems = append(problems, fmt.Sprintf("%s: %s=%s; %s is one of %s", where, k, it.Attributes[k], k,
|
||||
strings.Join(place.Values, ", ")))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case it.IfCapability == "":
|
||||
case !capabilityName.MatchString(it.IfCapability):
|
||||
problems = append(problems, fmt.Sprintf("%s: if-capability %q is not a capability's name", where, it.IfCapability))
|
||||
case !oneOf(KnownCapabilities, it.IfCapability):
|
||||
problems = append(problems, fmt.Sprintf("%s: if-capability %q is one no machine reports; the node-engine "+
|
||||
"detects %s", where, it.IfCapability, strings.Join(KnownCapabilities, ", ")))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// AttributeFits is whether an item's attribute is of the shape its collection allows: words of lower case, except
|
||||
// the environment's `name`, which is a variable's name (design 57 L3, issue 795). The same rule is mesh-catalog's
|
||||
// provisioner client's, and the inventory's.
|
||||
func attributeFits(collection, k, v string) bool {
|
||||
if collection == EnvironmentCollection && k == "name" {
|
||||
return variableName.MatchString(v)
|
||||
}
|
||||
return attributeKey.MatchString(k) && attributeValue.MatchString(v)
|
||||
}
|
||||
|
||||
// environmentItemProblems is what is wrong with an item of the environment collection: typed by `kind`, a variable
|
||||
// with its `name`, a PATH entry `at` the start or the end, and no attribute besides `order`.
|
||||
func environmentItemProblems(where string, it Item) []string {
|
||||
var problems []string
|
||||
for _, k := range sortedKeys(it.Attributes) {
|
||||
if !slices.Contains([]string{"kind", "name", "at", OrderAttribute}, k) {
|
||||
problems = append(problems, fmt.Sprintf("%s: %s is typed by kind, name and at, and %q is none of them",
|
||||
where, it.Collection, k))
|
||||
}
|
||||
}
|
||||
switch kind := it.Attributes["kind"]; kind {
|
||||
case "variable":
|
||||
if it.Attributes["name"] == "" || it.Attributes["at"] != "" {
|
||||
problems = append(problems, fmt.Sprintf("%s: a variable names itself with name=<NAME> and has no at", where))
|
||||
} else if it.Attributes["name"] == "PATH" {
|
||||
problems = append(problems, fmt.Sprintf("%s: PATH is not a variable; a PATH entry is kind=path", where))
|
||||
}
|
||||
case "path":
|
||||
if it.Attributes["name"] != "" || (it.Attributes["at"] != PathAtStart && it.Attributes["at"] != PathAtEnd) {
|
||||
problems = append(problems, fmt.Sprintf("%s: a PATH entry is at=%s or at=%s and has no name", where,
|
||||
PathAtStart, PathAtEnd))
|
||||
}
|
||||
default:
|
||||
problems = append(problems, fmt.Sprintf("%s: an item of %s is kind=variable or kind=path, not %q", where,
|
||||
it.Collection, kind))
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// itemTemplateProblem is what is wrong with an item's template path, or "".
|
||||
func itemTemplateProblem(m Manifest, name string) string {
|
||||
dir := m.TemplatesDir()
|
||||
if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, dir+"/") ||
|
||||
strings.Contains(name, "\\") {
|
||||
return fmt.Sprintf("the template %q is not a file under %s/ of its own repository: a template is %q, a "+
|
||||
"plain relative path, and the build packs only %s/", name, dir, dir+"/<name>", dir)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// itemTextProblems is what an item's text may not name: a secret or a bound value, since an item is kept in a
|
||||
// bucket and no secret is (ADR 0349 clause 7), and a collection, since an item is not a collector. And every
|
||||
// placeholder no pass fills, as in a file's content.
|
||||
func itemTextProblems(module string, it Item) []string {
|
||||
var problems []string
|
||||
for _, f := range templatePlaceholder.FindAllStringSubmatch(it.Content, -1) {
|
||||
namespace, key := f[1], f[2]
|
||||
switch _, collection := collectionOfPlaceholder(namespace, key); {
|
||||
case namespace == "secret" || namespace == "bound":
|
||||
problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is kept in a bucket, where "+
|
||||
"no secret is (novox/hq ADR 0349 §7)", module, it.Collection, f[0]))
|
||||
case collection:
|
||||
problems = append(problems, fmt.Sprintf("%s's item of %s names %s: an item is not a collector",
|
||||
module, it.Collection, f[0]))
|
||||
}
|
||||
}
|
||||
return append(problems, unconsumedPlaceholders(module, map[string]any{"id": "item:" + it.key(),
|
||||
"type": "file", "content": it.Content})...)
|
||||
}
|
||||
|
||||
// isOld is whether a collection's name is an old namespace's.
|
||||
func isOld(name string) bool {
|
||||
_, old, _ := oldCollection(name, true)
|
||||
return old
|
||||
}
|
||||
|
||||
// collectedPlaceholderProblems is what is wrong with one resource's ${collected:…}: placed only in a file's content,
|
||||
// and naming a collection the module declares it collects.
|
||||
func collectedPlaceholderProblems(m Manifest, r map[string]any) []string {
|
||||
var problems []string
|
||||
for _, field := range sortedKeys(r) {
|
||||
v, ok := r[field].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, f := range ofCollected.FindAllStringSubmatch(v, -1) {
|
||||
base, selector := splitSelector(f[1])
|
||||
if _, err := selectorOf(selector); err != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], err))
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case field != "content":
|
||||
problems = append(problems, fmt.Sprintf("%s's resource %v names %s in its %s; a collection is placed "+
|
||||
"only in a file's content", m.Module, r["id"], f[0], field))
|
||||
case isOld(base):
|
||||
if _, _, wrong := oldCollection(base, false); wrong != nil {
|
||||
problems = append(problems, fmt.Sprintf("%s's resource %v names %s: %v", m.Module, r["id"], f[0], wrong))
|
||||
}
|
||||
case !slices.ContainsFunc(m.Collections, func(c Collection) bool { return c.Name == base }):
|
||||
problems = append(problems, fmt.Sprintf("%s's resource %v names %s, and %s does not collect %q: "+
|
||||
"declare it under collections (novox/hq ADR 0349 §13)", m.Module, r["id"], f[0], m.Module, base))
|
||||
}
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// collectionComment is the comment of a collection a module's file names: declared, or the old namespace's.
|
||||
func collectionComment(m Manifest, name string) (string, error) {
|
||||
if comment, old, wrong := oldCollection(name, false); old {
|
||||
return comment, wrong
|
||||
}
|
||||
for _, c := range m.Collections {
|
||||
if c.Name == name {
|
||||
return c.Comment, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("%s names ${collected:%s} and does not collect it", m.Module, name)
|
||||
}
|
||||
|
||||
// itemsLeftOut says each item of a collection a file is filled with the old way that the controller could not render
|
||||
// there (heldItemText): the collector is not handed to the provisioner on this machine (ADR 0349 clause 17). Said,
|
||||
// never dropped quietly.
|
||||
func itemsLeftOut(modules []Manifest, holder Manifest, collection string, facts map[string]string) []string {
|
||||
return itemsLeftOutFor(modules, holder, collection, nil, facts, nil)
|
||||
}
|
||||
|
||||
// itemsLeftOutFor is itemsLeftOut for a placeholder that selects by attributes (where), on a machine with these
|
||||
// capabilities: an item the selector does not take, or whose capability the machine lacks, is not left out but not
|
||||
// wanted here.
|
||||
func itemsLeftOutFor(modules []Manifest, holder Manifest, collection string, where map[string]string,
|
||||
facts map[string]string, caps map[string]bool) []string {
|
||||
var out []string
|
||||
for _, m := range inModuleOrder(modules) {
|
||||
for _, it := range itemsInOrder(m.Items) {
|
||||
if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) {
|
||||
continue
|
||||
}
|
||||
if _, rendered := heldText(it, facts); rendered && renderedOldWay(collection) {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s's item of %s is not placed: %s is filled the old way on this "+
|
||||
"machine, from contributions, and an item only by the provisioner's client (design 57 L1)",
|
||||
m.Module, it.key(), holder.Module))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// oldFormsLeftOut says each piece of an old form a handed collector's collections would have held: the collector is
|
||||
// filled from items alone, and the piece's module contributes none (ADR 0349 clause 17).
|
||||
func oldFormsLeftOut(modules []Manifest, holder Manifest, collections []PlanCollection) []string {
|
||||
var out []string
|
||||
for _, c := range collections {
|
||||
namespace, key, _ := strings.Cut(c.Name, ":")
|
||||
for _, m := range inModuleOrder(modules) {
|
||||
old := false
|
||||
switch namespace {
|
||||
case "shell":
|
||||
target, slot, _ := strings.Cut(key, ":")
|
||||
old = slices.ContainsFunc(m.Shell, func(s ShellCode) bool { return s.For == target && s.Slot == slot })
|
||||
case "environment":
|
||||
old = m.Environment != nil && (len(m.Environment.Variables) > 0 || len(m.Environment.Path) > 0)
|
||||
case "contribution":
|
||||
seat, kind, _ := strings.Cut(key, ":")
|
||||
old = slices.ContainsFunc(m.allContributions(), func(s SeatContribution) bool {
|
||||
return canonicalSeat(s.Seat) == canonicalSeat(seat) && s.Kind == kind
|
||||
})
|
||||
}
|
||||
if old {
|
||||
out = append(out, fmt.Sprintf("%s's %s is not placed: %s is filled by the provisioner's client on "+
|
||||
"this machine, from items, and %s contributes it the old way (design 57 L1)", m.Module, c.Name,
|
||||
holder.Module, m.Module))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldItemText is a moved contributor's items of an old namespace's collection as the controller renders them into a
|
||||
// collector not handed on this machine (design 57 L2, issue 681): the text the mesh holds, read back from the
|
||||
// contributor's templates archive, with the machine's facts filled, each ending its line. As a moved module's file
|
||||
// keeps its text for a machine that cannot take it handed (handed.go), an item keeps its place in a collector filled
|
||||
// the old way, so the file stays what its old form made it. Only `shell:` and `contribution:` collections, which the
|
||||
// controller renders as text; false for a module with no such item, or an item whose text the mesh does not hold or
|
||||
// which names anything but a machine fact this machine has, which is then said as not placed.
|
||||
func heldItemText(m Manifest, collection string, facts map[string]string) (string, bool) {
|
||||
items := heldItems(m, collection, nil, facts, nil)
|
||||
var b strings.Builder
|
||||
for _, h := range items {
|
||||
b.WriteString(h.text)
|
||||
}
|
||||
return b.String(), len(items) > 0
|
||||
}
|
||||
|
||||
// renderedOldWay is whether the controller renders a collection's items into a collector not handed.
|
||||
func renderedOldWay(collection string) bool {
|
||||
namespace, _, _ := strings.Cut(collection, ":")
|
||||
return namespace == "shell" || namespace == "contribution" || namespace == EnvironmentCollection
|
||||
}
|
||||
|
||||
// heldItem is one item of a module rendered the old way.
|
||||
type heldItem struct {
|
||||
item Item
|
||||
text string
|
||||
}
|
||||
|
||||
// heldItems is a module's items of the collection that the selector takes and the machine's capabilities allow, by
|
||||
// order then name, each rendered as heldText does; the ones it cannot render are left out (itemsLeftOutFor says so).
|
||||
func heldItems(m Manifest, collection string, where map[string]string, facts map[string]string,
|
||||
caps map[string]bool) []heldItem {
|
||||
if !renderedOldWay(collection) {
|
||||
return nil
|
||||
}
|
||||
var out []heldItem
|
||||
for _, it := range itemsInOrder(m.Items) {
|
||||
if it.Collection != collection || !it.matches(where) || (it.IfCapability != "" && !caps[it.IfCapability]) {
|
||||
continue
|
||||
}
|
||||
if text, ok := heldText(it, facts); ok {
|
||||
out = append(out, heldItem{item: it, text: text})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldText is an item's text with the machine's facts filled, ending its line; false for an item whose text the mesh
|
||||
// does not hold or which names anything but a machine fact this machine has.
|
||||
func heldText(it Item, facts map[string]string) (string, bool) {
|
||||
if it.Content == "" {
|
||||
return "", false
|
||||
}
|
||||
text := it.Content
|
||||
for _, f := range templatePlaceholder.FindAllStringSubmatch(text, -1) {
|
||||
value, has := facts[f[2]]
|
||||
if f[1] != "machine" || !has {
|
||||
return "", false
|
||||
}
|
||||
text = strings.ReplaceAll(text, f[0], value)
|
||||
}
|
||||
if !strings.HasSuffix(text, "\n") {
|
||||
text += "\n"
|
||||
}
|
||||
return text, true
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A bar's blocks are a collection of text (design 57 L3, issue 781): the holder of node-bar names the kind as
|
||||
// `${collected:contribution:node-bar:block:bar=<b>,place=<p>}`, which a handed collector has its client fill from the
|
||||
// items and an unhanded one has the controller fill as it fills the old placeholder.
|
||||
|
||||
const (
|
||||
collectedBarsText = "top\n${collected:contribution:node-bar:block:bar=top,place=status}" +
|
||||
"bottom\n${collected:contribution:node-bar:block:bar=bottom,place=status}"
|
||||
collectedBars = `{"module":"bars","version":"1","moved":true,
|
||||
"claims":[{"name":"node-bar","scope":"node","renders":{"block":"{{.shows}}"}}],
|
||||
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
|
||||
"resources":[{"id":"conf","type":"file","path":"/etc/bars.conf","template":"files/bars"}]}`
|
||||
itemsOfPower = `{"module":"power","version":"1","moved":true,
|
||||
"items":[{"collection":"contribution:node-bar:block","name":"battery","if-capability":"battery","attributes":{"bar":"bottom","place":"status","order":"40"},"template":"files/battery"},
|
||||
{"collection":"contribution:node-bar:block","name":"draw","attributes":{"bar":"bottom","place":"status","order":"45"},"template":"files/draw"}],
|
||||
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
|
||||
)
|
||||
|
||||
func collectedBarsOn(t *testing.T, caps map[string]bool) Resolution {
|
||||
t.Helper()
|
||||
bars := held(t, collectedBars, builtWith("bars", map[string]string{"bars": collectedBarsText}))
|
||||
power := held(t, itemsOfPower, builtWith("power", map[string]string{"battery": "B\n", "draw": "D ${machine:name}\n"}))
|
||||
r, err := Resolve(shelf(bars, power), []string{"bars", "power"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Account, r.Capabilities = "op", caps
|
||||
return r
|
||||
}
|
||||
|
||||
func TestACollectorMayNameTheBarsBlocksAsACollectionAndNoOtherDataKind(t *testing.T) {
|
||||
bars := held(t, collectedBars, builtWith("bars", map[string]string{"bars": collectedBarsText}))
|
||||
if _, err := collectedBarsOn(t, nil).Compose(handingTo("store:5000")); err != nil {
|
||||
t.Fatalf("the handed collector is refused: %v", err)
|
||||
}
|
||||
if got := collectedPlaceholderProblems(bars, map[string]any{"id": "x",
|
||||
"content": "${collected:contribution:node-bar:block:bar=top,place=status}"}); len(got) != 0 {
|
||||
t.Fatalf("the check refuses it: %v", got)
|
||||
}
|
||||
// A data kind whose shape does not say its items are text stays refused.
|
||||
_, plain, _ := receivable(BarSeat, BarKindBlock)
|
||||
plain.Shape = &Shape{}
|
||||
if collectableAsText(plain) {
|
||||
t.Fatal("a shape not marked as text is collectable")
|
||||
}
|
||||
if _, _, err := oldCollection("contribution:node-bar:block", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBarsCollectionIsFilledTheOldWayWhereTheCollectorIsNotHanded(t *testing.T) {
|
||||
for _, caps := range []map[string]bool{nil, {"battery": true}} {
|
||||
got := composedFile(t, collectedBarsOn(t, caps), "bars.conf")
|
||||
want := "top\nbottom\n# power\nD workstation\n"
|
||||
if caps["battery"] {
|
||||
want = "top\nbottom\n# power\nB\n# power\nD workstation\n"
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("caps %v: the file is %q, want %q", caps, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBarsCollectionIsNamedInTheHandedPlan(t *testing.T) {
|
||||
c, _ := composeWith(t, collectedBarsOn(t, map[string]bool{"battery": true}), handingTo("store:5000"))
|
||||
if !reflect.DeepEqual(c.Plans["bars"].Collections, []PlanCollection{{Name: "contribution:node-bar:block", Comment: "#"}}) {
|
||||
t.Fatalf("the plan collects %+v", c.Plans["bars"].Collections)
|
||||
}
|
||||
if n := len(c.Plans["power"].Items); n != 2 {
|
||||
t.Fatalf("the contributor's plan names %d items", n)
|
||||
}
|
||||
}
|
||||
|
||||
// The bars of the catalogue's i3status-rust, with the blocks of power's items, compose to the files the old shaped
|
||||
// contribution made (issue 781). testdata/bars holds the catalogue's bars and block texts as of the move and the bars as
|
||||
// composed before it, for a machine with neither capability, the battery, the power meter and both. Where the
|
||||
// collector is not handed the controller fills the files; where it is, it names the items for the client, which
|
||||
// fills the same files (mesh-catalog's provisioner client holds that half against the same golden files).
|
||||
func TestTheBarsOfTheCatalogueAreTheSameHandedOrNot(t *testing.T) {
|
||||
read := func(name string) string {
|
||||
b, err := os.ReadFile(filepath.Join("testdata", "bars", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
collector := `{"module":"bars","version":"1","moved":true,"claims":[{"name":"node-bar","scope":"node"}],
|
||||
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},
|
||||
"resources":[{"id":"top","type":"file","path":"/t","template":"files/top"},
|
||||
{"id":"bottom","type":"file","path":"/b","template":"files/bottom"}]}`
|
||||
power := `{"module":"power","version":"1","moved":true,
|
||||
"items":[{"collection":"contribution:node-bar:block","name":"battery","if-capability":"battery","attributes":{"bar":"bottom","place":"status","order":"40"},"template":"files/battery"},
|
||||
{"collection":"contribution:node-bar:block","name":"power-draw","if-capability":"power-meter","attributes":{"bar":"bottom","place":"status","order":"45"},"template":"files/draw"}],
|
||||
"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`
|
||||
for name, caps := range map[string]map[string]bool{"none": nil, "battery": {"battery": true},
|
||||
"power-meter": {"power-meter": true}, "both": {"battery": true, "power-meter": true}} {
|
||||
bars := held(t, collector, builtWith("bars", map[string]string{"top": read("top-bar.toml"), "bottom": read("bottom-bar.toml")}))
|
||||
blocks := held(t, power, builtWith("power", map[string]string{"battery": read("bar-battery"), "draw": read("bar-power-draw")}))
|
||||
r, err := Resolve(shelf(bars, blocks), []string{"bars", "power"}, workstation(), World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Account, r.Capabilities = "op", caps
|
||||
// Not handed: the controller fills them.
|
||||
for id, which := range map[string]string{"bars.top": "top", "bars.bottom": "bottom"} {
|
||||
if got := composedFile(t, r, id); got != read(name+"."+which+"-bar.toml") {
|
||||
t.Errorf("%s, %s bar, not handed:\n%s\nwant\n%s", name, which, got, read(name+"."+which+"-bar.toml"))
|
||||
}
|
||||
}
|
||||
// Handed: the plan names the items the machine's capabilities allow, for the client to fill.
|
||||
c, _ := composeWith(t, r, handingTo("store:5000"))
|
||||
var names []string
|
||||
for _, it := range c.Plans["power"].Items {
|
||||
names = append(names, it.Name)
|
||||
}
|
||||
want := []string{}
|
||||
if caps["battery"] {
|
||||
want = append(want, "battery")
|
||||
}
|
||||
if caps["power-meter"] {
|
||||
want = append(want, "power-draw")
|
||||
}
|
||||
if !reflect.DeepEqual(append([]string{}, names...), want) {
|
||||
t.Errorf("%s: the plan names %v, want %v", name, names, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user