jschoubben
77d738d973
The indexer wrote mesh-catalog's merge 9cdac09a back over the later 69056948, rolling kernel and mesh-build-image back on the nodes: a merged message's commit was written whatever the held one was. Before writing or deleting a module's value, the indexer now asks the forge whether the merge contains the held commit, keeps the held value when it does not and says so, and leaves the message for redelivery when the forge cannot say. Each applied merged message is said in one line with the modules written. No tool showed what the indexer heard of FORGE nor a modules key's history with times, so the cause could not be proven: the indexer now records each FORGE message it handles (sequence, deliveries, times, commit, outcome) in its state heard, read by indexer_heard, and indexer_module_history reads modules.<module>'s kept values with their times. Quick fix (Src order in writeOverlay) rejected: Src is 0 once the source moved on, as here. Thorough fix chosen: the order is the forge's, on every path that writes a module (merged message, take-in).
326 lines
13 KiB
Go
326 lines
13 KiB
Go
package manifest
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"regexp"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// The controller's manifest reader is mesh-controller internal/catalogue (ParseManifest), an internal package of another
|
|
// repository, which this module cannot import. These tests therefore decode the manifest strictly into the fields the
|
|
// controller's Manifest, Build and Artifact types name (internal/catalogue/manifest.go:475, :602, :659, :667, :737,
|
|
// :809, :821, :892, :1010, :1065-:1090), as its parseManifest does (DisallowUnknownFields, manifest.go:1744), and check
|
|
// the rules the issue names, and two rules of the controller's module check (a module with templates packs a files
|
|
// archive; a module contributing an item is moved). **The merge gate's module check is what proves the controller
|
|
// takes this manifest (1134/AC7); these tests hold the rules so a unit run catches a refusal early.**
|
|
|
|
type artifact struct {
|
|
Name string `json:"name"`
|
|
Kind string `json:"kind"`
|
|
Language string `json:"language"`
|
|
System string `json:"system"`
|
|
From string `json:"from"`
|
|
Binary string `json:"binary"`
|
|
Loads []string `json:"loads"`
|
|
Context *struct {
|
|
Repository string `json:"repository"`
|
|
} `json:"context"`
|
|
}
|
|
|
|
type item struct {
|
|
Collection string `json:"collection"`
|
|
Template string `json:"template"`
|
|
}
|
|
|
|
type state struct {
|
|
Name string `json:"name"`
|
|
History int `json:"history,omitempty"`
|
|
}
|
|
|
|
type manifest struct {
|
|
State []state `json:"state"`
|
|
Module string `json:"module"`
|
|
Moved bool `json:"moved"`
|
|
Version string `json:"version"`
|
|
Slug string `json:"slug"`
|
|
Reads []string `json:"reads"`
|
|
Uses []string `json:"uses"`
|
|
Tools []string `json:"tools"`
|
|
Emits []string `json:"emits"`
|
|
OwnSecrets map[string]string `json:"own-secrets"`
|
|
SecretsOwner string `json:"secrets-owner"`
|
|
Resources []map[string]any `json:"resources"`
|
|
Items []item `json:"items"`
|
|
Build struct {
|
|
Artifacts []artifact `json:"artifacts"`
|
|
} `json:"build"`
|
|
}
|
|
|
|
func load(t *testing.T) (manifest, map[string]json.RawMessage) {
|
|
t.Helper()
|
|
raw, err := os.ReadFile("module.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var m manifest
|
|
dec := json.NewDecoder(bytes.NewReader(raw))
|
|
dec.DisallowUnknownFields()
|
|
if err := dec.Decode(&m); err != nil {
|
|
t.Fatalf("the controller's reader refuses a key this manifest has: %v", err)
|
|
}
|
|
var top map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &top); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return m, top
|
|
}
|
|
|
|
func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
|
m, top := load(t)
|
|
if m.Module != "mesh-delivery-view" {
|
|
t.Errorf("module = %q", m.Module)
|
|
}
|
|
want := []string{"mesh-build-go.pauses", "mesh-build-go.steps", "mesh-build-image.pauses", "mesh-build-image.steps", "mesh-build-node.pauses", "mesh-build-node.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-delivery-assigner.asks", "mesh-delivery-assigner.differences", "mesh-delivery-assigner.wants", "mesh-delivery-builder.builds", "mesh-delivery-checker.tries", "mesh-delivery-gate.asks", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.heard", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.placement-candidates", "mesh-delivery-indexer.taken", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-delivery-watchman.stuck", "node-engine.have"} // indexer_placement reads the placement candidates and every <node>.node (ADR 0559 §7); build_usage reads the build modules' steps and pauses, mesh-config's capacities and the checker's tries; 1168/AC1, 1241/AC4, 1345/AC3, 1297/B8, B9
|
|
got := slices.Clone(m.Reads)
|
|
slices.Sort(got)
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("reads = %v, want exactly %v", got, want)
|
|
}
|
|
for _, k := range []string{"consumes", "provides", "claims"} {
|
|
if _, ok := top[k]; ok {
|
|
t.Errorf("the manifest declares %q: the view writes only its answers", k)
|
|
}
|
|
}
|
|
// The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves.
|
|
if !reflect.DeepEqual(m.Emits, []string{"delivered", "moving", "settled", "stuck"}) {
|
|
t.Errorf("emits = %v, want exactly delivered, moving, settled and stuck (1906/D9, D10)", m.Emits)
|
|
}
|
|
// The view uses no build seat: it reads their workers by the broker's named grant and publishes no ask (ADR 0525 §2).
|
|
if len(m.Uses) != 0 {
|
|
t.Errorf("uses = %v, want none", m.Uses)
|
|
}
|
|
// The states the view writes: the operator's answers to the asks that need no proof (ADR 0528 clause 6, 1297/B9), which the
|
|
// teller reads, and how far each repository reached (ADR 0556 clause 2, issue 1280), which the gate reads.
|
|
if want := []state{{Name: "answers", History: 5}, {Name: "reach", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
|
t.Errorf("state = %+v, want %+v", m.State, want)
|
|
}
|
|
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
|
t.Errorf("own-secrets = %v, want exactly broker (or no credential is issued)", m.OwnSecrets)
|
|
}
|
|
if m.SecretsOwner != "mesh-delivery-view" {
|
|
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
|
}
|
|
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
|
|
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
|
|
if want := []string{"indexer_module", "indexer_modules", "indexer_placement", "indexer_candidate", "indexer_candidates", "builder_build", "builder_builds", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "build_usage", "view_reach", "indexer_module_history", "indexer_heard"}; !reflect.DeepEqual(m.Tools, want) {
|
|
t.Errorf("tools = %v, want %v", m.Tools, want)
|
|
}
|
|
for _, a := range m.Build.Artifacts {
|
|
// A bundle compiled to a binary is one executable: what the runtime loads is that binary's name, or the controller's
|
|
// module check refuses the manifest (mesh-sdk catalogue build.go, ADR 0193).
|
|
if a.Name == "tools" && !reflect.DeepEqual(a.Loads, []string{a.Binary}) {
|
|
t.Errorf("the tools artifact loads %v, want its binary %q", a.Loads, a.Binary)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestManifestIsWholeInItsRepository(t *testing.T) { // 1134/AC7
|
|
m, _ := load(t)
|
|
if len(m.Build.Artifacts) != 4 {
|
|
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
|
}
|
|
for _, a := range m.Build.Artifacts {
|
|
if a.Name == "files" {
|
|
continue
|
|
}
|
|
if a.Context != nil {
|
|
t.Errorf("artifact %s names a context of another repository", a.Name)
|
|
}
|
|
if a.Kind != "bundle" || a.Language != "go" || a.System != "arch" {
|
|
t.Errorf("artifact %s = %+v, want a go bundle for arch", a.Name, a)
|
|
}
|
|
if st, err := os.Stat(a.From); err != nil || !st.IsDir() {
|
|
t.Errorf("artifact %s builds from %q, which is not a directory of this module", a.Name, a.From)
|
|
}
|
|
if a.Binary == "" || !strings.HasSuffix(a.From, a.Binary) {
|
|
t.Errorf("artifact %s binary %q does not match %q", a.Name, a.Binary, a.From)
|
|
}
|
|
}
|
|
names := map[string]string{}
|
|
for _, a := range m.Build.Artifacts {
|
|
names[a.Name] = a.Binary
|
|
}
|
|
if names["program"] != "mesh-delivery-view" || names["cli"] != "mesh-delivery-cli" || names["tools"] != "mesh-delivery-view-tools" {
|
|
t.Errorf("artifacts = %v", names)
|
|
}
|
|
}
|
|
|
|
func TestManifestResources(t *testing.T) { // 1134/AC7
|
|
m, _ := load(t)
|
|
byType := map[string][]map[string]any{}
|
|
for _, r := range m.Resources {
|
|
byType[r["type"].(string)] = append(byType[r["type"].(string)], r)
|
|
}
|
|
for _, ty := range []string{"user", "directory", "process", "archive"} {
|
|
if len(byType[ty]) == 0 {
|
|
t.Fatalf("no %s resource", ty)
|
|
}
|
|
}
|
|
user := byType["user"][0]["name"]
|
|
if user != "mesh-delivery-view" {
|
|
t.Errorf("user = %v", user)
|
|
}
|
|
p := byType["process"][0]
|
|
if p["user"] != user || p["artifact"] != "program" {
|
|
t.Errorf("process = %v", p)
|
|
}
|
|
env, _ := p["env"].(map[string]any)
|
|
if env["MESH_BROKER_FILE"] != "${dir:state}/broker" || m.OwnSecrets["broker"] != "${dir:state}/broker" {
|
|
t.Errorf("the process reads %v, the own-secret is written at %v", env["MESH_BROKER_FILE"], m.OwnSecrets["broker"])
|
|
}
|
|
if h, _ := p["health"].(map[string]any); h["kind"] != "unit" {
|
|
t.Errorf("health = %v", p["health"])
|
|
}
|
|
var state, run bool
|
|
for _, d := range byType["directory"] {
|
|
if d["id"] == "state" && d["mode"] == "0700" && d["owner"] == user {
|
|
state = true
|
|
}
|
|
if d["path"] == "/var/lib/mesh-delivery-view-cli" && d["mode"] == "0755" && d["owner"] == user {
|
|
run = true
|
|
}
|
|
}
|
|
if !state || !run {
|
|
t.Errorf("state directory 0700 owned by the user: %v; run directory for the socket 0755: %v", state, run)
|
|
}
|
|
a := byType["archive"][0]
|
|
if a["artifact"] != "cli" || a["path"] != "/usr/local/lib/mesh-delivery-view" {
|
|
t.Errorf("archive = %v, want the cli unpacked into its own directory", a)
|
|
}
|
|
}
|
|
|
|
func TestModuleCheckRules(t *testing.T) { // 1134/AC7
|
|
m, _ := load(t)
|
|
if len(m.Items) > 0 && !m.Moved {
|
|
t.Error("a module contributing an item is marked moved: true")
|
|
}
|
|
var files bool
|
|
for _, a := range m.Build.Artifacts {
|
|
files = files || (a.Name == "files" && a.Kind == "archive" && a.From == "files")
|
|
}
|
|
if len(m.Items) > 0 && !files {
|
|
t.Error("a module naming templates packs a files archive from files/")
|
|
}
|
|
}
|
|
|
|
func TestAliasItem(t *testing.T) { // 1137/AC1
|
|
m, _ := load(t)
|
|
if !reflect.DeepEqual(m.Items, []item{{Collection: "shell:zsh:normal", Template: "files/zsh-normal"}}) {
|
|
t.Fatalf("items = %+v", m.Items)
|
|
}
|
|
raw, err := os.ReadFile("files/zsh-normal")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(raw) != "alias nox-delivery=mesh-delivery-cli\n" {
|
|
t.Fatalf("files/zsh-normal = %q, want the one line", raw)
|
|
}
|
|
}
|
|
|
|
// The controller refuses two modules declaring one path (internal/catalogue/resolve.go:1160-1168). mesh-cli (mesh-catalog
|
|
// modules/mesh-cli/module.json) declares /usr/local/bin whole, so this module may place only a file below a shared
|
|
// directory, never the directory itself.
|
|
func TestNoSharedDirectoryIsClaimed(t *testing.T) { // 1134/AC7
|
|
m, _ := load(t)
|
|
shared := map[string]bool{"/usr/local/bin": true, "/usr/local/lib": true, "/usr/bin": true, "/usr/local": true, "/usr/lib": true, "/etc": true, "/opt": true}
|
|
var binary bool
|
|
for _, r := range m.Resources {
|
|
p, _ := r["path"].(string)
|
|
if shared[strings.TrimRight(p, "/")] {
|
|
t.Errorf("resource %v declares the shared directory %s, which mesh-cli and others declare too", r["id"], p)
|
|
}
|
|
if p == "/usr/local/bin/mesh-delivery-cli" && r["type"] == "file" && r["template"] == "files/mesh-delivery-cli" {
|
|
binary = true
|
|
}
|
|
}
|
|
if !binary {
|
|
t.Error("the command is placed as the file /usr/local/bin/mesh-delivery-cli")
|
|
}
|
|
raw, err := os.ReadFile("files/mesh-delivery-cli")
|
|
if err != nil || !strings.Contains(string(raw), "exec /usr/local/lib/mesh-delivery-view/mesh-delivery-cli") {
|
|
t.Errorf("files/mesh-delivery-cli = %q, %v", raw, err)
|
|
}
|
|
}
|
|
|
|
// The node-engine never places a module-owned directory below /etc, /usr, /run or /var/run: they are root's
|
|
// (mesh-host internal/apply/placement_guard.go:80, rootsOnly; novox/hq issue 339).
|
|
func TestNoDirectoryBelowRootsTrees(t *testing.T) { // 1134/AC10
|
|
m, _ := load(t)
|
|
for _, r := range m.Resources {
|
|
if r["type"] != "directory" {
|
|
continue
|
|
}
|
|
p, _ := r["path"].(string)
|
|
for _, root := range []string{"/etc", "/usr", "/run", "/var/run"} {
|
|
if p == root || strings.HasPrefix(p, root+"/") {
|
|
t.Errorf("directory %v at %s is below %s, which is root's", r["id"], p, root)
|
|
}
|
|
}
|
|
}
|
|
var socketDir bool
|
|
for _, r := range m.Resources {
|
|
if r["path"] == "/var/lib/mesh-delivery-view-cli" && r["type"] == "directory" && r["mode"] == "0755" && r["owner"] == "mesh-delivery-view" {
|
|
socketDir = true
|
|
}
|
|
}
|
|
if !socketDir {
|
|
t.Error("the socket's directory is /var/lib/mesh-delivery-view-cli, 0755, owned by mesh-delivery-view")
|
|
}
|
|
}
|
|
|
|
// TestEveryStateTheCommandsReadIsIssued holds every literal stdio.State name in cmd/ to what the mesh issues this
|
|
// module: a state of its own by its bare name, another module's by <module>.<name> as `reads` lists it. The tools read
|
|
// their own reach as "mesh-delivery-view.reach", which the mesh never issued, and view_reach failed on the machines
|
|
// (issue 1280, note 13).
|
|
func TestEveryStateTheCommandsReadIsIssued(t *testing.T) {
|
|
m, _ := load(t)
|
|
issued := map[string]bool{}
|
|
for _, s := range m.State {
|
|
issued[s.Name] = true
|
|
}
|
|
for _, r := range m.Reads {
|
|
issued[r] = true
|
|
}
|
|
literal := regexp.MustCompile(`stdio\.State\("([^"]+)"\)`)
|
|
seen := 0
|
|
err := filepath.WalkDir("cmd", func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") {
|
|
return err
|
|
}
|
|
src, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, match := range literal.FindAllSubmatch(src, -1) {
|
|
seen++
|
|
if name := string(match[1]); !issued[name] {
|
|
t.Errorf("%s reads the state %q, which the mesh does not issue this module (its own by bare name, another's as `reads` lists it)", path, name)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if seen == 0 {
|
|
t.Fatal("no stdio.State literal found under cmd/: the scan reads nothing")
|
|
}
|
|
}
|