mesh-host/cmd/mesh-host/main_test.go

package main

import (
	"bytes"
	"context"
	"crypto/ed25519"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/apply"
	"github.com/novox/mesh-host/internal/bundle"
	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/identity"
	"github.com/novox/mesh-host/internal/link"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// Argument handling gets tests because it already failed silently once: `mesh-host inventory
// --json` printed text. The standard library stops parsing at the first non-flag argument, so
// the flag sat unread in the positional arguments and the command exited 0 having ignored what
// the user asked for.
//
// Silently doing something other than what was asked, and reporting success, is the fault this
// project exists to name — so it gets defended here rather than remembered.

func TestAFlagAfterTheCommandIsRead(t *testing.T) {
	command, opts, err := parseArgs([]string{"inventory", "--json"})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if command != "inventory" {
		t.Errorf("command = %q, want inventory", command)
	}
	if !opts.json {
		t.Error("--json after the subcommand was ignored")
	}
}

func TestFlagsAreReadInEitherPosition(t *testing.T) {
	for _, args := range [][]string{
		{"profile", "--json", "--timeout", "3s"},
		{"profile", "--timeout=3s", "--json"},
	} {
		_, opts, err := parseArgs(args)
		if err != nil {
			t.Fatalf("%v: unexpected error: %v", args, err)
		}
		if !opts.json || opts.timeout != 3*time.Second {
			t.Errorf("%v parsed as json=%v timeout=%s", args, opts.json, opts.timeout)
		}
	}
}

func TestAMistypedFlagIsRefusedNotIgnored(t *testing.T) {
	// The cost of getting this wrong is asymmetric: an error is a moment's annoyance, and a
	// silently dropped flag is a report that answers a question nobody asked.
	if _, _, err := parseArgs([]string{"profile", "--jsom"}); err == nil {
		t.Fatal("a mistyped flag was accepted")
	}
}

func TestAnUnexpectedArgumentIsRefused(t *testing.T) {
	if _, _, err := parseArgs([]string{"profile", "extra"}); err == nil {
		t.Fatal("a stray argument was ignored rather than refused")
	}
}

func TestTheDefaultsAreTheDocumentedOnes(t *testing.T) {
	// The usage text promises 10s. A default that drifts from what is printed is a small lie
	// that costs someone an afternoon.
	_, opts, err := parseArgs([]string{"profile"})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if opts.timeout != 10*time.Second {
		t.Errorf("default timeout is %s; the usage text says 10s", opts.timeout)
	}
	if opts.json {
		t.Error("json output is on by default; the usage text says it is a flag")
	}
}

func TestNoCommandIsNotAnError(t *testing.T) {
	// Running the binary with no arguments prints usage and exits 0. A host that returns
	// failure for "tell me what you do" is noise in every script that probes it.
	command, _, err := parseArgs(nil)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if command != "" {
		t.Errorf("command = %q, want empty", command)
	}
}

func TestApplyNeedsExactlyOneDeclaration(t *testing.T) {
	// `apply` takes a file where every other command takes nothing, so the leftover-argument
	// rule has an exception — and an exception is where a parser stops refusing things it
	// should. Both directions are checked.
	if _, _, err := parseArgs([]string{"apply"}); err == nil {
		t.Error("apply with no file was accepted")
	}
	if _, _, err := parseArgs([]string{"apply", "a.json", "b.json"}); err == nil {
		t.Error("apply with two files was accepted")
	}

	command, opts, err := parseArgs([]string{"apply", "decl.json", "--dry-run"})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if command != "apply" || opts.file != "decl.json" || !opts.dryRun {
		t.Errorf("parsed as command=%q file=%q dry-run=%v", command, opts.file, opts.dryRun)
	}
}

func TestTheStateHasADocumentedDefault(t *testing.T) {
	// A host that wrote its state somewhere unexpected would forget what it owns on the next
	// run, and then leave everything it had applied behind forever.
	_, opts, err := parseArgs([]string{"owned"})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if opts.state != store.DefaultPath {
		t.Errorf("default state path is %q, not the documented %q", opts.state, store.DefaultPath)
	}
}

func TestAFlagAfterAPositionalIsRead(t *testing.T) {
	// The same fault as TestAFlagAfterTheCommandIsRead, one level down. Taking the subcommand
	// off the front fixed the flag after the COMMAND and not the flag after its ARGUMENT: the
	// standard library stops at the first non-flag argument wherever that argument is.
	for _, args := range [][]string{
		{"apply", "decl.json", "--dry-run", "--json"},
		{"apply", "--dry-run", "decl.json", "--json"},
		{"apply", "--dry-run", "--json", "decl.json"},
	} {
		command, opts, err := parseArgs(args)
		if err != nil {
			t.Errorf("%v: unexpected error: %v", args, err)
			continue
		}
		if command != "apply" || opts.file != "decl.json" || !opts.dryRun || !opts.json {
			t.Errorf("%v parsed as file=%q dry-run=%v json=%v",
				args, opts.file, opts.dryRun, opts.json)
		}
	}
}

// Defends novox/hq ADR 0100: a reconcile on an adopted node speaks unasked only when what it holds
// or its firewall changed — which is how a predecessor still writing is caught, without a report
// every five minutes saying nothing new.
func TestAReconcileSpeaksOnlyWhenWhatIsHeldChanged(t *testing.T) {
	w := &adoptionWatch{}
	held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page"}, {ID: "hello-web.server"}}}
	if !w.changed(held) {
		t.Fatal("the first report of a hold was not said")
	}
	again := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.server"}, {ID: "hello-web.page"}}}
	if w.changed(again) {
		t.Error("the same holds in another order were said again")
	}
	rewritten := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}, {ID: "hello-web.server"}}}
	if !w.changed(rewritten) {
		t.Error("a held file rewritten by something else was not said")
	}
	if !w.changed(link.Report{Firewall: "none", Held: rewritten.Held}) {
		t.Error("a changed firewall was not said")
	}
}

func TestWhatTheLinkPublishedCountsAsSaid(t *testing.T) {
	w := &adoptionWatch{}
	report := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "a"}}}
	applier := w.noting(func(context.Context, []byte, []byte) link.Report { return report })
	applier(context.Background(), nil, nil)
	if w.changed(report) {
		t.Error("a reconcile repeated what the link had just published")
	}
}

func TestAReconcileSpeaksWhenWhatIsReachableChanged(t *testing.T) {
	// The controller previews a flip from what the node last said is reachable; a port that opened
	// since must reach it without waiting for the next delivery (novox/hq ADR 0100).
	w := &adoptionWatch{}
	before := link.Report{Firewall: "ufw", Reachable: []link.Reach{
		{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
	if !w.changed(before) {
		t.Fatal("the first report was not said")
	}
	reordered := link.Report{Firewall: "ufw", Reachable: []link.Reach{
		{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}}
	if w.changed(reordered) {
		t.Error("the same reachable set was said again")
	}
	opened := link.Report{Firewall: "ufw", Reachable: append(before.Reachable,
		link.Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true, ContainerPort: 80})}
	if !w.changed(opened) {
		t.Error("a newly published port was not said")
	}
}

// Defends the node's own record: the link and the reconcile loop both apply, and each reads the
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
// what the other recorded.
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
	// A host is built for one system at link time, and a test binary has no link time: this asks
	// the machine it runs on, and stands aside where the answer is no.
	built, err := system.For("arch")
	if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
		t.Skip("this machine is not one these tests can apply on")
	}
	was := builtFor
	builtFor = "arch"
	t.Cleanup(func() { builtFor = was })
	dir := t.TempDir()
	opts := options{state: filepath.Join(dir, "state.json")}
	raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
		filepath.Join(dir, "a.conf") + `","content":"x\n"}]}`)

	// Whatever else is applying — the link, while this is the reconcile — this waits for it.
	applying.Lock()
	done := make(chan link.Report, 1)
	go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil, nil) }()
	select {
	case report := <-done:
		applying.Unlock()
		t.Fatalf("an apply ran while another held the node: %+v", report)
	case <-time.After(50 * time.Millisecond):
	}
	if _, err := os.Stat(filepath.Join(dir, "a.conf")); !errors.Is(err, os.ErrNotExist) {
		applying.Unlock()
		t.Fatal("the waiting apply had already touched the machine")
	}
	applying.Unlock()

	select {
	case report := <-done:
		if report.Refused != "" {
			t.Fatalf("refused: %s", report.Refused)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("the apply never ran once the node was free")
	}
	known, loadErr := store.Load(opts.state)
	if loadErr != nil || len(known.Resources) != 1 {
		t.Errorf("the apply recorded %d resource(s): %v", len(known.Resources), loadErr)
	}
}

// Defends novox/hq ADR 0100: a change is counted as said only once the mesh has been told. Queued
// and lost — the link down when the reconcile spoke — it must be said again.
func TestAChangeThatNeverReachedTheMeshIsSaidAgain(t *testing.T) {
	w := &adoptionWatch{}
	held := link.Report{Firewall: "ufw", Held: []link.Held{{ID: "hello-web.page", Changed: "rewritten"}}}
	if !w.differs(held) {
		t.Fatal("the first report of a change was not new")
	}
	// The link was down: nothing published it, so nothing says it was said.
	if !w.differs(held) {
		t.Error("a change that never reached the mesh was counted as said")
	}
	w.said(held)
	if w.differs(held) {
		t.Error("a change the mesh was told was said again")
	}
}

// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
// the point of application, whichever command delivered it, naming both — an adopted control-node
// once applied its converged genesis bundle and closed itself for forty-five minutes.

func stateWithMode(t *testing.T, mode string) options {
	t.Helper()
	dir := t.TempDir()
	opts := options{state: filepath.Join(dir, "state.json"), out: &bytes.Buffer{}}
	if err := store.Save(opts.state, store.State{Mode: mode}); err != nil {
		t.Fatal(err)
	}
	return opts
}

func TestAConvergedDeclarationIsRefusedOnAnAdoptedNode(t *testing.T) {
	opts := stateWithMode(t, store.ModeAdopted)
	path := filepath.Join(filepath.Dir(opts.state), "filter.conf")
	raw := []byte(`{"declaration":1,"resources":[{"id":"filter","type":"file","path":"` + path +
		`","content":"table inet filter { chain input { policy drop; } }\n"}]}`)
	d, err := declaration.ParseFileTrusted(raw)
	if err != nil {
		t.Fatal(err)
	}
	for _, from := range []provenance{fromFile, fromBundle} {
		err := runApply(context.Background(), opts, d, raw, from)
		if err == nil {
			t.Fatalf("a converged declaration was applied to an adopted node (from %d)", from)
		}
		want := "this node is adopted; the declaration says converged"
		if !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "`converge`") {
			t.Errorf("the refusal does not name both modes and the act that changes it: %v", err)
		}
	}
	if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
		t.Error("the refused declaration touched the machine")
	}
}

func TestTheKeptDeclarationRepairsARecordThatDisagrees(t *testing.T) {
	// What a node kept is signed by the mesh and verified on load; the state's note of the mode is
	// the host's own. A genesis re-run after `converge`, or a state saved when the kept declaration
	// could not be, leaves them apart — and a loop that refused every five minutes would hold the
	// node off what the mesh said until a delivery that comes only when something changes. The
	// kept declaration wins, and the repair is said.
	opts := stateWithMode(t, store.ModeConverged)
	raw := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"` +
		filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
	var said []string
	report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(line string) { said = append(said, line) })
	if strings.Contains(report.Refused, "the declaration says") {
		t.Fatalf("what the node kept was refused against its own note: %s", report.Refused)
	}
	if len(said) != 1 || !strings.Contains(said[0], "record said converged") ||
		!strings.Contains(said[0], "says adopted") || !strings.Contains(said[0], "repaired") {
		t.Errorf("the repair was not said: %q", said)
	}
}

func TestTheMeshItselfMayChangeTheMode(t *testing.T) {
	// The flip is a declaration: `converge` on the controller records the mode and sends the
	// first converged declaration. Delivered by the link, signed, it is not held to the record —
	// it becomes it. (With no system linked in, the apply is refused later for that; what this
	// checks is that the refusal is not the mode's.)
	opts := stateWithMode(t, store.ModeAdopted)
	raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
		filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
	report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil, nil)
	if strings.Contains(report.Refused, "the declaration says") {
		t.Errorf("the mesh's own flip was refused for its mode: %s", report.Refused)
	}
}

// Defends novox/hq issue 104: a declaration older than what the mesh last said is refused, naming
// both — and `apply FILE` is refused altogether once the mesh has spoken, the last declaration
// itself included: from a file it is applied as the bundle is, which would record the mesh's
// resources as this machine's own and remove the foundation as undeclared.
func TestAnOlderDeclarationIsRefused(t *testing.T) {
	opts := stateWithMode(t, "")
	genesis := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"genesis\n"}]}`)
	since := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"since\n"}]}`)
	other := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"other\n"}]}`)
	if err := store.Save(opts.state, store.State{Genesis: &store.Genesis{Digest: apply.DigestOf(genesis),
		At: time.Now(), Rewritten: true}}); err != nil {
		t.Fatal(err)
	}
	if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: since,
		Signature: []byte("unverified here")}); err != nil {
		t.Fatal(err)
	}
	parsed := func(raw []byte) *declaration.Declaration {
		d, err := declaration.ParseFileTrusted(raw)
		if err != nil {
			t.Fatal(err)
		}
		return d
	}

	err := runApply(context.Background(), opts, parsed(genesis), genesis, fromFile)
	if err == nil {
		t.Fatal("the bundle genesis consumed was applied over what the mesh said since")
	}
	for _, want := range []string{"older", short(apply.DigestOf(genesis)), short(apply.DigestOf(since))} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("the refusal does not say %q: %v", want, err)
		}
	}

	err = runApply(context.Background(), opts, parsed(other), other, fromFile)
	if err == nil {
		t.Fatal("a declaration that is not what the mesh last said was applied")
	}
	if !strings.Contains(err.Error(), "for a machine the mesh has not spoken to") ||
		!strings.Contains(err.Error(), short(apply.DigestOf(since))) {
		t.Errorf("the refusal does not say what a file is for and what was last said: %v", err)
	}

	// The very declaration the mesh last sent, from a file: still a file.
	err = runApply(context.Background(), opts, parsed(since), since, fromFile)
	if err == nil || !strings.Contains(err.Error(), "applied as the bundle is") {
		t.Errorf("the last declaration, from a file, was not refused as a file: %v", err)
	}
	if known, _ := store.Load(opts.state); len(known.Resources) != 0 {
		t.Errorf("a refused file recorded %d resource(s)", len(known.Resources))
	}

	// A bundle other than the one genesis consumed: what genesis applied was rewritten for this
	// machine, so the carried bytes never are.
	err = runApply(context.Background(), opts, parsed(other), other, fromBundle)
	if err == nil || !strings.Contains(err.Error(), "consumed") ||
		!strings.Contains(err.Error(), short(apply.DigestOf(genesis))) {
		t.Errorf("a bundle other than the consumed one was not refused naming it: %v", err)
	}
}

// Defends novox/hq issue 104: what an apply would change is said before anything is, and
// `--dry-run` is that and nothing else — an action listed as the action it is.
func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
	opts := stateWithMode(t, "")
	opts.dryRun = true
	out := &bytes.Buffer{}
	opts.out = out
	path := filepath.Join(filepath.Dir(opts.state), "a.conf")
	raw := []byte(`{"declaration":1,"resources":[
	  {"id":"a","type":"file","path":"` + path + `","content":"x\n"},
	  {"id":"init","type":"action","command":["createdb","mesh"],"verify":["psql","-c","select 1"]}]}`)
	d, err := declaration.ParseFileTrusted(raw)
	if err != nil {
		t.Fatal(err)
	}
	if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
		t.Fatalf("a dry run failed: %v", err)
	}
	if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
		t.Error("a dry run wrote the file")
	}
	for _, want := range []string{"would change", "create   file       a", "run      action     init",
		"`createdb mesh`", "--dry-run: nothing applied"} {
		if !strings.Contains(out.String(), want) {
			t.Errorf("the preview does not say %q:\n%s", want, out.String())
		}
	}
	if strings.Contains(out.String(), "applying:") {
		t.Errorf("a dry run went on to apply:\n%s", out.String())
	}

	// Machine-readable, the same shape as an apply's: the plan, and no report.
	out.Reset()
	opts.json = true
	if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
		t.Fatal(err)
	}
	var shape struct {
		Plan   []apply.Step     `json:"plan"`
		Report *json.RawMessage `json:"report"`
	}
	if err := json.Unmarshal(out.Bytes(), &shape); err != nil || len(shape.Plan) != 2 || shape.Report != nil {
		t.Errorf("a json dry run is not {plan} alone: %v\n%s", err, out.String())
	}
}

// Defends novox/hq issue 104: once the mesh has told this node anything, `reconcile` holds it to
// that — never to the bundle the host carries, which genesis consumed.
func TestReconcileAfterAControllerDeclarationDoesNotReapplyTheBundle(t *testing.T) {
	was := builtFor
	builtFor = "arch"
	t.Cleanup(func() { builtFor = was })
	opts := stateWithMode(t, store.ModeAdopted)

	controllerPublic, controllerPrivate, err := ed25519.GenerateKey(nil)
	if err != nil {
		t.Fatal(err)
	}
	said := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"/tmp/a","content":"x\n"}]}`)
	if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: said,
		Signature: ed25519.Sign(controllerPrivate, said)}); err != nil {
		t.Fatal(err)
	}

	// Told, and unable to prove by whom: refused, and the bundle is not applied in its place.
	_, _, _, err = reconcileSource(opts)
	if err == nil || !strings.Contains(err.Error(), "not applied in its place") {
		t.Errorf("a node that cannot prove what it was told fell back to something: %v", err)
	}

	mine, err := identity.Generate("workstation")
	if err != nil {
		t.Fatal(err)
	}
	mine.Membership = identity.Membership{Broker: "198.51.100.10:5671", Fingerprint: "sha256:0",
		Signer: controllerPublic, Password: "issued"}
	if err := identity.Save(identity.Path(opts.state), mine); err != nil {
		t.Fatal(err)
	}
	d, raw, from, err := reconcileSource(opts)
	if err != nil {
		t.Fatal(err)
	}
	if from != fromDeclared || !bytes.Equal(raw, said) || d.Adoption == nil {
		t.Errorf("reconcile chose %d with %d bytes, not what the mesh last said", from, len(raw))
	}

	// And before the mesh has said anything: the bundle, as it always was. A test binary carries
	// only the placeholder, and asking for it is what proves the path.
	if err := os.Remove(store.DeclaredPath(opts.state)); err != nil {
		t.Fatal(err)
	}
	_, _, _, err = reconcileSource(opts)
	if !errors.Is(err, bundle.ErrEmpty) {
		t.Errorf("with nothing said, reconcile did not reach for the carried bundle: %v", err)
	}
}