mesh-host/cmd/mesh-host/main_test.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/declaration"
	"github.com/novox/mesh-host/internal/apply"
	"github.com/novox/mesh-host/internal/link"
	"github.com/novox/mesh-host/internal/store"
	"github.com/novox/mesh-host/internal/system"
)

// Argument handling gets tests because it already failed silently once: `mesh-host inventory
// --json` printed text. The standard library stops parsing at the first non-flag argument, so
// the flag sat unread in the positional arguments and the command exited 0 having ignored what
// the user asked for.
//
// Silently doing something other than what was asked, and reporting success, is the fault this
// project exists to name — so it gets defended here rather than remembered.

func TestAFlagAfterTheCommandIsRead(t *testing.T) {
	command, opts, err := parseArgs([]string{"inventory", "--json"})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if command != "inventory" {
		t.Errorf("command = %q, want inventory", command)
	}
	if !opts.json {
		t.Error("--json after the subcommand was ignored")
	}
}

func TestFlagsAreReadInEitherPosition(t *testing.T) {
	for _, args := range [][]string{
		{"profile", "--json", "--timeout", "3s"},
		{"profile", "--timeout=3s", "--json"},
	} {
		_, opts, err := parseArgs(args)
		if err != nil {
			t.Fatalf("%v: unexpected error: %v", args, err)
		}
		if !opts.json || opts.timeout != 3*time.Second {
			t.Errorf("%v parsed as json=%v timeout=%s", args, opts.json, opts.timeout)
		}
	}
}

func TestAMistypedFlagIsRefusedNotIgnored(t *testing.T) {
	// The cost of getting this wrong is asymmetric: an error is a moment's annoyance, and a
	// silently dropped flag is a report that answers a question nobody asked.
	if _, _, err := parseArgs([]string{"profile", "--jsom"}); err == nil {
		t.Fatal("a mistyped flag was accepted")
	}
}

func TestAnUnexpectedArgumentIsRefused(t *testing.T) {
	if _, _, err := parseArgs([]string{"profile", "extra"}); err == nil {
		t.Fatal("a stray argument was ignored rather than refused")
	}
}

func TestTheDefaultsAreTheDocumentedOnes(t *testing.T) {
	// The usage text promises 10s. A default that drifts from what is printed is a small lie
	// that costs someone an afternoon.
	_, opts, err := parseArgs([]string{"profile"})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if opts.timeout != 10*time.Second {
		t.Errorf("default timeout is %s; the usage text says 10s", opts.timeout)
	}
	if opts.json {
		t.Error("json output is on by default; the usage text says it is a flag")
	}
}

func TestNoCommandIsNotAnError(t *testing.T) {
	// Running the binary with no arguments prints usage and exits 0. A host that returns
	// failure for "tell me what you do" is noise in every script that probes it.
	command, _, err := parseArgs(nil)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if command != "" {
		t.Errorf("command = %q, want empty", command)
	}
}

func TestTheStateHasADocumentedDefault(t *testing.T) {
	// A host that wrote its state somewhere unexpected would forget what it owns on the next
	// run, and then leave everything it had applied behind forever.
	_, opts, err := parseArgs([]string{"owned"})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if opts.state != store.DefaultPath {
		t.Errorf("default state path is %q, not the documented %q", opts.state, store.DefaultPath)
	}
}

func TestAFlagAfterAPositionalIsRead(t *testing.T) {
	// The same fault as TestAFlagAfterTheCommandIsRead, one level down. Taking the subcommand
	// off the front fixed the flag after the COMMAND and not the flag after its ARGUMENT: the
	// standard library stops at the first non-flag argument wherever that argument is.
	for _, args := range [][]string{
		{"hand-over", "decl.json", "--dry-run", "--json"},
		{"hand-over", "--dry-run", "decl.json", "--json"},
		{"hand-over", "--dry-run", "--json", "decl.json"},
	} {
		command, opts, err := parseArgs(args)
		if err != nil {
			t.Errorf("%v: unexpected error: %v", args, err)
			continue
		}
		if command != "hand-over" || opts.file != "decl.json" || !opts.dryRun || !opts.json {
			t.Errorf("%v parsed as file=%q dry-run=%v json=%v",
				args, opts.file, opts.dryRun, opts.json)
		}
	}
}

// Defends the node's own record: the link and the reconcile loop both apply, and each reads the
// state, acts, and writes it back — so they must not run at the same time, or the last save loses
// what the other recorded.
func TestOnlyOneApplyRunsAtATime(t *testing.T) {
	// Issue 1585: the engine's stop waits for the apply in flight to end and save, and then lets no save start; before,
	// the process ended under it and a save cut before its rename left its temporary file and the state before it.
	t.Run("the stop waits for the apply in flight and its save", func(t *testing.T) {
		path := filepath.Join(t.TempDir(), "state.json")
		applying.Lock()
		var said []string
		var mu sync.Mutex
		waiting := make(chan struct{})
		say := func(line string) {
			mu.Lock()
			said = append(said, line)
			mu.Unlock()
			if strings.Contains(line, "waiting up to") {
				close(waiting)
			}
		}
		stopped := make(chan func(), 1)
		go func() { stopped <- stopBetweenApplies(say, time.Hour) }()
		// Said before it blocks, and it cannot end before the apply lets go: so it has not ended now.
		<-waiting
		select {
		case release := <-stopped:
			release()
			applying.Unlock()
			t.Fatal("the stop did not wait for the apply in flight")
		default:
		}
		if err := store.Save(path, store.State{Resources: []store.Applied{{ID: "nats.program-snapshot", Type: "archive"}}}); err != nil {
			t.Fatal(err)
		}
		applying.Unlock()
		release := <-stopped
		saved := make(chan error, 1)
		go func() { saved <- store.Save(path, store.State{}) }()
		// A save after the stop blocks; nothing says it blocks, so this one wait is a bounded look.
		select {
		case <-saved:
			release()
			t.Fatal("a save started after the stop")
		case <-time.After(50 * time.Millisecond):
		}
		release()
		if err := <-saved; err != nil {
			t.Fatal(err)
		}
		mu.Lock()
		defer mu.Unlock()
		if len(said) != 2 || !strings.Contains(said[0], "waiting up to 1h0m0s") || !strings.Contains(said[1], "its state is saved") {
			t.Errorf("said %q", said)
		}
	})
	// A host is built for one system at link time, and a test binary has no link time: this asks
	// the machine it runs on, and stands aside where the answer is no.
	built, err := system.For("arch")
	if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil {
		t.Skip("this machine is not one these tests can apply on")
	}
	was := builtFor
	builtFor = "arch"
	t.Cleanup(func() { builtFor = was })
	dir := t.TempDir()
	opts := options{state: filepath.Join(dir, "state.json")}
	raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
		filepath.Join(dir, "a.conf") + `","content":"x\n"}]}`)

	// Whatever else is applying — the link, while this is the reconcile — this waits for it.
	applying.Lock()
	done := make(chan link.Report, 1)
	go func() { done <- applyAndKeep(context.Background(), opts, raw, nil, nil, nil) }()
	select {
	case report := <-done:
		applying.Unlock()
		t.Fatalf("an apply ran while another held the node: %+v", report)
	case <-time.After(50 * time.Millisecond):
	}
	if _, err := os.Stat(filepath.Join(dir, "a.conf")); !errors.Is(err, os.ErrNotExist) {
		applying.Unlock()
		t.Fatal("the waiting apply had already touched the machine")
	}
	applying.Unlock()

	select {
	case report := <-done:
		if report.Refused != "" {
			t.Fatalf("refused: %s", report.Refused)
		}
	case <-time.After(10 * time.Second):
		t.Fatal("the apply never ran once the node was free")
	}
	known, loadErr := store.Load(opts.state)
	if loadErr != nil || len(known.Resources) != 1 {
		t.Errorf("the apply recorded %d resource(s): %v", len(known.Resources), loadErr)
	}
}

// Defends novox/hq issue 104: a declaration for the other mode than this node is in is refused at
// the point of application, whichever command delivered it, naming both — an adopted control-node
// once applied its converged genesis bundle and closed itself for forty-five minutes.

func stateWithMode(t *testing.T, mode string) options {
	t.Helper()
	dir := t.TempDir()
	opts := options{state: filepath.Join(dir, "state.json"), out: &bytes.Buffer{}}
	if err := store.Save(opts.state, store.State{Mode: mode}); err != nil {
		t.Fatal(err)
	}
	return opts
}

func TestAConvergedDeclarationIsRefusedOnAnAdoptedNode(t *testing.T) {
	opts := stateWithMode(t, store.ModeAdopted)
	path := filepath.Join(filepath.Dir(opts.state), "filter.conf")
	raw := []byte(`{"declaration":1,"resources":[{"id":"filter","type":"file","path":"` + path +
		`","content":"table inet filter { chain input { policy drop; } }\n"}]}`)
	d, err := declaration.ParseFileTrusted(raw)
	if err != nil {
		t.Fatal(err)
	}
	for _, from := range []provenance{fromFile, fromBundle} {
		err := runApply(context.Background(), opts, d, raw, from)
		if err == nil {
			t.Fatalf("a converged declaration was applied to an adopted node (from %d)", from)
		}
		want := "this node is adopted; the declaration says converged"
		if !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "`converge`") {
			t.Errorf("the refusal does not name both modes and the act that changes it: %v", err)
		}
	}
	if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
		t.Error("the refused declaration touched the machine")
	}
}

func TestTheKeptDeclarationRepairsARecordThatDisagrees(t *testing.T) {
	// What a node kept is signed by the mesh and verified on load; the state's note of the mode is
	// the host's own. A genesis re-run after `converge`, or a state saved when the kept declaration
	// could not be, leaves them apart — and a loop that refused every five minutes would hold the
	// node off what the mesh said until a delivery that comes only when something changes. The
	// kept declaration wins, and the repair is said.
	opts := stateWithMode(t, store.ModeConverged)
	raw := []byte(`{"declaration":1,"adoption":{"taken":[]},"resources":[{"id":"a","type":"file","path":"` +
		filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
	var said []string
	report := applyAndKeep(context.Background(), opts, raw, nil, nil, func(line string) { said = append(said, line) })
	if strings.Contains(report.Refused, "the declaration says") {
		t.Fatalf("what the node kept was refused against its own note: %s", report.Refused)
	}
	if len(said) != 1 || !strings.Contains(said[0], "record said converged") ||
		!strings.Contains(said[0], "says adopted") || !strings.Contains(said[0], "repaired") {
		t.Errorf("the repair was not said: %q", said)
	}
}

func TestTheMeshItselfMayChangeTheMode(t *testing.T) {
	// The flip is a declaration: `converge` on the controller records the mode and sends the
	// first converged declaration. Delivered by the link, signed, it is not held to the record —
	// it becomes it. (With no system linked in, the apply is refused later for that; what this
	// checks is that the refusal is not the mode's.)
	opts := stateWithMode(t, store.ModeAdopted)
	raw := []byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"` +
		filepath.Join(filepath.Dir(opts.state), "a.conf") + `","content":"x\n"}]}`)
	report := applyAndKeep(context.Background(), opts, raw, &store.Declared{Declaration: raw}, nil, nil)
	if strings.Contains(report.Refused, "the declaration says") {
		t.Errorf("the mesh's own flip was refused for its mode: %s", report.Refused)
	}
}

// Defends novox/hq issue 104: a declaration older than what the mesh last said is refused, naming
// both — and `apply FILE` is refused altogether once the mesh has spoken, the last declaration
// itself included: from a file it is applied as the bundle is, which would record the mesh's
// resources as this machine's own and remove the foundation as undeclared.
func TestAnOlderDeclarationIsRefused(t *testing.T) {
	opts := stateWithMode(t, "")
	genesis := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"genesis\n"}]}`)
	since := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"since\n"}]}`)
	other := []byte(`{"declaration":1,"resources":[{"id":"g","type":"file","path":"/tmp/g","content":"other\n"}]}`)
	if err := store.Save(opts.state, store.State{Genesis: &store.Genesis{Digest: apply.DigestOf(genesis),
		At: time.Now(), Rewritten: true}}); err != nil {
		t.Fatal(err)
	}
	if err := store.SaveDeclared(store.DeclaredPath(opts.state), store.Declared{Declaration: since,
		Signature: []byte("unverified here")}); err != nil {
		t.Fatal(err)
	}
	parsed := func(raw []byte) *declaration.Declaration {
		d, err := declaration.ParseFileTrusted(raw)
		if err != nil {
			t.Fatal(err)
		}
		return d
	}

	err := runApply(context.Background(), opts, parsed(genesis), genesis, fromFile)
	if err == nil {
		t.Fatal("the bundle genesis consumed was applied over what the mesh said since")
	}
	for _, want := range []string{"older", short(apply.DigestOf(genesis)), short(apply.DigestOf(since))} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("the refusal does not say %q: %v", want, err)
		}
	}

	err = runApply(context.Background(), opts, parsed(other), other, fromFile)
	if err == nil {
		t.Fatal("a declaration that is not what the mesh last said was applied")
	}
	if !strings.Contains(err.Error(), "for a machine the mesh has not spoken to") ||
		!strings.Contains(err.Error(), short(apply.DigestOf(since))) {
		t.Errorf("the refusal does not say what a file is for and what was last said: %v", err)
	}

	// The very declaration the mesh last sent, from a file: still a file.
	err = runApply(context.Background(), opts, parsed(since), since, fromFile)
	if err == nil || !strings.Contains(err.Error(), "applied as the bundle is") {
		t.Errorf("the last declaration, from a file, was not refused as a file: %v", err)
	}
	if known, _ := store.Load(opts.state); len(known.Resources) != 0 {
		t.Errorf("a refused file recorded %d resource(s)", len(known.Resources))
	}

	// A bundle other than the one genesis consumed: what genesis applied was rewritten for this
	// machine, so the carried bytes never are.
	err = runApply(context.Background(), opts, parsed(other), other, fromBundle)
	if err == nil || !strings.Contains(err.Error(), "consumed") ||
		!strings.Contains(err.Error(), short(apply.DigestOf(genesis))) {
		t.Errorf("a bundle other than the consumed one was not refused naming it: %v", err)
	}
}

// Defends novox/hq issue 104: what an apply would change is said before anything is, and
// `--dry-run` is that and nothing else — an action listed as the action it is.
func TestADryRunChangesNothingAndListsTheActions(t *testing.T) {
	opts := stateWithMode(t, "")
	opts.dryRun = true
	out := &bytes.Buffer{}
	opts.out = out
	path := filepath.Join(filepath.Dir(opts.state), "a.conf")
	raw := []byte(`{"declaration":1,"resources":[
	  {"id":"a","type":"file","path":"` + path + `","content":"x\n"},
	  {"id":"init","type":"action","command":["createdb","mesh"],"verify":["psql","-c","select 1"]}]}`)
	d, err := declaration.ParseFileTrusted(raw)
	if err != nil {
		t.Fatal(err)
	}
	if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
		t.Fatalf("a dry run failed: %v", err)
	}
	if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
		t.Error("a dry run wrote the file")
	}
	for _, want := range []string{"would change", "create   file       a", "run      action     init",
		"`createdb mesh`", "--dry-run: nothing applied"} {
		if !strings.Contains(out.String(), want) {
			t.Errorf("the preview does not say %q:\n%s", want, out.String())
		}
	}
	if strings.Contains(out.String(), "applying:") {
		t.Errorf("a dry run went on to apply:\n%s", out.String())
	}

	// Machine-readable, the same shape as an apply's: the plan, and no report.
	out.Reset()
	opts.json = true
	if err := runApply(context.Background(), opts, d, raw, fromFile); err != nil {
		t.Fatal(err)
	}
	var shape struct {
		Plan   []apply.Step     `json:"plan"`
		Report *json.RawMessage `json:"report"`
	}
	if err := json.Unmarshal(out.Bytes(), &shape); err != nil || len(shape.Plan) != 2 || shape.Report != nil {
		t.Errorf("a json dry run is not {plan} alone: %v\n%s", err, out.String())
	}
}

// **A host running as a service says what its apply did.**
//
// The serving path passed nil where the apply writes its detail, and nil is silence. The one-shot path
// has always passed a real function, so everything the apply says was visible when a person ran it by
// hand and discarded in the way the host actually runs. Measured before this was written: a machine was
// converged, its found firewall was not retired, and what the host decided was unrecoverable because it
// had been said to nobody (novox/hq 04-ISSUES/143).
//
// This asserts only that the apply's log is never nil and that a line reaches what the caller gave.
// **It cannot catch the fault it was written for** — a call site passing nil directly — because that is
// wiring, and wiring is only proved by running the thing. That proof is a deployed host whose journal
// carries the apply's detail, which is how this fix was verified.
func TestTheApplysLogIsNeverNil(t *testing.T) {
	if announceOr(nil) == nil {
		t.Fatal("a host with nowhere to say things got a nil log, which the apply will call")
	}
	announceOr(nil)("this goes nowhere and must not panic")

	var said []string
	announceOr(func(line string) { said = append(said, line) })("  disabled ufw")
	if len(said) != 1 || !strings.Contains(said[0], "disabled ufw") {
		t.Fatalf("the apply's detail did not reach the caller's announce: %v", said)
	}
}