mesh-host/internal/bootstrap/verify_test.go

package bootstrap

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
)

func substrate(t *testing.T) *declaration.Declaration {
	t.Helper()
	out, err := Rewrite(theRealBundle(t), held)
	if err != nil {
		t.Fatal(err)
	}
	return out.Declaration
}

// **A container that is up is not a control plane that replies**, and this project has paid for
// that distinction more than once. A runtime reports a container running from the moment its
// process starts — before it has opened a database, and before it has failed to.
func TestAContainerThatIsUpIsNotAControlPlaneThatReplies(t *testing.T) {
	previous := answerEvery
	answerEvery = time.Millisecond
	defer func() { answerEvery = previous }()

	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		switch args[0] {
		case "inspect":
			return "true running\n", nil
		case "exec":
			// Up, and saying nothing. The program inside is not answering.
			return "", errors.New("exit status 1")
		}
		return "", fmt.Errorf("unexpected command: %v", args)
	}}

	_, err := Verify(context.Background(), substrate(t), runtime.run,
		time.Second, 0, func(string) {})
	if err == nil {
		t.Fatal("every container was running, nothing answered, and the substrate was reported up")
	}
	for _, wanted := range []string{"mesh-control", "Running is not replying", "docker logs"} {
		if !strings.Contains(err.Error(), wanted) {
			t.Errorf("the failure does not mention %q:\n%v", wanted, err)
		}
	}
}

// Exit zero and silence is not an answer either. A program that returns nothing has not been asked
// anything, and treating it as success is the same fault one level down.
func TestAControlPlaneThatSaysNothingHasNotAnswered(t *testing.T) {
	previous := answerEvery
	answerEvery = time.Millisecond
	defer func() { answerEvery = previous }()

	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		if args[0] == "inspect" {
			return "true running\n", nil
		}
		return "   \n", nil
	}}

	if _, err := Verify(context.Background(), substrate(t), runtime.run,
		time.Second, 0, func(string) {}); err == nil {
		t.Fatal("a control plane that exited zero without saying anything was accepted")
	}
}

// The substrate answering is the whole point, and what it said is reported rather than asserted.
func TestASubstrateThatIsUpAndAnsweringIsAccepted(t *testing.T) {
	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		if args[0] == "inspect" {
			return "true running\n", nil
		}
		return "1 node, 0 waiting\n", nil
	}}

	verified, err := Verify(context.Background(), substrate(t), runtime.run,
		time.Second, 0, func(string) {})
	if err != nil {
		t.Fatal(err)
	}
	// Three long-running containers: the store, the broker and the TEMPORARY control plane. The
	// run-once and scheduled shapes are excluded on purpose — a step that has exited is not a
	// fault. The name is `temp-mesh-control` because the permanent one is a module and takes the
	// plain name (novox/hq ADR 0067), which is what makes the two of them coexist at all.
	want := []string{"mesh-store", "mesh-broker", "temp-mesh-control"}
	if len(verified.Running) != len(want) {
		t.Fatalf("confirmed %v running, want %v", verified.Running, want)
	}
	for i := range want {
		if verified.Running[i] != want[i] {
			t.Errorf("confirmed %v running, want %v", verified.Running, want)
		}
	}
	if verified.Answered != "1 node, 0 waiting" {
		t.Errorf("the control plane's reply is reported as %q", verified.Answered)
	}
}

// A control plane that is still opening its stores is waited for, not refused. Refusing on the
// first attempt would make a correct bootstrap fail for being observed too early.
func TestAControlPlaneThatIsStillStartingIsWaitedFor(t *testing.T) {
	previous := answerEvery
	answerEvery = time.Millisecond
	defer func() { answerEvery = previous }()

	attempts := 0
	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		if args[0] == "inspect" {
			return "true running\n", nil
		}
		attempts++
		if attempts < 3 {
			return "", errors.New("exit status 1")
		}
		return "1 node\n", nil
	}}

	if _, err := Verify(context.Background(), substrate(t), runtime.run,
		time.Second, time.Second, func(string) {}); err != nil {
		t.Fatalf("a control plane that answered on the third ask was refused: %v", err)
	}
}

// A container that exited is named with what it IS, so somebody can go and read its logs rather
// than being told only that something is not what it should be.
func TestAContainerThatExitedIsNamedWithItsState(t *testing.T) {
	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		if args[0] == "inspect" && args[len(args)-1] == "mesh-broker" {
			return "false exited\n", nil
		}
		if args[0] == "inspect" {
			return "true running\n", nil
		}
		return "", fmt.Errorf("unexpected command: %v", args)
	}}

	_, err := Verify(context.Background(), substrate(t), runtime.run,
		time.Second, 0, func(string) {})
	if err == nil {
		t.Fatal("a container that had exited was reported as part of a running substrate")
	}
	if !strings.Contains(err.Error(), "mesh-broker") || !strings.Contains(err.Error(), "exited") {
		t.Errorf("the failure does not say which container is in what state: %v", err)
	}
}

// The control plane is asked by running the binary in its own image directly, because the image is
// `FROM scratch` and has no shell for a command line to be interpreted by.
func TestTheControlPlaneIsAskedByRunningItsOwnBinary(t *testing.T) {
	runtime := &asked{answer: func(_ string, args []string) (string, error) {
		if args[0] == "inspect" {
			return "true running\n", nil
		}
		return "1 node\n", nil
	}}
	if _, err := Verify(context.Background(), substrate(t), runtime.run,
		time.Second, 0, func(string) {}); err != nil {
		t.Fatal(err)
	}
	if !runtime.ran("docker exec " + TempPrefix + "mesh-control " + controlPlaneBinary + " status") {
		t.Errorf("the control plane was never asked anything: %v", runtime.commands)
	}
}