mesh-host/internal/filling/filling_test.go

package filling

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
)

// Defends novox/hq ADR 0349 §4 (issue 668): the node-engine writes a filled file on request only for the provisioner
// client's account, only for a clean path the current declaration names as filled, and one at a time with the apply.

const provisionerUID = 4242

func declared(t *testing.T, raw string) *declaration.Declaration {
	t.Helper()
	d, err := declaration.Parse([]byte(raw))
	if err != nil {
		t.Fatal(err)
	}
	return d
}

const withAccount = `{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/m/m.conf"],
  "resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"},
  {"id":"m.other","type":"file","path":"/etc/m/other.conf","content":"y"}]}`

const withoutAccount = `{"declaration":1,"filled_files":["/etc/m/m.conf"],
  "resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"}]}`

// engine is an Engine over a fixed declaration, recording what it was asked to place.
type engine struct {
	Engine
	mu     sync.Mutex
	placed []string
}

func newEngine(t *testing.T, raw string, lock sync.Locker) *engine {
	d := declared(t, raw)
	e := &engine{}
	e.Engine = Engine{
		Declared: func() (*declaration.Declaration, error) { return d, nil },
		UIDOf: func(name string) (uint32, error) {
			if name == "mesh-node-provisioner" {
				return provisionerUID, nil
			}
			return 0, errors.New("no such account")
		},
		Lock: func() (func(), error) { lock.Lock(); return lock.Unlock, nil },
		Place: func(_ context.Context, _ *declaration.Declaration, path string, content []byte) (string, string, error) {
			e.mu.Lock()
			defer e.mu.Unlock()
			e.placed = append(e.placed, path)
			return "updated", "digest", nil
		},
	}
	return e
}

func TestAWriteFromTheProvisionersAccountOfAFilledPathIsPlaced(t *testing.T) {
	e := newEngine(t, withAccount, &sync.Mutex{})
	if err := e.Accepts(provisionerUID); err != nil {
		t.Fatal(err)
	}
	a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
	if a.Refused != "" || a.Failed != "" || a.Action != "updated" || len(e.placed) != 1 {
		t.Fatalf("answer %+v, placed %v", a, e.placed)
	}
}

func TestAnUndeclaredPathIsRefused(t *testing.T) {
	e := newEngine(t, withAccount, &sync.Mutex{})
	for _, path := range []string{"/etc/m/other.conf", "/etc/m/none.conf", "/etc/passwd"} {
		a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("evil")})
		if a.Refused == "" || !strings.Contains(a.Refused, "not a file this node's declaration names as filled") {
			t.Errorf("%s: %+v", path, a)
		}
	}
	if len(e.placed) != 0 {
		t.Errorf("placed %v", e.placed)
	}
}

func TestAnotherAccountIsRefused(t *testing.T) {
	e := newEngine(t, withAccount, &sync.Mutex{})
	for _, uid := range []uint32{0, 1000, provisionerUID + 1} {
		if err := e.Accepts(uid); err == nil {
			t.Errorf("user id %d was accepted", uid)
		}
		a := e.Write(context.Background(), uid, Request{Path: "/etc/m/m.conf", Content: []byte("evil")})
		if a.Refused == "" || !strings.Contains(a.Refused, "is not the provisioner client's account") {
			t.Errorf("user id %d: %+v", uid, a)
		}
	}
	if len(e.placed) != 0 {
		t.Errorf("placed %v", e.placed)
	}
}

func TestEveryWriteIsRefusedWhileTheDeclarationNamesNoAccount(t *testing.T) {
	e := newEngine(t, withoutAccount, &sync.Mutex{})
	if err := e.Accepts(provisionerUID); err == nil || !strings.Contains(err.Error(), "names no provisioner") {
		t.Errorf("accepted with no account named: %v", err)
	}
	a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
	if a.Refused == "" || len(e.placed) != 0 {
		t.Errorf("written with no account named: %+v", a)
	}
}

func TestAnAccountThatIsRootHereIsRefused(t *testing.T) {
	e := newEngine(t, withAccount, &sync.Mutex{})
	e.UIDOf = func(string) (uint32, error) { return 0, nil }
	if err := e.Accepts(0); err == nil || !strings.Contains(err.Error(), "root") {
		t.Errorf("an account that is root here was accepted: %v", err)
	}
}

func TestAnUncleanPathIsRefused(t *testing.T) {
	e := newEngine(t, withAccount, &sync.Mutex{})
	for _, path := range []string{"", "etc/m/m.conf", "m.conf", "/etc//m/m.conf", "/etc/m/../m/m.conf",
		"/etc/m/./m.conf", "/etc/m/m.conf/"} {
		a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("x")})
		if a.Refused == "" {
			t.Errorf("%q was not refused: %+v", path, a)
		}
	}
	if len(e.placed) != 0 {
		t.Errorf("placed %v", e.placed)
	}
}

// A write waits for an apply holding the apply's lock, and writes only once it is let go.
func TestAWriteWaitsForTheApply(t *testing.T) {
	var applying sync.Mutex
	e := newEngine(t, withAccount, &applying)
	applying.Lock()
	done := make(chan Answer)
	go func() {
		done <- e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
	}()
	select {
	case a := <-done:
		t.Fatalf("written while the apply held the lock: %+v", a)
	case <-time.After(100 * time.Millisecond):
	}
	e.mu.Lock()
	n := len(e.placed)
	e.mu.Unlock()
	if n != 0 {
		t.Fatalf("placed while the apply held the lock")
	}
	applying.Unlock()
	if a := <-done; a.Action != "updated" {
		t.Fatalf("after the apply: %+v", a)
	}
}

// fixed is a Writer that accepts one user id and answers every write.
type fixed struct {
	uid    uint32
	mu     sync.Mutex
	writes []Request
}

func (f *fixed) Accepts(uid uint32) error {
	if uid != f.uid {
		return errors.New("not the provisioner client's account")
	}
	return nil
}

func (f *fixed) Write(_ context.Context, _ uint32, req Request) Answer {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.writes = append(f.writes, req)
	return Answer{Path: req.Path, Action: "updated", Digest: "d"}
}

// serve serves w on a socket in a fresh directory and returns its path, and what was said.
func serve(t *testing.T, w Writer) (string, func() []string) {
	t.Helper()
	dir, err := os.MkdirTemp("", "fill")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { os.RemoveAll(dir) })
	socket := filepath.Join(dir, "fill.sock")
	ctx, cancel := context.WithCancel(context.Background())
	var mu sync.Mutex
	var said []string
	stopped := make(chan error)
	go func() {
		stopped <- Serve(ctx, socket, w, func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() })
	}()
	t.Cleanup(func() {
		cancel()
		if err := <-stopped; err != nil {
			t.Error(err)
		}
	})
	for i := 0; i < 100; i++ {
		if _, err := os.Stat(socket); err == nil {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	return socket, func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), said...) }
}

func ask(t *testing.T, socket string, req Request) Answer {
	t.Helper()
	conn, err := net.Dial("unix", socket)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	// Bounded: a socket that never answers fails the test instead of hanging it.
	_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
	if err := json.NewEncoder(conn).Encode(req); err != nil {
		t.Fatal(err)
	}
	var a Answer
	if err := json.NewDecoder(conn).Decode(&a); err != nil {
		t.Fatal(err)
	}
	return a
}

// Over a real socket, the kernel says who is asking: this test's own account is the provisioner's here.
func TestTheSocketWritesForTheAccountTheKernelSays(t *testing.T) {
	w := &fixed{uid: uint32(os.Getuid())}
	socket, said := serve(t, w)
	a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("secret value")})
	if a.Refused != "" || a.Action != "updated" || len(w.writes) != 1 || string(w.writes[0].Content) != "secret value" {
		t.Fatalf("answer %+v, writes %+v", a, w.writes)
	}
	for _, line := range said() {
		if strings.Contains(line, "secret value") {
			t.Errorf("the content was said: %q", line)
		}
	}
}

func TestTheSocketRefusesAnotherAccountBeforeReadingItsRequest(t *testing.T) {
	w := &fixed{uid: uint32(os.Getuid()) + 1}
	socket, said := serve(t, w)
	conn, err := net.Dial("unix", socket)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	// Closed at once, its request never read: the write may meet a closed socket, and the refusal is there to read.
	_ = json.NewEncoder(conn).Encode(Request{Path: "/etc/m/m.conf", Content: []byte("secret value")})
	var a Answer
	if err := json.NewDecoder(conn).Decode(&a); err != nil {
		t.Fatal(err)
	}
	if a.Refused == "" || len(w.writes) != 0 {
		t.Fatalf("another account was written for: %+v", a)
	}
	for _, line := range said() {
		if strings.Contains(line, "secret value") {
			t.Errorf("the content was said: %q", line)
		}
	}
}

// floodersAreAnother has the kernel's word say another account for every connection a test binds to a name holding
// "flood": the test runs as one account, and the socket must tell the client from everyone else.
func floodersAreAnother(t *testing.T) {
	t.Helper()
	was := peerOf
	peerOf = func(conn *net.UnixConn) (uint32, int32, error) {
		uid, pid, err := was(conn)
		if a, ok := conn.RemoteAddr().(*net.UnixAddr); ok && a != nil && strings.Contains(a.Name, "flood") {
			uid++
		}
		return uid, pid, err
	}
	t.Cleanup(func() { peerOf = was })
}

// A flood of connections from another account, each saying nothing and held open, does not stop the client's write:
// each is refused and closed before it holds anything.
func TestAFloodFromAnotherAccountDoesNotStopTheClient(t *testing.T) {
	floodersAreAnother(t)
	w := &fixed{uid: uint32(os.Getuid())}
	socket, _ := serve(t, w)
	from := t.TempDir()
	var flood []*net.UnixConn
	t.Cleanup(func() {
		for _, c := range flood {
			c.Close()
		}
	})
	for i := 0; i < 3*atOnce+busyWaitingCap(); i++ {
		c, err := net.DialUnix("unix", &net.UnixAddr{Name: filepath.Join(from, fmt.Sprintf("flood-%d", i)), Net: "unix"},
			&net.UnixAddr{Name: socket, Net: "unix"})
		if err != nil {
			t.Fatal(err)
		}
		flood = append(flood, c)
	}
	start := time.Now()
	a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
	if a.Refused != "" || a.Action != "updated" {
		t.Fatalf("the client's write during a flood: %+v", a)
	}
	if took := time.Since(start); took > refusedWithin/2 {
		t.Errorf("the client's write waited %s behind the flood", took)
	}
	// And every flooder was closed: it reads its refusal and then the end.
	for _, c := range flood {
		_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
		var got Answer
		if err := json.NewDecoder(c).Decode(&got); err != nil || got.Refused == "" {
			t.Fatalf("a flooder was not refused at once: %+v %v", got, err)
		}
	}
}

func busyWaitingCap() int { return busyWaitingAtOnce }

// A client that connects and says nothing holds one slot only until within, and another write goes on meanwhile.
func TestASilentClientDoesNotHoldTheSocket(t *testing.T) {
	was := within
	within = 300 * time.Millisecond
	t.Cleanup(func() { within = was })
	w := &fixed{uid: uint32(os.Getuid())}
	socket, _ := serve(t, w)
	silent, err := net.Dial("unix", socket)
	if err != nil {
		t.Fatal(err)
	}
	defer silent.Close()
	if a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")}); a.Action != "updated" {
		t.Fatalf("a write beside a silent client: %+v", a)
	}
	_ = silent.SetReadDeadline(time.Now().Add(3 * time.Second))
	var got Answer
	if err := json.NewDecoder(silent).Decode(&got); err != nil || got.Refused == "" {
		t.Errorf("the silent client was not let go after within: %+v %v", got, err)
	}
}

// A request carrying more than the path and the content is refused: it says what the declaration says.
func TestARequestWithAnUnknownFieldIsRefused(t *testing.T) {
	w := &fixed{uid: uint32(os.Getuid())}
	socket, _ := serve(t, w)
	conn, err := net.Dial("unix", socket)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	if _, err := conn.Write([]byte(`{"path":"/etc/m/m.conf","content":"eA==","mode":"0777"}` + "\n")); err != nil {
		t.Fatal(err)
	}
	var a Answer
	if err := json.NewDecoder(conn).Decode(&a); err != nil {
		t.Fatal(err)
	}
	if a.Refused == "" || !strings.Contains(a.Refused, "mode") || len(w.writes) != 0 {
		t.Errorf("a request with an unknown field: %+v, writes %v", a, w.writes)
	}
}

// An accept that fails is said, waited for and tried again: the socket goes on serving.
func TestTheSocketGoesOnAfterAnAcceptFails(t *testing.T) {
	wasAccept, wasBackoff := acceptFrom, acceptBackoff
	failures := 3
	var mu sync.Mutex
	acceptFrom = func(l *net.UnixListener) (*net.UnixConn, error) {
		mu.Lock()
		defer mu.Unlock()
		if failures > 0 {
			failures--
			return nil, errors.New("accept: too many open files")
		}
		return l.AcceptUnix()
	}
	acceptBackoff = time.Millisecond
	t.Cleanup(func() { acceptFrom, acceptBackoff = wasAccept, wasBackoff })
	w := &fixed{uid: uint32(os.Getuid())}
	socket, said := serve(t, w)
	if a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")}); a.Action != "updated" {
		t.Fatalf("after failed accepts: %+v", a)
	}
	if !strings.Contains(strings.Join(said(), "\n"), "too many open files") {
		t.Errorf("the failed accepts were not said: %v", said())
	}
}

// The socket's directory is given the mode the socket needs, as mesh-cli's is.
func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) {
	dir, err := os.MkdirTemp("", "fill")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { os.RemoveAll(dir) })
	if err := os.Chmod(dir, 0o700); err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	stopped := make(chan error)
	go func() { stopped <- Serve(ctx, filepath.Join(dir, "fill.sock"), &fixed{}, nil) }()
	for i := 0; i < 100; i++ {
		if _, err := os.Stat(filepath.Join(dir, "fill.sock")); err == nil {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	cancel()
	if err := <-stopped; err != nil {
		t.Fatal(err)
	}
	info, err := os.Stat(dir)
	if err != nil {
		t.Fatal(err)
	}
	if info.Mode().Perm() != 0o755 {
		t.Errorf("the socket's directory is %o", info.Mode().Perm())
	}
}

// The first refusal of each kind for each account is said at once; what follows within the time is counted, and the
// count is said when the time is up, even when the account has stopped.
func TestRefusalsAreSaidOnceAndTheirCountIsNeverLost(t *testing.T) {
	var mu sync.Mutex
	var said []string
	r := &refusalWords{say: func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }}
	for i := 0; i < 3; i++ {
		r.said(1000, "account", "not the provisioner client's account")
	}
	// Another kind from the same account is said at once, not hidden behind the first.
	r.said(1000, "namespace", "another namespace")
	if len(said) != 2 || !strings.Contains(said[0], "(account)") || !strings.Contains(said[1], "(namespace)") {
		t.Fatalf("said at once: %q", said)
	}
	// The account stops; the time is up; the two counted are said.
	r.flush()
	if len(said) != 3 || !strings.Contains(said[2], "2 more") || !strings.Contains(said[2], "user id 1000") {
		t.Fatalf("after the time: %q", said)
	}
	// And the next refusal is said at once again.
	r.said(1000, "account", "not the provisioner client's account")
	if len(said) != 4 {
		t.Fatalf("after the count was said: %q", said)
	}
}

// refusingSocket serves a socket that refuses this test's account, with counted refusals said every every. It returns
// how to be refused n times, everything said so far, and how to stop it.
func refusingSocket(t *testing.T, every time.Duration) (func(int), func() string, func()) {
	t.Helper()
	was := refusalsSaidEvery
	refusalsSaidEvery = every
	dir, err := os.MkdirTemp("", "fill")
	if err != nil {
		t.Fatal(err)
	}
	socket := filepath.Join(dir, "fill.sock")
	ctx, cancel := context.WithCancel(context.Background())
	var mu sync.Mutex
	var said []string
	all := func() string { mu.Lock(); defer mu.Unlock(); return strings.Join(said, "\n") }
	stopped := make(chan error)
	go func() {
		stopped <- Serve(ctx, socket, &fixed{uid: uint32(os.Getuid()) + 1},
			func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() })
	}()
	for i := 0; i < 100; i++ {
		if _, err := os.Stat(socket); err == nil {
			break
		}
		time.Sleep(10 * time.Millisecond)
	}
	refuse := func(n int) {
		for i := 0; i < n; i++ {
			conn, err := net.Dial("unix", socket)
			if err != nil {
				t.Fatal(err)
			}
			_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
			var a Answer
			_ = json.NewDecoder(conn).Decode(&a)
			conn.Close()
		}
	}
	var once sync.Once
	stop := func() {
		once.Do(func() {
			cancel()
			if err := <-stopped; err != nil {
				t.Error(err)
			}
			refusalsSaidEvery = was
			os.RemoveAll(dir)
		})
	}
	t.Cleanup(stop)
	return refuse, all, stop
}

// Refusals counted while the socket serves are said on its timer, though the account has stopped asking.
func TestTheSocketSaysItsCountedRefusalsOnATimer(t *testing.T) {
	refuse, all, _ := refusingSocket(t, 200*time.Millisecond)
	refuse(3)
	// "more writes", not a number: a tick may fall between two refusals and split the count.
	deadline := time.Now().Add(3 * time.Second)
	for !strings.Contains(all(), "more writes") && time.Now().Before(deadline) {
		time.Sleep(20 * time.Millisecond)
	}
	if !strings.Contains(all(), "more writes") {
		t.Fatalf("the counted refusals were not said on the timer: %s", all())
	}
}

// Refusals counted when the socket stops are said as it stops.
func TestTheSocketSaysItsCountedRefusalsWhenItStops(t *testing.T) {
	refuse, all, stop := refusingSocket(t, time.Hour)
	refuse(5)
	if strings.Contains(all(), "4 more") {
		t.Fatalf("said before the socket stopped: %s", all())
	}
	stop()
	if !strings.Contains(all(), "4 more") {
		t.Errorf("the refusals counted when the socket stopped were lost: %s", all())
	}
}

// A connection over the bound of those waiting to be judged is closed unread, and said.
func TestAConnectionOverTheWaitingBoundIsClosedUnread(t *testing.T) {
	was := judgeWaiting
	judgeWaiting = 0
	t.Cleanup(func() { judgeWaiting = was })
	w := &fixed{uid: uint32(os.Getuid())}
	socket, said := serve(t, w)
	conn, err := net.Dial("unix", socket)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
	var a Answer
	if err := json.NewDecoder(conn).Decode(&a); err == nil {
		t.Fatalf("answered over the bound: %+v", a)
	}
	if len(w.writes) != 0 || !strings.Contains(strings.Join(said(), "\n"), "(overflow)") {
		t.Errorf("writes %v, said %q", w.writes, said())
	}
}