mesh-host/cmd/mesh-host/filling_test.go

package main

import (
	"context"
	"crypto/ed25519"
	"encoding/json"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/provisions"

	"github.com/novox/mesh-host/internal/filling"
	"github.com/novox/mesh-host/internal/store"
)

// Defends novox/hq ADR 0349 §4 (issue 668): a write of a filled file and an apply are one at a time — the write
// takes the lock the apply holds, and writes only once the apply let it go — and the write is recorded in the
// node's state as an apply's outcome is.
func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
	was := builtFor
	builtFor = "arch"
	t.Cleanup(func() { builtFor = was })
	me, err := user.Current()
	if err != nil {
		t.Fatal(err)
	}
	if me.Uid == "0" {
		t.Skip("run as root, whose account is never the provisioner's")
	}
	dir := t.TempDir()
	opts := options{state: filepath.Join(dir, "state.json")}
	public, private, err := ed25519.GenerateKey(nil)
	if err != nil {
		t.Fatal(err)
	}
	path := filepath.Join(dir, "m.conf")
	raw, err := json.Marshal(map[string]any{"declaration": 1, "provisioner_account": me.Username,
		"filled_files": []string{path}, "resources": []any{map[string]any{"id": "m.conf", "type": "file",
			"path": path, "content": "from the declaration\n", "owner-at-runtime": "owned", "owner": me.Username}}})
	if err != nil {
		t.Fatal(err)
	}
	if err := store.SaveDeclared(store.DeclaredPath(opts.state),
		store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
		t.Fatal(err)
	}
	// The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked:
	// the opener reads the box's "v").
	provision := map[string]string{}
	vaultName := "provision/n2/db"
	secrets := provisionSecrets{node: "n1",
		list: func(_ context.Context, prefix string) (map[string][]byte, error) {
			out := map[string][]byte{}
			for k, v := range provision {
				if strings.HasPrefix(k, prefix) {
					out[k] = []byte(v)
				}
			}
			return out, nil
		},
		own: func(_ context.Context, args map[string]any) (json.RawMessage, error) {
			raw, _ := json.Marshal(args)
			var a struct{ Credentials []map[string]any }
			_ = json.Unmarshal(raw, &a)
			c := a.Credentials[0]
			c["name"], c["generation"], c["sealed"] = vaultName, 2, map[string]string{"v": "s3cret"}
			return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}})
		},
		open: func(sealed string) ([]byte, error) {
			var b struct{ V string }
			err := json.Unmarshal([]byte(sealed), &b)
			return []byte(b.V), err
		}}
	e := fillingEngine(opts, public, secrets, func(string) {})
	uid := uint32(os.Getuid())
	if err := e.Accepts(uid); err != nil {
		t.Fatal(err)
	}

	// An apply holds the lock.
	applying.Lock()
	done := make(chan filling.Answer)
	go func() {
		done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("filled\n")})
	}()
	select {
	case a := <-done:
		applying.Unlock()
		t.Fatalf("written while an apply held the lock: %+v", a)
	case <-time.After(200 * time.Millisecond):
	}
	if _, err := os.Stat(path); err == nil {
		applying.Unlock()
		t.Fatal("the file was written while an apply held the lock")
	}
	applying.Unlock()
	a := <-done
	if a.Refused != "" || a.Failed != "" || a.Action != "created" {
		t.Fatalf("the write after the apply: %+v", a)
	}
	got, err := os.ReadFile(path)
	if err != nil || string(got) != "filled\n" {
		t.Fatalf("the file holds %q (%v)", got, err)
	}
	known, err := store.Load(opts.state)
	if err != nil {
		t.Fatal(err)
	}
	if rec, ok := known.Find("m.conf"); !ok || rec.Wrote != a.Digest || rec.Target != path {
		t.Errorf("the write is not recorded in the node's state: %+v, answered %q", rec, a.Digest)
	}

	// And the cross-process lock beside the state too: a write waits for a `reconcile` run by hand.
	unlock, err := store.Lock(opts.state, nil)
	if err != nil {
		t.Fatal(err)
	}
	go func() {
		done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("again\n")})
	}()
	select {
	case a := <-done:
		unlock()
		t.Fatalf("written while another process held the apply's lock: %+v", a)
	case <-time.After(200 * time.Millisecond):
	}
	unlock()
	if a := <-done; a.Action != "updated" {
		t.Fatalf("the second write: %+v", a)
	}

	// ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write
	// from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it.
	keyed := func(state string, ref bool, id, local string) string {
		p := provisions.Provision{Provision: "db", Local: local, Consumer: provisions.End{Module: "m", Node: "n1"},
			Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()}
		if state != provisions.Provided {
			p.Why = "the provider has not answered"
		}
		if ref {
			p.Reference = &provisions.Reference{ID: id, Generation: 2}
		}
		raw, err := provisions.Encode(p)
		if err != nil {
			t.Fatal(err)
		}
		return string(raw)
	}
	for _, tc := range []struct {
		name, state string
		ref         bool
		content     string
		want        string // the file after; "" keeps what was there
		mode        os.FileMode
		vault       string // the vault's credential name, when not the one the provision names
		refID       string // the reference's id, when not the provision's shared value
		local       string // the provision's local name (ADR 0094): the placeholder names it
	}{
		{name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"},
		{name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"},
		{name: "a reference to no credential of the provision is refused", state: provisions.Provided, ref: true,
			refID: "provision/n9/db", content: "password=${secret:db}\n"},
		{name: "a name no provision goes by is refused", state: provisions.Provided, ref: true, content: "password=${secret:other}\n"},
		{name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true,
			vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"},
		{name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true,
			content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600},
		{name: "a local name is the provision's by its key", state: provisions.Provided, ref: true, local: "admin",
			content: "admin=${secret:admin}\n", want: "admin=s3cret\n", mode: 0o600},
		{name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644},
	} {
		before, _ := os.ReadFile(path)
		if tc.state != "" {
			id := "provision/n2/db"
			if tc.refID != "" {
				id = tc.refID
			}
			clear(provision)
			key := "n1.m.db"
			if tc.local != "" {
				key += "." + tc.local
			}
			provision[key] = keyed(tc.state, tc.ref, id, tc.local)
		}
		vaultName = "provision/n2/db"
		if tc.vault != "" {
			vaultName = tc.vault
		}
		a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)})
		got, _ := os.ReadFile(path)
		if tc.want == "" {
			if a.Failed == "" && a.Refused == "" || string(got) != string(before) {
				t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got)
			}
			continue
		}
		info, _ := os.Stat(path)
		if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode {
			t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode())
		}
	}
}