150 lines
6.7 KiB
Go
150 lines
6.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
|
"github.com/novox/mesh-host/internal/agent"
|
|
"github.com/novox/mesh-host/internal/apply"
|
|
"github.com/novox/mesh-host/internal/link"
|
|
)
|
|
|
|
func backupEvent(node, module string, ok bool, at time.Time, why string) []byte {
|
|
raw, _ := json.Marshal(map[string]any{"node": node, "module": module, "ok": ok, "at": at, "error": why})
|
|
return raw
|
|
}
|
|
|
|
// A backup is done when the holder says so for this node and module, held on its words when it says it failed, and held
|
|
// when it says nothing within the bound; another node's or module's event, or one from before the ask, is not the answer
|
|
// (R6-A A5, novox/hq ADR 0429 §14).
|
|
func TestABackupEndsWhenItsEventSaysSo(t *testing.T) {
|
|
since := time.Now()
|
|
run := func(within time.Duration, events ...[]byte) (done bool, words string) {
|
|
ch := make(chan []byte, len(events))
|
|
for _, e := range events {
|
|
ch <- e
|
|
}
|
|
got := awaitBackup(context.Background(), ch, "n1", "db", since, within)
|
|
return got.Done, got.Words
|
|
}
|
|
if done, _ := run(time.Second,
|
|
backupEvent("n2", "db", true, since, ""), backupEvent("n1", "web", true, since, ""),
|
|
backupEvent("n1", "db", false, since.Add(-time.Hour), "an old night"),
|
|
backupEvent("n1", "db", true, since.Add(time.Second), "")); !done {
|
|
t.Error("the event of this node and module was not the answer")
|
|
}
|
|
if done, words := run(time.Second, backupEvent("n1", "db", false, since, "the repository is locked")); done ||
|
|
!strings.Contains(words, "repository is locked") {
|
|
t.Errorf("a failed backup: done=%v words=%q", done, words)
|
|
}
|
|
if done, words := run(50 * time.Millisecond); done || !strings.Contains(words, "did not say it was done") {
|
|
t.Errorf("silence within the bound: done=%v words=%q", done, words)
|
|
}
|
|
// The node agent stopping while it waits is no failure of the backup: said as interrupted, so its next start backs
|
|
// the module up again (postgres on novox, 2026-10-31).
|
|
stopped, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if got := awaitBackup(stopped, make(chan []byte), "n1", "db", since, time.Second); got.Done || !got.Interrupted ||
|
|
got.Words != agent.BackupInterrupted {
|
|
t.Errorf("a stopped node agent: %+v", got)
|
|
}
|
|
if got := awaitBackup(context.Background(), make(chan []byte), "n1", "db", since, 10*time.Millisecond); got.Interrupted {
|
|
t.Errorf("silence within the bound taken as interrupted: %+v", got)
|
|
}
|
|
|
|
// The ask that starts it (sweep 2026-10-30): one that never reached the holder (nothing on this machine answers it
|
|
// while the tool runner restarts) is asked again within its bound, and past it fails saying so; one that may have
|
|
// reached it is never asked twice.
|
|
notAsked := fmt.Errorf("node-backup.now could not be asked: %w: nothing on this machine answers "+
|
|
"mesh.seat.node-backup.tool.now.ace", link.ErrNotAsked)
|
|
for _, row := range []struct {
|
|
name string
|
|
answers []error
|
|
within time.Duration
|
|
asked int
|
|
words string
|
|
}{
|
|
{"heard after the tool runner is back", []error{notAsked, notAsked, nil}, time.Second, 3, ""},
|
|
// Asked again until the bound, however slow the machine: at least twice, and fewer than it would answer.
|
|
{"never heard within the bound", []error{notAsked, notAsked, notAsked, notAsked, notAsked, nil}, 25 * time.Millisecond,
|
|
0, "times over"},
|
|
{"a timeout may have started it", []error{errors.New("no answer from x: context deadline exceeded"), nil},
|
|
time.Second, 1, "deadline exceeded"},
|
|
{"a refusal in words", []error{errors.New("restic is not on this machine"), nil}, time.Second, 1, "not on this"},
|
|
} {
|
|
asked := 0
|
|
_, err := askUntilHeard(context.Background(), func(context.Context) (json.RawMessage, error) {
|
|
asked++
|
|
return json.RawMessage(`{}`), row.answers[asked-1]
|
|
}, row.within, func(int) time.Duration { return 10 * time.Millisecond })
|
|
if row.asked == 0 && (asked < 2 || asked >= len(row.answers)) || row.asked != 0 && asked != row.asked || (row.words == "") != (err == nil) || err != nil && !strings.Contains(err.Error(), row.words) {
|
|
t.Errorf("%s: asked %d times, %v", row.name, asked, err)
|
|
}
|
|
}
|
|
|
|
// The events of the same kind (ENG-11): the status view's settled is told to the agent by the module that settled, on
|
|
// whichever node; a body with no module, or no JSON, tells nothing; the end of the link ends the hearing.
|
|
told := &settledRecorder{}
|
|
events := make(chan []byte, 4)
|
|
for _, body := range []string{`{"node":"n2","module":"distribution"}`, `{"node":"n1"}`, `not json`, `{"module":"gitea"}`} {
|
|
events <- []byte(body)
|
|
}
|
|
linked, end := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
go func() { defer close(done); listen(linked, events, func() settledTold { return told }, func(string) {}) }()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for len(told.heard()) < 2 && time.Now().Before(deadline) {
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
end()
|
|
<-done
|
|
if got := strings.Join(told.heard(), ","); got != "distribution,gitea" {
|
|
t.Errorf("the agent was told %q settled, want distribution,gitea", got)
|
|
}
|
|
// And the images an apply could not pull are said with the holder the declaration names beside each, none for a public
|
|
// one (ENG-8, ENG-10).
|
|
d, err := declaration.ParseFileTrusted([]byte(`{"declaration":1,"resources":[` +
|
|
`{"id":"a.first","type":"container","name":"first","image":"reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","image-seat":"mesh-artifact-store","image-holder":"distribution"},` +
|
|
`{"id":"a.second","type":"container","name":"second","image":"public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}`))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pull := func(image string) error {
|
|
return &apply.Error{Resource: "x", Err: &apply.PullError{Image: image, Err: errors.New("manifest unknown")}}
|
|
}
|
|
if got := pullsOf(d, pull("reg/x@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); len(got) != 1 || got[0].Seat != "mesh-artifact-store" || got[0].Holder != "distribution" {
|
|
t.Errorf("a store's image is said as %+v", got)
|
|
}
|
|
if got := pullsOf(d, pull("public/y@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); len(got) != 1 || got[0].Holder != "" || got[0].Seat != "" {
|
|
t.Errorf("a public image is said as %+v", got)
|
|
}
|
|
if got := pullsOf(d, errors.New("another failure")); len(got) != 0 {
|
|
t.Errorf("a failure that is no pull is said as %+v", got)
|
|
}
|
|
}
|
|
|
|
// settledRecorder is the agent a hearing tells.
|
|
type settledRecorder struct {
|
|
mu sync.Mutex
|
|
list []string
|
|
}
|
|
|
|
func (r *settledRecorder) Settled(holder string) {
|
|
r.mu.Lock()
|
|
r.list = append(r.list, holder)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
func (r *settledRecorder) heard() []string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return append([]string(nil), r.list...)
|
|
}
|