Merge branch 'feat/47-G-recover-host' into feat/47-G-join8
# Conflicts: # internal/selfcompose/entries.go # module.json
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
)
|
||||
|
||||
// The artifact store's holder takes its own new build from the build node that kept it (design 47): its done build names the
|
||||
// node and the digest, the node's build seat answers where the bytes are as an object on the bus, and the node-engine
|
||||
// reads them and checks them against the digest. Never another module's files: the seat's verb is the way.
|
||||
|
||||
// keptAskWithin bounds one ask of a build seat; keptObjectWithin one read of the object.
|
||||
var (
|
||||
keptAskWithin = 30 * time.Second
|
||||
keptObjectWithin = 10 * time.Minute
|
||||
)
|
||||
|
||||
// keptBuildSeats are the three build seats, each of which serves the verb kept-artifact: the node that kept the build
|
||||
// holds the seat of the language it was built in, and the others do not answer.
|
||||
var keptBuildSeats = []string{nodebuildgo.Seat, nodebuildnode.Seat, nodebuildimage.Seat}
|
||||
|
||||
// keptArtifactFetch is the apply's fetch of what a module's done build kept on a build node.
|
||||
func keptArtifactFetch(buildOf func(context.Context, string, string) (string, string, bool, error),
|
||||
ask func(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error),
|
||||
object func(ctx context.Context, store, name string, limit int64) ([]byte, error)) apply.KeptFetch {
|
||||
return func(ctx context.Context, module, digest string) ([]byte, bool, error) {
|
||||
node, fp, kept, err := buildOf(ctx, module, digest)
|
||||
if err != nil || !kept {
|
||||
return nil, false, err
|
||||
}
|
||||
hex := strings.TrimPrefix(digest, "sha256:")
|
||||
var notHeld []string
|
||||
for _, seat := range keptBuildSeats {
|
||||
raw, err := ask(ctx, node, seat, nodebuildgo.VerbKeptArtifact,
|
||||
nodebuildgo.KeptArtifactInput{Module: module, Fp: fp, Digest: hex}, keptAskWithin)
|
||||
if err != nil {
|
||||
notHeld = append(notHeld, seat+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
var got nodebuildgo.KeptArtifactAnswer
|
||||
if json.Unmarshal(raw, &got) != nil {
|
||||
return nil, true, fmt.Errorf("%s of %s answered with something that is not an answer of kept-artifact", seat, node)
|
||||
}
|
||||
switch {
|
||||
case got.Error != "":
|
||||
return nil, true, fmt.Errorf("%s of %s could not give the kept build of %s: %s", seat, node, module, got.Error)
|
||||
case got.Refused != "":
|
||||
return nil, true, fmt.Errorf("%s of %s refuses the kept build of %s: %s %s", seat, node, module, got.Refused, got.Words)
|
||||
case got.Store == "" || got.Object == "" || got.Digest != hex:
|
||||
return nil, true, fmt.Errorf("%s of %s answered for %s without the store and the object of the digest %s", seat, node, module, hex)
|
||||
}
|
||||
octx, cancel := context.WithTimeout(ctx, keptObjectWithin)
|
||||
body, err := object(octx, got.Store, got.Object, got.Size+1)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, true, fmt.Errorf("the build of %s kept on %s could not be read from %s: %w", module, node, got.Store, err)
|
||||
}
|
||||
return body, true, nil
|
||||
}
|
||||
return nil, true, fmt.Errorf("%s keeps the build of %s for the artifact store and none of its build seats answers (%s)",
|
||||
node, module, strings.Join(notHeld, "; "))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A module's done build that kept its artifact on a build node is fetched through that node's build seat (design 47).
|
||||
func TestAKeptArtifactIsFetchedThroughTheBuildSeatOfItsNode(t *testing.T) {
|
||||
hex := strings.Repeat("a", 64)
|
||||
stored := func(string) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"module":"distribution","fp":"f1","digest":"` + hex + `","size":5,"store":"kept-builds","object":"` + hex + `"}`), nil
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
kept bool
|
||||
answer func(seat string) (json.RawMessage, error)
|
||||
object error
|
||||
wantKept bool
|
||||
wantErr string
|
||||
wantSeat string
|
||||
}{
|
||||
{"a build kept nothing is not asked", false, nil, nil, false, "", ""},
|
||||
{"the seat of the language answers", true, func(seat string) (json.RawMessage, error) {
|
||||
if seat != "node-build-node" {
|
||||
return nil, errors.New("no responder")
|
||||
}
|
||||
return stored(seat)
|
||||
}, nil, true, "", "node-build-node"},
|
||||
{"no build seat of the node answers", true, func(string) (json.RawMessage, error) { return nil, errors.New("no responder") },
|
||||
nil, true, "novox keeps the build of distribution", ""},
|
||||
{"the seat refuses it", true, func(string) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"module":"distribution","fp":"f1","refused":"not kept","words":"swept"}`), nil
|
||||
}, nil, true, "refuses the kept build of distribution: not kept", ""},
|
||||
{"the object cannot be read", true, stored, errors.New("no object"), true, "could not be read from kept-builds", "node-build-go"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
var seats []string
|
||||
fetch := keptArtifactFetch(
|
||||
func(context.Context, string, string) (string, string, bool, error) { return "novox", "f1", row.kept, nil },
|
||||
func(_ context.Context, node, seat, verb string, _ any, _ time.Duration) (json.RawMessage, error) {
|
||||
if node != "novox" || verb != "kept-artifact" {
|
||||
t.Errorf("asked %s of %s on %s", verb, seat, node)
|
||||
}
|
||||
seats = append(seats, seat)
|
||||
return row.answer(seat)
|
||||
},
|
||||
func(_ context.Context, store, name string, limit int64) ([]byte, error) {
|
||||
if store != "kept-builds" || name != hex || limit < 5 {
|
||||
t.Errorf("read %s/%s up to %d", store, name, limit)
|
||||
}
|
||||
return []byte("bytes"), row.object
|
||||
})
|
||||
body, kept, err := fetch(context.Background(), "distribution", "sha256:"+hex)
|
||||
if kept != row.wantKept {
|
||||
t.Fatalf("kept = %v", kept)
|
||||
}
|
||||
if row.wantErr == "" && (err != nil || (kept && string(body) != "bytes")) {
|
||||
t.Fatalf("got %q, %v", body, err)
|
||||
}
|
||||
if row.wantErr != "" && (err == nil || !strings.Contains(err.Error(), row.wantErr)) {
|
||||
t.Fatalf("the error is %v, want %q", err, row.wantErr)
|
||||
}
|
||||
if !row.kept && len(seats) > 0 {
|
||||
t.Errorf("a build that kept nothing asked %v", seats)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/novox/mesh-host/internal/seatwait"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup
|
||||
@@ -89,7 +90,7 @@ func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.SealedAt}, true
|
||||
Sealed: got.SealedAt, Wait: got.Wait}, true
|
||||
}
|
||||
|
||||
// ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
|
||||
@@ -107,7 +108,7 @@ func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, b
|
||||
return agent.Have{}, false, err
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.SealedAt}, true, nil
|
||||
Sealed: got.SealedAt, Wait: got.Wait}, true, nil
|
||||
}
|
||||
|
||||
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
@@ -124,7 +125,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
}
|
||||
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
|
||||
Failed: phase == string(nodeengineclient.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
|
||||
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait})
|
||||
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait, Wait: x.Wait})
|
||||
}
|
||||
|
||||
func (h sessionHaves) Removed(ctx context.Context, module string) {
|
||||
@@ -448,6 +449,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
}
|
||||
began := time.Now()
|
||||
node := session.Membership.Node
|
||||
var keptFetch atomic.Pointer[apply.KeptFetch] // the composer's, set once it exists
|
||||
composedBucket, err := compositions.BucketFor(node)
|
||||
if err != nil {
|
||||
cancel()
|
||||
@@ -550,6 +552,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
Backoff: link.Backoff,
|
||||
Apply: func(ctx context.Context, c compositions.Composed, others []compositions.Composed) agent.Result {
|
||||
actx, done := applyContext(session.WithLink(ctx))
|
||||
if kf := keptFetch.Load(); kf != nil {
|
||||
actx = apply.WithKept(actx, *kf)
|
||||
}
|
||||
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
||||
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
||||
held, herr := users.Hold(c)
|
||||
@@ -574,6 +579,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
sayFacts(ctx, ag)
|
||||
return res
|
||||
},
|
||||
Stop: func(ctx context.Context, module string) agent.Result {
|
||||
return stopComposition(ctx, opts, module)
|
||||
},
|
||||
Remove: func(ctx context.Context, module string, others []compositions.Composed) agent.Result {
|
||||
actx, done := applyContext(session.WithLink(ctx))
|
||||
res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
|
||||
@@ -680,7 +688,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
||||
go followRunning(run, session, composer, say)
|
||||
apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527)
|
||||
env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday
|
||||
env.Recovery = recoveryEnv(session, composer)
|
||||
env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting
|
||||
keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject)
|
||||
keptFetch.Store(&keptFetchFn)
|
||||
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
||||
session.NoteComposes(linked, nodeengineclient.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
|
||||
ag = agent.New(env)
|
||||
@@ -851,6 +862,30 @@ func removeComposition(ctx context.Context, opts options, module string, others
|
||||
return agent.Result{}
|
||||
}
|
||||
|
||||
// stopComposition stops a module's units and containers from the node's record and removes nothing (design 47 G A-98,
|
||||
// §297): what the module kept stays, so a revert restores it before the reverted build starts.
|
||||
func stopComposition(ctx context.Context, opts options, module string) agent.Result {
|
||||
applying.Lock()
|
||||
defer applying.Unlock()
|
||||
unlock, err := store.Lock(opts.state, nil)
|
||||
if err != nil {
|
||||
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||
}
|
||||
defer unlock()
|
||||
known, err := store.Load(opts.state)
|
||||
if err != nil {
|
||||
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||
}
|
||||
built, err := system.For(builtFor)
|
||||
if err != nil {
|
||||
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||
}
|
||||
if _, err := apply.StopModule(ctx, built, known, module, opts.run()); err != nil {
|
||||
return agent.Result{Failed: true, Detail: shorten(err.Error())}
|
||||
}
|
||||
return agent.Result{}
|
||||
}
|
||||
|
||||
// busyModules are the modules whose last apply left a container running because it said busy.
|
||||
func busyModules() []string {
|
||||
busyMu.Lock()
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
)
|
||||
|
||||
// recoveryEnv is what the node agent asks of the recovery module and the manifests to move a module's kept data (design 47 G §296,
|
||||
// §297, §554, §560, §630, A-77, A-98, A-108): the seat's verbs asked in the node-engine's own name, the endings heard from the
|
||||
// seat's events, the bound of a wait kept as a key of the state bounds, the catch-up called as the module's tool.
|
||||
func recoveryEnv(session *link.Session, composer *selfcompose.Composer) *agent.Recovery {
|
||||
ask := func(ctx context.Context, verb string, in, out any) error {
|
||||
raw, err := session.AskMeshSeat(ctx, meshrecovery.Seat, verb, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(raw, out)
|
||||
}
|
||||
return &agent.Recovery{
|
||||
FormChanges: composer.FormChanges,
|
||||
SnapshotKept: func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (a meshrecovery.SnapshotKeptAnswer, err error) {
|
||||
return a, ask(ctx, meshrecovery.VerbSnapshotKept, in, &a)
|
||||
},
|
||||
RestoreKept: func(ctx context.Context, in meshrecovery.RestoreKeptInput) (a meshrecovery.RestoreKeptAnswer, err error) {
|
||||
return a, ask(ctx, meshrecovery.VerbRestoreKept, in, &a)
|
||||
},
|
||||
KeptPoints: func(ctx context.Context, in meshrecovery.KeptPointsInput) (a meshrecovery.KeptPointsAnswer, err error) {
|
||||
return a, ask(ctx, meshrecovery.VerbKeptPoints, in, &a)
|
||||
},
|
||||
Hear: func(_ context.Context, kind, module, kept string) (<-chan agent.Ending, func(), error) {
|
||||
heard, stop, err := session.HearRecoveryEnding(kind, module, kept)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out := make(chan agent.Ending, 64)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case ev := <-heard:
|
||||
select {
|
||||
case out <- agent.Ending(ev):
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, func() { stop(); close(done) }, nil
|
||||
},
|
||||
Bound: func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (<-chan struct{}, func(), error) {
|
||||
return session.KeepBound(ctx, what, id, module, since, d)
|
||||
},
|
||||
CatchUp: func(ctx context.Context, module string, c meshrecovery.CatchUp) error {
|
||||
return session.CallTool(ctx, module, c.Tool, c.Arguments)
|
||||
},
|
||||
WaitMinutes: recoveryWaitMinutes(),
|
||||
}
|
||||
}
|
||||
|
||||
// recoveryWaitFile is the file the manifest declares with the content ${config:recovery-wait-minutes} (design 47 G §633, A-112): the
|
||||
// node-engine is no bundle under a runtime, so it reads the setting from its file at its start, as the builder does (A-71).
|
||||
var recoveryWaitFile = "/var/lib/mesh-host/recovery-wait-minutes"
|
||||
|
||||
// recoveryWaitMinutes is the bound of the wait for the recovery module, in minutes: the file's number, the design's 90 when the
|
||||
// file is missing, empty or not a positive number (design 47 G §554, A-112).
|
||||
func recoveryWaitMinutes() int {
|
||||
raw, err := os.ReadFile(recoveryWaitFile)
|
||||
if err != nil {
|
||||
return agent.RecoveryWaitMinutes
|
||||
}
|
||||
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||
if err != nil || n <= 0 {
|
||||
return agent.RecoveryWaitMinutes
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/nodegrant"
|
||||
)
|
||||
|
||||
// Design 47 G §560, §630, A-77, A-92, A-108: the node agent's recovery step over a real bus. Its session asks the seat's verbs, hears
|
||||
// the last `restored` already said and the next, keeps a wait's bound as a key of the state bounds whose time ends with the bus's
|
||||
// marker, and calls the catch-up as the module's tool with its arguments. The recovery module is a stand-in that answers the verbs and
|
||||
// says the events; the grants part composes the node's subjects from the manifest, so the test adds them to the node's grant.
|
||||
func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) {
|
||||
perms := nodegrant.For(itNode)
|
||||
perms.Publish = append(perms.Publish, "$JS.API.>", "$KV.mesh-host_bounds.>", "mesh.seat.mesh-recovery.call.*.node~"+itNode,
|
||||
"mesh.mod.db.tool.rebuild."+itNode)
|
||||
perms.Subscribe = append(perms.Subscribe, "mesh.seat.mesh-recovery.event.>", "$KV.mesh-host_bounds.>", "_INBOX.>", "$JS.ACK.>", "$JS.FC.>")
|
||||
b := newItBedAs(t, perms)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, s := range []struct{ stream, subject string }{{meshrecovery.StreamEnded, meshrecovery.SubjectEndedAll},
|
||||
{meshrecovery.StreamRestored, meshrecovery.SubjectRestoredAll}} {
|
||||
if _, err := b.js.AddStream(&nats.StreamConfig{Name: s.stream, Subjects: []string{s.subject}, MaxMsgsPerSubject: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
js, err := jetstream.New(b.conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: link.BoundsBucket, History: 1, LimitMarkerTTL: time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
say := func(act, outcome string) {
|
||||
raw, _ := json.Marshal(meshrecovery.RestoredEvent{Act: act, Module: "db", Kept: "cache", Outcome: outcome, At: time.Now().UTC()})
|
||||
if _, err := b.js.Publish(meshrecovery.SubjectRestored("db", "cache"), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var restoreAsked meshrecovery.RestoreKeptInput
|
||||
for verb, answer := range map[string]func(m *nats.Msg) any{
|
||||
meshrecovery.VerbKeptPoints: func(*nats.Msg) any {
|
||||
return meshrecovery.KeptPointsAnswer{Points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1",
|
||||
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}}
|
||||
},
|
||||
meshrecovery.VerbRestoreKept: func(m *nats.Msg) any {
|
||||
_ = json.Unmarshal(m.Data, &restoreAsked)
|
||||
say("r1", meshrecovery.RestoredEventOutcomeDone)
|
||||
return meshrecovery.RestoreKeptAnswer{Accepted: "r1"}
|
||||
},
|
||||
} {
|
||||
answer := answer
|
||||
sub, err := b.conn.Subscribe("mesh.seat.mesh-recovery.call."+verb+".*", func(m *nats.Msg) {
|
||||
out, _ := json.Marshal(map[string]any{"result": answer(m)})
|
||||
_ = m.Respond(out)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
var toolArgs map[string]any
|
||||
toolSub, err := b.conn.Subscribe(link.ToolSubject("db", "rebuild", itNode), func(m *nats.Msg) {
|
||||
_ = json.Unmarshal(m.Data, &toolArgs)
|
||||
_ = m.Respond([]byte(`{"result":{}}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = toolSub.Unsubscribe() })
|
||||
|
||||
// A-112: the bound is the number in the file the manifest declares, 90 where it is missing or not a number.
|
||||
for content, want := range map[string]int{"45\n": 45, "": 90, "soon\n": 90, "0\n": 90} {
|
||||
recoveryWaitFile = filepath.Join(t.TempDir(), "w")
|
||||
if err := os.WriteFile(recoveryWaitFile, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recoveryWaitMinutes(); got != want {
|
||||
t.Errorf("the file %q gives %d minutes, want %d", content, got, want)
|
||||
}
|
||||
}
|
||||
recoveryWaitFile = filepath.Join(t.TempDir(), "missing")
|
||||
if got := recoveryWaitMinutes(); got != 90 {
|
||||
t.Errorf("no file gives %d minutes, want 90", got)
|
||||
}
|
||||
|
||||
say("old", meshrecovery.RestoredEventOutcomeFailed) // said before anyone listens: the last message of the subject
|
||||
session := b.startSession()
|
||||
r := recoveryEnv(session, nil)
|
||||
|
||||
events, stop, err := r.Hear(ctx, "restored", "db", "cache")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
if first := <-events; first.Act != "old" || first.Outcome != "failed" {
|
||||
t.Fatalf("the first event heard is %+v, want the last one said before", first)
|
||||
}
|
||||
pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: "db", Kept: "cache"})
|
||||
if err != nil || len(pts.Points) != 1 {
|
||||
t.Fatalf("kept-points: %+v, %v", pts, err)
|
||||
}
|
||||
got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: pts.Points[0].Snapshot, Why: "w"})
|
||||
if err != nil || got.Accepted != "r1" || restoreAsked.Snapshot != "snap-1" {
|
||||
t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, restoreAsked)
|
||||
}
|
||||
if ev := <-events; ev.Act != "r1" || ev.Outcome != "done" {
|
||||
t.Fatalf("the ending heard is %+v", ev)
|
||||
}
|
||||
if err := r.CatchUp(ctx, "db", *pts.Points[0].CatchUp); err != nil || toolArgs["all"] != true {
|
||||
t.Fatalf("catch-up: %v, arguments %v", err, toolArgs)
|
||||
}
|
||||
expired, end, err := r.Bound(ctx, "recovery", "db.restored.cache", "db", time.Now(), time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer end()
|
||||
select {
|
||||
case <-expired:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the bound's end, the bus's marker, was not heard")
|
||||
}
|
||||
}
|
||||
|
||||
// startSession is a session on the node's own connection, with no node agent run on it.
|
||||
func (b *itBed) startSession() *link.Session {
|
||||
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
||||
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
||||
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
||||
return b.session
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
// The node agent's Stop stops a module's container from the node's record, asks the machine nothing else and leaves the
|
||||
// record, so the module's kept data and its record survive (design 47 G A-98).
|
||||
func TestTheNodeAgentsStopStopsAModuleAndRemovesNothing(t *testing.T) {
|
||||
was := builtFor
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
path := filepath.Join(t.TempDir(), "state.json")
|
||||
var s store.State
|
||||
s.Record(store.Applied{ID: "app.box", Type: "container", Target: "app-box"})
|
||||
s.Record(store.Applied{ID: "other.box", Type: "container", Target: "other-box"})
|
||||
if err := store.Save(path, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var asked []string
|
||||
opts := options{state: path, runner: func(_ context.Context, name string, args ...string) (string, error) {
|
||||
asked = append(asked, name+" "+strings.Join(args, " "))
|
||||
return "", nil
|
||||
}}
|
||||
if res := stopComposition(context.Background(), opts, "app"); res.Failed {
|
||||
t.Fatalf("stop failed: %s", res.Detail)
|
||||
}
|
||||
if !strings.Contains(strings.Join(asked, "\n"), "stop app-box") || strings.Contains(strings.Join(asked, "\n"), "other-box") ||
|
||||
strings.Contains(strings.Join(asked, "\n"), " rm ") {
|
||||
t.Fatalf("the machine was asked %v", asked)
|
||||
}
|
||||
if after, err := store.Load(path); err != nil || len(after.Resources) != 2 {
|
||||
t.Fatalf("the record changed: %v, %v", after.Resources, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user