jschoubben
22a37b758c
On 2026-11-08 23:25 UTC the node-engine replaced distribution's store container and only then fetched distribution's program from the store it had just cut off: the module was left half applied and the artifact store unreachable from every other machine. An apply now fetches and checks every archive and process first, as it already pulls a module's images first; one that cannot be had gates its module, so none of its resources is replaced and none of its orphans removed. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5
347 lines
13 KiB
Go
347 lines
13 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
// Issue 1742: the engine removed a running container and only then found the replacement image
|
|
// unobtainable, leaving the service down. The replacement image comes first.
|
|
func TestARunningContainerIsKeptWhenItsReplacementImageCannotBeObtained(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
var removed, created bool
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
return "", errors.New("no such image")
|
|
case "pull":
|
|
return "", errors.New("manifest unknown")
|
|
case "rm":
|
|
removed = true
|
|
return "", nil
|
|
case "run":
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if removed || created {
|
|
t.Fatalf("the running container was touched although its image is unobtainable (removed=%v created=%v)", removed, created)
|
|
}
|
|
if err == nil {
|
|
t.Fatal("an unobtainable image failed nothing")
|
|
}
|
|
if !strings.Contains(err.Error(), pinned) {
|
|
t.Errorf("the failure does not name the image %s: %v", pinned, err)
|
|
}
|
|
var pull *PullError
|
|
if !errors.As(err, &pull) || pull.Image != pinned {
|
|
t.Errorf("the failure is not a pull error naming %s: %v", pinned, err)
|
|
}
|
|
|
|
// design 51, ENG-7: a module of two containers whose second new image cannot be pulled replaces neither, and says the
|
|
// image; with both pullable, both are pulled before the first is set aside.
|
|
second := strings.Replace(pinned, "postgres", "other", 1)
|
|
two := `{"declaration":1,"resources":[
|
|
{"id":"a.first","type":"container","name":"first","image":"` + pinned + `","env":{"X":"1"}},
|
|
{"id":"a.second","type":"container","name":"second","image":"` + second + `","env":{"X":"2"}}
|
|
]}`
|
|
for _, c := range []struct {
|
|
name string
|
|
unpullable string
|
|
wantTouched bool
|
|
wantPulled string
|
|
}{
|
|
{"the second image cannot be pulled", second, false, "pull " + pinned},
|
|
{"both can", "", true, "pull " + pinned + "; pull " + second},
|
|
} {
|
|
d := parseTrusted(t, two)
|
|
var steps []string
|
|
present := map[string]bool{}
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
if present[args[len(args)-1]] {
|
|
return "", nil
|
|
}
|
|
return "", errors.New("no such image")
|
|
case "pull":
|
|
if args[1] == c.unpullable {
|
|
return "", errors.New("manifest unknown")
|
|
}
|
|
present[args[1]] = true
|
|
steps = append(steps, "pull "+args[1])
|
|
case "rm", "rename", "stop", "run":
|
|
steps = append(steps, args[0]+" "+strings.Join(args[1:], " "))
|
|
}
|
|
return "", nil
|
|
}
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
got := strings.Join(steps, "; ")
|
|
touched := strings.Contains(got, "rename") || strings.Contains(got, "run ")
|
|
if touched != c.wantTouched || !strings.HasPrefix(got, c.wantPulled) {
|
|
t.Errorf("%s: steps %q (touched %v), want pulls %q first and touched=%v", c.name, got, touched, c.wantPulled, c.wantTouched)
|
|
}
|
|
if !c.wantTouched && (err == nil || !strings.Contains(err.Error(), second)) {
|
|
t.Errorf("%s: the failure does not name %s: %v", c.name, second, err)
|
|
}
|
|
}
|
|
|
|
// Issue 1966 (2026-11-08 23:25 UTC): a module whose container serves the artifact store had it replaced, and then
|
|
// could not fetch its program from the store it had cut off. Every archive and process is fetched before anything of
|
|
// the module is replaced or removed: with the store away, its container and its orphan stay as they were, and the
|
|
// failure names the program's source.
|
|
t.Run("an artifact that cannot be fetched replaces nothing of its module", func(t *testing.T) {
|
|
gone := httptest.NewServer(http.NotFoundHandler())
|
|
source := gone.URL + "/program.tar.gz"
|
|
gone.Close() // the store is cut off: refused
|
|
was := storeAwayAtMost
|
|
storeAwayAtMost = 0 // no patience: the refusal fails at once, without waiting
|
|
t.Cleanup(func() { storeAwayAtMost = was })
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"distribution.store","type":"container","name":"store","image":"`+pinned+`","env":{"X":"new"}},
|
|
{"id":"distribution.program","type":"process","name":"program","source":"`+source+`",
|
|
"digest":"sha256:`+strings.Repeat("0", 64)+`","run":["program"]}
|
|
]}`)
|
|
known := store.State{}
|
|
known.Record(store.Applied{ID: "distribution.old", Type: string(declaration.TypeContainer), Target: "old",
|
|
Module: "distribution"})
|
|
var steps []string
|
|
removed := map[string]bool{}
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
if name == "systemctl" {
|
|
steps = append(steps, "systemctl "+strings.Join(args, " "))
|
|
return "", nil
|
|
}
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
if removed[args[len(args)-1]] {
|
|
return "", errors.New("no such container")
|
|
}
|
|
return "true\tsome-older-spec", nil
|
|
case "rm", "rename", "stop", "run":
|
|
steps = append(steps, args[0]+" "+strings.Join(args[1:], " "))
|
|
if args[0] == "rm" {
|
|
removed[args[len(args)-1]] = true
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
report, after, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
|
if len(steps) > 0 {
|
|
t.Errorf("the module was touched though its program could not be fetched: %q", steps)
|
|
}
|
|
if err == nil || !strings.Contains(err.Error(), source) {
|
|
t.Errorf("the failure does not name the program's source %s: %v", source, err)
|
|
}
|
|
if _, still := after.Find("distribution.old"); !still {
|
|
t.Error("the module's orphan was removed though its program could not be fetched")
|
|
}
|
|
if o := outcomeOf(report, "distribution.store"); o.Action != "skipped" {
|
|
t.Errorf("the container's outcome is %+v, want skipped", o)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
// The seat and holder a routed image names (ENG-10) are no part of what a container is: another holder is the same spec,
|
|
// so a declaration composed again for it replaces no container.
|
|
for _, holder := range []string{"distribution", "elsewhere"} {
|
|
named := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","image-seat":"mesh-artifact-store","image-holder":"`+holder+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
if got := containerSpec(named.Resources[0].(*declaration.Container), inputs{}); got != want {
|
|
t.Errorf("a container naming the holder %s has another spec than one naming none", holder)
|
|
}
|
|
}
|
|
var order []string
|
|
created := false
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
if created {
|
|
return "true\t" + want, nil
|
|
}
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
order = append(order, "inspect")
|
|
if len(order) == 1 {
|
|
return "", errors.New("no such image")
|
|
}
|
|
return "", nil
|
|
case "pull":
|
|
order = append(order, "pull")
|
|
case "rm":
|
|
order = append(order, "rm")
|
|
case "run":
|
|
order = append(order, "run")
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
|
|
t.Fatalf("apply failed: %v", err)
|
|
}
|
|
got := strings.Join(order, ",")
|
|
if !strings.HasPrefix(got, "inspect,pull,inspect,rm,run") {
|
|
t.Errorf("order of runtime calls = %s, want the image pulled before rm", got)
|
|
}
|
|
}
|
|
|
|
// asideRun is a runtime whose container "store" runs from an older spec, recording the steps a replacement takes; run
|
|
// fails when failRun, and the new container reads back as declared otherwise.
|
|
func asideRun(t *testing.T, want string, failRun bool, order *[]string) Runner {
|
|
created := false
|
|
return func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
if created {
|
|
return "true\t" + want, nil
|
|
}
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
return "", nil
|
|
case "rm", "rename", "stop", "start":
|
|
*order = append(*order, strings.Join(args, " "))
|
|
return "", nil
|
|
case "run":
|
|
*order = append(*order, "run")
|
|
if failRun {
|
|
return "", errors.New("port is already allocated")
|
|
}
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
}
|
|
|
|
// A container whose replacement does not come up is put back: the running one is set aside (renamed and stopped), never
|
|
// removed before the new one runs, and started again under its name when the new one fails, the apply failing loudly.
|
|
// Removed first, the bus's own container left novox without a bus when its replacement was never made (issue 1945,
|
|
// live on 2026-10-25 at 11:23).
|
|
func TestAContainerWhoseReplacementFailsIsPutBack(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
var order []string
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, asideRun(t, want, true, &order), nil, nil)
|
|
if err == nil || !strings.Contains(err.Error(), "started again") {
|
|
t.Fatalf("a failed replacement: %v", err)
|
|
}
|
|
got := strings.Join(order, "; ")
|
|
if !strings.Contains(got, "rename store store.replaced; stop store.replaced; run") ||
|
|
!strings.Contains(got, "rename store.replaced store; start store") {
|
|
t.Fatalf("the steps were %q", got)
|
|
}
|
|
if strings.Contains(got, "rm -f store;") && strings.Index(got, "rm -f store;") < strings.Index(got, "run") {
|
|
t.Fatalf("the running container was removed before its replacement ran: %q", got)
|
|
}
|
|
}
|
|
|
|
// A replacement that runs removes the container set aside, and only after it runs.
|
|
func TestAReplacementThatRunsRemovesTheContainerSetAside(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
var order []string
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, asideRun(t, want, false, &order), nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := strings.Join(order, "; ")
|
|
run, removed := strings.Index(got, "run"), strings.LastIndex(got, "rm -f store.replaced")
|
|
if run < 0 || removed < run || strings.Contains(got, "start store") {
|
|
t.Fatalf("the steps were %q: the set-aside container is removed after the replacement runs", got)
|
|
}
|
|
}
|
|
|
|
// A container left set aside by an engine that stopped between setting it aside and running its replacement is the
|
|
// set-aside one of the next apply: put back when the replacement fails, removed once it runs (the review of issue
|
|
// 1945).
|
|
func TestAContainerLeftAsideIsPutBackOrRemovedByTheNextApply(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
for _, failRun := range []bool{true, false} {
|
|
var order []string
|
|
created := false
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
switch args[len(args)-1] {
|
|
case "store":
|
|
if created {
|
|
return "true\t" + want, nil
|
|
}
|
|
return "", errors.New("no such container")
|
|
case "store.replaced":
|
|
return "false\tsome-older-spec", nil
|
|
}
|
|
return "", errors.New("no such container")
|
|
case "image":
|
|
return "", nil
|
|
case "rm", "rename", "stop", "start":
|
|
order = append(order, strings.Join(args, " "))
|
|
return "", nil
|
|
case "run":
|
|
order = append(order, "run")
|
|
if failRun {
|
|
return "", errors.New("port is already allocated")
|
|
}
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
got := strings.Join(order, "; ")
|
|
if failRun {
|
|
if err == nil || !strings.Contains(got, "rename store.replaced store; start store") {
|
|
t.Fatalf("a failed run beside a container left aside: %v; steps %q", err, got)
|
|
}
|
|
continue
|
|
}
|
|
if err != nil || !strings.Contains(got, "run; rm -f store.replaced") {
|
|
t.Fatalf("a run beside a container left aside: %v; steps %q", err, got)
|
|
}
|
|
}
|
|
}
|