jschoubben
5173f2a31c
A want that names a node in kept-on, for an artifact its parts name, is fetched through kept-artifact of that node's build seat and read as an object on the bus, checked against the digest; with no kept-on it is fetched from the store, and the builder's builds are read no more (the apply hook, the fetch and the object read from bc8fbc7, its node and fingerprint from the want). Builds: design 47 G §271, §526, §421 (A-50) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
388 lines
15 KiB
Go
388 lines
15 KiB
Go
package apply
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
|
"github.com/novox/mesh-host/internal/store"
|
|
)
|
|
|
|
// Issue 1742: the engine removed a running container and only then found the replacement image
|
|
// unobtainable, leaving the service down. The replacement image comes first.
|
|
func TestARunningContainerIsKeptWhenItsReplacementImageCannotBeObtained(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
var removed, created bool
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
return "", errors.New("no such image")
|
|
case "pull":
|
|
return "", errors.New("manifest unknown")
|
|
case "rm":
|
|
removed = true
|
|
return "", nil
|
|
case "run":
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if removed || created {
|
|
t.Fatalf("the running container was touched although its image is unobtainable (removed=%v created=%v)", removed, created)
|
|
}
|
|
if err == nil {
|
|
t.Fatal("an unobtainable image failed nothing")
|
|
}
|
|
if !strings.Contains(err.Error(), pinned) {
|
|
t.Errorf("the failure does not name the image %s: %v", pinned, err)
|
|
}
|
|
var pull *PullError
|
|
if !errors.As(err, &pull) || pull.Image != pinned {
|
|
t.Errorf("the failure is not a pull error naming %s: %v", pinned, err)
|
|
}
|
|
|
|
// design 51, ENG-7: a module of two containers whose second new image cannot be pulled replaces neither, and says the
|
|
// image; with both pullable, both are pulled before the first is set aside.
|
|
second := strings.Replace(pinned, "postgres", "other", 1)
|
|
two := `{"declaration":1,"resources":[
|
|
{"id":"a.first","type":"container","name":"first","image":"` + pinned + `","env":{"X":"1"}},
|
|
{"id":"a.second","type":"container","name":"second","image":"` + second + `","env":{"X":"2"}}
|
|
]}`
|
|
for _, c := range []struct {
|
|
name string
|
|
unpullable string
|
|
wantTouched bool
|
|
wantPulled string
|
|
}{
|
|
{"the second image cannot be pulled", second, false, "pull " + pinned},
|
|
{"both can", "", true, "pull " + pinned + "; pull " + second},
|
|
} {
|
|
d := parseTrusted(t, two)
|
|
var steps []string
|
|
present := map[string]bool{}
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
if present[args[len(args)-1]] {
|
|
return "", nil
|
|
}
|
|
return "", errors.New("no such image")
|
|
case "pull":
|
|
if args[1] == c.unpullable {
|
|
return "", errors.New("manifest unknown")
|
|
}
|
|
present[args[1]] = true
|
|
steps = append(steps, "pull "+args[1])
|
|
case "rm", "rename", "stop", "run":
|
|
steps = append(steps, args[0]+" "+strings.Join(args[1:], " "))
|
|
}
|
|
return "", nil
|
|
}
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
got := strings.Join(steps, "; ")
|
|
touched := strings.Contains(got, "rename") || strings.Contains(got, "run ")
|
|
if touched != c.wantTouched || !strings.HasPrefix(got, c.wantPulled) {
|
|
t.Errorf("%s: steps %q (touched %v), want pulls %q first and touched=%v", c.name, got, touched, c.wantPulled, c.wantTouched)
|
|
}
|
|
if !c.wantTouched && (err == nil || !strings.Contains(err.Error(), second)) {
|
|
t.Errorf("%s: the failure does not name %s: %v", c.name, second, err)
|
|
}
|
|
}
|
|
|
|
// Issue 1966 (2026-11-08 23:25 UTC): a module whose container serves the artifact store had it replaced, and then
|
|
// could not fetch its program from the store it had cut off. Every archive and process is fetched before anything of
|
|
// the module is replaced or removed: with the store away, its container and its orphan stay as they were, and the
|
|
// failure names the program's source.
|
|
t.Run("an artifact that cannot be fetched replaces nothing of its module", func(t *testing.T) {
|
|
gone := httptest.NewServer(http.NotFoundHandler())
|
|
source := gone.URL + "/program.tar.gz"
|
|
gone.Close() // the store is cut off: refused
|
|
was := storeAwayAtMost
|
|
storeAwayAtMost = 0 // no patience: the refusal fails at once, without waiting
|
|
t.Cleanup(func() { storeAwayAtMost = was })
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"distribution.store","type":"container","name":"store","image":"`+pinned+`","env":{"X":"new"}},
|
|
{"id":"distribution.program","type":"process","name":"program","source":"`+source+`",
|
|
"digest":"sha256:`+strings.Repeat("0", 64)+`","run":["program"]}
|
|
]}`)
|
|
known := store.State{}
|
|
known.Record(store.Applied{ID: "distribution.old", Type: string(declaration.TypeContainer), Target: "old",
|
|
Module: "distribution"})
|
|
var steps []string
|
|
removed := map[string]bool{}
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
if name == "systemctl" {
|
|
steps = append(steps, "systemctl "+strings.Join(args, " "))
|
|
return "", nil
|
|
}
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
if removed[args[len(args)-1]] {
|
|
return "", errors.New("no such container")
|
|
}
|
|
return "true\tsome-older-spec", nil
|
|
case "rm", "rename", "stop", "run":
|
|
steps = append(steps, args[0]+" "+strings.Join(args[1:], " "))
|
|
if args[0] == "rm" {
|
|
removed[args[len(args)-1]] = true
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
report, after, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil)
|
|
if len(steps) > 0 {
|
|
t.Errorf("the module was touched though its program could not be fetched: %q", steps)
|
|
}
|
|
if err == nil || !strings.Contains(err.Error(), source) {
|
|
t.Errorf("the failure does not name the program's source %s: %v", source, err)
|
|
}
|
|
if _, still := after.Find("distribution.old"); !still {
|
|
t.Error("the module's orphan was removed though its program could not be fetched")
|
|
}
|
|
if o := outcomeOf(report, "distribution.store"); o.Action != "skipped" {
|
|
t.Errorf("the container's outcome is %+v, want skipped", o)
|
|
}
|
|
})
|
|
|
|
// Design 47 (W13.8): the holder's own program is fetched from the build node that kept it, not from the store; when it
|
|
// cannot be had, or is not the declared digest, nothing of the module is replaced and the failure says so.
|
|
for _, row := range []struct {
|
|
name string
|
|
fetch KeptFetch
|
|
wantErr string
|
|
}{
|
|
{"the node that keeps it is away", func(context.Context, string, string) ([]byte, bool, error) {
|
|
return nil, true, errors.New("novox keeps it and does not answer")
|
|
}, "novox keeps it and does not answer"},
|
|
{"what the node kept is not the declared digest", func(context.Context, string, string) ([]byte, bool, error) {
|
|
return []byte("other"), true, nil
|
|
}, "what arrived is"},
|
|
} {
|
|
t.Run("a kept artifact: "+row.name, func(t *testing.T) {
|
|
gone := httptest.NewServer(http.NotFoundHandler())
|
|
source := gone.URL + "/program.tar.gz"
|
|
gone.Close()
|
|
was := storeAwayAtMost
|
|
storeAwayAtMost = 0
|
|
t.Cleanup(func() { storeAwayAtMost = was })
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"distribution.program","type":"process","name":"program","source":"`+source+`",
|
|
"digest":"sha256:`+strings.Repeat("0", 64)+`","run":["program"]}
|
|
]}`)
|
|
var asked []string
|
|
fetch := func(ctx context.Context, module, digest string) ([]byte, bool, error) {
|
|
asked = append(asked, module+" "+digest)
|
|
return row.fetch(ctx, module, digest)
|
|
}
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
|
_, _, err := Apply(WithKept(context.Background(), fetch), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
if err == nil || !strings.Contains(err.Error(), row.wantErr) {
|
|
t.Errorf("the failure is %v, want it to say %q", err, row.wantErr)
|
|
}
|
|
if len(asked) != 1 || asked[0] != "distribution sha256:"+strings.Repeat("0", 64) {
|
|
t.Errorf("the kept fetch was asked %q", asked)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
// The seat and holder a routed image names (ENG-10) are no part of what a container is: another holder is the same spec,
|
|
// so a declaration composed again for it replaces no container.
|
|
for _, holder := range []string{"distribution", "elsewhere"} {
|
|
named := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","image-seat":"mesh-artifact-store","image-holder":"`+holder+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
if got := containerSpec(named.Resources[0].(*declaration.Container), inputs{}); got != want {
|
|
t.Errorf("a container naming the holder %s has another spec than one naming none", holder)
|
|
}
|
|
}
|
|
var order []string
|
|
created := false
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
args = oldShape(name, args)
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
if created {
|
|
return "true\t" + want, nil
|
|
}
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
order = append(order, "inspect")
|
|
if len(order) == 1 {
|
|
return "", errors.New("no such image")
|
|
}
|
|
return "", nil
|
|
case "pull":
|
|
order = append(order, "pull")
|
|
case "rm":
|
|
order = append(order, "rm")
|
|
case "run":
|
|
order = append(order, "run")
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil {
|
|
t.Fatalf("apply failed: %v", err)
|
|
}
|
|
got := strings.Join(order, ",")
|
|
if !strings.HasPrefix(got, "inspect,pull,inspect,rm,run") {
|
|
t.Errorf("order of runtime calls = %s, want the image pulled before rm", got)
|
|
}
|
|
}
|
|
|
|
// asideRun is a runtime whose container "store" runs from an older spec, recording the steps a replacement takes; run
|
|
// fails when failRun, and the new container reads back as declared otherwise.
|
|
func asideRun(t *testing.T, want string, failRun bool, order *[]string) Runner {
|
|
created := false
|
|
return func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
if created {
|
|
return "true\t" + want, nil
|
|
}
|
|
return "true\tsome-older-spec", nil
|
|
case "image":
|
|
return "", nil
|
|
case "rm", "rename", "stop", "start":
|
|
*order = append(*order, strings.Join(args, " "))
|
|
return "", nil
|
|
case "run":
|
|
*order = append(*order, "run")
|
|
if failRun {
|
|
return "", errors.New("port is already allocated")
|
|
}
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
}
|
|
|
|
// A container whose replacement does not come up is put back: the running one is set aside (renamed and stopped), never
|
|
// removed before the new one runs, and started again under its name when the new one fails, the apply failing loudly.
|
|
// Removed first, the bus's own container left novox without a bus when its replacement was never made (issue 1945,
|
|
// live on 2026-10-25 at 11:23).
|
|
func TestAContainerWhoseReplacementFailsIsPutBack(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
var order []string
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, asideRun(t, want, true, &order), nil, nil)
|
|
if err == nil || !strings.Contains(err.Error(), "started again") {
|
|
t.Fatalf("a failed replacement: %v", err)
|
|
}
|
|
got := strings.Join(order, "; ")
|
|
if !strings.Contains(got, "rename store store.replaced; stop store.replaced; run") ||
|
|
!strings.Contains(got, "rename store.replaced store; start store") {
|
|
t.Fatalf("the steps were %q", got)
|
|
}
|
|
if strings.Contains(got, "rm -f store;") && strings.Index(got, "rm -f store;") < strings.Index(got, "run") {
|
|
t.Fatalf("the running container was removed before its replacement ran: %q", got)
|
|
}
|
|
}
|
|
|
|
// A replacement that runs removes the container set aside, and only after it runs.
|
|
func TestAReplacementThatRunsRemovesTheContainerSetAside(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
var order []string
|
|
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, asideRun(t, want, false, &order), nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := strings.Join(order, "; ")
|
|
run, removed := strings.Index(got, "run"), strings.LastIndex(got, "rm -f store.replaced")
|
|
if run < 0 || removed < run || strings.Contains(got, "start store") {
|
|
t.Fatalf("the steps were %q: the set-aside container is removed after the replacement runs", got)
|
|
}
|
|
}
|
|
|
|
// A container left set aside by an engine that stopped between setting it aside and running its replacement is the
|
|
// set-aside one of the next apply: put back when the replacement fails, removed once it runs (the review of issue
|
|
// 1945).
|
|
func TestAContainerLeftAsideIsPutBackOrRemovedByTheNextApply(t *testing.T) {
|
|
d := parseTrusted(t, `{"declaration":1,"resources":[
|
|
{"id":"store","type":"container","name":"store","image":"`+pinned+`","env":{"PGDATA":"/data"}}
|
|
]}`)
|
|
want := containerSpec(d.Resources[0].(*declaration.Container), inputs{})
|
|
for _, failRun := range []bool{true, false} {
|
|
var order []string
|
|
created := false
|
|
run := func(ctx context.Context, name string, args ...string) (string, error) {
|
|
switch args[0] {
|
|
case "info":
|
|
return "27.0\n", nil
|
|
case "container":
|
|
switch args[len(args)-1] {
|
|
case "store":
|
|
if created {
|
|
return "true\t" + want, nil
|
|
}
|
|
return "", errors.New("no such container")
|
|
case "store.replaced":
|
|
return "false\tsome-older-spec", nil
|
|
}
|
|
return "", errors.New("no such container")
|
|
case "image":
|
|
return "", nil
|
|
case "rm", "rename", "stop", "start":
|
|
order = append(order, strings.Join(args, " "))
|
|
return "", nil
|
|
case "run":
|
|
order = append(order, "run")
|
|
if failRun {
|
|
return "", errors.New("port is already allocated")
|
|
}
|
|
created = true
|
|
return "deadbeef\n", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
_, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil)
|
|
got := strings.Join(order, "; ")
|
|
if failRun {
|
|
if err == nil || !strings.Contains(got, "rename store.replaced store; start store") {
|
|
t.Fatalf("a failed run beside a container left aside: %v; steps %q", err, got)
|
|
}
|
|
continue
|
|
}
|
|
if err != nil || !strings.Contains(got, "run; rm -f store.replaced") {
|
|
t.Fatalf("a run beside a container left aside: %v; steps %q", err, got)
|
|
}
|
|
}
|
|
}
|