novox/mesh-host
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Recreate a container when the content of a file it reads at creation changes

Browse Source 
The host decided whether a container was still the one declared by a digest
of its declaration, and the declaration names an env-file's path and a
mount's path — never what is in them. So when the store was given a new
port, the host rewrote the forge's and the analytics service's environment
files, correctly, and left both containers running with the old port in
their environment: a container reads its env-file when it is CREATED, and
`docker restart` hands it the same environment again. Both looked healthy
until they answered 502.

What a running container takes in at creation is now part of its spec, by
content: every env-file, a file bind-mounted into it, and every file this
host wrote at or under a directory bind-mounted into it — the secrets,
bindings and configs under a module's state directories. The digest is the
one the store already records for a file the host wrote (`wrote`), read
from the state as it stands when the container is reached, so a file
rewritten earlier in the same apply is already the new one; a file the host
has no record of — an env-file a predecessor left, the superuser secret
genesis writes before any declaration names it — is read from disk, which
is what keeps adopting a running store in place a reconcile and not a
recreate.

Deliberately not part of it: what else is in a bind-mounted directory,
which is the service's own data and changes while it runs; a named volume;
a seed created once, which digests as the seed the host wrote and not as
what has grown in it; and a step — a run-once or scheduled container reads
its files when it runs and runs fresh each time. On an adopted node a held
container is held before any of this is looked at.

The host records what each container was created reading, per file, so
the recreate can say which file changed — "recreated: <file> changed" in
the report and, now with its detail, in the log. A container made before
this record existed is recreated once and says so.

novox/hq 04-ISSUES/103
This commit is contained in:
jschoubben
2026-09-23 23:12:52 +02:00
parent 9176aea6c4
commit c0d94e04f3
7 changed files with 423 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/apply/apply.go
+151 -11
View File
@@ -52,6 +52,9 @@ type Outcome struct {
wrote string
// into is what a file written into held before the mesh's keys (novox/hq ADR 0102).
into *store.Into
// reads is, for a container, the digest of each file it was created reading, by path — so
// the next apply can say which one changed (novox/hq 04-ISSUES/103).
reads map[string]string
}
// Report is what an apply did, in the order it did it.
@@ -266,9 +269,16 @@ func ApplyKeeping(
// machine reports success, and what is inside is using a credential the mesh has replaced
// (novox/hq 04-ISSUES/045). Folding these into the container's spec makes the comparison a
// standing one instead.
declares := map[string]string{}
//
// And what each file a container reads at creation holds — its env-files and what is mounted
// into it — by the digest this host recorded when it wrote the file, read from `known` as it
// stands when the container is reached, so a file rewritten earlier in this same apply is
// already the new one (novox/hq 04-ISSUES/103). That needs the file applied before the
// container, which is the declared order; a container declared ahead of its file sees the
// change one apply late, and never misses it.
in := inputs{declares: map[string]string{}, known: &known}
for _, resource := range d.Resources {
declares[resource.Identity()] = declaredDigest(resource)
in.declares[resource.Identity()] = declaredDigest(resource)
}
// Everything is attempted, and every failure is reported.
@@ -340,7 +350,7 @@ func ApplyKeeping(
!known.Recorded(string(declaration.TypeFile), f.Path) {
keepFound = keep
}
outcome, err = applyOne(ctx, sys, resource, run, changed, declares, was, unseal, keepFound)
outcome, err = applyOne(ctx, sys, resource, run, changed, in, was, unseal, keepFound)
}
if err != nil {
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
@@ -386,6 +396,7 @@ func ApplyKeeping(
Target: outcome.Target, AppliedAt: time.Now().UTC(),
Wrote: outcome.wrote,
Into: outcome.into,
Reads: outcome.reads,
Holds: holds(resource),
})
// Its module has been taken, and what was held for it is now the mesh's.
@@ -396,7 +407,14 @@ func ApplyKeeping(
report.Outcomes = append(report.Outcomes, outcome)
if outcome.Action != "unchanged" {
changed[resource.Identity()] = true
log(fmt.Sprintf(" %s %s (%s)", outcome.Action, outcome.ID, outcome.Target))
// With the detail, when there is one: "updated app" says a container was replaced;
// which file made that happen is what somebody reading the log at the time needs
// (novox/hq 04-ISSUES/103).
line := fmt.Sprintf(" %s %s (%s)", outcome.Action, outcome.ID, outcome.Target)
if outcome.Detail != "" {
line += ": " + outcome.Detail
}
log(line)
}
}
@@ -440,7 +458,7 @@ const guardPrefix = declaration.AdoptionPrefix + "guard"
type Unseal func(sealed string) ([]byte, error)
func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner,
changed map[string]bool, declares map[string]string, previous store.Applied,
changed map[string]bool, in inputs, previous store.Applied,
unseal Unseal, keepFound Keep) (Outcome, error) {
switch res := r.(type) {
case *declaration.Directory:
@@ -452,7 +470,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru
case *declaration.Package:
return applyPackage(ctx, sys, res, run)
case *declaration.Container:
return applyContainer(ctx, res, run, changed, declares, previous)
return applyContainer(ctx, res, run, changed, in, previous)
case *declaration.User:
return applyUser(ctx, sys, res, run)
case *declaration.Archive:
@@ -1129,9 +1147,99 @@ const (
idLabel = "mesh-host.id"
)
// inputs is what a container takes in when it is created beyond its own declaration: what each
// resource it names under restart-on currently declares, and what the files it reads hold.
type inputs struct {
// declares is each resource's declared digest, by id (declaredDigest).
declares map[string]string
// known is the node's state as it stands when the container is reached — so a file applied
// earlier in the same pass is already its new self. Nil where nothing was written: a test,
// or a scheduled fire, which reads no file at creation.
known *store.State
}
// fileDigest is what a file the container reads holds, by digest.
//
// **What this host wrote when it has a record of writing it, and what is on disk when it has
// not.** The record is preferred because it is what the host means by the file: a seed created
// once digests as the seed, not as whatever the service has grown in it, and a file written into
// digests as the mesh's keys, not the machine's (novox/hq ADR 0102). A file the host never wrote
// — an env-file a predecessor left, the superuser secret genesis writes before any declaration
// names it — is read, so the digest is the same one the host records when it later writes the
// same bytes there, and adopting a running store in place stays a reconcile rather than a
// recreate (bootstrap phase three). Empty when there is nothing readable there: the runtime
// refuses an absent env-file itself, with a better message than this could give.
func (in inputs) fileDigest(path string) string {
if in.known != nil {
for _, f := range in.known.FilesUnder(path) {
if f.Target == path {
return f.Wrote
}
}
}
info, err := os.Stat(path)
if err != nil || !info.Mode().IsRegular() {
return ""
}
content, err := os.ReadFile(path)
if err != nil {
return ""
}
return digestOf(string(content))
}
// reads is every file a running container takes in when it is created, by path and digest
// (novox/hq 04-ISSUES/103).
//
// - every env-file: the runtime reads it once, at create, and `docker restart` hands the
// container the same environment it had.
// - a file bind-mounted into it, by its content — and, for a directory bind-mounted into it,
// every file THIS HOST wrote at or beneath the source: the secrets, bindings and configs it
// put under the module's state directories. What else is in a mounted directory is the
// service's own data, which changes while it runs and is nothing to recreate it for.
//
// A step is not here: a run-once or scheduled container reads its files when it runs, and
// runs fresh each time. Only a container that stays running holds what it read.
func (in inputs) reads(r *declaration.Container) map[string]string {
if r.RunOnce || r.Schedule != "" {
return nil
}
out := map[string]string{}
for _, path := range r.EnvFile {
out[path] = in.fileDigest(path)
}
for _, v := range r.Volumes {
src := mountSource(v)
if !strings.HasPrefix(src, "/") {
continue // a named volume: the runtime's, holding data
}
if in.known != nil {
for _, f := range in.known.FilesUnder(src) {
out[f.Target] = f.Wrote
}
}
if _, recorded := out[src]; !recorded {
if digest := in.fileDigest(src); digest != "" {
out[src] = digest
}
}
}
if len(out) == 0 {
return nil
}
return out
}
// containerSpec is the identity of a declared container: everything that, if changed, means
// the running container is no longer what was asked for.
func containerSpec(r *declaration.Container, declares map[string]string) string {
func containerSpec(r *declaration.Container, in inputs) string {
return containerSpecReading(r, in.declares, in.reads(r))
}
// containerSpecReading is containerSpec with what the container reads already read — so an
// applier that also records those digests reads each file once, and the label and the record
// cannot disagree about a file that moved between two reads.
func containerSpecReading(r *declaration.Container, declares, reads map[string]string) string {
keys := make([]string, 0, len(r.Env))
for k := range r.Env {
keys = append(keys, k)
@@ -1171,6 +1279,15 @@ func containerSpec(r *declaration.Container, declares map[string]string) string
for _, id := range depends {
b.WriteString("reads " + id + "=" + declares[id] + "\n")
}
// And what the files it reads at creation hold — not only their paths, which `volume` and the
// env-file arguments already name. The spec named the env-file's path and not its content,
// so the host rewrote two environment files with the store's new port and left both
// containers running with the old one, healthy-looking, until they answered 502 (novox/hq
// 04-ISSUES/103). Added only when there is something read, so a container that reads nothing
// keeps the digest it had.
for _, path := range sortedKeys(reads) {
b.WriteString("file " + path + "=" + reads[path] + "\n")
}
return fmt.Sprintf("%x", sha256.Sum256([]byte(b.String())))
}
@@ -1236,9 +1353,12 @@ func applyNetwork(ctx context.Context, r *declaration.Network, run Runner) (Outc
}
func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
changed map[string]bool, declares map[string]string, previous store.Applied) (Outcome, error) {
changed map[string]bool, in inputs, previous store.Applied) (Outcome, error) {
out := begin(r)
want := containerSpec(r, declares)
// Read once, so the spec and the record agree on what was read even if a file moves under them.
reads := in.reads(r)
want := containerSpecReading(r, in.declares, reads)
out.reads = reads
cri, err := containerRuntime(ctx, run)
if err != nil {
@@ -1279,6 +1399,16 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
// already has.
reasons := restartedBy(r.RestartOn, changed)
// Which of the files it reads no longer hold what it was created reading. The spec label says
// only that SOMETHING moved; the record of what was read says what — and that is the line a
// person needs when a service went stale without a word (novox/hq 04-ISSUES/103).
var changedFiles []string
for _, path := range sortedKeys(previous.Reads) {
if now, still := reads[path]; still && now != previous.Reads[path] {
changedFiles = append(changedFiles, path)
}
}
switch {
case existed && before.Spec == want && before.Running && len(reasons) == 0:
out.Action = "unchanged"
@@ -1340,9 +1470,19 @@ func applyContainer(ctx context.Context, r *declaration.Container, run Runner,
out.Action = "created"
if existed {
out.Action = "updated"
if len(reasons) > 0 {
switch {
case len(changedFiles) > 0:
out.Detail = "recreated: " + strings.Join(changedFiles, ", ") + " changed"
if len(reasons) > 0 {
out.Detail += "; and to pick up " + strings.Join(reasons, ", ")
}
case len(reasons) > 0:
out.Detail = "recreated to pick up " + strings.Join(reasons, ", ")
} else {
case previous.Reads == nil && len(reads) > 0:
// A container made before this host kept what it read: whether the files it holds
// are the ones on disk cannot be known, so it is recreated once and from now on can.
out.Detail = "recreated: what it reads was not on record, so what it holds could not be checked"
default:
out.Detail = "replaced; a container's configuration is fixed when it is created"
}
}