review: a unit whose file the mesh wrote is stopped when undeclared, and what was found survives a failed first apply (hq ADR 0118)
Records written before Found existed left the adoption guard and the converge filter loaded on undeclare, then deleted their unit files from under them; a unit whose own file the mesh created is now the mesh's, whatever its record says. Found is kept apart the moment it is read, so a first apply that enabled and then failed is not read back as the machine's; boot is found the first time the mesh sets it; a service once stateless, or moved to another unit, is found afresh (the old unit given back). The unit is read after the reload that loads a file written in the same apply, and removal reports what it actually did.
This commit is contained in:
jochen
+201
-46
@@ -170,6 +170,14 @@ func ApplyKeeping(
|
||||
// as the service that took it over is (novox/hq ADR 0105).
|
||||
declared[takeOverID(svc)] = true
|
||||
}
|
||||
// What an unfinished apply found a unit as is kept only while its service is declared: one
|
||||
// declared again later is read afresh, as any resource with no record is (novox/hq ADR 0118).
|
||||
known.DropFoundUndeclared(declared, origin)
|
||||
|
||||
// Which units the mesh made, read before any removal can take a unit file's record away — so
|
||||
// whichever order a module declared a unit and its file in, the unit is known for the mesh's
|
||||
// when its service goes (novox/hq ADR 0118).
|
||||
made := meshMadeUnits(known)
|
||||
|
||||
// Which firewall is found here, before anything else, since an unsupported one refuses the
|
||||
// whole declaration (novox/hq ADR 0100). Nothing for a converged node.
|
||||
@@ -188,7 +196,7 @@ func ApplyKeeping(
|
||||
if declaration.Type(orphan.Type) == declaration.TypeOpening {
|
||||
action, detail, err = removeOpening(ctx, orphan, run, known.Firewall)
|
||||
} else {
|
||||
action, detail, err = remove(ctx, sys, orphan, run)
|
||||
action, detail, err = remove(ctx, sys, orphan, run, made)
|
||||
}
|
||||
if err != nil {
|
||||
return &Error{Resource: orphan.ID, Err: err, Done: report}
|
||||
@@ -385,6 +393,14 @@ func ApplyKeeping(
|
||||
}
|
||||
|
||||
was, _ := known.Find(resource.Identity())
|
||||
// What an earlier apply of this service found its unit as and could not yet record is
|
||||
// newer than anything the record says — the record's own finding with what was read
|
||||
// since — so it is what this apply goes on from (novox/hq ADR 0118).
|
||||
previous := was
|
||||
if p, ok := known.FoundFirst[resource.Identity()]; ok {
|
||||
f := p.FoundUnit
|
||||
previous.Found = &f
|
||||
}
|
||||
var outcome Outcome
|
||||
var err error
|
||||
if o, isOpening := resource.(*declaration.Opening); isOpening {
|
||||
@@ -397,9 +413,15 @@ func ApplyKeeping(
|
||||
!known.Recorded(string(declaration.TypeFile), f.Path) {
|
||||
keepFound = keep
|
||||
}
|
||||
outcome, err = applyOne(ctx, sys, resource, run, changed, in, was, unseal, keepFound)
|
||||
outcome, err = applyOne(ctx, sys, resource, run, changed, in, previous, unseal, keepFound)
|
||||
}
|
||||
if err != nil {
|
||||
// **What was found is kept whatever the apply then did** (novox/hq ADR 0118). The
|
||||
// failure may have come after the mesh enabled or started the unit, and the next apply
|
||||
// would otherwise read that as the machine's own.
|
||||
if outcome.found != nil {
|
||||
known.KeepFound(resource.Identity(), origin, *outcome.found)
|
||||
}
|
||||
failed := &Error{Resource: resource.Identity(), Err: err, Done: report}
|
||||
failures = append(failures, failed)
|
||||
log(fmt.Sprintf(" failed %s (%s): %v", resource.Identity(), outcome.Target, err))
|
||||
@@ -470,6 +492,9 @@ func ApplyKeeping(
|
||||
Found: outcome.found,
|
||||
Holds: holds(resource),
|
||||
})
|
||||
if outcome.found != nil {
|
||||
known.DropFound(resource.Identity())
|
||||
}
|
||||
// Its module has been taken, and what was held for it is now the mesh's. A file written
|
||||
// into, or a service whose lifecycle is the machine's, replaced nothing that was found, so
|
||||
// its outcome says what the apply did, not that a cutover happened; a hold from when it was
|
||||
@@ -930,29 +955,6 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
||||
return reflectOnly(ctx, sys, r, run, changed)
|
||||
}
|
||||
out := begin(r)
|
||||
|
||||
// **What the unit was before the mesh touched it**, read once — the first time this host
|
||||
// applies it — and carried in the record from then on (novox/hq ADR 0118). Undeclared, the
|
||||
// unit is given back to exactly this: it is the one fact that separates the container runtime,
|
||||
// running before the mesh arrived and to be left running, from the mesh's packet filter,
|
||||
// stopped until the mesh started it and to be stopped again. A record that predates this
|
||||
// field is not read now: the unit's state by then is the mesh's doing, not what was found.
|
||||
switch {
|
||||
case previous.Found != nil:
|
||||
out.found = previous.Found
|
||||
case previous.ID == "":
|
||||
state, err := sys.ServiceState(ctx, run, r.Unit)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
found := &store.FoundUnit{State: state}
|
||||
if r.Boot != "" {
|
||||
if found.Boot, err = sys.ServiceBoot(ctx, run, r.Unit); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
out.found = found
|
||||
}
|
||||
var changes []string
|
||||
|
||||
// A file the service reflects changed, and it may be the unit's own file or a drop-in: the
|
||||
@@ -966,6 +968,21 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
||||
}
|
||||
}
|
||||
|
||||
// **What the unit was before the mesh touched it**, read once and carried in the record from
|
||||
// then on (novox/hq ADR 0118). Undeclared, the unit is given back to exactly this: it is the one
|
||||
// fact that separates the container runtime, running before the mesh arrived and to be left
|
||||
// running, from the mesh's packet filter, stopped until the mesh started it and to be stopped
|
||||
// again. Read after the reload above, never before it: a unit whose file this same apply wrote
|
||||
// is not a unit the service manager knows until then, and reading it first would find nothing.
|
||||
found, gaveBack, err := foundAs(ctx, sys, r, run, previous)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.found = found
|
||||
if gaveBack != "" {
|
||||
changes = append(changes, gaveBack)
|
||||
}
|
||||
|
||||
// Boot first. A unit asked to be running and enabled should survive this apply failing
|
||||
// half way in the more useful direction: enabled-and-stopped comes back at the next boot,
|
||||
// where running-and-disabled does not.
|
||||
@@ -974,6 +991,15 @@ func applyService(ctx context.Context, sys system.System, r *declaration.Service
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
// Whether it started at boot is found the first time the mesh is about to change that —
|
||||
// which is not always the first apply: a declaration that said nothing about boot never
|
||||
// touched it, so what is there when one first does is still the machine's (novox/hq ADR
|
||||
// 0118). Kept before the change, so a failure after it still has it.
|
||||
if out.found != nil && out.found.Boot == "" {
|
||||
f := *out.found
|
||||
f.Boot = bootBefore
|
||||
out.found = &f
|
||||
}
|
||||
if bootBefore != r.Boot {
|
||||
if err := sys.SetServiceBoot(ctx, run, r.Unit, r.Boot); err != nil {
|
||||
return out, fmt.Errorf("setting %s to %s at boot: %w", r.Unit, r.Boot, err)
|
||||
@@ -1138,7 +1164,10 @@ func reflectOnly(ctx context.Context, sys system.System, r *declaration.Service,
|
||||
// It returns the action rather than assuming "removed", because for half the vocabulary the
|
||||
// honest word is "forgotten". A host that reported a package removed when it left the package
|
||||
// installed would be describing an effect it declined to have.
|
||||
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) (string, string, error) {
|
||||
//
|
||||
// made is the units whose unit file this host wrote where there was none (meshMadeUnits).
|
||||
func remove(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
made map[string]bool) (string, string, error) {
|
||||
switch declaration.Type(a.Type) {
|
||||
case declaration.TypeDirectory:
|
||||
// **A directory with anything left in it is kept, and that is the rule that protects
|
||||
@@ -1186,7 +1215,7 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner)
|
||||
return "removed", "no longer declared", nil
|
||||
|
||||
case declaration.TypeService:
|
||||
return removeService(ctx, sys, a, run)
|
||||
return removeService(ctx, sys, a, run, made[a.Target])
|
||||
|
||||
case declaration.TypeProcess:
|
||||
// The other side of the same line: a process's unit is the host's own — it wrote the unit
|
||||
@@ -2036,45 +2065,171 @@ func declaredDigest(r declaration.Resource) string {
|
||||
// network stopped the container runtime and every container with it, how unassigning sshd would
|
||||
// have stopped ssh, and how an uplink module would have taken a machine off its only link
|
||||
// (novox/hq issue 130). It means undoing what the mesh did to it: a unit found running is left
|
||||
// running; a unit the mesh started — the packet filter a converge loaded, which returning to
|
||||
// adopted must unload — is stopped again, and disabled again if the mesh enabled it.
|
||||
// running; a unit the mesh started is stopped again, and disabled again if the mesh enabled it.
|
||||
//
|
||||
// **A unit whose file the mesh wrote is the mesh's, whatever was found** — made is that. The
|
||||
// adoption guard and the converge filter are units of exactly this kind: their unit files are the
|
||||
// mesh's own `file` resources, written where there was none, and records written before the host
|
||||
// kept what it found say nothing about them. Forgetting one would leave its table loaded — the
|
||||
// guard beside a converged node's own filter, or the filter beside the predecessor's firewall
|
||||
// re-enabled — and its unit file then deleted from under a running unit. So it is stopped and
|
||||
// disabled at boot, as a process's unit is, before its file goes: orphans are removed newest
|
||||
// first, and a unit's file is declared before the service that starts it.
|
||||
//
|
||||
// **Never started on the way out.** A unit the mesh stopped is not started again when its
|
||||
// declaration goes: starting something is a decision, and the operator makes it.
|
||||
func removeService(ctx context.Context, sys system.System, a store.Applied, run Runner) (string, string, error) {
|
||||
// declaration goes: starting something is a decision, and the operator makes it. The report says
|
||||
// what was actually done — a unit already as it was found is forgotten, not "restored".
|
||||
func removeService(ctx context.Context, sys system.System, a store.Applied, run Runner,
|
||||
made bool) (string, string, error) {
|
||||
if a.Stateless {
|
||||
// Declared with no state (novox/hq ADR 0117): its lifecycle was never the mesh's, and the
|
||||
// declaration that said so is the operator's word to hold to, a file of the mesh's or not.
|
||||
return "forgotten", "its state was never the mesh's", nil
|
||||
}
|
||||
if a.Found == nil {
|
||||
if !made && a.Found == nil {
|
||||
// Recorded before the host kept what it found. Not knowing, it leaves the unit as it is:
|
||||
// a unit left running can be stopped by the operator, and one stopped by mistake may be
|
||||
// the link the operator would use to do it.
|
||||
return "forgotten", "the unit is the machine's; left as it is", nil
|
||||
return "forgotten", "recorded before the host kept what it found; left as it is", nil
|
||||
}
|
||||
if a.Found.State != "stopped" && (a.Found.Boot == "" || a.Found.Boot == "enabled") {
|
||||
stop := made || a.Found.State == "stopped"
|
||||
disable := made || a.Found.Boot == "disabled"
|
||||
|
||||
if !stop && !disable {
|
||||
// Found running, and not disabled by anyone but the mesh: nothing to give back. What the
|
||||
// mesh did since is said, so a unit left stopped is not reported as the machine's doing —
|
||||
// read as well as the machine answers, since nothing here acts on the answer.
|
||||
state, err := sys.ServiceState(ctx, run, a.Target)
|
||||
if err != nil && strings.Contains(err.Error(), "does not exist on this machine") {
|
||||
return "forgotten", "the unit no longer exists", nil
|
||||
}
|
||||
var did []string
|
||||
if err == nil && state == "stopped" {
|
||||
did = append(did, "stopped it")
|
||||
}
|
||||
if a.Found.Boot == "enabled" {
|
||||
if boot, err := sys.ServiceBoot(ctx, run, a.Target); err == nil && boot == "disabled" {
|
||||
did = append(did, "disabled it at boot")
|
||||
}
|
||||
}
|
||||
if len(did) > 0 {
|
||||
return "forgotten", "left as it is; the mesh " + strings.Join(did, " and ") +
|
||||
" and does not start anything on the way out", nil
|
||||
}
|
||||
return "forgotten", "it was running before the mesh; left as it is", nil
|
||||
}
|
||||
// Something is to be given back, so the unit must still be there to give it to. One since
|
||||
|
||||
// Something may be given back, so the unit must still be there to give it to. One since
|
||||
// uninstalled is already as far from the mesh as it can be, and saying so keeps a record of it
|
||||
// from failing every apply.
|
||||
if _, err := sys.ServiceState(ctx, run, a.Target); err != nil {
|
||||
state, err := sys.ServiceState(ctx, run, a.Target)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "does not exist on this machine") {
|
||||
return "forgotten", "the unit no longer exists", nil
|
||||
}
|
||||
return "", "", err
|
||||
}
|
||||
var gave []string
|
||||
if a.Found.State == "stopped" {
|
||||
if err := sys.SetServiceState(ctx, run, a.Target, "stopped"); err != nil {
|
||||
return "", "", fmt.Errorf("stopping %s, which the mesh started: %w", a.Target, err)
|
||||
var did, already []string
|
||||
if stop {
|
||||
if state != "stopped" {
|
||||
if err := sys.SetServiceState(ctx, run, a.Target, "stopped"); err != nil {
|
||||
return "", "", fmt.Errorf("stopping %s, which the mesh started: %w", a.Target, err)
|
||||
}
|
||||
did = append(did, "stopped")
|
||||
} else {
|
||||
already = append(already, "stopped")
|
||||
}
|
||||
gave = append(gave, "stopped again")
|
||||
}
|
||||
if a.Found.Boot == "disabled" {
|
||||
if err := sys.SetServiceBoot(ctx, run, a.Target, "disabled"); err != nil {
|
||||
return "", "", fmt.Errorf("disabling %s, which the mesh enabled: %w", a.Target, err)
|
||||
if disable {
|
||||
// Disabled unless it reads as disabled: a unit the service manager cannot say a boot state
|
||||
// for — one with no install section — takes a disable as a no-op, and asking is how the
|
||||
// host stays sure the mesh's enable did not outlive it.
|
||||
if boot, err := sys.ServiceBoot(ctx, run, a.Target); err == nil && boot == "disabled" {
|
||||
already = append(already, "disabled at boot")
|
||||
} else {
|
||||
if err := sys.SetServiceBoot(ctx, run, a.Target, "disabled"); err != nil {
|
||||
return "", "", fmt.Errorf("disabling %s, which the mesh enabled: %w", a.Target, err)
|
||||
}
|
||||
did = append(did, "disabled at boot")
|
||||
}
|
||||
gave = append(gave, "disabled at boot again")
|
||||
}
|
||||
return "restored", strings.Join(gave, ", ") + ", as the host found it", nil
|
||||
|
||||
if made {
|
||||
if len(did) == 0 {
|
||||
return "forgotten", "already stopped and disabled at boot; the mesh wrote its unit file", nil
|
||||
}
|
||||
return "removed", strings.Join(did, ", ") + "; the mesh wrote its unit file, so the unit is the mesh's own", nil
|
||||
}
|
||||
if len(did) == 0 {
|
||||
return "forgotten", "already as the host found it (" + strings.Join(already, ", ") + ")", nil
|
||||
}
|
||||
detail := strings.Join(did, ", ") + ", as the host found it"
|
||||
if len(already) > 0 {
|
||||
detail += "; already " + strings.Join(already, ", ")
|
||||
}
|
||||
return "restored", detail, nil
|
||||
}
|
||||
|
||||
// foundAs is what a service's unit was before the mesh touched it, as far as this host can know:
|
||||
// what an earlier apply recorded, or — the first time the mesh is about to act on the unit — what
|
||||
// is there now (novox/hq ADR 0118). Nil when it cannot be known: a record written before the host
|
||||
// kept this has had the mesh acting on the unit since, and a reading now would be the mesh's doing.
|
||||
//
|
||||
// Read now as well, besides on a first apply, where the mesh has never acted on this unit's state
|
||||
// although a record exists: the record is of a service declared with no state (novox/hq ADR 0117),
|
||||
// which the mesh never starts or stops, or of another unit altogether. What was found about one
|
||||
// unit says nothing about another, so a service moved to a new unit gives the old one back, just as
|
||||
// if it had been undeclared, and is read afresh for the new one; gave says what that gave back.
|
||||
func foundAs(ctx context.Context, sys system.System, r *declaration.Service, run Runner,
|
||||
previous store.Applied) (found *store.FoundUnit, gave string, err error) {
|
||||
if f := previous.Found; f != nil {
|
||||
unit := f.Unit
|
||||
if unit == "" {
|
||||
unit = previous.Target
|
||||
}
|
||||
if unit == "" || unit == r.Unit {
|
||||
return f, "", nil
|
||||
}
|
||||
// Given back as if undeclared, never as the mesh's own: whether the mesh wrote the old
|
||||
// unit's file is known to the removal of orphans, and that file's own record goes with it.
|
||||
action, detail, err := removeService(ctx, sys,
|
||||
store.Applied{Type: string(declaration.TypeService), Target: unit, Found: f}, run, false)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("giving %s back as the host found it, now that %s is declared instead: %w",
|
||||
unit, r.Unit, err)
|
||||
}
|
||||
if action == "restored" {
|
||||
gave = unit + " " + detail
|
||||
}
|
||||
} else if previous.ID != "" && !previous.Stateless && previous.Target == r.Unit {
|
||||
return nil, "", nil
|
||||
}
|
||||
state, err := sys.ServiceState(ctx, run, r.Unit)
|
||||
if err != nil {
|
||||
return nil, gave, err
|
||||
}
|
||||
return &store.FoundUnit{Unit: r.Unit, State: state}, gave, nil
|
||||
}
|
||||
|
||||
// meshMadeUnits is every unit whose unit file this host wrote whole where there was none: a `file`
|
||||
// record with no kept original and not written into, at a unit's own path under a directory the
|
||||
// service manager loads administrators' units from — or the one the host writes a process's unit
|
||||
// in. Such a unit is the mesh's, whatever was found (novox/hq ADR 0118); see removeService.
|
||||
//
|
||||
// A drop-in is not the unit's own file, and a file the host wrote over is the machine's unit with
|
||||
// the mesh's text in it: its original is kept, and put back when the file's record goes.
|
||||
func meshMadeUnits(known store.State) map[string]bool {
|
||||
made := map[string]bool{}
|
||||
dirs := map[string]bool{"/etc/systemd/system": true, "/run/systemd/system": true,
|
||||
filepath.Clean(unitDir): true}
|
||||
for _, r := range known.Resources {
|
||||
if r.Type != string(declaration.TypeFile) || r.Kept != "" || r.Into != nil {
|
||||
continue
|
||||
}
|
||||
path := filepath.Clean(r.Target)
|
||||
if dirs[filepath.Dir(path)] {
|
||||
made[filepath.Base(path)] = true
|
||||
}
|
||||
}
|
||||
return made
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user