A scheduled step may hold its module's own containers still (hq ADR 0189)
while-stopped names resource ids of the same module's containers; the host stops them before the run and starts them again after it, in reverse order, whatever the step did. The restart is deferred before the first stop and runs on its own context, because the one real risk of this field is a window that never closes. Scheduled steps only: at apply the declaration is applied in order and a run-once step already gates what follows.
This commit is contained in:
@@ -65,6 +65,29 @@ type scheduledJob struct {
|
||||
container *declaration.Container
|
||||
next time.Time // the next minute at which it is due
|
||||
running bool // a run is in flight — the next due run is skipped rather than stacked
|
||||
// hold is the runtime names of the containers held still for the duration of a run, in the
|
||||
// order the step named them (novox/hq ADR 0189).
|
||||
hold []string
|
||||
}
|
||||
|
||||
// heldStillFor is the runtime names of the containers a step holds still, resolved from ids.
|
||||
func heldStillFor(step *declaration.Container, d *declaration.Declaration) []string {
|
||||
if len(step.WhileStopped) == 0 {
|
||||
return nil
|
||||
}
|
||||
byID := map[string]string{}
|
||||
for _, r := range d.Resources {
|
||||
if c, ok := r.(*declaration.Container); ok {
|
||||
byID[c.Identity()] = c.Name
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(step.WhileStopped))
|
||||
for _, id := range step.WhileStopped {
|
||||
if name := byID[id]; name != "" {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewScheduler builds a scheduler. A nil clock is the system clock; a nil log says nothing.
|
||||
@@ -123,15 +146,20 @@ func (s *Scheduler) Sync(d *declaration.Declaration, held map[string]bool) {
|
||||
// on its cadence rather than staying running to be restarted — so its identity cannot
|
||||
// depend on another resource's content and there is nothing to pass.
|
||||
spec := containerSpec(c, inputs{})
|
||||
// The runtime stops containers by name; the declaration names them by id. Resolved here,
|
||||
// against the declaration this job was armed from, so a fire never has to look anything up
|
||||
// (novox/hq ADR 0189). The parser has already refused an id that is not a container here.
|
||||
hold := heldStillFor(c, d)
|
||||
if existing := s.jobs[c.Identity()]; existing != nil && existing.spec == spec {
|
||||
// Unchanged: keep where it is in its cadence, refresh the declaration pointer only.
|
||||
existing.container = c
|
||||
existing.hold = hold
|
||||
continue
|
||||
}
|
||||
// New or changed: arm it for the next due minute after now.
|
||||
next, _ := cron.Next(s.clock.Now())
|
||||
s.jobs[c.Identity()] = &scheduledJob{
|
||||
id: c.Identity(), spec: spec, cron: cron, container: c, next: next,
|
||||
id: c.Identity(), spec: spec, cron: cron, container: c, next: next, hold: hold,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +230,15 @@ func (s *Scheduler) fire(ctx context.Context, j *scheduledJob) {
|
||||
return
|
||||
}
|
||||
|
||||
// **The window opens here and closes in the defer, whatever happens** (novox/hq ADR 0189).
|
||||
// Deferred before the first stop so a panic, a failing step or a step that runs long all end
|
||||
// the same way: the service running. The one real risk of this field is a window that never
|
||||
// closes, and the only defence against it is that closing is not conditional on anything.
|
||||
if len(j.hold) > 0 {
|
||||
defer s.letRun(ctx, cri, j)
|
||||
s.holdStill(ctx, cri, j)
|
||||
}
|
||||
|
||||
// A container by this name left exited by the previous run would collide with --name. Removing
|
||||
// one that is not there is the state we want, so its error is ignored — the same as run-once.
|
||||
_, _ = s.run(ctx, cri, "rm", "-f", j.container.Name)
|
||||
@@ -260,3 +297,51 @@ func (s *Scheduler) Run(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// holdStill stops the containers this step runs instead of, in the order it named them.
|
||||
//
|
||||
// A stop that fails is said and not fatal. The step runs anyway: for the case this exists for —
|
||||
// a collector walking storage nothing must be writing to — a writer that would not stop is worth
|
||||
// knowing about, and refusing to run would mean the work never happens and the log says nothing
|
||||
// new each night. What must not be skipped is the restart, and it is not: it is deferred.
|
||||
func (s *Scheduler) holdStill(ctx context.Context, cri string, j *scheduledJob) {
|
||||
for _, name := range j.hold {
|
||||
if _, err := s.run(ctx, cri, "stop", name); err != nil {
|
||||
s.log(fmt.Sprintf("scheduled step %s: could not stop %s for the run: %v", j.id, name, err))
|
||||
continue
|
||||
}
|
||||
s.log(fmt.Sprintf("scheduled step %s: %s held still for the run", j.id, name))
|
||||
}
|
||||
}
|
||||
|
||||
// letRun starts them again, in the reverse of the order they were stopped, and says so loudly if
|
||||
// one does not come back.
|
||||
//
|
||||
// **Reverse order**, because stopping walks a dependency the other way: a module that holds two
|
||||
// containers still names the one that depends on the other first, and bringing them back the same
|
||||
// way would start a dependant before what it depends on.
|
||||
//
|
||||
// Given its own context, because this runs in a defer and the one the run used may already be
|
||||
// cancelled — a host shutting down mid-window would otherwise leave the service stopped, which is
|
||||
// precisely the outcome this field must never have.
|
||||
func (s *Scheduler) letRun(_ context.Context, cri string, j *scheduledJob) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), closingWindow)
|
||||
defer cancel()
|
||||
for i := len(j.hold) - 1; i >= 0; i-- {
|
||||
name := j.hold[i]
|
||||
if _, err := s.run(ctx, cri, "start", name); err != nil {
|
||||
// Said as loudly as this host says anything: a service the mesh stopped for a
|
||||
// maintenance window and could not start again is down, and nothing else will notice
|
||||
// until the next apply compares it.
|
||||
s.log(fmt.Sprintf(
|
||||
"scheduled step %s: %s was held still for the run and WILL NOT START AGAIN: %v",
|
||||
j.id, name, err))
|
||||
continue
|
||||
}
|
||||
s.log(fmt.Sprintf("scheduled step %s: %s running again", j.id, name))
|
||||
}
|
||||
}
|
||||
|
||||
// closingWindow is how long the host will spend putting back what it stopped. Generous: this is
|
||||
// the half that must not be given up on.
|
||||
const closingWindow = 5 * time.Minute
|
||||
|
||||
Reference in New Issue
Block a user