303 lines
11 KiB
Go
303 lines
11 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-host/internal/declaration"
|
|
"github.com/novox/mesh-host/internal/system"
|
|
)
|
|
|
|
// Retired is what step 10 did.
|
|
type Retired struct {
|
|
// Container is the temporary control plane that was dropped.
|
|
Container string
|
|
// Gone is true when the machine no longer has it.
|
|
Gone bool
|
|
// Already is true when it was gone before this step ran.
|
|
Already bool
|
|
// Bundle is where the bundle without it was written.
|
|
Bundle string
|
|
// Removed is how many resources the re-apply reported removing.
|
|
Removed int
|
|
}
|
|
|
|
// RetireTheTemporaryControlPlane drops it from the bundle and lets the host take it away.
|
|
//
|
|
// **Destruction by omission, which is the host's ordinary behaviour and not a new mechanism.** The
|
|
// host owns what it has applied and removes what it owns and is no longer declared. So retiring the
|
|
// temporary control plane is not a verb anybody had to invent: the bundle stops declaring it, the
|
|
// bundle is applied again, and the removal pass does what it does for every other resource that
|
|
// leaves a declaration.
|
|
//
|
|
// That is the whole of why the rename at step 3 mattered. Had the foundation and the module both
|
|
// called their container `mesh-controller`, this apply would have removed the module's container —
|
|
// the host would have been asked to take away something it believed it owned, and it would have
|
|
// been right. Two names, two owners, and the removal is unambiguous.
|
|
//
|
|
// **It is the last step for a reason.** Until step 9 has a control plane that answers, the
|
|
// temporary one is the only thing that can tell this machine anything, and a machine left with no
|
|
// control plane cannot be fixed remotely (novox/hq ADR 0067). So this runs after the permanent one
|
|
// has been proved, and a run interrupted before it leaves two control planes, which is untidy and
|
|
// harmless — a re-run reaches this step and finishes.
|
|
func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.System,
|
|
produced []byte, run Runner, say func(string)) (Retired, error) {
|
|
|
|
out := Retired{Bundle: o.Out}
|
|
|
|
current, err := declaration.ParseFileTrusted(produced)
|
|
if err != nil {
|
|
return out, fmt.Errorf("the bundle this installer produced is not a declaration: %w", err)
|
|
}
|
|
temporary, err := controlPlaneIn(current)
|
|
if err != nil {
|
|
// The bundle already declares no control plane, which is what a re-run after this step
|
|
// finds. Nothing to drop, and nothing to be alarmed about.
|
|
say(" already dropped the bundle declares no temporary control plane")
|
|
out.Already = true
|
|
return out, nil
|
|
}
|
|
out.Container = temporary.Name
|
|
|
|
// Textual, for the reason the rewrite at step 3 is textual: the produced bundle is meant to be
|
|
// READ, and a person coming to a machine after a pivot should be able to open the file the
|
|
// installer applied and see the foundation they recognise with the control plane gone from it.
|
|
// Re-serialising a parsed declaration would drop every comment in it.
|
|
bundle, err := removeResource(produced, ControlPlaneID)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
without, err := declaration.ParseFileTrusted(bundle)
|
|
if err != nil {
|
|
return out, fmt.Errorf(
|
|
"taking the temporary control plane out of the bundle broke it: %w", err)
|
|
}
|
|
if _, err := controlPlaneIn(without); err == nil {
|
|
return out, fmt.Errorf(
|
|
"the bundle still declares %q after it was taken out, so nothing was removed and the "+
|
|
"apply below would change nothing", ControlPlaneID)
|
|
}
|
|
if err := writeBundleFile(o.Out, bundle); err != nil {
|
|
return out, err
|
|
}
|
|
say(fmt.Sprintf(" wrote %s (%d resources) — without %s",
|
|
o.Out, len(without.Resources), temporary.Name))
|
|
|
|
// Applied the same way everything else here is applied, under the same origin, against the
|
|
// same state file. What makes this a removal rather than a no-op is that the state file
|
|
// records the container as something this installer applied, and the declaration no longer
|
|
// asks for it.
|
|
report, err := ApplyBundle(ctx, o, sys, without, run, say)
|
|
if err != nil {
|
|
return out, fmt.Errorf(
|
|
"%w\n\nThe permanent control plane is running and the temporary one is still here. "+
|
|
"That is untidy and it is not broken: two control planes on one mesh are both "+
|
|
"stateless and both correct. Run this installer again to finish", err)
|
|
}
|
|
for _, outcome := range report.Outcomes {
|
|
if outcome.Action == "removed" {
|
|
out.Removed++
|
|
}
|
|
}
|
|
|
|
// Read back. A removal that reported success and left the container running would leave two
|
|
// control planes consuming the same broker queues for ever, which is the state this step
|
|
// exists to end.
|
|
gone, err := isGone(ctx, run, o.Timeout, o.Wait, temporary.Name)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
out.Gone = gone
|
|
if !gone {
|
|
return out, fmt.Errorf(
|
|
"the apply reported the temporary control plane removed and %q is still running.\n"+
|
|
"Two control planes are consuming this mesh's broker queues. Neither is wrong and "+
|
|
"the mesh is not damaged, but the pivot is not finished: `docker rm -f %s` ends "+
|
|
"it, and this installer will then agree", temporary.Name, temporary.Name)
|
|
}
|
|
say(" gone " + temporary.Name)
|
|
return out, nil
|
|
}
|
|
|
|
// removeResource takes one resource out of a bundle's text, comments and all.
|
|
//
|
|
// It walks the `resources` array counting braces, skipping over strings and comments so that a
|
|
// `//` inside a connection string is not read as the start of one — the foundation's own bundle
|
|
// contains `postgres://…` several times, and a scanner that did not know the difference would
|
|
// treat the rest of the line as a comment and lose a brace.
|
|
//
|
|
// What is removed is the element AND whatever precedes it back to the previous element, which is
|
|
// where the comment explaining it lives. A comment that outlives the thing it describes is worse
|
|
// than no comment: it is the file telling somebody the machine has a control plane it does not.
|
|
func removeResource(bundle []byte, id string) ([]byte, error) {
|
|
previous, from, to, err := resourceAt(bundle, id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return cut(bundle, previous, from, to), nil
|
|
}
|
|
|
|
// resourceAt finds one resource's object in a bundle's text by its id: where the one before it
|
|
// ended, and where it starts and ends — comments and strings skipped, so an id quoted in a comment
|
|
// or a command is never mistaken for the resource.
|
|
func resourceAt(bundle []byte, id string) (previous, from, to int, err error) {
|
|
array := indexOutsideStrings(bundle, `"resources"`)
|
|
if array < 0 {
|
|
return 0, 0, 0, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it")
|
|
}
|
|
open := indexOutsideStrings(bundle[array:], "[")
|
|
if open < 0 {
|
|
return 0, 0, 0, fmt.Errorf("this bundle's resources are not a list")
|
|
}
|
|
open += array
|
|
|
|
depth := 0
|
|
from, previous = -1, open
|
|
inString, escaped, inLine, inBlock := false, false, false, false
|
|
for i := open + 1; i < len(bundle); i++ {
|
|
c := bundle[i]
|
|
switch {
|
|
case escaped:
|
|
escaped = false
|
|
case inString && c == '\\':
|
|
escaped = true
|
|
case inString:
|
|
if c == '"' {
|
|
inString = false
|
|
}
|
|
case inLine:
|
|
if c == '\n' {
|
|
inLine = false
|
|
}
|
|
case inBlock:
|
|
if c == '*' && i+1 < len(bundle) && bundle[i+1] == '/' {
|
|
inBlock, i = false, i+1
|
|
}
|
|
case c == '"':
|
|
inString = true
|
|
case c == '/' && i+1 < len(bundle) && bundle[i+1] == '/':
|
|
inLine, i = true, i+1
|
|
case c == '/' && i+1 < len(bundle) && bundle[i+1] == '*':
|
|
inBlock, i = true, i+1
|
|
case c == '{':
|
|
if depth == 0 {
|
|
from = i
|
|
}
|
|
depth++
|
|
case c == '}':
|
|
depth--
|
|
if depth != 0 {
|
|
break
|
|
}
|
|
if isResource(bundle[from:i+1], id) {
|
|
return previous, from, i + 1, nil
|
|
}
|
|
previous = i + 1
|
|
from = -1
|
|
case c == ']' && depth == 0:
|
|
return 0, 0, 0, fmt.Errorf(
|
|
"this bundle declares no %q, so there is nothing to take out of it", id)
|
|
}
|
|
}
|
|
return 0, 0, 0, fmt.Errorf("this bundle's resources list does not end")
|
|
}
|
|
|
|
// isResource reports whether one resource's text is the one wanted.
|
|
//
|
|
// Whitespace-insensitive on the pair, quotes included, so `"id": "control-plane"` and
|
|
// `"id":"control-plane"` are the same answer and `"id": "control-planes"` is not.
|
|
func isResource(resource []byte, id string) bool {
|
|
var tight []byte
|
|
for _, c := range resource {
|
|
if c != ' ' && c != '\t' && c != '\n' && c != '\r' {
|
|
tight = append(tight, c)
|
|
}
|
|
}
|
|
return bytes.Contains(tight, []byte(`"id":"`+id+`"`))
|
|
}
|
|
|
|
// cut removes an element and what leads up to it, leaving the list valid.
|
|
//
|
|
// Whether the comma before or the comma after goes depends on where the element sits: an element
|
|
// with something before it takes the comma that joined them, and the first element takes the one
|
|
// after it. Getting this wrong produces a trailing comma, which is JSON nothing will parse —
|
|
// caught by the re-parse either way, and better not produced.
|
|
func cut(bundle []byte, previous, from, to int) []byte {
|
|
start := from
|
|
for i := previous; i < from; i++ {
|
|
if bundle[i] == ',' {
|
|
start = i
|
|
break
|
|
}
|
|
}
|
|
end := to
|
|
if start == from {
|
|
// Nothing before it, so the comma that follows is the one that would be left dangling.
|
|
for i := to; i < len(bundle); i++ {
|
|
if bundle[i] == ',' {
|
|
end = i + 1
|
|
break
|
|
}
|
|
if bundle[i] == ']' {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
out := make([]byte, 0, len(bundle))
|
|
out = append(out, bundle[:start]...)
|
|
return append(out, bundle[end:]...)
|
|
}
|
|
|
|
// indexOutsideStrings finds a fragment that is not inside a JSON string.
|
|
func indexOutsideStrings(haystack []byte, needle string) int {
|
|
inString, escaped := false, false
|
|
for i := 0; i < len(haystack); i++ {
|
|
switch {
|
|
case escaped:
|
|
escaped = false
|
|
continue
|
|
case haystack[i] == '\\' && inString:
|
|
escaped = true
|
|
continue
|
|
case haystack[i] == '"':
|
|
// The needle may itself start with a quote, so the match is tried before the quote is
|
|
// consumed.
|
|
if !inString && bytes.HasPrefix(haystack[i:], []byte(needle)) {
|
|
return i
|
|
}
|
|
inString = !inString
|
|
continue
|
|
case inString:
|
|
continue
|
|
}
|
|
if bytes.HasPrefix(haystack[i:], []byte(needle)) {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// isGone waits for a container to stop existing.
|
|
//
|
|
// Waited for rather than asked once, because a container being removed is a container that is
|
|
// stopping first, and a runtime answers about it until it has finished.
|
|
func isGone(ctx context.Context, run Runner, probe, wait time.Duration, name string) (bool, error) {
|
|
deadline := time.Now().Add(wait)
|
|
for {
|
|
if _, err := containerRunning(ctx, run, probe, name); err != nil {
|
|
// The runtime does not know it. That is the answer being waited for.
|
|
return true, nil
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return false, nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return false, ctx.Err()
|
|
case <-time.After(answerEvery):
|
|
}
|
|
}
|
|
}
|