mesh-host/internal/bootstrap/retire.go

package bootstrap

import (
	"bytes"
	"context"
	"fmt"
	"time"

	"github.com/novox/mesh-host/internal/declaration"
	"github.com/novox/mesh-host/internal/system"
)

// Retired is what step 10 did.
type Retired struct {
	// Container is the temporary control plane that was dropped.
	Container string
	// Gone is true when the machine no longer has it.
	Gone bool
	// Already is true when it was gone before this step ran.
	Already bool
	// Bundle is where the bundle without it was written.
	Bundle string
	// Removed is how many resources the re-apply reported removing.
	Removed int
}

// RetireTheTemporaryControlPlane drops it from the bundle and lets the host take it away.
//
// **Destruction by omission, which is the host's ordinary behaviour and not a new mechanism.** The
// host owns what it has applied and removes what it owns and is no longer declared. So retiring the
// temporary control plane is not a verb anybody had to invent: the bundle stops declaring it, the
// bundle is applied again, and the removal pass does what it does for every other resource that
// leaves a declaration.
//
// That is the whole of why the rename at step 3 mattered. Had the foundation and the module both
// called their container `mesh-controller`, this apply would have removed the module's container —
// the host would have been asked to take away something it believed it owned, and it would have
// been right. Two names, two owners, and the removal is unambiguous.
//
// **It is the last step for a reason.** Until step 9 has a control plane that answers, the
// temporary one is the only thing that can tell this machine anything, and a machine left with no
// control plane cannot be fixed remotely (novox/hq ADR 0067). So this runs after the permanent one
// has been proved, and a run interrupted before it leaves two control planes, which is untidy and
// harmless — a re-run reaches this step and finishes.
func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.System,
	produced []byte, run Runner, say func(string)) (Retired, error) {

	out := Retired{Bundle: o.Out}

	current, err := declaration.ParseFileTrusted(produced)
	if err != nil {
		return out, fmt.Errorf("the bundle this installer produced is not a declaration: %w", err)
	}
	temporary, err := controlPlaneIn(current)
	if err != nil {
		// The bundle already declares no control plane, which is what a re-run after this step
		// finds. Nothing to drop, and nothing to be alarmed about.
		say("  already dropped   the bundle declares no temporary control plane")
		out.Already = true
		return out, nil
	}
	out.Container = temporary.Name

	// Textual, for the reason the rewrite at step 3 is textual: the produced bundle is meant to be
	// READ, and a person coming to a machine after a pivot should be able to open the file the
	// installer applied and see the foundation they recognise with the control plane gone from it.
	// Re-serialising a parsed declaration would drop every comment in it.
	bundle, err := removeResource(produced, ControlPlaneID)
	if err != nil {
		return out, err
	}
	without, err := declaration.ParseFileTrusted(bundle)
	if err != nil {
		return out, fmt.Errorf(
			"taking the temporary control plane out of the bundle broke it: %w", err)
	}
	if _, err := controlPlaneIn(without); err == nil {
		return out, fmt.Errorf(
			"the bundle still declares %q after it was taken out, so nothing was removed and the "+
				"apply below would change nothing", ControlPlaneID)
	}
	if err := writeBundleFile(o.Out, bundle); err != nil {
		return out, err
	}
	say(fmt.Sprintf("  wrote             %s (%d resources) — without %s",
		o.Out, len(without.Resources), temporary.Name))

	// Applied the same way everything else here is applied, under the same origin, against the
	// same state file. What makes this a removal rather than a no-op is that the state file
	// records the container as something this installer applied, and the declaration no longer
	// asks for it.
	report, err := ApplyBundle(ctx, o, sys, without, bundle, run, say)
	if err != nil {
		return out, fmt.Errorf(
			"%w\n\nThe permanent control plane is running and the temporary one is still here. "+
				"That is untidy and it is not broken: two control planes on one mesh are both "+
				"stateless and both correct. Run this installer again to finish", err)
	}
	for _, outcome := range report.Outcomes {
		if outcome.Action == "removed" {
			out.Removed++
		}
	}

	// Read back. A removal that reported success and left the container running would leave two
	// control planes consuming the same broker queues for ever, which is the state this step
	// exists to end.
	gone, err := isGone(ctx, run, o.Timeout, o.Wait, temporary.Name)
	if err != nil {
		return out, err
	}
	out.Gone = gone
	if !gone {
		return out, fmt.Errorf(
			"the apply reported the temporary control plane removed and %q is still running.\n"+
				"Two control planes are consuming this mesh's broker queues. Neither is wrong and "+
				"the mesh is not damaged, but the pivot is not finished: `docker rm -f %s` ends "+
				"it, and this installer will then agree", temporary.Name, temporary.Name)
	}
	say("  gone              " + temporary.Name)
	return out, nil
}

// removeResource takes one resource out of a bundle's text, comments and all.
//
// It walks the `resources` array counting braces, skipping over strings and comments so that a
// `//` inside a connection string is not read as the start of one — the foundation's own bundle
// contains `postgres://…` several times, and a scanner that did not know the difference would
// treat the rest of the line as a comment and lose a brace.
//
// What is removed is the element AND whatever precedes it back to the previous element, which is
// where the comment explaining it lives. A comment that outlives the thing it describes is worse
// than no comment: it is the file telling somebody the machine has a control plane it does not.
func removeResource(bundle []byte, id string) ([]byte, error) {
	previous, from, to, err := resourceAt(bundle, id)
	if err != nil {
		return nil, err
	}
	return cut(bundle, previous, from, to), nil
}

// resourceAt finds one resource's object in a bundle's text by its id: where the one before it
// ended, and where it starts and ends — comments and strings skipped, so an id quoted in a comment
// or a command is never mistaken for the resource.
func resourceAt(bundle []byte, id string) (previous, from, to int, err error) {
	array := indexOutsideStrings(bundle, `"resources"`)
	if array < 0 {
		return 0, 0, 0, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it")
	}
	open := indexOutsideStrings(bundle[array:], "[")
	if open < 0 {
		return 0, 0, 0, fmt.Errorf("this bundle's resources are not a list")
	}
	open += array

	depth := 0
	from, previous = -1, open
	inString, escaped, inLine, inBlock := false, false, false, false
	for i := open + 1; i < len(bundle); i++ {
		c := bundle[i]
		switch {
		case escaped:
			escaped = false
		case inString && c == '\\':
			escaped = true
		case inString:
			if c == '"' {
				inString = false
			}
		case inLine:
			if c == '\n' {
				inLine = false
			}
		case inBlock:
			if c == '*' && i+1 < len(bundle) && bundle[i+1] == '/' {
				inBlock, i = false, i+1
			}
		case c == '"':
			inString = true
		case c == '/' && i+1 < len(bundle) && bundle[i+1] == '/':
			inLine, i = true, i+1
		case c == '/' && i+1 < len(bundle) && bundle[i+1] == '*':
			inBlock, i = true, i+1
		case c == '{':
			if depth == 0 {
				from = i
			}
			depth++
		case c == '}':
			depth--
			if depth != 0 {
				break
			}
			if isResource(bundle[from:i+1], id) {
				return previous, from, i + 1, nil
			}
			previous = i + 1
			from = -1
		case c == ']' && depth == 0:
			return 0, 0, 0, fmt.Errorf(
				"this bundle declares no %q, so there is nothing to take out of it", id)
		}
	}
	return 0, 0, 0, fmt.Errorf("this bundle's resources list does not end")
}

// isResource reports whether one resource's text is the one wanted.
//
// Whitespace-insensitive on the pair, quotes included, so `"id": "control-plane"` and
// `"id":"control-plane"` are the same answer and `"id": "control-planes"` is not.
func isResource(resource []byte, id string) bool {
	var tight []byte
	for _, c := range resource {
		if c != ' ' && c != '\t' && c != '\n' && c != '\r' {
			tight = append(tight, c)
		}
	}
	return bytes.Contains(tight, []byte(`"id":"`+id+`"`))
}

// cut removes an element and what leads up to it, leaving the list valid.
//
// Whether the comma before or the comma after goes depends on where the element sits: an element
// with something before it takes the comma that joined them, and the first element takes the one
// after it. Getting this wrong produces a trailing comma, which is JSON nothing will parse —
// caught by the re-parse either way, and better not produced.
func cut(bundle []byte, previous, from, to int) []byte {
	start := from
	for i := previous; i < from; i++ {
		if bundle[i] == ',' {
			start = i
			break
		}
	}
	end := to
	if start == from {
		// Nothing before it, so the comma that follows is the one that would be left dangling.
		for i := to; i < len(bundle); i++ {
			if bundle[i] == ',' {
				end = i + 1
				break
			}
			if bundle[i] == ']' {
				break
			}
		}
	}
	out := make([]byte, 0, len(bundle))
	out = append(out, bundle[:start]...)
	return append(out, bundle[end:]...)
}

// indexOutsideStrings finds a fragment that is not inside a JSON string.
func indexOutsideStrings(haystack []byte, needle string) int {
	inString, escaped := false, false
	for i := 0; i < len(haystack); i++ {
		switch {
		case escaped:
			escaped = false
			continue
		case haystack[i] == '\\' && inString:
			escaped = true
			continue
		case haystack[i] == '"':
			// The needle may itself start with a quote, so the match is tried before the quote is
			// consumed.
			if !inString && bytes.HasPrefix(haystack[i:], []byte(needle)) {
				return i
			}
			inString = !inString
			continue
		case inString:
			continue
		}
		if bytes.HasPrefix(haystack[i:], []byte(needle)) {
			return i
		}
	}
	return -1
}

// isGone waits for a container to stop existing.
//
// Waited for rather than asked once, because a container being removed is a container that is
// stopping first, and a runtime answers about it until it has finished.
func isGone(ctx context.Context, run Runner, probe, wait time.Duration, name string) (bool, error) {
	deadline := time.Now().Add(wait)
	for {
		if _, err := containerRunning(ctx, run, probe, name); err != nil {
			// The runtime does not know it. That is the answer being waited for.
			return true, nil
		}
		if time.Now().After(deadline) {
			return false, nil
		}
		select {
		case <-ctx.Done():
			return false, ctx.Err()
		case <-time.After(answerEvery):
		}
	}
}