bootstrap: the rest of the pivot — enrol, registry, publish, reinstall, retire
Steps 6 to 10, which turn a substrate into a mesh that can maintain itself
(novox/hq ADR 0067).
6 enrol a node record, a token, `mesh-host enrol`, and the host agent
running. Proved by the mesh having HEARD from the node, not by a
process existing: a host that cannot reach the broker looks exactly
like a successful install until the first push applies nothing.
7 registry the module that gives this mesh an image store, registered from a
--catalog checkout, assigned and pushed. Its image is upstream and
never built (04-ISSUES/029) — a placeholder digest there is refused.
Verified by asking `/v2/`, because a container that is up is not a
registry that serves.
8 publish the carried image pushed into that registry, which assigns it the
first manifest digest it has ever had. This is the hinge: without
it the mesh works and can never upgrade itself.
9 control the control plane registered as an ordinary module pinned to that
digest, with the substrate's own store connections delivered
through `secret accept` — read out of the bundle that made them,
because the mesh cannot invent a credential that predates it.
10 retire the temporary control plane dropped from the bundle and removed by
the host's ordinary removal pass.
Every step asks before it acts and reports "already done". No step leaves the
machine without a control plane: steps 9 and 10 overlap deliberately, and two
stateless control planes are untidy rather than broken.
mesh-control's `internal/builder`.PublishImage is mirrored rather than imported —
tier 0 depends on nothing that must be installed first — with one correction: the
digest is chosen from RepoDigests by repository instead of taken as element zero,
so an image pushed to two registries cannot silently pin this mesh to the wrong
one.
Claude-Session: https://claude.ai/code/session_01LrgweAeERJYBg88c5cKDzF
This commit is contained in:
@@ -0,0 +1,291 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-host/internal/declaration"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
)
|
||||
|
||||
// Retired is what step 10 did.
|
||||
type Retired struct {
|
||||
// Container is the temporary control plane that was dropped.
|
||||
Container string
|
||||
// Gone is true when the machine no longer has it.
|
||||
Gone bool
|
||||
// Already is true when it was gone before this step ran.
|
||||
Already bool
|
||||
// Bundle is where the bundle without it was written.
|
||||
Bundle string
|
||||
// Removed is how many resources the re-apply reported removing.
|
||||
Removed int
|
||||
}
|
||||
|
||||
// RetireTheTemporaryControlPlane drops it from the bundle and lets the host take it away.
|
||||
//
|
||||
// **Destruction by omission, which is the host's ordinary behaviour and not a new mechanism.** The
|
||||
// host owns what it has applied and removes what it owns and is no longer declared. So retiring the
|
||||
// temporary control plane is not a verb anybody had to invent: the bundle stops declaring it, the
|
||||
// bundle is applied again, and the removal pass does what it does for every other resource that
|
||||
// leaves a declaration.
|
||||
//
|
||||
// That is the whole of why the rename at step 3 mattered. Had the substrate and the module both
|
||||
// called their container `mesh-control`, this apply would have removed the module's container —
|
||||
// the host would have been asked to take away something it believed it owned, and it would have
|
||||
// been right. Two names, two owners, and the removal is unambiguous.
|
||||
//
|
||||
// **It is the last step for a reason.** Until step 9 has a control plane that answers, the
|
||||
// temporary one is the only thing that can tell this machine anything, and a machine left with no
|
||||
// control plane cannot be fixed remotely (novox/hq ADR 0067). So this runs after the permanent one
|
||||
// has been proved, and a run interrupted before it leaves two control planes, which is untidy and
|
||||
// harmless — a re-run reaches this step and finishes.
|
||||
func RetireTheTemporaryControlPlane(ctx context.Context, o Options, sys system.System,
|
||||
produced []byte, run Runner, say func(string)) (Retired, error) {
|
||||
|
||||
out := Retired{Bundle: o.Out}
|
||||
|
||||
current, err := declaration.ParseFileTrusted(produced)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the bundle this installer produced is not a declaration: %w", err)
|
||||
}
|
||||
temporary, err := controlPlaneIn(current)
|
||||
if err != nil {
|
||||
// The bundle already declares no control plane, which is what a re-run after this step
|
||||
// finds. Nothing to drop, and nothing to be alarmed about.
|
||||
say(" already dropped the bundle declares no temporary control plane")
|
||||
out.Already = true
|
||||
return out, nil
|
||||
}
|
||||
out.Container = temporary.Name
|
||||
|
||||
// Textual, for the reason the rewrite at step 3 is textual: the produced bundle is meant to be
|
||||
// READ, and a person coming to a machine after a pivot should be able to open the file the
|
||||
// installer applied and see the substrate they recognise with the control plane gone from it.
|
||||
// Re-serialising a parsed declaration would drop every comment in it.
|
||||
bundle, err := removeResource(produced, ControlPlaneID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
without, err := declaration.ParseFileTrusted(bundle)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf(
|
||||
"taking the temporary control plane out of the bundle broke it: %w", err)
|
||||
}
|
||||
if _, err := controlPlaneIn(without); err == nil {
|
||||
return out, fmt.Errorf(
|
||||
"the bundle still declares %q after it was taken out, so nothing was removed and the "+
|
||||
"apply below would change nothing", ControlPlaneID)
|
||||
}
|
||||
if err := writeBundleFile(o.Out, bundle); err != nil {
|
||||
return out, err
|
||||
}
|
||||
say(fmt.Sprintf(" wrote %s (%d resources) — without %s",
|
||||
o.Out, len(without.Resources), temporary.Name))
|
||||
|
||||
// Applied the same way everything else here is applied, under the same origin, against the
|
||||
// same state file. What makes this a removal rather than a no-op is that the state file
|
||||
// records the container as something this installer applied, and the declaration no longer
|
||||
// asks for it.
|
||||
report, err := ApplyBundle(ctx, o, sys, without, run, say)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf(
|
||||
"%w\n\nThe permanent control plane is running and the temporary one is still here. "+
|
||||
"That is untidy and it is not broken: two control planes on one mesh are both "+
|
||||
"stateless and both correct. Run this installer again to finish", err)
|
||||
}
|
||||
for _, outcome := range report.Outcomes {
|
||||
if outcome.Action == "removed" {
|
||||
out.Removed++
|
||||
}
|
||||
}
|
||||
|
||||
// Read back. A removal that reported success and left the container running would leave two
|
||||
// control planes consuming the same broker queues for ever, which is the state this step
|
||||
// exists to end.
|
||||
gone, err := isGone(ctx, run, o.Timeout, o.Wait, temporary.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Gone = gone
|
||||
if !gone {
|
||||
return out, fmt.Errorf(
|
||||
"the apply reported the temporary control plane removed and %q is still running.\n"+
|
||||
"Two control planes are consuming this mesh's broker queues. Neither is wrong and "+
|
||||
"the mesh is not damaged, but the pivot is not finished: `docker rm -f %s` ends "+
|
||||
"it, and this installer will then agree", temporary.Name, temporary.Name)
|
||||
}
|
||||
say(" gone " + temporary.Name)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// removeResource takes one resource out of a bundle's text, comments and all.
|
||||
//
|
||||
// It walks the `resources` array counting braces, skipping over strings and comments so that a
|
||||
// `//` inside a connection string is not read as the start of one — the substrate's own bundle
|
||||
// contains `postgres://…` several times, and a scanner that did not know the difference would
|
||||
// treat the rest of the line as a comment and lose a brace.
|
||||
//
|
||||
// What is removed is the element AND whatever precedes it back to the previous element, which is
|
||||
// where the comment explaining it lives. A comment that outlives the thing it describes is worse
|
||||
// than no comment: it is the file telling somebody the machine has a control plane it does not.
|
||||
func removeResource(bundle []byte, id string) ([]byte, error) {
|
||||
array := indexOutsideStrings(bundle, `"resources"`)
|
||||
if array < 0 {
|
||||
return nil, fmt.Errorf("this bundle has no resources array, so there is nothing to take out of it")
|
||||
}
|
||||
open := indexOutsideStrings(bundle[array:], "[")
|
||||
if open < 0 {
|
||||
return nil, fmt.Errorf("this bundle's resources are not a list")
|
||||
}
|
||||
open += array
|
||||
|
||||
depth, from := 0, -1
|
||||
previous := open
|
||||
inString, escaped, inLine, inBlock := false, false, false, false
|
||||
for i := open + 1; i < len(bundle); i++ {
|
||||
c := bundle[i]
|
||||
switch {
|
||||
case escaped:
|
||||
escaped = false
|
||||
case inString && c == '\\':
|
||||
escaped = true
|
||||
case inString:
|
||||
if c == '"' {
|
||||
inString = false
|
||||
}
|
||||
case inLine:
|
||||
if c == '\n' {
|
||||
inLine = false
|
||||
}
|
||||
case inBlock:
|
||||
if c == '*' && i+1 < len(bundle) && bundle[i+1] == '/' {
|
||||
inBlock, i = false, i+1
|
||||
}
|
||||
case c == '"':
|
||||
inString = true
|
||||
case c == '/' && i+1 < len(bundle) && bundle[i+1] == '/':
|
||||
inLine, i = true, i+1
|
||||
case c == '/' && i+1 < len(bundle) && bundle[i+1] == '*':
|
||||
inBlock, i = true, i+1
|
||||
case c == '{':
|
||||
if depth == 0 {
|
||||
from = i
|
||||
}
|
||||
depth++
|
||||
case c == '}':
|
||||
depth--
|
||||
if depth != 0 {
|
||||
break
|
||||
}
|
||||
if isResource(bundle[from:i+1], id) {
|
||||
return cut(bundle, previous, from, i+1), nil
|
||||
}
|
||||
previous = i + 1
|
||||
from = -1
|
||||
case c == ']' && depth == 0:
|
||||
return nil, fmt.Errorf(
|
||||
"this bundle declares no %q, so there is nothing to take out of it", id)
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("this bundle's resources list does not end")
|
||||
}
|
||||
|
||||
// isResource reports whether one resource's text is the one wanted.
|
||||
//
|
||||
// Whitespace-insensitive on the pair, quotes included, so `"id": "control-plane"` and
|
||||
// `"id":"control-plane"` are the same answer and `"id": "control-planes"` is not.
|
||||
func isResource(resource []byte, id string) bool {
|
||||
var tight []byte
|
||||
for _, c := range resource {
|
||||
if c != ' ' && c != '\t' && c != '\n' && c != '\r' {
|
||||
tight = append(tight, c)
|
||||
}
|
||||
}
|
||||
return bytes.Contains(tight, []byte(`"id":"`+id+`"`))
|
||||
}
|
||||
|
||||
// cut removes an element and what leads up to it, leaving the list valid.
|
||||
//
|
||||
// Whether the comma before or the comma after goes depends on where the element sits: an element
|
||||
// with something before it takes the comma that joined them, and the first element takes the one
|
||||
// after it. Getting this wrong produces a trailing comma, which is JSON nothing will parse —
|
||||
// caught by the re-parse either way, and better not produced.
|
||||
func cut(bundle []byte, previous, from, to int) []byte {
|
||||
start := from
|
||||
for i := previous; i < from; i++ {
|
||||
if bundle[i] == ',' {
|
||||
start = i
|
||||
break
|
||||
}
|
||||
}
|
||||
end := to
|
||||
if start == from {
|
||||
// Nothing before it, so the comma that follows is the one that would be left dangling.
|
||||
for i := to; i < len(bundle); i++ {
|
||||
if bundle[i] == ',' {
|
||||
end = i + 1
|
||||
break
|
||||
}
|
||||
if bundle[i] == ']' {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make([]byte, 0, len(bundle))
|
||||
out = append(out, bundle[:start]...)
|
||||
return append(out, bundle[end:]...)
|
||||
}
|
||||
|
||||
// indexOutsideStrings finds a fragment that is not inside a JSON string.
|
||||
func indexOutsideStrings(haystack []byte, needle string) int {
|
||||
inString, escaped := false, false
|
||||
for i := 0; i < len(haystack); i++ {
|
||||
switch {
|
||||
case escaped:
|
||||
escaped = false
|
||||
continue
|
||||
case haystack[i] == '\\' && inString:
|
||||
escaped = true
|
||||
continue
|
||||
case haystack[i] == '"':
|
||||
// The needle may itself start with a quote, so the match is tried before the quote is
|
||||
// consumed.
|
||||
if !inString && bytes.HasPrefix(haystack[i:], []byte(needle)) {
|
||||
return i
|
||||
}
|
||||
inString = !inString
|
||||
continue
|
||||
case inString:
|
||||
continue
|
||||
}
|
||||
if bytes.HasPrefix(haystack[i:], []byte(needle)) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// isGone waits for a container to stop existing.
|
||||
//
|
||||
// Waited for rather than asked once, because a container being removed is a container that is
|
||||
// stopping first, and a runtime answers about it until it has finished.
|
||||
func isGone(ctx context.Context, run Runner, probe, wait time.Duration, name string) (bool, error) {
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
if _, err := containerRunning(ctx, run, probe, name); err != nil {
|
||||
// The runtime does not know it. That is the answer being waited for.
|
||||
return true, nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return false, nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false, ctx.Err()
|
||||
case <-time.After(answerEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user