package bootstrap import ( "context" "fmt" "strings" "time" "github.com/novox/mesh-host/internal/declaration" ) // controlPlaneBinary is where the control plane's program lives in its own image. // // A path rather than a shell command, because the image is `FROM scratch` and holds one static // binary and nothing else — no shell to invoke, nothing to interpret a command line // (mesh-controller's Dockerfile, novox/hq ADR 0006). That is a property of the image this installer // carries, which is why the path can be written down here. const controlPlaneBinary = "/mesh-controller" // answerEvery is how often the control plane is asked again while it is starting. var answerEvery = 2 * time.Second // Verified is what the foundation was found to be. type Verified struct { // Running is every long-running container the bundle declares, confirmed up. Running []string // Answered is the control plane's own first words back, so the report shows the reply rather // than asserting there was one. Answered string } // Verify proves the foundation is up and the control plane replies. // // **A container that is up is not a control plane that replies**, and this project has paid for // that distinction more than once: a runtime reports a container running from the moment the // process starts, which is before it has opened a database, before it has read its configuration, // and before it has failed to. So the containers are checked, and then the program inside one of // them is asked a question and has to answer it. // // The question is `status`, and it is chosen rather than convenient: answering it means the // control plane opened all three of its stores from the environment the bundle gave it. A reply // therefore proves the image runs, that `network: host` really does reach the store on this // machine, and that the contexts' schemas migrated — the three things the steps before this were // for. There is no HTTP endpoint to curl: `serve` is a broker consumer, not a web server. // // It waits. A control plane that is not answering yet and a control plane that will never answer // look identical for the first few seconds, and refusing on the first attempt would make a correct // bootstrap fail for being observed too early. func Verify(ctx context.Context, d *declaration.Declaration, run Runner, probe, wait time.Duration, say func(string)) (Verified, error) { var out Verified control, err := controlPlaneIn(d) if err != nil { return out, err } for _, container := range longRunning(d) { state, err := containerRunning(ctx, run, probe, container) if err != nil { return out, err } if !state.running { return out, fmt.Errorf( "the container %q is %s, not running.\n"+ "The apply reported success, so it was created — what it did afterwards is "+ "in `docker logs %s`", container, state.status, container) } out.Running = append(out.Running, container) say(" running " + container) } answer, err := waitForTheControlPlane(ctx, run, probe, wait, control.Name, say) if err != nil { return out, err } out.Answered = answer return out, nil } func waitForTheControlPlane(ctx context.Context, run Runner, probe, wait time.Duration, container string, say func(string)) (string, error) { deadline := time.Now().Add(wait) var last error for { asking, cancel := context.WithTimeout(ctx, probe) out, err := run(asking, "docker", "exec", container, controlPlaneBinary, "status") cancel() answer := strings.TrimSpace(firstLineOf(out)) switch { case err != nil: last = err case answer == "": // Exit zero and nothing said. Treated as no answer rather than as success: a program // that returns silence is not one that has been asked anything. last = fmt.Errorf("it exited without saying anything") default: say(" replies " + container + ": " + answer) return answer, nil } if time.Now().After(deadline) { break } select { case <-ctx.Done(): return "", ctx.Err() case <-time.After(answerEvery): } } return "", fmt.Errorf( "the container %q is running and the control plane in it does not answer, after waiting "+ "%s: %v\n"+ "Running is not replying. `status` opens this mesh's three stores, so what failed is "+ "most likely the store or the schemas rather than the control plane itself — "+ "`docker logs %s` says which", container, wait, last, container) } type containerState struct { running bool status string } func containerRunning(ctx context.Context, run Runner, probe time.Duration, name string) (containerState, error) { asking, cancel := context.WithTimeout(ctx, probe) defer cancel() // Both facts in one answer, so a container that is not running is reported with what it IS // rather than with the absence of what it should be. out, err := run(asking, "docker", "inspect", "--format", "{{.State.Running}} {{.State.Status}}", name) if err != nil { return containerState{}, fmt.Errorf( "the container %q is not there at all, and the apply reported it applied: %w", name, err) } running, status, _ := strings.Cut(strings.TrimSpace(firstLineOf(out)), " ") if status == "" { status = "in a state the runtime did not name" } return containerState{running: running == "true", status: status}, nil } // longRunning is every container the bundle expects to still be there afterwards. // // A run-once step has exited by design and a scheduled step has deliberately never been started // (novox/hq ADR 0052, ADR 0053), so asking either of them to be running would be asking the // foundation to be something other than what it declared. func longRunning(d *declaration.Declaration) []string { var names []string for _, r := range d.Resources { container, ok := r.(*declaration.Container) if !ok || container.RunOnce || container.Schedule != "" { continue } names = append(names, container.Name) } return names } func firstLineOf(s string) string { if i := strings.IndexByte(s, '\n'); i >= 0 { return s[:i] } return s }