package replays import ( "bufio" "context" "crypto/tls" "encoding/json" "fmt" "net" "net/http" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "time" ) // The bed (novox/hq ADR 0240 rule 7, to-be 48 §8, Phase D): **a declaration is proved before it is trusted.** // // Two of the nineteen image checks the catalogue's containers ship read *unhealthy* while working in the // mesh's hands — the studio's asked an address its program does not bind, the flow editor's read settings // from a path the module mounted elsewhere. Read without proof, they would have put back two good builds. // So the catalogue's merge check starts every resource whose declared check or image the change touches, // **alone**, on a throwaway runtime, and requires its check to see the program within its grace. // // **What it proves is the check's wiring**: that it can see the program working — a property of the image // and the declaration, not of the mesh. A resource is started with what the module declares and nothing of // the mesh's: its literal environment, its arguments, its files where the module writes them in full, and an // empty directory for every other place it mounts; no secret, no binding, no provider. So: // // - a check naming a provision in `needs` gets no provider here, and must only reach the program — an // http answer of any status, a connect; // - a program that does not stay up alone — it needs what the mesh gives it — is said as not proved here, // and judged on the first machine's gate, never passed as proved and never failed; // - a program that stays up while its check does not see it **fails**: that is the studio's fault. // // A tool check and a unit's own readiness need the mesh and a machine; they are said, not proved. // BedResource is one resource the bed proves: its module, its id, and the resource as the manifest says it. type BedResource struct { Module string ID string Resource map[string]any // Listens are the module's endpoints, by name: the container's own port for each. Listens map[string]int } // BedVerdict is what the bed found of one resource. type BedVerdict struct { Proved bool // Failed says the program stayed up and its check did not see it: the change's fault. Failed bool Said string } // BedLook is how often the bed looks: often, because it proves the wiring, not the timing. var BedLook = 2 * time.Second // BedFloor is the least the bed waits for a check to pass, whatever the grace: a program needs a moment to // listen even when the module declared no grace. var BedFloor = 30 * time.Second // ChangedHealth is every long-running container resource of the catalogue at root that declares `health` // and whose resource changed against base (a git ref in that checkout): its declaration, its image, or // anything else of it. A manifest new on this branch counts whole. Read through git; an error when the // base cannot be read. func ChangedHealth(root, base string, show func(ref, path string) ([]byte, error)) ([]BedResource, error) { paths, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) if err != nil { return nil, err } var out []BedResource for _, p := range paths { raw, err := os.ReadFile(p) if err != nil { return nil, err } rel, _ := filepath.Rel(root, p) was := map[string]string{} if before, err := show(base, rel); err == nil { for _, r := range resourcesOf(before) { was[fmt.Sprint(r["id"])] = canonical(r) } } module, listens, resources := manifestParts(raw) for _, r := range resources { if _, declared := r["health"]; !declared || fmt.Sprint(r["type"]) != "container" || !stays(r) { continue } if was[fmt.Sprint(r["id"])] == canonical(r) { continue } out = append(out, BedResource{Module: module, ID: fmt.Sprint(r["id"]), Resource: r, Listens: listens}) } } sort.Slice(out, func(a, b int) bool { return out[a].Module+"."+out[a].ID < out[b].Module+"."+out[b].ID }) return out, nil } func stays(r map[string]any) bool { once, _ := r["run-once"].(bool) return !once && r["schedule"] == nil } func canonical(r map[string]any) string { raw, _ := json.Marshal(r) return string(raw) } func resourcesOf(raw []byte) []map[string]any { _, _, rs := manifestParts(raw) return rs } func manifestParts(raw []byte) (string, map[string]int, []map[string]any) { var m struct { Module string `json:"module"` Listens []struct { Name string `json:"name"` Port int `json:"port"` } `json:"listens"` Resources []map[string]any `json:"resources"` } _ = json.Unmarshal(raw, &m) listens := map[string]int{} for _, l := range m.Listens { if l.Name != "" { listens[l.Name] = l.Port } } return m.Module, listens, m.Resources } // placeholder is anything the mesh fills in on a machine: a value holding one is not the module's alone. var placeholder = regexp.MustCompile(`\$\{[^}]*\}`) // Prove starts one resource alone on the runtime and looks at it with its declared check until the check // sees it, the program stops, or its grace (at least BedFloor) runs out. Everything it made is removed. func (d *Docker) Prove(ctx context.Context, r BedResource, files map[string]string) BedVerdict { h, _ := r.Resource["health"].(map[string]any) kind := fmt.Sprint(h["kind"]) switch kind { case "tool", "unit": return BedVerdict{Said: fmt.Sprintf("%s.%s: a %s check needs the mesh and a machine; judged on the first "+ "machine's gate", r.Module, r.ID, kind)} } image, _ := r.Resource["image"].(string) if image == "" || placeholder.MatchString(image) { return BedVerdict{Said: fmt.Sprintf("%s.%s: its image is built by the mesh (%v); judged on the first machine's gate", r.Module, r.ID, r.Resource["artifact"])} } if err := d.Pull(ctx, image); err != nil { return BedVerdict{Said: fmt.Sprintf("%s.%s: its image could not be fetched here: %v", r.Module, r.ID, oneLine(err.Error()))} } scratch, err := os.MkdirTemp("", "mesh-bed-") if err != nil { return BedVerdict{Said: err.Error()} } defer os.RemoveAll(scratch) // Its literal environment, and every env-file the module writes in full. var env []string if e, ok := r.Resource["env"].(map[string]any); ok { for k, v := range e { if s := fmt.Sprint(v); !placeholder.MatchString(s) { env = append(env, k+"="+s) } } } if list, ok := r.Resource["env-file"].([]any); ok { for _, f := range list { if content, ok := files[fmt.Sprint(f)]; ok { env = append(env, envLines(content)...) } } } sort.Strings(env) // Its mounts: a file the module writes in full, or an empty place of its own. var binds []string if list, ok := r.Resource["volumes"].([]any); ok { for i, v := range list { src, dst, rest := splitMount(fmt.Sprint(v)) if dst == "" || !strings.HasPrefix(dst, "/") { continue } local := filepath.Join(scratch, "m"+strconv.Itoa(i)) if content, ok := files[src]; ok { if err := os.WriteFile(local, []byte(content), 0o644); err != nil { continue } } else if err := os.MkdirAll(local, 0o777); err != nil { continue } _ = os.Chmod(local, 0o777) binds = append(binds, local+":"+dst+rest) } } var args []string if list, ok := r.Resource["args"].([]any); ok { for _, a := range list { args = append(args, fmt.Sprint(a)) } } network := fmt.Sprintf("mesh-bed-%d", time.Now().UnixNano()) netID, err := d.Network(ctx, network) if err != nil { return BedVerdict{Said: "the bed's network could not be made: " + err.Error()} } defer d.RemoveNetwork(context.Background(), netID) grace, _ := time.ParseDuration(fmt.Sprint(h["grace"])) if _, said := h["grace"]; !said { grace = 60 * time.Second } if grace < BedFloor { grace = BedFloor } config := map[string]any{"Image": image, "Env": env, "Labels": map[string]string{d.Label: "bed"}} if len(args) > 0 { config["Cmd"] = args } switch kind { case "exec": config["Healthcheck"] = map[string]any{"Test": []string{"CMD-SHELL", fmt.Sprint(h["command"])}, "Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, "Retries": 1} case "runtime": // The image's own command, adopted by name: an empty Test keeps it. config["Healthcheck"] = map[string]any{"Interval": BedLook.Nanoseconds(), "Timeout": BedLook.Nanoseconds() - 1, "Retries": 1} } config["HostConfig"] = map[string]any{"Binds": binds, "NetworkMode": network} name := fmt.Sprintf("mesh-bed-%s-%s-%d", r.Module, r.ID, time.Now().UnixNano()) var made struct{ ID string } if err := d.call(ctx, http.MethodPost, "/containers/create?name="+name, config, &made); err != nil { return BedVerdict{Said: fmt.Sprintf("%s.%s could not be made here: %v", r.Module, r.ID, oneLine(err.Error()))} } defer d.Remove(context.Background(), made.ID) if err := d.call(ctx, http.MethodPost, "/containers/"+made.ID+"/start", nil, nil); err != nil { return BedVerdict{Said: fmt.Sprintf("%s.%s could not be started here: %v", r.Module, r.ID, oneLine(err.Error()))} } port := r.Listens[fmt.Sprint(h["endpoint"])] needs := fmt.Sprint(h["needs"]) != "" && h["needs"] != nil deadline := time.Now().Add(grace) last := "it was never looked at" for { var seen struct { State struct { Running bool Health *struct { Status string Log []struct{ Output string } } } NetworkSettings struct { Networks map[string]struct{ IPAddress string } } } if err := d.call(ctx, http.MethodGet, "/containers/"+made.ID+"/json", nil, &seen); err != nil { return BedVerdict{Said: err.Error()} } if !seen.State.Running { return BedVerdict{Said: fmt.Sprintf("%s.%s does not stay up alone — it needs what the mesh gives it — so its "+ "check is judged on the first machine's gate: %s", r.Module, r.ID, oneLine(lastLine(d.Logs(ctx, made.ID))))} } ok := false switch kind { case "exec", "runtime": hs := seen.State.Health switch { case hs == nil: last = "the container carries no check: its image ships none to adopt" case hs.Status == "healthy": ok = true default: last = "the runtime says " + hs.Status if n := len(hs.Log); n > 0 { last += ": " + oneLine(hs.Log[n-1].Output) } } case "http", "tcp": address := "" for _, n := range seen.NetworkSettings.Networks { address = n.IPAddress } ok, last = look(ctx, kind, address, port, h, needs) default: return BedVerdict{Said: fmt.Sprintf("%s.%s declares a %s check the bed does not know", r.Module, r.ID, kind)} } if ok { return BedVerdict{Proved: true, Said: fmt.Sprintf("%s.%s: its %s check sees the program", r.Module, r.ID, kind)} } if time.Now().After(deadline) || ctx.Err() != nil { return BedVerdict{Failed: true, Said: fmt.Sprintf("%s.%s stays up and its %s check does not see it within %s: %s", r.Module, r.ID, kind, grace, last)} } select { case <-ctx.Done(): case <-time.After(BedLook): } } } // look is one http or tcp look at the program, from outside it, as the node-engine makes it. A check that // needs a provider only has to reach the program: any answer, a connect. func look(ctx context.Context, kind, address string, port int, h map[string]any, needs bool) (bool, string) { if address == "" || port == 0 { return false, "it has no address on the bed's network yet" } at := net.JoinHostPort(address, strconv.Itoa(port)) ctx, cancel := context.WithTimeout(ctx, BedLook) defer cancel() if kind == "tcp" { c, err := (&net.Dialer{}).DialContext(ctx, "tcp", at) if err != nil { return false, oneLine(err.Error()) } c.Close() return true, "" } scheme, _ := h["scheme"].(string) if scheme == "" { scheme = "http" } path, _ := h["path"].(string) if path == "" { path = "/" } req, _ := http.NewRequestWithContext(ctx, http.MethodGet, scheme+"://"+at+path, nil) res, err := (&http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, Transport: insecure()}).Do(req) if err != nil { return false, oneLine(err.Error()) } res.Body.Close() if needs { return true, "" } want, _ := h["status"].(float64) switch { case want != 0 && res.StatusCode != int(want): return false, fmt.Sprintf("answered %d, expected %d", res.StatusCode, int(want)) case want == 0 && res.StatusCode >= 400: return false, fmt.Sprintf("answered %d", res.StatusCode) } return true, "" } func splitMount(v string) (src, dst, rest string) { parts := strings.SplitN(v, ":", 3) if len(parts) < 2 { return "", "", "" } if len(parts) == 3 { rest = ":" + parts[2] } return parts[0], parts[1], rest } func envLines(content string) []string { var out []string s := bufio.NewScanner(strings.NewReader(content)) for s.Scan() { line := strings.TrimSpace(s.Text()) if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") || placeholder.MatchString(line) { continue } out = append(out, line) } return out } func lastLine(s string) string { lines := strings.Split(strings.TrimSpace(s), "\n") return lines[len(lines)-1] } // FilesOf is every file a module writes in full — its content declared, nothing the mesh fills in — by the // path it writes it at, as the module names it: what the bed gives a container that mounts or reads it. func FilesOf(manifest []byte) map[string]string { _, _, rs := manifestParts(manifest) out := map[string]string{} for _, r := range rs { if fmt.Sprint(r["type"]) != "file" { continue } path, _ := r["path"].(string) content, ok := r["content"].(string) if path == "" || !ok || placeholder.MatchString(content) { continue } out[path] = content } return out } // insecure is a transport that asks whether a program answers, not whom to trust. func insecure() *http.Transport { return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, DisableKeepAlives: true} } // oneLine is the first line of what was said, short. func oneLine(s string) string { line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") if len(line) > 300 { line = line[:300] + "…" } return line }