Fail a resource when the container runtime cannot answer, instead of reading silence as nothing there (hq ADR 0100)

internal/apply/hold.go

@@ -51,11 +51,21 @@ func KeepIn(dir string) Keep {
 // record (novox/hq ADR 0103). Looked at first, because the apply itself makes such things — a
 // file's parent directory, a unit file a module writes, a package that brings its unit — and what
 // the mesh made in this apply was not found.
-type foundBefore map[string]bool
+type foundBefore struct {
+	is map[string]bool
+	// trouble is what could not be asked about, by the same key, so the resource that would need
+	// the answer fails rather than proceeding as if the machine had nothing there.
+	trouble map[string]string
+}
+
+func (f foundBefore) has(key string) bool { return f.is[key] }
+
+// why is the reason a key could not be settled, and empty when there was none.
+func (f foundBefore) why(key string) string { return f.trouble[key] }
 
 func lookBefore(ctx context.Context, sys system.System, d *declaration.Declaration, known store.State,
 	run Runner) foundBefore {
-	seen := foundBefore{}
+	seen := foundBefore{is: map[string]bool{}, trouble: map[string]string{}}
 	if d.Adoption == nil {
 		return seen
 	}
@@ -70,25 +80,25 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati
 		switch res := r.(type) {
 		case *declaration.Directory:
 			if present(res.Path) && !recordedPath(known, res.Path) {
-				seen["path:"+res.Path] = true
+				seen.is["path:"+res.Path] = true
 			}
 		case *declaration.Archive:
 			// Unpacking over it, and re-owning it recursively, would change the predecessor's
 			// files.
 			if present(res.Path) && !recordedPath(known, res.Path) {
-				seen["path:"+res.Path] = true
+				seen.is["path:"+res.Path] = true
 			}
 		case *declaration.Process:
 			// Its unit would be written over and restarted.
 			if !known.Recorded(string(declaration.TypeProcess), res.Name) &&
 				present(filepath.Join(unitDir, res.Name+".service")) {
-				seen["unit-file:"+res.Name] = true
+				seen.is["unit-file:"+res.Name] = true
 			}
 		case *declaration.User:
 			// Its shell and groups would be changed.
 			if !known.Recorded(string(declaration.TypeUser), res.Name) {
 				if _, exists, err := system.LookUpUser(ctx, run, res.Name); err == nil && exists {
-					seen["user:"+res.Name] = true
+					seen.is["user:"+res.Name] = true
 				}
 			}
 		case *declaration.Service:
@@ -105,7 +115,7 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati
 			}
 			boot, _ := sys.ServiceBoot(ctx, run, res.Unit)
 			if state == "running" || boot == "enabled" {
-				seen["unit:"+res.Unit] = true
+				seen.is["unit:"+res.Unit] = true
 			}
 		case *declaration.Container:
 			if known.Recorded(string(declaration.TypeContainer), res.Name) {
@@ -117,7 +127,7 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati
 				case src == "":
 				case strings.HasPrefix(src, "/"):
 					if !systemPath(src) && present(src) && !recordedPath(known, src) {
-						seen["path:"+src] = true
+						seen.is["path:"+src] = true
 					}
 				default:
 					if !asked {
@@ -128,7 +138,10 @@ func lookBefore(ctx context.Context, sys system.System, d *declaration.Declarati
 						continue
 					}
 					if _, err := run(ctx, cri, "volume", "inspect", src); err == nil {
-						seen["volume:"+src] = true
+						seen.is["volume:"+src] = true
+					} else if !absent(err) {
+						seen.trouble["volume:"+src] = fmt.Sprintf(
+							"the container runtime could not say whether the volume %s is here: %v", src, err)
 					}
 				}
 			}
@@ -287,21 +300,28 @@ func holdOnAdopted(ctx context.Context, sys system.System, r declaration.Resourc
 				if strings.HasPrefix(src, "/") {
 					key = "path:" + src
 				}
-				if src != "" && before[key] {
+				if src == "" {
+					continue
+				}
+				if trouble := before.why(key); trouble != "" {
+					return false, false, begin(r), fmt.Errorf(
+						"%s, so it is not safe to create a container that would mount it", trouble)
+				}
+				if before.has(key) {
 					isFound, why = true, "would mount "+src+", found on the machine"
 					break
 				}
 			}
 		case *declaration.Directory:
-			isFound = before["path:"+res.Path]
+			isFound = before.has("path:" + res.Path)
 		case *declaration.Archive:
-			isFound = before["path:"+res.Path]
+			isFound = before.has("path:" + res.Path)
 		case *declaration.Process:
-			isFound = before["unit-file:"+res.Name]
+			isFound = before.has("unit-file:" + res.Name)
 		case *declaration.User:
-			isFound = before["user:"+res.Name]
+			isFound = before.has("user:" + res.Name)
 		case *declaration.Service:
-			isFound = before["unit:"+res.Unit]
+			isFound = before.has("unit:" + res.Unit)
 		}
 	}
 	if !isFound {
@@ -373,7 +393,16 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
 	out, err := run(ctx, cri, "inspect", "--format",
 		"{{.Id}}\t{{.State.Running}}\t{{index .Config.Labels \""+specLabel+"\"}}", name)
 	if err != nil {
-		return foundContainer{}, false, nil
+		if absent(err) {
+			return foundContainer{}, false, nil
+		}
+		// **A runtime that could not answer is not a machine with nothing there.** Read as
+		// absence, a daemon that is down or a permission denied would let the mesh create its own
+		// container over a predecessor's — the one thing an adopted node must never do
+		// (novox/hq ADR 0100).
+		return foundContainer{}, false, fmt.Errorf(
+			"the container runtime could not say whether %s is here, so it is not safe to make one: %w",
+			name, err)
 	}
 	parts := strings.Split(strings.TrimSpace(out), "\t")
 	for len(parts) < 3 {
@@ -386,6 +415,19 @@ func inspectFound(ctx context.Context, name string, run Runner) (foundContainer,
 	return foundContainer{id: strings.TrimSpace(parts[0]), running: parts[1] == "true", spec: spec}, true, nil
 }
 
+// absent is whether a runtime said the thing is not there, rather than failing to answer. Its own
+// words: docker and podman both say "No such object", "No such container" or "No such volume".
+func absent(err error) bool {
+	said := strings.ToLower(err.Error())
+	for _, missing := range []string{"no such object", "no such container", "no such volume",
+		"no such image"} {
+		if strings.Contains(said, missing) {
+			return true
+		}
+	}
+	return false
+}
+
 // hold keeps a found file or container as it is, and reports it — the first time by recording
 // what was found, every time after by comparing against that. Nothing is reverted, restarted or
 // created: a held target that disappears stays held and gone until its module is taken.