review: rebuild a file the mesh once wrote whole, keep links, give back a missing line end, and release a hold only after the write (hq issue 128)

internal/apply/block.go

@@ -3,6 +3,7 @@ package apply
 import (
 	"errors"
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,17 +29,71 @@ import (
 // mesh created goes only if nothing but whitespace is left.
 
 // applyBlock writes a file's declared lines into its region of the file already at its path.
+//
+// **A link stays a link.** Where the path is a symbolic link — a hosts file some distributions keep
+// elsewhere and link into /etc — the file read, written and renamed over is the one it points to,
+// so the link and whatever manages it are left as they were. A file written whole, or into JSON,
+// still replaces a link with a file; that is unchanged here.
 func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
 	out := begin(r)
 	opening, closing := declaration.BlockMarkers(r.ID)
 	want := blockBody(r.Content)
 
-	raw, err := os.ReadFile(r.Path)
+	real, err := realPath(r.Path)
+	if err != nil {
+		return out, err
+	}
+	raw, err := os.ReadFile(real)
 	existed := err == nil
 	if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return out, err
 	}
+	// What the file is, taken once with what it holds: its mode and owner are the machine's and
+	// go back onto what is written. A file read and then not there to stat is a failure, never a
+	// file with no owner.
+	var info os.FileInfo
+	if existed {
+		if info, err = os.Stat(real); err != nil {
+			return out, fmt.Errorf("read %s and cannot see it: %w", r.Path, err)
+		}
+	}
 	existing := string(raw)
+
+	rec := store.Into{Format: declaration.IntoBlock}
+	var note string
+	rebuilt := false
+
+	// **A file the mesh once wrote whole** (novox/hq issue 128). The resource keeps its id when its
+	// module moves from writing the file whole to writing into it, and the file on the machine is
+	// then the mesh's own old write — its header, its loopback lines, its names. Adding the region
+	// after that would leave the old names above the new ones, and a resolver takes the first
+	// line that answers: the region would be shadowed by what it replaced. So the file is rebuilt:
+	// the original the mesh kept before its first write, with the region in it; or, where the mesh
+	// made the file itself, the loopback lines every machine needs, kept as the machine's, with the
+	// region beside them. Changed since the mesh wrote it, the file is somebody's again and is
+	// written into as it stands, and the outcome says so.
+	if existed && previous.Into == nil && previous.Wrote != "" {
+		if digestOf(existing) == previous.Wrote {
+			if previous.Kept != "" {
+				original, err := os.ReadFile(previous.Kept)
+				if err != nil {
+					return out, fmt.Errorf("%s was written whole by the mesh over an original kept at %s, "+
+						"which cannot be read to give it back: %w; it was left as it is", r.Path, previous.Kept, err)
+				}
+				existing = string(original)
+				note = "the mesh's old whole file replaced by the original kept at " + previous.Kept + ", with the region in it"
+			} else {
+				existing = loopbackOf(existing)
+				rec.Created = true
+				note = "the mesh's old whole file replaced by its loopback lines and the region"
+			}
+			// Not what was read: the whole of it was the mesh's, and the file is written afresh.
+			rebuilt = true
+		} else {
+			note = "a file the mesh once wrote whole, changed since; its old lines were kept"
+		}
+	}
+
 	lines := linesOf(existing)
 	at, found, err := regionIn(lines, opening, closing)
 	if err != nil {
@@ -49,14 +104,14 @@ func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
 
 	// A record of a block is carried; anything else — no record, a file once written whole, one
 	// once written into as JSON — is a file the host is seeing for the first time as a block.
-	rec := store.Into{Format: declaration.IntoBlock}
 	recorded := previous.Into != nil && previous.Into.Format == declaration.IntoBlock
 	if recorded && existed {
 		rec.Created = previous.Into.Created
 		rec.Region = previous.Into.Region
 		rec.Separated = previous.Into.Separated
 		rec.At = previous.Into.At
-	} else {
+		rec.Ended = previous.Into.Ended
+	} else if !rebuilt {
 		// A file gone since the last apply is made again, and made by the mesh: what it held
 		// before went with it, so there is nothing to give back but the file's absence.
 		rec.Created = !existed
@@ -89,7 +144,7 @@ func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
 	case r.At == declaration.AtStart:
 		// Above everything, and one blank line between the region and the machine's first line
 		// unless there is one already — a line in some files means what the lines above it say.
-		rec.At, rec.Separated = declaration.AtStart, false
+		rec.At, rec.Separated, rec.Ended = declaration.AtStart, false, false
 		next = regionOf(opening, closing, want)
 		if existing != "" && !strings.HasPrefix(existing, "\n") {
 			next += "\n"
@@ -99,10 +154,11 @@ func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
 	default:
 		// At the end, apart from whatever is there: the file's last line is ended if it was not,
 		// and one blank line separates the region from the machine's lines unless there is one.
-		rec.At, rec.Separated = "", false
+		rec.At, rec.Separated, rec.Ended = "", false, false
 		next = existing
 		if next != "" && !strings.HasSuffix(next, "\n") {
 			next += "\n"
+			rec.Ended = true
 		}
 		if next != "" && next != "\n" && !strings.HasSuffix(next, "\n\n") {
 			next += "\n"
@@ -110,45 +166,60 @@ func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
 		}
 		next += regionOf(opening, closing, want)
 	}
+	// What was not the mesh's is what it was. By construction — and checked, because a slip in
+	// splicing lines is exactly the fault this mode exists to prevent, and it must never be written.
+	if found {
+		after := linesOf(next)
+		if where, ok, err := regionIn(after, opening, closing); err != nil || !ok || outside(after, where) != outside(lines, at) {
+			return out, fmt.Errorf("%s: writing the region would change lines outside it; it was left as it is", r.Path)
+		}
+	}
 
-	same := existed && next == existing
+	same := existed && next == string(raw)
 	if !same {
-		var info os.FileInfo
 		mode := os.FileMode(0o644)
-		if info, err = os.Stat(r.Path); err == nil {
+		if info != nil {
 			mode = info.Mode().Perm() // the machine's file keeps the machine's mode
 		} else if mode, err = modeOf(r.Mode, mode); err != nil {
 			return out, err
 		}
-		if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil {
+		if err := os.MkdirAll(filepath.Dir(real), 0o755); err != nil {
 			return out, err
 		}
-		if err := writeAtomically(r.Path, []byte(next), mode); err != nil {
+		if err := writeAtomically(real, []byte(next), mode); err != nil {
 			return out, err
 		}
-		if existed {
+		if info != nil {
 			// The write is a new file renamed over the old, so it belongs to whoever wrote it. The
 			// machine's file keeps the machine's owner, as it keeps its mode.
-			if err := keepOwner(r.Path, info); err != nil {
+			if err := keepOwner(real, info); err != nil {
 				return out, err
 			}
-		} else if err := own(r.Path, r.Owner); err != nil {
+		} else if err := own(real, r.Owner); err != nil {
 			return out, err
 		}
 	}
 
-	// Read back: the region holds what was declared, and nothing outside it moved.
-	written, err := os.ReadFile(r.Path)
+	// Read back: the region holds what was declared. Only the region — another tool writing its
+	// own lines in the moment after the rename is not a failed write. What remains is the moment
+	// between reading the file and renaming over it: a line another tool writes there is lost, and
+	// found again at its next write. Nothing short of a lock every writer honours closes that, and
+	// the other writers of a hosts file honour none.
+	written, err := os.ReadFile(real)
 	if err != nil {
 		return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err)
 	}
-	if string(written) != next {
-		return out, fmt.Errorf("%s does not hold the mesh's region as written after writing into it", r.Path)
+	back := linesOf(string(written))
+	if where, ok, err := regionIn(back, opening, closing); err != nil || !ok || where.body(back) != want {
+		return out, fmt.Errorf("%s does not hold the mesh's region after writing into it", r.Path)
 	}
 
 	out.into = &rec
 	out.wrote = digestOf(want)
 	switch {
+	case note != "" && !same:
+		out.Action = "updated"
+		out.Detail = note
 	case !existed:
 		out.Action = "created"
 		out.Detail = "written into; the file was not there"
@@ -173,16 +244,20 @@ func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) {
 
 // removeBlock gives back what a file written into a block held before the mesh's region.
 func removeBlock(a store.Applied) (string, string, error) {
-	raw, err := os.ReadFile(a.Target)
+	real, err := realPath(a.Target)
+	if err != nil {
+		return "", "", err
+	}
+	raw, err := os.ReadFile(real)
 	if errors.Is(err, os.ErrNotExist) {
 		return "forgotten", "no longer there", nil
 	}
 	if err != nil {
 		return "", "", err
 	}
-	info, err := os.Stat(a.Target)
+	info, err := os.Stat(real)
 	if err != nil {
-		return "", "", err
+		return "", "", fmt.Errorf("read %s and cannot see it: %w", a.Target, err)
 	}
 	opening, closing := declaration.BlockMarkers(a.ID)
 	lines := linesOf(string(raw))
@@ -198,8 +273,10 @@ func removeBlock(a store.Applied) (string, string, error) {
 		action, detail = "restored", "no longer declared; the region was given back what it held"
 	case found:
 		from, to := at.begin, at.end+1
-		// The blank line the host put beside the region, and only that one: if what stands there
-		// now is not blank, it is somebody's, and it stays.
+		// The blank line the host added beside the region, when a blank line still stands there.
+		// Whether it is the same one the host added cannot be known from the file; a blank line
+		// is the one line whose going changes nothing any program reads, so it is taken. A line
+		// that is not blank is never taken, whoever put it there.
 		if a.Into.Separated {
 			if a.Into.At == declaration.AtStart {
 				if to < len(lines) && lines[to] == "\n" {
@@ -210,11 +287,15 @@ func removeBlock(a store.Applied) (string, string, error) {
 			}
 		}
 		next = strings.Join(lines[:from], "") + strings.Join(lines[to:], "")
+		// And the line end the host gave the machine's last line, if that line is still last.
+		if a.Into.Ended && strings.Join(lines[to:], "") == "" {
+			next = strings.TrimSuffix(next, "\n")
+		}
 		action, detail = "restored", "no longer declared; the mesh's region was taken out and every other line kept"
 	}
 
-	if a.Into.Created && strings.TrimSpace(next) == "" {
-		if err := os.Remove(a.Target); err != nil {
+	if a.Into.Created && strings.TrimSpace(next) == "" && real == a.Target {
+		if err := os.Remove(real); err != nil {
 			return "", "", err
 		}
 		return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil
@@ -222,15 +303,54 @@ func removeBlock(a store.Applied) (string, string, error) {
 	if next == string(raw) {
 		return action, detail, nil
 	}
-	if err := writeAtomically(a.Target, []byte(next), info.Mode().Perm()); err != nil {
+	if err := writeAtomically(real, []byte(next), info.Mode().Perm()); err != nil {
 		return "", "", err
 	}
-	if err := keepOwner(a.Target, info); err != nil {
+	if err := keepOwner(real, info); err != nil {
 		return "", "", err
 	}
 	return action, detail, nil
 }
 
+// realPath is the file a path names, through any links; a path that is not there yet is itself.
+// A link to nothing is refused: writing through it would replace the link with a file.
+func realPath(path string) (string, error) {
+	real, err := filepath.EvalSymlinks(path)
+	if err == nil {
+		return real, nil
+	}
+	if _, lerr := os.Lstat(path); errors.Is(lerr, os.ErrNotExist) {
+		return path, nil
+	}
+	return "", fmt.Errorf("%s is a link the host cannot follow to a file: %w; it was left as it is", path, err)
+}
+
+// loopbackOf is the lines of a file that answer for the machine itself — localhost, its own name on
+// 127.0.1.1, ::1 — and nothing else: what the mesh's old whole hosts file carried that the machine
+// needs, without the mesh's header or its names.
+func loopbackOf(text string) string {
+	var b strings.Builder
+	for _, line := range linesOf(text) {
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		if ip := net.ParseIP(fields[0]); ip != nil && ip.IsLoopback() {
+			b.WriteString(strings.TrimSuffix(line, "\n") + "\n")
+		}
+	}
+	return b.String()
+}
+
+// outside is every line of a file but the mesh's region, markers included, as one string.
+func outside(lines []string, at region) string {
+	end := at.end + 1
+	if end > len(lines) {
+		end = len(lines)
+	}
+	return strings.Join(lines[:at.begin], "") + "\x00" + strings.Join(lines[end:], "")
+}
+
 // blockBody is the declared lines as they stand in the region: ending in exactly one line end, or
 // nothing at all when there are no lines.
 func blockBody(content string) string {