From 1cb895346dc4c150ce10e5bcd7e87e875818dec8 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 26 Sep 2026 23:51:36 +0200 Subject: [PATCH] Write into a marked block of a text file instead of over it, so a shared hosts file keeps every line that is not the mesh's (hq issue 128) --- internal/apply/apply.go | 6 + internal/apply/block.go | 313 ++++++++++++++++++ internal/apply/block_test.go | 471 ++++++++++++++++++++++++++++ internal/declaration/block_test.go | 51 +++ internal/declaration/declaration.go | 83 ++++- internal/store/store.go | 15 +- 6 files changed, 932 insertions(+), 7 deletions(-) create mode 100644 internal/apply/block.go create mode 100644 internal/apply/block_test.go create mode 100644 internal/declaration/block_test.go diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 132fa8c..8ec6c02 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -669,6 +669,9 @@ func applyAccess(r *declaration.Access) (Outcome, error) { // keepFound, when not nil, is where the original of a file this host has no record of is kept // before it is written over (novox/hq ADR 0100): once, never overwritten, and named in the outcome. func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepFound Keep) (Outcome, error) { + if r.Into == declaration.IntoBlock { + return applyBlock(r, previous) + } if r.Into != "" { return applyInto(r, previous) } @@ -1049,6 +1052,9 @@ func remove(ctx context.Context, sys system.System, a store.Applied, run Runner) return "removed", "no longer declared, and empty", nil case declaration.TypeFile: + if a.Into != nil && a.Into.Format == declaration.IntoBlock { + return removeBlock(a) + } if a.Into != nil { return removeInto(a) } diff --git a/internal/apply/block.go b/internal/apply/block.go new file mode 100644 index 0000000..7115cda --- /dev/null +++ b/internal/apply/block.go @@ -0,0 +1,313 @@ +package apply + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// A file written into a marked block, never over (novox/hq issue 128, ADR 0102). +// +// **The file is the machine's; the mesh owns lines in it.** The machine's hosts file is the case +// that needed it. The mesh wrote it whole — its own header, localhost, the machine's name and every +// name in the mesh — and on a workstation that file is shared: the distribution's lines, a local +// development tool's own marked blocks rewritten whenever its projects change, the operator's +// hand-added names. Written whole, all of those went at the next change to the mesh's names, with +// no failure anywhere: the tool believed it had written its block, and the mesh believed it owned +// the file. It is ADR 0102's failure exactly, in a file ADR 0102's JSON verb cannot speak. +// +// So the host finds the lines between `# BEGIN mesh ` and `# END mesh `, rewrites those and +// nothing else, and records what they held before. Every line outside the markers is kept byte for +// byte — including another tool's `# BEGIN …` blocks, which are that tool's. Undeclared, the region +// is given back what it held, or taken out with its markers when it held nothing, and a file the +// mesh created goes only if nothing but whitespace is left. + +// applyBlock writes a file's declared lines into its region of the file already at its path. +func applyBlock(r *declaration.File, previous store.Applied) (Outcome, error) { + out := begin(r) + opening, closing := declaration.BlockMarkers(r.ID) + want := blockBody(r.Content) + + raw, err := os.ReadFile(r.Path) + existed := err == nil + if err != nil && !errors.Is(err, os.ErrNotExist) { + return out, err + } + existing := string(raw) + lines := linesOf(existing) + at, found, err := regionIn(lines, opening, closing) + if err != nil { + // Refused, never guessed at: markers the host cannot pair are markers it cannot write + // between without risking lines that are not the mesh's. + return out, fmt.Errorf("%s: %w; it was left as it is", r.Path, err) + } + + // A record of a block is carried; anything else — no record, a file once written whole, one + // once written into as JSON — is a file the host is seeing for the first time as a block. + rec := store.Into{Format: declaration.IntoBlock} + recorded := previous.Into != nil && previous.Into.Format == declaration.IntoBlock + if recorded && existed { + rec.Created = previous.Into.Created + rec.Region = previous.Into.Region + rec.Separated = previous.Into.Separated + rec.At = previous.Into.At + } else { + // A file gone since the last apply is made again, and made by the mesh: what it held + // before went with it, so there is nothing to give back but the file's absence. + rec.Created = !existed + if found { + // **What the host may have written itself is not the machine's** — the same reasoning + // as a key in a JSON file (novox/hq ADR 0102). With no record, a region already holding + // exactly the declared lines cannot be told from one this host wrote a moment ago and + // died before saving; remembered as the machine's, it would be put back on undeclare + // for ever. So it is the mesh's, and undeclaring takes it out. + if held := at.body(lines); held != want { + rec.Region = &held + } + } + } + + // Drift: the machine no longer holds, between the mesh's markers, what this host last put + // there. Judged only against a record of a block: a digest of a whole file says nothing about + // a region of it. + drifted := recorded && previous.Wrote != "" && existed && + (!found || digestOf(at.body(lines)) != previous.Wrote) + + var next string + switch { + case !existed: + next = regionOf(opening, closing, want) + case found: + // Where it is, whatever At says: the region is never moved, because moving it moves the + // machine's lines around it. + next = strings.Join(lines[:at.begin+1], "") + want + strings.Join(lines[at.end:], "") + case r.At == declaration.AtStart: + // Above everything, and one blank line between the region and the machine's first line + // unless there is one already — a line in some files means what the lines above it say. + rec.At, rec.Separated = declaration.AtStart, false + next = regionOf(opening, closing, want) + if existing != "" && !strings.HasPrefix(existing, "\n") { + next += "\n" + rec.Separated = true + } + next += existing + default: + // At the end, apart from whatever is there: the file's last line is ended if it was not, + // and one blank line separates the region from the machine's lines unless there is one. + rec.At, rec.Separated = "", false + next = existing + if next != "" && !strings.HasSuffix(next, "\n") { + next += "\n" + } + if next != "" && next != "\n" && !strings.HasSuffix(next, "\n\n") { + next += "\n" + rec.Separated = true + } + next += regionOf(opening, closing, want) + } + + same := existed && next == existing + if !same { + var info os.FileInfo + mode := os.FileMode(0o644) + if info, err = os.Stat(r.Path); err == nil { + mode = info.Mode().Perm() // the machine's file keeps the machine's mode + } else if mode, err = modeOf(r.Mode, mode); err != nil { + return out, err + } + if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { + return out, err + } + if err := writeAtomically(r.Path, []byte(next), mode); err != nil { + return out, err + } + if existed { + // The write is a new file renamed over the old, so it belongs to whoever wrote it. The + // machine's file keeps the machine's owner, as it keeps its mode. + if err := keepOwner(r.Path, info); err != nil { + return out, err + } + } else if err := own(r.Path, r.Owner); err != nil { + return out, err + } + } + + // Read back: the region holds what was declared, and nothing outside it moved. + written, err := os.ReadFile(r.Path) + if err != nil { + return out, fmt.Errorf("wrote into %s and cannot read it back: %w", r.Path, err) + } + if string(written) != next { + return out, fmt.Errorf("%s does not hold the mesh's region as written after writing into it", r.Path) + } + + out.into = &rec + out.wrote = digestOf(want) + switch { + case !existed: + out.Action = "created" + out.Detail = "written into; the file was not there" + case same: + out.Action = "unchanged" + case drifted: + out.Action = "corrected" + out.Detail = "the mesh's region had been changed on the machine; every line outside it was kept" + case !found: + out.Action = "updated" + where := "end" + if rec.At == declaration.AtStart { + where = "start" + } + out.Detail = "the mesh's region added at the " + where + "; every other line kept as it was" + default: + out.Action = "updated" + out.Detail = "the mesh's region rewritten; every line outside it kept as it was" + } + return out, nil +} + +// removeBlock gives back what a file written into a block held before the mesh's region. +func removeBlock(a store.Applied) (string, string, error) { + raw, err := os.ReadFile(a.Target) + if errors.Is(err, os.ErrNotExist) { + return "forgotten", "no longer there", nil + } + if err != nil { + return "", "", err + } + info, err := os.Stat(a.Target) + if err != nil { + return "", "", err + } + opening, closing := declaration.BlockMarkers(a.ID) + lines := linesOf(string(raw)) + at, found, err := regionIn(lines, opening, closing) + if err != nil { + return "kept", err.Error() + ", so nothing was taken out of it; remove the mesh's region by hand", nil + } + + next, action, detail := string(raw), "forgotten", "the mesh's region was no longer in it" + switch { + case found && a.Into.Region != nil: + next = strings.Join(lines[:at.begin+1], "") + *a.Into.Region + strings.Join(lines[at.end:], "") + action, detail = "restored", "no longer declared; the region was given back what it held" + case found: + from, to := at.begin, at.end+1 + // The blank line the host put beside the region, and only that one: if what stands there + // now is not blank, it is somebody's, and it stays. + if a.Into.Separated { + if a.Into.At == declaration.AtStart { + if to < len(lines) && lines[to] == "\n" { + to++ + } + } else if from > 0 && lines[from-1] == "\n" { + from-- + } + } + next = strings.Join(lines[:from], "") + strings.Join(lines[to:], "") + action, detail = "restored", "no longer declared; the mesh's region was taken out and every other line kept" + } + + if a.Into.Created && strings.TrimSpace(next) == "" { + if err := os.Remove(a.Target); err != nil { + return "", "", err + } + return "removed", "no longer declared; the mesh had created it and nothing else was in it", nil + } + if next == string(raw) { + return action, detail, nil + } + if err := writeAtomically(a.Target, []byte(next), info.Mode().Perm()); err != nil { + return "", "", err + } + if err := keepOwner(a.Target, info); err != nil { + return "", "", err + } + return action, detail, nil +} + +// blockBody is the declared lines as they stand in the region: ending in exactly one line end, or +// nothing at all when there are no lines. +func blockBody(content string) string { + trimmed := strings.TrimRight(content, "\n") + if trimmed == "" { + return "" + } + return trimmed + "\n" +} + +func regionOf(begin, end, body string) string { + return begin + "\n" + body + end + "\n" +} + +// linesOf splits text into lines that keep their line ends, so joining them again gives back +// exactly the bytes that were read — a last line without one included. +func linesOf(text string) []string { + return strings.SplitAfter(text, "\n") +} + +// region is where the mesh's markers stand, as indices into the lines of a file. +type region struct{ begin, end int } + +// body is what stands between the markers. +func (r region) body(lines []string) string { + return strings.Join(lines[r.begin+1:r.end], "") +} + +// regionIn finds the mesh's markers for one resource. A line is a marker only if it is exactly the +// marker, so another tool's block and another resource's region are never it. Markers that do not +// form one pair — a begin with no end, an end before its begin, either twice — are an error rather +// than a best guess, because a guess is how the host would rewrite lines that are not its own. +func regionIn(lines []string, begin, end string) (region, bool, error) { + at := region{begin: -1, end: -1} + for i, line := range lines { + switch strings.TrimSuffix(line, "\n") { + case begin: + if at.begin >= 0 { + return at, false, fmt.Errorf("%q is in it more than once", begin) + } + at.begin = i + case end: + if at.end >= 0 { + return at, false, fmt.Errorf("%q is in it more than once", end) + } + at.end = i + } + } + switch { + case at.begin < 0 && at.end < 0: + return at, false, nil + case at.begin < 0: + return at, false, fmt.Errorf("%q is in it with no %q before it", end, begin) + case at.end < 0: + return at, false, fmt.Errorf("%q is in it with no %q after it", begin, end) + case at.end < at.begin: + return at, false, fmt.Errorf("%q stands before %q", end, begin) + } + return at, true, nil +} + +// keepOwner gives a file rewritten through a new one back to whoever owned what it replaced. +// Changed only where it differs, so a host that is not root can still write a file it owns. +func keepOwner(path string, was os.FileInfo) error { + uid, gid, ok := ownerOf(was) + if !ok { + return nil + } + now, err := os.Stat(path) + if err != nil { + return err + } + if u, g, ok := ownerOf(now); ok && u == uid && g == gid { + return nil + } + if err := os.Chown(path, uid, gid); err != nil { + return fmt.Errorf("cannot give %s back to its owner %d:%d: %w", path, uid, gid, err) + } + return nil +} diff --git a/internal/apply/block_test.go b/internal/apply/block_test.go new file mode 100644 index 0000000..2dfefab --- /dev/null +++ b/internal/apply/block_test.go @@ -0,0 +1,471 @@ +package apply + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq issue 128 and ADR 0102: a text file the mesh shares with software it did not +// install is written into a marked block, never over — every line outside the mesh's markers is +// the machine's and is kept byte for byte, and undeclaring gives the file back. + +const namesID = "mesh-wireguard.fact-node-names" + +func blockDecl(t *testing.T, path, content string, extra ...string) string { + t.Helper() + more := "" + for _, e := range extra { + more += "," + e + } + return fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":%q,"type":"file","path":%q,"into":"block","content":%q%s} + ]}`, namesID, path, content, more) +} + +func applyBlockDecl(t *testing.T, raw string, known store.State) (Report, store.State) { + t.Helper() + report, state, err := Apply(context.Background(), archHost(t), parse(t, raw), known, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + return report, state +} + +func undeclare(t *testing.T, known store.State) (Report, store.State) { + t.Helper() + report, state, err := Apply(context.Background(), archHost(t), somethingElse(t), known, store.OriginDeclared, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + return report, state +} + +func readText(t *testing.T, path string) string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func marked(id, body string) string { + return "# BEGIN mesh " + id + "\n" + body + "# END mesh " + id + "\n" +} + +// A workstation's hosts file, the way issue 128 found it: the distribution's lines, a development +// tool's own marked blocks, and the operator's hand-added names. +const workstationHosts = "127.0.0.1\tlocalhost\n" + + "127.0.1.1\tg14.localdomain g14\n" + + "\n" + + "# BEGIN devtool project-a\n" + + "127.0.0.1 a.test api.a.test\n" + + "# END devtool project-a\n" + + "# BEGIN devtool project-b\n" + + "127.0.0.1 b.test\n" + + "# END devtool project-b\n" + + "192.168.1.20 printer # the operator's\n" + +const meshNames = "10.42.0.1 ace\n10.42.0.2 novox\n" + +func TestABlockKeepsEveryLineOutsideItsMarkers(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + if err := os.WriteFile(path, []byte(workstationHosts), 0o640); err != nil { + t.Fatal(err) + } + report, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + want := workstationHosts + "\n" + marked(namesID, meshNames) + if got := readText(t, path); got != want { + t.Fatalf("the file after writing into it:\n%q\nwant\n%q", got, want) + } + if got := report.Outcomes[0].Action; got != "updated" { + t.Errorf("adding the region was %q", got) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o640 { + t.Errorf("the machine's file mode was changed to %o", info.Mode().Perm()) + } + rec, _ := state.Find(namesID) + if rec.Into == nil || rec.Into.Format != "block" || rec.Into.Region != nil || rec.Into.Created { + t.Fatalf("recorded as %+v", rec.Into) + } + + // Again, with nothing changed: nothing written. + report, state = applyBlockDecl(t, blockDecl(t, path, meshNames), state) + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("a second apply was %q", got) + } + + // The development tool rewrites its block, and the operator adds a line after the mesh's + // region; the mesh's names change. Only the region moves. + edited := strings.Replace(readText(t, path), "127.0.0.1 b.test\n", "127.0.0.1 b.test c.test\n", 1) + + "10.0.0.5 nas # added after\n" + _ = os.WriteFile(path, []byte(edited), 0o640) + changed := meshNames + "10.42.0.3 shanks\n" + report, state = applyBlockDecl(t, blockDecl(t, path, changed), state) + want = strings.Replace(edited, marked(namesID, meshNames), marked(namesID, changed), 1) + if got := readText(t, path); got != want { + t.Fatalf("rewriting the region moved something else:\n%q\nwant\n%q", got, want) + } + if got := report.Outcomes[0].Action; got != "updated" { + t.Errorf("rewriting the region was %q", got) + } + + // Undeclared: the region, its markers and the blank line the host put before it go; every + // other line is where it was. + report, _ = undeclare(t, state) + want = strings.Replace(edited, "\n"+marked(namesID, meshNames), "", 1) + if got := readText(t, path); got != want { + t.Fatalf("undeclaring left:\n%q\nwant\n%q", got, want) + } + if got := report.Outcomes[0].Action; got != "restored" { + t.Errorf("undeclaring was %q", got) + } +} + +func TestUndeclaringABlockAddedAtTheEndGivesTheFileBackExactly(t *testing.T) { + for name, original := range map[string]string{ + "ending in a line": "127.0.0.1 localhost\n", + "ending in a blank line": "127.0.0.1 localhost\n\n", + "empty": "", + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + _ = os.WriteFile(path, []byte(original), 0o644) + _, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + undeclare(t, state) + if got := readText(t, path); got != original { + t.Errorf("undeclaring left %q, the machine had %q", got, original) + } + }) + } +} + +func TestABlockAddedAtTheEndIsSetApartFromTheMachinesLines(t *testing.T) { + for name, c := range map[string]struct{ before, after string }{ + "no line end": {"127.0.0.1 localhost", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)}, + "a line end": {"127.0.0.1 localhost\n", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)}, + "a blank line already": {"127.0.0.1 localhost\n\n", "127.0.0.1 localhost\n\n" + marked(namesID, meshNames)}, + "empty": {"", marked(namesID, meshNames)}, + "only a blank line": {"\n", "\n" + marked(namesID, meshNames)}, + "content without an end": {"x\n\n", "x\n\n" + marked(namesID, meshNames)}, + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + _ = os.WriteFile(path, []byte(c.before), 0o644) + applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + if got := readText(t, path); got != c.after { + t.Errorf("got %q, want %q", got, c.after) + } + }) + } +} + +func TestTheRegionEndsInExactlyOneLineEnd(t *testing.T) { + for content, body := range map[string]string{ + "10.42.0.1 ace": "10.42.0.1 ace\n", + "10.42.0.1 ace\n": "10.42.0.1 ace\n", + "10.42.0.1 ace\n\n\n": "10.42.0.1 ace\n", + "": "", + "\n\n": "", + } { + path := filepath.Join(t.TempDir(), "hosts") + _, state := applyBlockDecl(t, blockDecl(t, path, content), store.State{}) + if got := readText(t, path); got != marked(namesID, body) { + t.Errorf("content %q was written as %q", content, got) + } + // And the same content again is not a change. + report, _ := applyBlockDecl(t, blockDecl(t, path, content), state) + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("content %q applied twice was %q", content, got) + } + } +} + +func TestARegionAlreadyThereIsRewrittenInPlaceAndGivenBack(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + before := "127.0.0.1 localhost\n" + after := "# BEGIN devtool x\n127.0.0.1 x.test\n# END devtool x\n192.168.1.20 printer\n" + found := "10.42.0.9 old-name\n" + original := before + marked(namesID, found) + after + _ = os.WriteFile(path, []byte(original), 0o644) + + report, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + if got := readText(t, path); got != before+marked(namesID, meshNames)+after { + t.Fatalf("the region was not rewritten in place: %q", got) + } + if got := report.Outcomes[0].Action; got != "updated" { + t.Errorf("rewriting a found region was %q", got) + } + rec, _ := state.Find(namesID) + if rec.Into.Region == nil || *rec.Into.Region != found { + t.Fatalf("what the region held before was recorded as %v", rec.Into.Region) + } + + // Undeclared: what the region held goes back, where it was. + report, _ = undeclare(t, state) + if got := readText(t, path); got != original { + t.Errorf("undeclaring left %q, the machine had %q", got, original) + } + if got := report.Outcomes[0].Action; got != "restored" { + t.Errorf("undeclaring was %q", got) + } +} + +func TestARegionWithNoRecordHoldingExactlyTheDeclaredLinesIsTheMeshs(t *testing.T) { + // A host that wrote the region and died before saving its state: what is between the markers + // is exactly what the mesh declares, and remembered as the machine's it would never go. + path := filepath.Join(t.TempDir(), "hosts") + original := "127.0.0.1 localhost\n" + _ = os.WriteFile(path, []byte(original+"\n"+marked(namesID, meshNames)), 0o644) + _, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + if rec, _ := state.Find(namesID); rec.Into.Region != nil { + t.Fatalf("the mesh's own lines were recorded as the machine's: %q", *rec.Into.Region) + } + undeclare(t, state) + if got := readText(t, path); !strings.HasPrefix(got, original) || strings.Contains(got, "BEGIN mesh") { + t.Errorf("undeclaring left the mesh's region behind: %q", got) + } +} + +func TestADriftedRegionIsCorrected(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + _ = os.WriteFile(path, []byte(workstationHosts), 0o644) + _, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + written := readText(t, path) + + _ = os.WriteFile(path, []byte(strings.Replace(written, "10.42.0.2 novox\n", "10.42.0.2 novox\n6.6.6.6 evil\n", 1)), 0o644) + report, _ := applyBlockDecl(t, blockDecl(t, path, meshNames), state) + if got := report.Outcomes[0].Action; got != "corrected" { + t.Errorf("a region edited on the machine was %q", got) + } + if got := readText(t, path); got != written { + t.Errorf("the region was not put back: %q", got) + } + + // The region taken out by hand is drift too, and it is put back. + _ = os.WriteFile(path, []byte(workstationHosts), 0o644) + report, _ = applyBlockDecl(t, blockDecl(t, path, meshNames), state) + if got := report.Outcomes[0].Action; got != "corrected" { + t.Errorf("a region removed on the machine was %q", got) + } + if got := readText(t, path); got != written { + t.Errorf("the region was not put back: %q", got) + } +} + +func TestTwoRegionsInOneFileAreEachTheirOwn(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + _ = os.WriteFile(path, []byte(workstationHosts), 0o644) + raw := fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"a.names","type":"file","path":%q,"into":"block","content":"10.42.0.1 ace\n"}, + {"id":"b.names","type":"file","path":%q,"into":"block","content":"10.43.0.1 lab\n"} + ]}`, path, path) + _, state := applyBlockDecl(t, raw, store.State{}) + want := workstationHosts + "\n" + marked("a.names", "10.42.0.1 ace\n") + "\n" + marked("b.names", "10.43.0.1 lab\n") + if got := readText(t, path); got != want { + t.Fatalf("two regions:\n%q\nwant\n%q", got, want) + } + report, state := applyBlockDecl(t, raw, state) + for _, o := range report.Outcomes { + if o.Action != "unchanged" { + t.Errorf("%s applied twice was %q", o.ID, o.Action) + } + } + + // One undeclared: only its region goes. + only := fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"b.names","type":"file","path":%q,"into":"block","content":"10.43.0.1 lab\n"} + ]}`, path) + applyBlockDecl(t, only, state) + want = workstationHosts + "\n" + marked("b.names", "10.43.0.1 lab\n") + if got := readText(t, path); got != want { + t.Errorf("undeclaring one region:\n%q\nwant\n%q", got, want) + } +} + +func TestABlockInAFileThatWasNotThereIsCreatedAndRemovedWithIt(t *testing.T) { + path := filepath.Join(t.TempDir(), "conf.d", "mesh.conf") + report, state := applyBlockDecl(t, blockDecl(t, path, meshNames, `"mode":"0600"`), store.State{}) + if got := readText(t, path); got != marked(namesID, meshNames) { + t.Fatalf("a created file holds %q", got) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { + t.Errorf("a created file is mode %o, declared 0600", info.Mode().Perm()) + } + if got := report.Outcomes[0].Action; got != "created" { + t.Errorf("creating was %q", got) + } + if rec, _ := state.Find(namesID); !rec.Into.Created { + t.Error("the mesh creating the file was not recorded") + } + report, _ = undeclare(t, state) + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("a file the mesh created, holding only its region, was left behind") + } + if got := report.Outcomes[0].Action; got != "removed" { + t.Errorf("undeclaring was %q", got) + } + + // Somebody else wrote into it meanwhile: it is no longer only the mesh's, and it stays. + _, state = applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + _ = os.WriteFile(path, []byte(readText(t, path)+"their = line\n"), 0o600) + undeclare(t, state) + if got := readText(t, path); got != "their = line\n" { + t.Errorf("undeclaring a created file somebody wrote into left %q", got) + } +} + +func TestMarkersThatDoNotPairAreRefusedAndLeftAlone(t *testing.T) { + for name, text := range map[string]string{ + "a begin with no end": "a\n# BEGIN mesh " + namesID + "\nb\n", + "an end with no begin": "a\n# END mesh " + namesID + "\n", + "an end before a begin": "# END mesh " + namesID + "\n# BEGIN mesh " + namesID + "\n", + "a begin twice": marked(namesID, "x\n") + "# BEGIN mesh " + namesID + "\n", + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + _ = os.WriteFile(path, []byte(text), 0o644) + if _, _, err := Apply(context.Background(), archHost(t), parse(t, blockDecl(t, path, meshNames)), + store.State{}, store.OriginDeclared, nil, nil, nil); err == nil { + t.Fatal("markers that do not pair were written between") + } + if got := readText(t, path); got != text { + t.Errorf("the file was changed: %q", got) + } + }) + } +} + +func TestAMarkerOfAnotherIDIsNotThisRegion(t *testing.T) { + // The id is part of the marker: a region whose id merely starts with this one is not it. + path := filepath.Join(t.TempDir(), "hosts") + other := marked(namesID+"-extra", "10.9.9.9 other\n") + _ = os.WriteFile(path, []byte(other), 0o644) + _, state := applyBlockDecl(t, blockDecl(t, path, meshNames), store.State{}) + if got := readText(t, path); got != other+"\n"+marked(namesID, meshNames) { + t.Fatalf("got %q", got) + } + undeclare(t, state) + if got := readText(t, path); got != other { + t.Errorf("undeclaring touched the other region: %q", got) + } +} + +func TestABlockAtTheStartStandsAboveEverything(t *testing.T) { + // dhcpcd scopes every line after `interface X` to that interface, so the mesh's global options + // go above all of it. + dhcpcd := "hostname\nduid\n\ninterface enp6s0\nstatic ip_address=192.168.1.5/24\n" + opts := "nohook resolv.conf\ndenyinterfaces mesh0\n" + for name, c := range map[string]struct{ before, after string }{ + "a file with content": {dhcpcd, marked(namesID, opts) + "\n" + dhcpcd}, + "an empty file": {"", marked(namesID, opts)}, + "a file opening blank": {"\n" + dhcpcd, marked(namesID, opts) + "\n" + dhcpcd}, + "a last line with no end": {"interface enp6s0", marked(namesID, opts) + "\ninterface enp6s0"}, + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "dhcpcd.conf") + _ = os.WriteFile(path, []byte(c.before), 0o644) + report, state := applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), store.State{}) + if got := readText(t, path); got != c.after { + t.Fatalf("got %q, want %q", got, c.after) + } + if got := report.Outcomes[0].Action; got != "updated" { + t.Errorf("adding the region was %q", got) + } + report, state = applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), state) + if got := report.Outcomes[0].Action; got != "unchanged" { + t.Errorf("a second apply was %q", got) + } + undeclare(t, state) + if got := readText(t, path); got != c.before { + t.Errorf("undeclaring left %q, the machine had %q", got, c.before) + } + }) + } + + t.Run("a file that was not there", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "dhcpcd.conf") + applyBlockDecl(t, blockDecl(t, path, opts, `"at":"start"`), store.State{}) + if got := readText(t, path); got != marked(namesID, opts) { + t.Errorf("got %q", got) + } + }) +} + +func TestARegionAlreadyThereIsNotMovedWhereverAtSaysItGoes(t *testing.T) { + for _, at := range []string{`"at":"start"`, `"at":"end"`} { + path := filepath.Join(t.TempDir(), "dhcpcd.conf") + original := "hostname\n" + marked(namesID, "old\n") + "interface enp6s0\n" + _ = os.WriteFile(path, []byte(original), 0o644) + applyBlockDecl(t, blockDecl(t, path, "nohook resolv.conf\n", at), store.State{}) + want := "hostname\n" + marked(namesID, "nohook resolv.conf\n") + "interface enp6s0\n" + if got := readText(t, path); got != want { + t.Errorf("%s: a found region was moved: %q", at, got) + } + } +} + +func TestAFileWrittenIntoABlockIsNeverHeldOnAnAdoptedNode(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + _ = os.WriteFile(path, []byte(workstationHosts), 0o644) + resource := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path, meshNames) + d := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource) + report, state := applyAdopted(t, d, store.State{}, &machine{}, t.TempDir()) + if got := outcomeOf(report, namesID).Action; got == "held" { + t.Fatal("a file written into a block was held, though it replaces nothing that was found") + } + if len(state.Held) != 0 { + t.Errorf("something was held: %+v", state.Held) + } + if got := readText(t, path); got != workstationHosts+"\n"+marked(namesID, meshNames) { + t.Errorf("the adopted node's file was not written into: %q", got) + } + + // Held from when it was declared whole, the hold does not keep the region out. + path2 := filepath.Join(t.TempDir(), "hosts") + _ = os.WriteFile(path2, []byte(workstationHosts), 0o644) + known := store.State{Held: []store.Held{{ID: namesID, Module: "mesh-wireguard", Kind: "file", Target: path2}}} + resource2 := fmt.Sprintf(`{"id":%q,"type":"file","path":%q,"into":"block","content":%q}`, namesID, path2, meshNames) + d2 := adopted(t, `{"taken":[],"untaken":{"mesh-wireguard":["`+namesID+`"]}}`, resource2) + report, state = applyAdopted(t, d2, known, &machine{}, t.TempDir()) + if got := outcomeOf(report, namesID).Action; got != "updated" { + t.Errorf("the file was %q, not written into", got) + } + if len(state.Held) != 0 { + t.Errorf("the old hold outlived the block declaration: %+v", state.Held) + } + + // And the preview says the same: written into, not held. + for _, s := range Plan(d2, known, store.OriginDeclared) { + if s.ID == namesID && s.Verb == "hold" { + t.Errorf("the preview holds a file written into a block: %+v", s) + } + } +} + +func TestTheRecordOfABlockSurvivesTheStateFile(t *testing.T) { + // What undeclaring needs is in the state a host saves, not only in memory. + path := filepath.Join(t.TempDir(), "hosts") + original := "a\n" + marked(namesID, "old\n") + _ = os.WriteFile(path, []byte(original), 0o644) + _, state := applyBlockDecl(t, blockDecl(t, path, meshNames, `"at":"start"`), store.State{}) + raw, err := json.Marshal(state) + if err != nil { + t.Fatal(err) + } + var back store.State + if err := json.Unmarshal(raw, &back); err != nil { + t.Fatal(err) + } + undeclare(t, back) + if got := readText(t, path); got != original { + t.Errorf("undeclaring from a saved state left %q", got) + } +} diff --git a/internal/declaration/block_test.go b/internal/declaration/block_test.go new file mode 100644 index 0000000..a9acdec --- /dev/null +++ b/internal/declaration/block_test.go @@ -0,0 +1,51 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq issue 128: a file written into a block carries only its lines, in content, +// never a line the host keeps as its own marker, and says where a new region goes only as a block. + +func TestAFileWrittenIntoABlockIsRefusedUnlessItIsOnlyItsLines(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "a begin marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# BEGIN mesh f\nb\n"}`, "# BEGIN mesh f"}, + "an end marker in content": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a\n# END mesh other\n"}`, "# END mesh other"}, + "a marker with a CR": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"# BEGIN mesh x\r\n"}`, "marker"}, + "sealed": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","sealed":"abc"}`, "not sealed"}, + "bytes": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","bytes":"YQ=="}`, "not sealed, bytes"}, + "secrets": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"${secret:s}","secrets":{"s":"abc"}}`, "secrets"}, + "create-once": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","create-once":true}`, "create-once"}, + "at on a whole file": {`{"id":"f","type":"file","path":"/etc/hosts","content":"a","at":"start"}`, `at "start"`}, + "at on a JSON file": {`{"id":"f","type":"file","path":"/etc/x.json","into":"json","content":"{}","at":"end"}`, `at "end"`}, + "at somewhere else": {`{"id":"f","type":"file","path":"/etc/hosts","into":"block","content":"a","at":"middle"}`, `"start" or "end"`}, + "an unknown format": {`{"id":"f","type":"file","path":"/etc/hosts","into":"lines","content":"a"}`, `"json" or "block"`}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } +} + +func TestAFileWrittenIntoABlockIsRead(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"mesh-wireguard.fact-node-names","type":"file","path":"/etc/hosts","into":"block","content":"10.42.0.1 ace\n# BEGIN devtool x\n"}, + {"id":"dhcpcd.options","type":"file","path":"/etc/dhcpcd.conf","into":"block","content":"nohook resolv.conf\n","at":"start"}, + {"id":"empty","type":"file","path":"/etc/x","into":"block","content":"","at":"end"} + ]}`)) + if err != nil { + t.Fatal(err) + } + if f := d.Resources[0].(*File); f.Into != IntoBlock || f.At != "" { + t.Errorf("read as into %q at %q", f.Into, f.At) + } + if f := d.Resources[1].(*File); f.At != AtStart { + t.Errorf("at was read as %q", f.At) + } + if begin, end := BlockMarkers("mesh-wireguard.fact-node-names"); begin != "# BEGIN mesh mesh-wireguard.fact-node-names" || + end != "# END mesh mesh-wireguard.fact-node-names" { + t.Errorf("the markers are %q and %q", begin, end) + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 5fa7f27..65765a8 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -161,8 +161,27 @@ type File struct { // "json" is spoken — the content is a JSON object whose keys the host sets in the file's // object, keeping every other key as it found it and recording what each of its keys held // before, so undeclaring the file gives those back. + // + // "block" is the same idea for a file that is not structured (novox/hq issue 128): the + // content is the mesh's lines, and the host owns only the region between `# BEGIN mesh ` + // and `# END mesh `, keeping every line outside it byte for byte. The machine's hosts file + // is the case that needed it — on a workstation the distribution, a local development tool and + // the operator all write into it, and the mesh writing it whole took their lines away at the + // next change to the mesh's names, silently. Marked blocks are the shape the other tools in + // that file already use, and `#` is the comment character of every file this serves. Into string `json:"into,omitempty"` + // At is where a file written into a block has its region added when the file does not hold + // one yet: "end", the default, or "start". A region already there stays where it is, whatever + // this says — moving it would move the lines around it, and those are the machine's. + // + // **Some files give a line its meaning by what stands above it.** dhcpcd's configuration scopes + // every line after `interface X` to that interface, and a real one ends with exactly that — an + // interface and its static address. A region added at the end would make the mesh's global + // options (`nohook resolv.conf`, `denyinterfaces mesh0`) options of one interface, and dhcpcd + // would read them without complaint. At the start, nothing stands above them. + At string `json:"at,omitempty"` + // Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver // without being able to read. // @@ -241,16 +260,45 @@ func (f *File) validate(where string, _ bool) []string { problems = append(problems, where+ ": a file written into JSON carries a JSON object of the keys it sets") } - if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { + case IntoBlock: + // The markers are how the host finds its region again. A marker in the content would be + // a second region, or the end of this one, the next time the file is read — and the host + // would then rewrite, or on undeclare take out, lines that were never the mesh's. + for _, line := range strings.Split(f.Content, "\n") { + if strings.HasPrefix(line, BlockBegin) || strings.HasPrefix(line, BlockEnd) { + problems = append(problems, fmt.Sprintf( + "%s: a file written into a block carries the mesh's lines, and %q is a marker the "+ + "host keeps for itself", where, strings.TrimRight(line, "\r"))) + break + } + } + // The id is written into the markers, so it has to stay on one line. + if strings.ContainsAny(f.ID, "\r\n") { problems = append(problems, where+ - ": a file written into says only its keys, in content — not sealed, bytes, "+ - "secrets or create-once") + ": a file written into a block names its region by its id, and this id spans lines") } default: problems = append(problems, fmt.Sprintf( - "%s: into %q; a file is written into \"json\", or omits it to be written whole", + "%s: into %q; a file is written into \"json\" or \"block\", or omits it to be written whole", where, f.Into)) } + switch { + case f.At == "": + case f.Into != IntoBlock: + problems = append(problems, fmt.Sprintf( + "%s: at %q; only a file written into a block has a place its region is added", where, f.At)) + case f.At != AtStart && f.At != AtEnd: + problems = append(problems, fmt.Sprintf( + "%s: at %q; a block is added at \"start\" or \"end\", or omits it to be added at the end", + where, f.At)) + } + if f.Into == IntoJSON || f.Into == IntoBlock { + if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { + problems = append(problems, where+ + ": a file written into says only its part, in content — not sealed, bytes, "+ + "secrets or create-once") + } + } var said []string for name, value := range map[string]string{ "content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes, @@ -670,8 +718,31 @@ func (s *Service) validate(where string, _ bool) []string { return problems } -// IntoJSON is the one structured format a file is written into. -const IntoJSON = "json" +// What a file is written into (novox/hq ADR 0102): a JSON object whose keys the mesh sets, or a +// text file in which the mesh owns one marked block of lines (novox/hq issue 128). +const ( + IntoJSON = "json" + IntoBlock = "block" +) + +// The lines that delimit the mesh's region in a file written into a block, each followed by the +// resource's id. Exact lines, never patterns: another tool's `# BEGIN …` block in the same file is +// that tool's, and a marker that merely resembled the mesh's must not be taken for it. +const ( + BlockBegin = "# BEGIN mesh " + BlockEnd = "# END mesh " +) + +// Where a file written into a block has its region added, when it has none yet. +const ( + AtStart = "start" + AtEnd = "end" +) + +// BlockMarkers are the two lines, without their line ends, that delimit a resource's region. +func BlockMarkers(id string) (begin, end string) { + return BlockBegin + id, BlockEnd + id +} // Opening is a port reachable on an adopted node, from where, and on which path. // diff --git a/internal/store/store.go b/internal/store/store.go index eb8264d..a33ba5d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -84,7 +84,7 @@ type Applied struct { Reads map[string]string `json:"reads,omitempty"` } -// Into is what a file written into held before the mesh's keys. +// Into is what a file written into held before the mesh's keys, or before the mesh's block. type Into struct { Format string `json:"format"` Before map[string]json.RawMessage `json:"before,omitempty"` @@ -94,6 +94,19 @@ type Into struct { // to the machine's list — never a member that was already there. Undeclared, only these go, // and drift is judged on these alone (novox/hq ADR 0102). Added map[string][]json.RawMessage `json:"added,omitempty"` + + // Region is, for a file written into a block (novox/hq issue 128), what the lines between the + // mesh's markers held before the mesh wrote them — nil when there was no region, which is + // different from a region that was there and empty. Undeclared, a recorded region is put back + // and an unrecorded one is taken out, markers and all. It is the block's "what each key held + // before": the one thing the host needs to give the file back. + Region *string `json:"region,omitempty"` + // Separated says the host put a blank line between the region and the machine's lines when it + // added the region — before it at the end, after it at the start, as At says — so taking the + // region out takes that line with it and nothing of the operator's. + Separated bool `json:"separated,omitempty"` + // At is where the host added the region: "start", or empty for the end. + At string `json:"at,omitempty"` } // State is the whole of what a node knows about what it has done.