claude-code: memory carrying never leaves a memory folder, never takes the unreadable for gone, and screens every write (issue 367, review of #276)
mesh/merge-gate head 305601d25ef4 merged into trunk a84d0094a0a3 — pass: builds claude-code → ace, g14, novox, shanks; no bus step; every machine composes …
mesh/repo-check head 305601d25ef4 merged into trunk a84d0094a0a3 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
mesh/merge-gate head 305601d25ef4 merged into trunk a84d0094a0a3 — pass: builds claude-code → ace, g14, novox, shanks; no bus step; every machine composes …
mesh/repo-check head 305601d25ef4 merged into trunk a84d0094a0a3 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A value in the state could name a project such as ../../.. and be written outside the memory folder; a file or folder that could not be read was carried as a removal; text kept over a removal or merged in a conflict reached the state unscreened; and a conflict whose file could not be written was merged into the state again on every look. Names are judged before any file is touched, what cannot be read is kept and listed, every state write is screened, a conflict is written to the file before the state and warns once when it cannot be, a file changed between reading and writing is not written over, non-UTF-8 text is not carried, and a node that was away takes a removal it holds no change against (ADR 0343 §9, as clarified).
This commit is contained in:
@@ -37,6 +37,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -121,6 +122,7 @@ type MemoryCarrier struct {
|
||||
rec memoryRecord
|
||||
loaded bool
|
||||
notCarried map[string]string
|
||||
failed map[string]bool
|
||||
pruned time.Time
|
||||
seen map[string]seenFile
|
||||
}
|
||||
@@ -175,6 +177,9 @@ func memoryProblem(text []byte) string {
|
||||
if len(text) > MemoryMostBytes {
|
||||
return fmt.Sprintf("it is %d KiB, over the %d KiB a memory file may be to be carried", len(text)/1024, MemoryMostBytes/1024)
|
||||
}
|
||||
if !utf8.Valid(text) {
|
||||
return "it is not text (not valid UTF-8), and the state carries text only"
|
||||
}
|
||||
// One value of the state holds at most 256 KiB (ADR 0201): the text as it is encoded, with room for the rest.
|
||||
if encoded, _ := json.Marshal(string(text)); len(encoded) > MemoryValueBytes {
|
||||
return "its text, as the state encodes it, would not fit in one value of the state"
|
||||
@@ -204,6 +209,12 @@ func (m *MemoryCarrier) load() {
|
||||
if m.rec.Conflicts == nil {
|
||||
m.rec.Conflicts = map[string]conflict{}
|
||||
}
|
||||
if m.notCarried == nil {
|
||||
m.notCarried = map[string]string{}
|
||||
}
|
||||
if m.failed == nil {
|
||||
m.failed = map[string]bool{}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MemoryCarrier) save() error {
|
||||
@@ -236,32 +247,60 @@ type localFile struct {
|
||||
|
||||
// memoryFolder is a project's memory folder, when it is a directory and not a symbolic link.
|
||||
func (m *MemoryCarrier) memoryFolder(project string) (string, bool) {
|
||||
if !entryName(project) {
|
||||
return "", false
|
||||
}
|
||||
dir := filepath.Join(m.projects(), project, "memory")
|
||||
info, err := fsys.Lstat(dir)
|
||||
return dir, err == nil && info.IsDir() && info.Mode()&fs.ModeSymlink == 0
|
||||
}
|
||||
|
||||
// scan reads every memory folder: the files it may carry, the folders that exist, and why any file is not carried.
|
||||
func (m *MemoryCarrier) scan() (map[string]localFile, map[string]bool, map[string]string) {
|
||||
files, folders, not := map[string]localFile{}, map[string]bool{}, map[string]string{}
|
||||
// sight is what one look saw of the memory folders.
|
||||
type sight struct {
|
||||
files map[string]localFile
|
||||
// folders are the memory folders seen whole: only in them is a carried file that is gone a removal.
|
||||
folders map[string]bool
|
||||
// unseen are the files that are there but could not be read on this look: kept as they are everywhere.
|
||||
unseen map[string]bool
|
||||
not map[string]string
|
||||
}
|
||||
|
||||
// scan reads every memory folder: the files it may carry, the folders seen whole, and why any file is not carried.
|
||||
// What cannot be read is never taken for gone: a file that cannot be read is unseen, and a folder that cannot be
|
||||
// listed whole is not seen at all on this look (ADR 0343 §9).
|
||||
func (m *MemoryCarrier) scan() (sight, error) {
|
||||
in := sight{files: map[string]localFile{}, folders: map[string]bool{}, unseen: map[string]bool{}, not: map[string]string{}}
|
||||
seen := map[string]seenFile{}
|
||||
defer func() { m.seen = seen }()
|
||||
projects, err := fsys.ReadDir(m.projects())
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return in, nil
|
||||
}
|
||||
if err != nil {
|
||||
return files, folders, not
|
||||
return in, fmt.Errorf("the projects directory could not be listed (%v); nothing was carried on this look", err)
|
||||
}
|
||||
for _, pe := range projects {
|
||||
if !pe.IsDir() {
|
||||
if !pe.IsDir() || !entryName(pe.Name()) {
|
||||
continue
|
||||
}
|
||||
project := pe.Name()
|
||||
dir, ok := m.memoryFolder(project)
|
||||
if !ok {
|
||||
if _, err := fsys.Lstat(dir); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
in.not[project+"/"] = fmt.Sprintf("its memory folder could not be looked at (%v); nothing of it was carried or removed on this look", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
folders[project] = true
|
||||
files, unseen, not := map[string]localFile{}, map[string]bool{}, map[string]string{}
|
||||
whole := true
|
||||
_ = fsys.WalkDir(dir, func(full string, d fs.DirEntry, err error) error {
|
||||
if err != nil || full == dir {
|
||||
if err != nil {
|
||||
// A folder that cannot be listed: this project is not seen on this look.
|
||||
whole = false
|
||||
in.not[project+"/"] = fmt.Sprintf("its memory folder could not be listed whole (%v); nothing of it was carried or removed on this look", err)
|
||||
return filepath.SkipAll
|
||||
}
|
||||
if full == dir {
|
||||
return nil
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, full)
|
||||
@@ -269,8 +308,13 @@ func (m *MemoryCarrier) scan() (map[string]localFile, map[string]bool, map[strin
|
||||
if strings.HasPrefix(d.Name(), stagePrefix) {
|
||||
return nil
|
||||
}
|
||||
if NameProblem(project, rel) != "" {
|
||||
return nil
|
||||
}
|
||||
key := MemoryKey(m.Kind, project, rel)
|
||||
if d.Type()&fs.ModeSymlink != 0 {
|
||||
not[project+"/"+rel] = "it is a symbolic link, which is never carried"
|
||||
unseen[key] = true
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() || !d.Type().IsRegular() {
|
||||
@@ -278,11 +322,14 @@ func (m *MemoryCarrier) scan() (map[string]localFile, map[string]bool, map[strin
|
||||
}
|
||||
info, err := fsys.Lstat(full)
|
||||
if err != nil {
|
||||
not[project+"/"+rel] = fmt.Sprintf("it could not be looked at here (%v); it is kept as it is everywhere", err)
|
||||
unseen[key] = true
|
||||
return nil
|
||||
}
|
||||
if info.Size() > MemoryMostBytes {
|
||||
not[project+"/"+rel] = fmt.Sprintf("it is %d KiB, over the %d KiB a memory file may be to be carried",
|
||||
info.Size()/1024, MemoryMostBytes/1024)
|
||||
unseen[key] = true
|
||||
return nil
|
||||
}
|
||||
// A file whose size and time did not change since the last look is not read again.
|
||||
@@ -290,23 +337,42 @@ func (m *MemoryCarrier) scan() (map[string]localFile, map[string]bool, map[strin
|
||||
if was, ok := m.seen[full]; ok && was.size == info.Size() && was.mod.Equal(info.ModTime()) {
|
||||
text = was.text
|
||||
} else if text, err = fsys.ReadFile(full); err != nil {
|
||||
not[project+"/"+rel] = fmt.Sprintf("it could not be read here (%v); it is kept as it is everywhere", err)
|
||||
unseen[key] = true
|
||||
return nil
|
||||
}
|
||||
seen[full] = seenFile{size: info.Size(), mod: info.ModTime(), text: text}
|
||||
if why := memoryProblem(text); why != "" {
|
||||
not[project+"/"+rel] = why
|
||||
unseen[key] = true
|
||||
return nil
|
||||
}
|
||||
files[MemoryKey(m.Kind, project, rel)] = localFile{Project: project, Path: rel, Text: text, Changed: info.ModTime()}
|
||||
files[key] = localFile{Project: project, Path: rel, Text: text, Changed: info.ModTime()}
|
||||
return nil
|
||||
})
|
||||
if !whole {
|
||||
continue
|
||||
}
|
||||
in.folders[project] = true
|
||||
for k, v := range files {
|
||||
in.files[k] = v
|
||||
}
|
||||
for k := range unseen {
|
||||
in.unseen[k] = true
|
||||
}
|
||||
for k, v := range not {
|
||||
in.not[k] = v
|
||||
}
|
||||
}
|
||||
return files, folders, not
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// readLocal reads one file of a memory folder: absent, or its text; a symbolic link or anything but a file is an
|
||||
// error, never followed.
|
||||
func (m *MemoryCarrier) readLocal(project, path string) ([]byte, time.Time, bool, error) {
|
||||
if err := badName(project, path); err != nil {
|
||||
return nil, time.Time{}, false, err
|
||||
}
|
||||
dir, ok := m.memoryFolder(project)
|
||||
if !ok {
|
||||
if _, err := fsys.Lstat(dir); err == nil {
|
||||
@@ -332,6 +398,31 @@ func (m *MemoryCarrier) readLocal(project, path string) ([]byte, time.Time, bool
|
||||
return text, info.ModTime(), err == nil, err
|
||||
}
|
||||
|
||||
// NameProblem says why a project and a path may not name a memory file, or "": the project is one entry of the
|
||||
// projects directory, and the path is relative, each of its parts a name — never empty, "." or "..". It is judged
|
||||
// before any file is touched and on every value taken from the state, so no value leads outside a memory folder.
|
||||
func NameProblem(project, path string) string {
|
||||
if !entryName(project) {
|
||||
return fmt.Sprintf("the project %q is not one entry of the projects directory", project)
|
||||
}
|
||||
if path == "" || strings.HasPrefix(path, "/") || strings.ContainsRune(path, 0) {
|
||||
return fmt.Sprintf("the path %q is not a relative path in a memory folder", path)
|
||||
}
|
||||
for _, part := range strings.Split(path, "/") {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return fmt.Sprintf("the path %q has an empty, \".\" or \"..\" part", path)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func badName(project, path string) error {
|
||||
if why := NameProblem(project, path); why != "" {
|
||||
return fmt.Errorf("%s; nothing is touched for it (novox/hq ADR 0343)", why)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func inside(dir, full string) bool {
|
||||
rel, err := filepath.Rel(dir, full)
|
||||
return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
@@ -339,6 +430,9 @@ func inside(dir, full string) bool {
|
||||
|
||||
// keep copies a version into the copies before it is replaced or removed: nothing of a memory folder is lost.
|
||||
func (m *MemoryCarrier) keep(project, path string, text []byte, why string) error {
|
||||
if err := badName(project, path); err != nil {
|
||||
return err
|
||||
}
|
||||
stamp := m.Now().UTC().Format("20060102T150405.000000000Z")
|
||||
full := filepath.Join(m.copies(), project, filepath.FromSlash(path)+"."+stamp+"."+why)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o700); err != nil {
|
||||
@@ -347,11 +441,21 @@ func (m *MemoryCarrier) keep(project, path string, text []byte, why string) erro
|
||||
return os.WriteFile(full, text, 0o600)
|
||||
}
|
||||
|
||||
// writeLocal writes one file of a memory folder: the version there kept first, then the new text staged in the
|
||||
// same folder and renamed into place, never through a symbolic link.
|
||||
func (m *MemoryCarrier) writeLocal(project, path string, text []byte, why string) error {
|
||||
// errChangedHere says a file changed on this node between its reading and its writing: it is not written, and the
|
||||
// next look judges the change, a conflict where there is one.
|
||||
var errChangedHere = errors.New("it changed here meanwhile, and is judged again on the next look")
|
||||
|
||||
// writeLocal writes one file of a memory folder over the version the caller read (expect: its digest, "" for none):
|
||||
// the version there kept first, then the new text staged in the same folder and renamed into place, never through
|
||||
// a symbolic link. A file that is no longer the version read is not written (errChangedHere).
|
||||
func (m *MemoryCarrier) writeLocal(project, path string, text []byte, why, expect string) error {
|
||||
if err := badName(project, path); err != nil {
|
||||
return err
|
||||
}
|
||||
if was, _, ok, err := m.readLocal(project, path); err != nil {
|
||||
return err
|
||||
} else if !ok && expect != "" || ok && digestOf(was) != expect {
|
||||
return fmt.Errorf("%s/%s: %w", project, path, errChangedHere)
|
||||
} else if ok {
|
||||
if bytes.Equal(was, text) {
|
||||
return nil
|
||||
@@ -389,6 +493,14 @@ func (m *MemoryCarrier) writeLocal(project, path string, text []byte, why string
|
||||
|
||||
// removeLocal moves one file of a memory folder into the copies.
|
||||
func (m *MemoryCarrier) removeLocal(project, path string, text []byte) error {
|
||||
if err := badName(project, path); err != nil {
|
||||
return err
|
||||
}
|
||||
if now, _, ok, err := m.readLocal(project, path); err != nil {
|
||||
return err
|
||||
} else if !ok || !bytes.Equal(now, text) {
|
||||
return fmt.Errorf("%s/%s: %w", project, path, errChangedHere)
|
||||
}
|
||||
if err := m.keep(project, path, text, "removed"); err != nil {
|
||||
return fmt.Errorf("%s/%s could not be kept, so it is not removed: %w", project, path, err)
|
||||
}
|
||||
@@ -499,7 +611,35 @@ func (m *MemoryCarrier) current(key string) (*MemoryVersion, uint64, error) {
|
||||
|
||||
// put writes a version at the revision read (0: there was none), and records it as carried once it is in the state.
|
||||
func (m *MemoryCarrier) put(key string, v MemoryVersion, revision uint64) error {
|
||||
if err := m.write(key, v, revision); err != nil {
|
||||
return err
|
||||
}
|
||||
m.record(key, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
// errNotCarried says a version was screened out before it reached the state.
|
||||
var errNotCarried = errors.New("not carried")
|
||||
|
||||
// write puts a version into the state at the revision read — the one way anything reaches the state — after
|
||||
// judging it as a file found here is judged (ADR 0343 §2): its project and path, and its text, on every path.
|
||||
func (m *MemoryCarrier) write(key string, v MemoryVersion, revision uint64) error {
|
||||
v.Kind = m.Kind
|
||||
if why := NameProblem(v.Project, v.Path); why != "" {
|
||||
return fmt.Errorf("%s: %w", why, errNotCarried)
|
||||
}
|
||||
if MemoryKey(m.Kind, v.Project, v.Path) != key {
|
||||
return fmt.Errorf("%s does not name %s/%s: %w", key, v.Project, v.Path, errNotCarried)
|
||||
}
|
||||
if !v.Removed {
|
||||
if why := memoryProblem([]byte(v.Text)); why != "" {
|
||||
m.notCarried[v.Project+"/"+v.Path] = why
|
||||
return fmt.Errorf("%s/%s is not carried, because %s: %w", v.Project, v.Path, why, errNotCarried)
|
||||
}
|
||||
if v.Digest != digestOf([]byte(v.Text)) {
|
||||
return fmt.Errorf("%s/%s: its digest does not name its text: %w", v.Project, v.Path, errNotCarried)
|
||||
}
|
||||
}
|
||||
var err error
|
||||
if revision == 0 {
|
||||
err = m.Store.Create(key, v)
|
||||
@@ -510,7 +650,6 @@ func (m *MemoryCarrier) put(key string, v MemoryVersion, revision uint64) error
|
||||
return fmt.Errorf("%s/%s was not carried, because the state changed meanwhile or did not answer; it is "+
|
||||
"carried on the next look: %w", v.Project, v.Path, err)
|
||||
}
|
||||
m.record(key, v)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -530,7 +669,11 @@ func (m *MemoryCarrier) Look() error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.load()
|
||||
files, folders, not := m.scan()
|
||||
in, err := m.scan()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files, folders, not := in.files, in.folders, in.not
|
||||
m.notCarried = not
|
||||
var problems []error
|
||||
// A key the state lost is carried again, never read as a removal (ADR 0343 §9): one listing per look.
|
||||
@@ -559,10 +702,8 @@ func (m *MemoryCarrier) Look() error {
|
||||
// A carried file gone from a folder that still exists is a removal; a folder gone removes nothing.
|
||||
gone := []string{}
|
||||
for key, c := range m.rec.Files {
|
||||
if _, here := files[key]; !here && folders[c.Project] && strings.HasPrefix(key, m.Kind+".") {
|
||||
if _, skipped := not[c.Project+"/"+c.Path]; !skipped {
|
||||
gone = append(gone, key)
|
||||
}
|
||||
if _, here := files[key]; !here && folders[c.Project] && !in.unseen[key] && strings.HasPrefix(key, m.Kind+".") {
|
||||
gone = append(gone, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(gone)
|
||||
@@ -624,14 +765,17 @@ func (m *MemoryCarrier) conflict(key, project, path string, local []byte, localA
|
||||
}
|
||||
v := MemoryVersion{Kind: m.Kind, Project: project, Path: path, Text: string(merged), Digest: digestOf(merged),
|
||||
Base: cur.Digest, Node: m.Node, Changed: m.Now().UTC()}
|
||||
if err := m.Store.Update(key, v, revision); err != nil {
|
||||
return fmt.Errorf("%s/%s changed on two nodes, and the state changed again meanwhile; it is judged again on "+
|
||||
"the next change: %w", project, path, err)
|
||||
}
|
||||
if err := m.writeLocal(project, path, merged, "before-merge"); err != nil {
|
||||
// The state holds both; this node's folder still holds its own, which the next look merges again.
|
||||
// The file first, over exactly the version merged: a write that fails leaves the state as it was, so the next
|
||||
// look merges the same two versions again and nothing grows. Only then the state, at the revision read; when
|
||||
// another node got there first, its version is judged against the merged file as it comes.
|
||||
if err := m.writeLocal(project, path, merged, "before-merge", digestOf(local)); err != nil {
|
||||
m.unwritten(key, project, path, err)
|
||||
return err
|
||||
}
|
||||
if err := m.write(key, v, revision); err != nil {
|
||||
return fmt.Errorf("%s/%s changed on two nodes; both are kept in it here, and the state changed again "+
|
||||
"meanwhile, so it is judged again on the next change: %w", project, path, err)
|
||||
}
|
||||
m.record(key, v)
|
||||
if m.Say != nil {
|
||||
m.Say("memory: %s/%s was changed on %s and on %s; both versions are kept in it, the newer first", project, path, m.Node, cur.Node)
|
||||
@@ -640,6 +784,30 @@ func (m *MemoryCarrier) conflict(key, project, path string, local []byte, localA
|
||||
return nil
|
||||
}
|
||||
|
||||
// unwritten says, once per file, that a memory file could not be written here: the operator is told, since a
|
||||
// file that cannot be written cannot follow them (ADR 0343 §8).
|
||||
func (m *MemoryCarrier) unwritten(key, project, path string, err error) {
|
||||
if errors.Is(err, errChangedHere) {
|
||||
return
|
||||
}
|
||||
if m.Say != nil {
|
||||
m.Say("memory: %s/%s could not be written here: %v", project, path, err)
|
||||
}
|
||||
if _, told := m.failed[key]; told || m.Notify == nil {
|
||||
return
|
||||
}
|
||||
m.failed[key] = true
|
||||
label := projectLabel(project)
|
||||
_ = m.Notify(map[string]any{
|
||||
"key": noticeKey(key) + ".unwritten", "severity": "warning", "subject": "claude-code",
|
||||
"summary": fmt.Sprintf("a memory file of project %s could not be written on %s", label, m.Node),
|
||||
"headline": "Claude memory not written on " + m.Node,
|
||||
"explanation": fmt.Sprintf("Needs you: make the memory folder of project %s writable on %s. Two versions of one "+
|
||||
"of its files are waiting to be kept together there, and nothing is lost meanwhile.", label, m.Node),
|
||||
"resolved": fmt.Sprintf("the memory of project %s is written on %s again", label, m.Node),
|
||||
})
|
||||
}
|
||||
|
||||
// OlderBeside is where the older of two versions too large to share one file is kept: beside the file, named for
|
||||
// the node it was changed on ("MEMORY.md" from shanks: "MEMORY.older-from-shanks.md").
|
||||
func OlderBeside(path, node string) string {
|
||||
@@ -662,16 +830,25 @@ func (m *MemoryCarrier) conflictBeside(key, project, path string, local []byte,
|
||||
if newerNode == m.Node {
|
||||
v.Node, v.Changed = m.Node, m.Now().UTC()
|
||||
}
|
||||
if err := m.Store.Update(key, v, revision); err != nil {
|
||||
return fmt.Errorf("%s/%s changed on two nodes, and the state changed again meanwhile; it is judged again on "+
|
||||
"the next change: %w", project, path, err)
|
||||
// The files first, then the state, as conflict does.
|
||||
besideNow := ""
|
||||
if was, _, ok, err := m.readLocal(project, beside); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
besideNow = digestOf(was)
|
||||
}
|
||||
if err := m.writeLocal(project, beside, older, "before-merge"); err != nil {
|
||||
if err := m.writeLocal(project, beside, older, "before-merge", besideNow); err != nil {
|
||||
m.unwritten(key, project, path, err)
|
||||
return err
|
||||
}
|
||||
if err := m.writeLocal(project, path, newer, "before-merge"); err != nil {
|
||||
if err := m.writeLocal(project, path, newer, "before-merge", digestOf(local)); err != nil {
|
||||
m.unwritten(key, project, path, err)
|
||||
return err
|
||||
}
|
||||
if err := m.write(key, v, revision); err != nil {
|
||||
return fmt.Errorf("%s/%s changed on two nodes; both are kept here, and the state changed again meanwhile, "+
|
||||
"so it is judged again on the next change: %w", project, path, err)
|
||||
}
|
||||
m.record(key, v)
|
||||
if m.Say != nil {
|
||||
m.Say("memory: %s/%s was changed on %s and on %s, too large to hold both: the newer is in it, the older beside it as %s",
|
||||
@@ -714,6 +891,9 @@ func (m *MemoryCarrier) Take(key, op string, value json.RawMessage, revision uin
|
||||
if v.Kind != m.Kind || MemoryKey(m.Kind, v.Project, v.Path) != key {
|
||||
return fmt.Errorf("the state's value for %s names another file or kind, and is not taken", key)
|
||||
}
|
||||
if err := badName(v.Project, v.Path); err != nil {
|
||||
return fmt.Errorf("the state's value for %s is not taken: %w", key, err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.load()
|
||||
@@ -734,6 +914,9 @@ func (m *MemoryCarrier) Take(key, op string, value json.RawMessage, revision uin
|
||||
|
||||
// apply writes one incoming version, read at revision, into this node's folder (ADR 0343 §6, §7, §9).
|
||||
func (m *MemoryCarrier) apply(key string, v MemoryVersion, revision uint64) error {
|
||||
if err := badName(v.Project, v.Path); err != nil {
|
||||
return fmt.Errorf("the state's value for %s is not taken: %w", key, err)
|
||||
}
|
||||
if v.Node == m.Node && !v.Removed {
|
||||
// This node's own version: what is here is either it, or newer, which the next look carries.
|
||||
if text, _, ok, _ := m.readLocal(v.Project, v.Path); ok && digestOf(text) == v.Digest {
|
||||
@@ -757,7 +940,10 @@ func (m *MemoryCarrier) apply(key string, v MemoryVersion, revision uint64) erro
|
||||
switch {
|
||||
case !here:
|
||||
delete(m.rec.Files, key)
|
||||
case localDigest == v.Base || (known && localDigest == rec.Digest && rec.Digest == v.Base):
|
||||
// The file here holds no change this node did not carry — it is the version removed, or the version this
|
||||
// node last carried, which the removal replaced whether or not this node saw the versions between — so the
|
||||
// removal takes it, into the copies. Anything else is a change made here, which outlives the removal (§9).
|
||||
case localDigest == v.Base || (known && localDigest == rec.Digest):
|
||||
if err := m.removeLocal(v.Project, v.Path, local); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -783,7 +969,8 @@ func (m *MemoryCarrier) apply(key string, v MemoryVersion, revision uint64) erro
|
||||
here && known && localDigest == rec.Digest,
|
||||
here && v.Base != "" && localDigest == v.Base:
|
||||
// Nothing here, or here still the version it replaced: written, the version here kept first.
|
||||
if err := m.writeLocal(v.Project, v.Path, []byte(v.Text), "replaced"); err != nil {
|
||||
if err := m.writeLocal(v.Project, v.Path, []byte(v.Text), "replaced", localDigest); err != nil {
|
||||
m.unwritten(key, v.Project, v.Path, err)
|
||||
return err
|
||||
}
|
||||
m.rec.Files[key] = carried{Project: v.Project, Path: v.Path, Digest: v.Digest, Node: v.Node, Changed: v.Changed}
|
||||
@@ -795,6 +982,12 @@ func (m *MemoryCarrier) apply(key string, v MemoryVersion, revision uint64) erro
|
||||
// Removed here, and the state still holds the version removed: the next look carries the removal.
|
||||
return nil
|
||||
default:
|
||||
if why := memoryProblem(local); why != "" {
|
||||
// What is here is never carried, so it is not merged into what is: both stay where they are.
|
||||
m.notCarried[v.Project+"/"+v.Path] = why
|
||||
return fmt.Errorf("%s/%s changed here and elsewhere, and what is here is not carried, because %s; both "+
|
||||
"stay as they are: %w", v.Project, v.Path, why, errNotCarried)
|
||||
}
|
||||
return m.conflict(key, v.Project, v.Path, local, localAt, v, revision)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -676,3 +676,229 @@ func TestTwoVersionsTooLargeForOneFileAreKeptInTwo(t *testing.T) {
|
||||
t.Fatalf("it is not listed as not carried: %v", not)
|
||||
}
|
||||
}
|
||||
|
||||
// stateHolds says whether any value of the state holds s.
|
||||
func (s *memState) holds(sub string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, v := range s.values {
|
||||
var mv MemoryVersion
|
||||
_ = json.Unmarshal(v, &mv)
|
||||
if strings.Contains(mv.Text, sub) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// putRaw puts a value straight into the state, as a holder of a tool runner's credential could.
|
||||
func (s *memState) putRaw(t *testing.T, v MemoryVersion) string {
|
||||
t.Helper()
|
||||
key := MemoryKey(v.Kind, v.Project, v.Path)
|
||||
s.mu.Lock()
|
||||
s.write(key, v)
|
||||
s.mu.Unlock()
|
||||
return key
|
||||
}
|
||||
|
||||
// Review of #276, 1: a value whose project or path leads out of a memory folder is never taken, and nothing is
|
||||
// written for it.
|
||||
func TestAValueLeadingOutOfAMemoryFolderIsNeverTaken(t *testing.T) {
|
||||
state, c, g14, shanks := twoNodes(t)
|
||||
root := filepath.Dir(filepath.Dir(shanks.Home))
|
||||
for _, bad := range []struct{ project, path string }{
|
||||
{"../../../escaped", "x.md"}, {"..", "x.md"}, {"a/b", "x.md"}, {"", "x.md"},
|
||||
{project, "../x.md"}, {project, "/abs.md"}, {project, "a//x.md"}, {project, "./x.md"}, {project, "a/../../x.md"},
|
||||
} {
|
||||
text := "planted\n"
|
||||
state.putRaw(t, MemoryVersion{Kind: HomeOperator, Project: bad.project, Path: bad.path, Text: text,
|
||||
Digest: digestOf([]byte(text)), Node: "elsewhere", Changed: c.now})
|
||||
if NameProblem(bad.project, bad.path) == "" {
|
||||
t.Errorf("%q %q is taken as a name", bad.project, bad.path)
|
||||
}
|
||||
}
|
||||
state.pump(t)
|
||||
g14.look(t)
|
||||
shanks.look(t)
|
||||
_ = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
|
||||
if err == nil && !d.IsDir() {
|
||||
if raw, _ := os.ReadFile(p); string(raw) == "planted\n" {
|
||||
t.Errorf("a planted value was written at %s", p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if NameProblem(project, "notes/a.md") != "" || NameProblem(project, "MEMORY.md") != "" {
|
||||
t.Fatal("a memory file's own name is refused")
|
||||
}
|
||||
}
|
||||
|
||||
// Review of #276, 2: a file or a folder that cannot be read is kept everywhere, never carried as a removal, and is
|
||||
// said as not carried.
|
||||
func TestWhatCannotBeReadIsNeverTakenForGone(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root reads a file whatever its mode")
|
||||
}
|
||||
state, c, g14, shanks := twoNodes(t)
|
||||
carriedOnBoth(t, state, c, g14, shanks, "MEMORY.md", "index\n")
|
||||
carriedOnBoth(t, state, c, g14, shanks, "notes/deep.md", "deep\n")
|
||||
file, folder := g14.path(project, "MEMORY.md"), filepath.Dir(g14.path(project, "notes/deep.md"))
|
||||
_ = os.Chmod(file, 0)
|
||||
_ = os.Chmod(folder, 0)
|
||||
t.Cleanup(func() { _ = os.Chmod(file, 0o644); _ = os.Chmod(folder, 0o755) })
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
for _, f := range []string{"MEMORY.md", "notes/deep.md"} {
|
||||
if _, ok := shanks.read(t, project, f); !ok {
|
||||
t.Errorf("%s, unreadable on g14, was removed on shanks", f)
|
||||
}
|
||||
}
|
||||
not, _ := g14.Status()["not carried"].(map[string]string)
|
||||
if len(not) == 0 {
|
||||
t.Fatalf("nothing is said as not carried: %v", not)
|
||||
}
|
||||
keys, _ := state.Keys()
|
||||
for _, k := range keys {
|
||||
raw, _, _, _ := state.Get(k)
|
||||
var v MemoryVersion
|
||||
_ = json.Unmarshal(raw, &v)
|
||||
if v.Removed {
|
||||
t.Errorf("%s/%s was carried as a removal", v.Project, v.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Review of #276, 3: no text reaches the state without being screened, on the removal and the conflict paths too.
|
||||
func TestNoUnscreenedTextReachesTheState(t *testing.T) {
|
||||
token := "ghp_" + strings.Repeat("A1b2C3d4", 5) + "\n"
|
||||
big := strings.Repeat("x", MemoryMostBytes+10) + "\n"
|
||||
for _, text := range []string{token, big} {
|
||||
// The removal path: removed on g14, changed on shanks to what is never carried.
|
||||
state, c, g14, shanks := twoNodes(t)
|
||||
carriedOnBoth(t, state, c, g14, shanks, "note.md", "v1\n")
|
||||
_ = os.Remove(g14.path(project, "note.md"))
|
||||
shanks.write(t, project, "note.md", text, c.now.Add(time.Minute))
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
if state.holds(text[:20]) {
|
||||
t.Errorf("the removal path put text that is never carried into the state")
|
||||
}
|
||||
if got, _ := shanks.read(t, project, "note.md"); got != text {
|
||||
t.Errorf("the change on shanks was not kept")
|
||||
}
|
||||
// The conflict path: changed on g14, and on shanks to what is never carried.
|
||||
state, c, g14, shanks = twoNodes(t)
|
||||
carriedOnBoth(t, state, c, g14, shanks, "note.md", "v1\n")
|
||||
g14.write(t, project, "note.md", "v2\n", c.now.Add(time.Minute))
|
||||
shanks.write(t, project, "note.md", text, c.now.Add(2*time.Minute))
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
shanks.look(t)
|
||||
state.pump(t)
|
||||
if state.holds(text[:20]) {
|
||||
t.Errorf("the conflict path put text that is never carried into the state")
|
||||
}
|
||||
if got, _ := shanks.read(t, project, "note.md"); got != text {
|
||||
t.Errorf("shanks's own version was not kept as it was")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Review of #276, 4: a conflict whose file cannot be written leaves the state as it was, says so once, and merges
|
||||
// once when the folder is writable again.
|
||||
func TestAConflictThatCannotBeWrittenIsNotMergedAgainAndAgain(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("root writes a folder whatever its mode")
|
||||
}
|
||||
state, c, g14, shanks := twoNodes(t)
|
||||
carriedOnBoth(t, state, c, g14, shanks, "MEMORY.md", "base\n")
|
||||
g14.write(t, project, "MEMORY.md", "base\nfrom g14\n", c.now.Add(time.Minute))
|
||||
shanks.write(t, project, "MEMORY.md", "base\nfrom shanks\n", c.now.Add(2*time.Minute))
|
||||
folder := filepath.Dir(shanks.path(project, "MEMORY.md"))
|
||||
_ = os.Chmod(folder, 0o500)
|
||||
t.Cleanup(func() { _ = os.Chmod(folder, 0o700) })
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
for i := 0; i < 5; i++ {
|
||||
_ = shanks.Look()
|
||||
state.pump(t)
|
||||
}
|
||||
if state.holds("<!-- novox mesh") {
|
||||
t.Fatal("a merge that could not be written here reached the state")
|
||||
}
|
||||
unwritten := 0
|
||||
for _, n := range shanks.notices {
|
||||
if strings.HasSuffix(n["key"].(string), ".unwritten") {
|
||||
unwritten++
|
||||
}
|
||||
}
|
||||
if unwritten != 1 {
|
||||
t.Fatalf("%d warnings that the file could not be written", unwritten)
|
||||
}
|
||||
_ = os.Chmod(folder, 0o700)
|
||||
shanks.look(t)
|
||||
state.pump(t)
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
for _, n := range []*memNode{g14, shanks} {
|
||||
got, _ := n.read(t, project, "MEMORY.md")
|
||||
if strings.Count(got, "<!-- novox mesh") != 1 || !strings.Contains(got, "from g14") || !strings.Contains(got, "from shanks") {
|
||||
t.Fatalf("%s does not hold both versions once:\n%s", n.Node, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Review of #276, minor: a file that changed between its reading and its writing is not written over; text that is
|
||||
// not UTF-8 is not carried.
|
||||
func TestAFileChangedBeforeItsWritingIsNotWrittenOver(t *testing.T) {
|
||||
state, c, g14, shanks := twoNodes(t)
|
||||
_ = state
|
||||
g14.write(t, project, "a.md", "read\n", c.now)
|
||||
g14.write(t, project, "a.md", "changed meanwhile\n", c.now)
|
||||
err := g14.writeLocal(project, "a.md", []byte("incoming\n"), "replaced", digestOf([]byte("read\n")))
|
||||
if !errors.Is(err, errChangedHere) {
|
||||
t.Fatalf("written over a change: %v", err)
|
||||
}
|
||||
if got, _ := g14.read(t, project, "a.md"); got != "changed meanwhile\n" {
|
||||
t.Fatalf("the change was replaced: %q", got)
|
||||
}
|
||||
g14.write(t, project, "latin1.md", "caf\xe9\n", c.now)
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
if _, ok := shanks.read(t, project, "latin1.md"); ok {
|
||||
t.Fatal("text that is not UTF-8 was carried")
|
||||
}
|
||||
not, _ := g14.Status()["not carried"].(map[string]string)
|
||||
if !strings.Contains(not[project+"/latin1.md"], "UTF-8") {
|
||||
t.Fatalf("it is not said why: %v", not)
|
||||
}
|
||||
}
|
||||
|
||||
// §9: a node that was away takes a removal of a version newer than the one it holds, when it holds no change of
|
||||
// its own: the removed file does not come back.
|
||||
func TestARemovalReachesANodeThatWasAway(t *testing.T) {
|
||||
state, c, g14, shanks := twoNodes(t)
|
||||
carriedOnBoth(t, state, c, g14, shanks, "old.md", "v1\n")
|
||||
state.watchers = []*MemoryCarrier{g14.MemoryCarrier}
|
||||
c.now = c.now.Add(time.Minute)
|
||||
g14.write(t, project, "old.md", "v2\n", c.now)
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
_ = os.Remove(g14.path(project, "old.md"))
|
||||
g14.look(t)
|
||||
state.pump(t)
|
||||
state.watchers = append(state.watchers, shanks.MemoryCarrier)
|
||||
state.replay(t, shanks)
|
||||
shanks.look(t)
|
||||
state.pump(t)
|
||||
g14.look(t)
|
||||
if _, ok := shanks.read(t, project, "old.md"); ok {
|
||||
t.Fatal("the removal did not reach the node that was away")
|
||||
}
|
||||
if _, ok := g14.read(t, project, "old.md"); ok {
|
||||
t.Fatal("the removed file came back")
|
||||
}
|
||||
if !holdsText(shanks.copies(t), "v1\n") {
|
||||
t.Fatal("the version removed on the node that was away was not kept")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user