enginereads: a statement older than an event heard does not stand (issue 1975)
Builds: issue 1975 note 4 (the lead's handover; no design line)
This commit is contained in:
@@ -251,8 +251,17 @@ type Keeper struct {
|
||||
again map[string]string
|
||||
// eventAt is, per node, when its event manifest-reads-changed was last heard (Statement.EventAt).
|
||||
eventAt map[string]time.Time
|
||||
// said is, per node, the newest event heard with what it says the engine states: a statement older than it is behind
|
||||
// (issue 1975's kind: an answer taken while the engine still stated its bootstrapped seats stood after the full list).
|
||||
said map[string]nodeengine.ManifestReadsChangedEvent
|
||||
// catching counts, per node, the asks made since the last event only to catch a statement up to it.
|
||||
catching map[string]int
|
||||
}
|
||||
|
||||
// MaxCatchUp is how many asks one event may cause beyond its own while the engine's answer stays behind it: an engine
|
||||
// that answers older than its own event is said loudly, never asked in a loop.
|
||||
const MaxCatchUp = 3
|
||||
|
||||
func (k *Keeper) logf(f string, a ...any) {
|
||||
if k.Logf != nil {
|
||||
k.Logf(f, a...)
|
||||
@@ -296,6 +305,58 @@ func (k *Keeper) AskNode(node, why string) {
|
||||
go k.ask(node, why)
|
||||
}
|
||||
|
||||
// HeardChanged keeps the newest event heard of a node: until the node's statement is at least as new, the node reads
|
||||
// as the event says (the engine's own word), and the node is asked again until its statement is (see ask). Call it before
|
||||
// AskNode for the same event.
|
||||
func (k *Keeper) HeardChanged(e nodeengine.ManifestReadsChangedEvent) {
|
||||
at, err := time.Parse(time.RFC3339Nano, e.At)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if k.said == nil {
|
||||
k.said, k.catching = map[string]nodeengine.ManifestReadsChangedEvent{}, map[string]int{}
|
||||
}
|
||||
if old, ok := k.said[e.Node]; ok {
|
||||
if oat, err := time.Parse(time.RFC3339Nano, old.At); err == nil && !at.After(oat) {
|
||||
return
|
||||
}
|
||||
}
|
||||
e.ManifestReads = slices.Clone(e.ManifestReads)
|
||||
k.said[e.Node] = e
|
||||
k.catching[e.Node] = 0
|
||||
}
|
||||
|
||||
// behindLocked says node's statement is older than the newest event heard of it: the engine said its reads changed
|
||||
// after the time the statement was made, and to something else. k.mu is held.
|
||||
func (k *Keeper) behindLocked(node string) bool {
|
||||
e, ok := k.said[node]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, e.At)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s := k.by[node]
|
||||
return s.StatedAt.Before(at) && !slices.Equal(sortedCopy(s.ManifestReads), sortedCopy(e.ManifestReads))
|
||||
}
|
||||
|
||||
func sortedCopy(in []string) []string {
|
||||
out := slices.Clone(in)
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// readsLocked is what node reads: its statement, or the newest event's reads while the statement is behind it.
|
||||
func (k *Keeper) readsLocked(node string) []string {
|
||||
if k.behindLocked(node) {
|
||||
return slices.Clone(k.said[node].ManifestReads)
|
||||
}
|
||||
return slices.Clone(k.by[node].ManifestReads)
|
||||
}
|
||||
|
||||
// AskAll asks every node of nodes once, each as AskNode does.
|
||||
func (k *Keeper) AskAll(nodes []string, why string) {
|
||||
for _, n := range nodes {
|
||||
@@ -319,6 +380,16 @@ func (k *Keeper) ask(node, why string) {
|
||||
k.mu.Lock()
|
||||
next, again := k.again[node]
|
||||
delete(k.again, node)
|
||||
if !again && k.behindLocked(node) {
|
||||
// The answer taken is older than an event heard: ask until the statement is at least as new.
|
||||
if k.catching[node] < MaxCatchUp {
|
||||
k.catching[node]++
|
||||
next, again = "its statement is older than the event heard", true
|
||||
} else {
|
||||
defer k.logf("%s on %s: the engine answers older than its own event said (%d asks); the node reads as the event says",
|
||||
Verb, node, MaxCatchUp)
|
||||
}
|
||||
}
|
||||
if !again {
|
||||
delete(k.asking, node)
|
||||
}
|
||||
@@ -471,6 +542,7 @@ func (k *Keeper) Forget(node string) {
|
||||
_, had := k.by[node]
|
||||
delete(k.by, node)
|
||||
delete(k.eventAt, node)
|
||||
delete(k.said, node)
|
||||
k.mu.Unlock()
|
||||
if had && k.Delete != nil {
|
||||
if err := k.Delete(node); err != nil {
|
||||
@@ -504,7 +576,7 @@ func (k *Keeper) Reads(nodes []string) map[string][]string {
|
||||
defer k.mu.Unlock()
|
||||
out := make(map[string][]string, len(nodes))
|
||||
for _, n := range nodes {
|
||||
out[n] = slices.Clone(k.by[n].ManifestReads)
|
||||
out[n] = k.readsLocked(n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -516,7 +588,7 @@ func (k *Keeper) Present(nodes []string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, n := range nodes {
|
||||
if s, ok := k.by[n]; ok && s.Answered {
|
||||
out[n] = slices.Clone(s.ManifestReads)
|
||||
out[n] = k.readsLocked(n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
Reference in New Issue
Block a user