A machine that wakes or moves says so, instead of waiting to be told
A suspended laptop's connection is dead the moment it wakes, and the socket looks perfectly healthy from inside the process — no error, no close, because nothing has tried to send anything. Heartbeats find out twenty or thirty seconds later. For that time the node believes it is in a mesh it has left, which is the one state this design says must never be indistinguishable from being connected. The machine knew immediately. So being roused ends the current attempt rather than only shortening the wait after it: shortening the wait would do nothing at all, because the process is not waiting — it is sitting inside a connection that will not return. A signal, because nothing may listen on a node (novox/hq ADR 0004). A socket for this would be a control surface on every machine, reachable by anything that can reach the machine, in exchange for saving twenty seconds — and the whole security argument rests on there not being one. Two rouses in the same instant are one: a machine suspending and resuming repeatedly must not build a backlog of reconnections to work through. And the backoff is not reset by being roused — that says the machine changed, not that whatever was refusing the connection has stopped, and a laptop woken on a network with no route would otherwise retry at full speed for as long as somebody keeps opening the lid. The dispatcher acts on the events that change where packets go and not on `down`: the link is already gone there, reconnecting will fail, and the backoff exists for exactly that.
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A machine that just woke does not wait to be told its link is dead.
|
||||
//
|
||||
// After a resume the socket looks perfectly healthy from inside the process — no error, no close,
|
||||
// because nothing has tried to send anything. Heartbeats discover it twenty or thirty seconds
|
||||
// later, and for that time the node believes it is in a mesh it has left, which is the one state
|
||||
// this design says must never be indistinguishable from being connected.
|
||||
func TestBeingRousedEndsTheCurrentAttemptRatherThanWaitingForATimeout(t *testing.T) {
|
||||
// A link that never returns on its own, which is exactly what a suspended connection is.
|
||||
held := make(chan context.Context, 4)
|
||||
running := func(ctx context.Context) error {
|
||||
held <- ctx
|
||||
<-ctx.Done()
|
||||
return errors.New("the link ended")
|
||||
}
|
||||
|
||||
rouse := make(chan struct{}, 1)
|
||||
said := &saidSoFar{}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
|
||||
finished := make(chan error, 1)
|
||||
go func() { finished <- holdWith(ctx, running, said.say, rouse) }()
|
||||
|
||||
first := <-held
|
||||
select {
|
||||
case <-first.Done():
|
||||
t.Fatal("the link ended before anything roused it")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
rouse <- struct{}{}
|
||||
select {
|
||||
case <-first.Done():
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the machine woke and the link was left running against a socket that is gone")
|
||||
}
|
||||
|
||||
// And it opens another one rather than stopping.
|
||||
select {
|
||||
case <-held:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the link was dropped and never opened again")
|
||||
}
|
||||
if !strings.Contains(said.all(), "woken or moved") {
|
||||
t.Fatalf("nothing was said about why the link was dropped:\n%s", said.all())
|
||||
}
|
||||
|
||||
stop()
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("it did not stop when asked")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine nothing ever rouses is every machine that does not suspend, and must behave as before.
|
||||
func TestAMachineNothingRousesIsUnaffected(t *testing.T) {
|
||||
attempts := make(chan struct{}, 4)
|
||||
running := func(context.Context) error {
|
||||
attempts <- struct{}{}
|
||||
return errors.New("the link ended")
|
||||
}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go func() { _ = holdWith(ctx, running, func(string) {}, nil) }()
|
||||
|
||||
// It keeps trying, which is the behaviour a node with no rouse has always had.
|
||||
for i := 0; i < 2; i++ {
|
||||
select {
|
||||
case <-attempts:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("a node with nothing to rouse it stopped reconnecting")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type saidSoFar struct {
|
||||
mu sync.Mutex
|
||||
said []string
|
||||
}
|
||||
|
||||
func (s *saidSoFar) say(line string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.said = append(s.said, line)
|
||||
}
|
||||
|
||||
func (s *saidSoFar) all() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return strings.Join(s.said, "\n")
|
||||
}
|
||||
+65
-1
@@ -57,7 +57,39 @@ type Announce func(string)
|
||||
// hours. Retrying every second for hours is a node shouting into nothing; waiting a minute after
|
||||
// a broker blip is a node that is needlessly late. So it starts fast and slows down, and resets
|
||||
// once a connection has actually held.
|
||||
// Roused is a channel that says the machine has reason to believe its link is stale — it woke
|
||||
// from suspend, or its network changed.
|
||||
//
|
||||
// **The machine knows before any timeout does.** A suspended laptop's connection is dead the
|
||||
// moment it wakes, and heartbeats find that out in twenty or thirty seconds; for that time the
|
||||
// node believes it is in the mesh and is not, which is the one state this design says must never
|
||||
// be indistinguishable from being connected. Nothing new listens on the node to arrange it — the
|
||||
// signal a service manager already sends is enough (novox/hq ADR 0004).
|
||||
//
|
||||
// Nil is allowed and means nothing ever rouses it, which is every machine that does not suspend.
|
||||
type Roused <-chan struct{}
|
||||
|
||||
func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeout time.Duration) error {
|
||||
return HoldRoused(ctx, m, apply, say, timeout, nil)
|
||||
}
|
||||
|
||||
// HoldRoused is Hold, told when the machine has reason to think its link is stale.
|
||||
func HoldRoused(ctx context.Context, m Membership, apply Applier, say Announce,
|
||||
timeout time.Duration, roused Roused) error {
|
||||
|
||||
return holdWith(ctx, func(ctx context.Context) error {
|
||||
return Run(ctx, m, apply, say, timeout)
|
||||
}, say, roused)
|
||||
}
|
||||
|
||||
// attempt is one try at holding the link open, returning when it ends for any reason.
|
||||
//
|
||||
// Named so the loop below can be driven without a broker. What the loop decides — when to wait,
|
||||
// how long, what being roused does — is the part with the reasoning in it, and it was reachable
|
||||
// only through a real connection before.
|
||||
type attempt func(context.Context) error
|
||||
|
||||
func holdWith(ctx context.Context, run attempt, say Announce, roused Roused) error {
|
||||
const (
|
||||
first = 2 * time.Second
|
||||
most = 2 * time.Minute
|
||||
@@ -70,7 +102,31 @@ func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeou
|
||||
|
||||
for {
|
||||
began := time.Now()
|
||||
err := Run(ctx, m, apply, say, timeout)
|
||||
|
||||
// The link runs under a context this loop can cancel, so being roused ends the current
|
||||
// attempt rather than only shortening the wait after it.
|
||||
//
|
||||
// **That is the whole of it.** After a resume the socket looks perfectly healthy from
|
||||
// inside this process — there is no error and no close, because nothing has tried to
|
||||
// send anything. It is heartbeats that eventually discover it, twenty or thirty seconds
|
||||
// later. A machine that knows it just woke does not have to wait to be told.
|
||||
//
|
||||
// A rouse that turns out to be spurious costs one reconnect, which is cheap and
|
||||
// idempotent: the node redeclares its queue and anything unacknowledged is redelivered.
|
||||
// The alternative costs half a minute of believing it is in a mesh it has left.
|
||||
trying, done := context.WithCancel(ctx)
|
||||
if roused != nil {
|
||||
go func() {
|
||||
select {
|
||||
case <-trying.Done():
|
||||
case <-roused:
|
||||
say("woken or moved — dropping the link and opening it again")
|
||||
done()
|
||||
}
|
||||
}()
|
||||
}
|
||||
err := run(trying)
|
||||
done()
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -95,6 +151,14 @@ func Hold(ctx context.Context, m Membership, apply Applier, say Announce, timeou
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-roused:
|
||||
// And it does not serve out a wait computed for a broker that was restarting, either.
|
||||
//
|
||||
// The backoff is not *reset* by this. Being roused says the machine changed, not that
|
||||
// whatever was refusing the connection has stopped — a laptop woken repeatedly on a
|
||||
// network with no route would otherwise retry at full speed for as long as somebody
|
||||
// keeps opening the lid.
|
||||
say("woken or moved — trying again now")
|
||||
case <-time.After(wait):
|
||||
}
|
||||
if wait *= 2; wait > most {
|
||||
|
||||
Reference in New Issue
Block a user