jochen
3c1ac6aef2
At 03:30 the store's collector held the registry still while the node-engine's reconcile on that machine was fetching bundle blobs from it: every archive failed 'connection refused' and the machine was held until the next pass. Other machines can meet the same window. A scheduled step now opens its window only once no apply is in flight here (the apply lock is taken just to write the record, so a push still never queues behind the window). An apply whose fetch the store does not answer waits for a window open on its own machine to close, and elsewhere retries with backoff within one bounded budget per apply, well past the window's length; an answer such as 404 still fails at once.
83 lines
3.2 KiB
Go
83 lines
3.2 KiB
Go
package store
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"syscall"
|
|
)
|
|
|
|
// One apply at a time on a machine, whoever asks.
|
|
//
|
|
// Three things apply here and each reads the state, acts, and writes the state back: the link
|
|
// and the reconcile loop inside `mesh-host run`, the host's own `reconcile` and `apply` run by
|
|
// hand, and the installer at genesis. Inside one process a mutex serialises them; across
|
|
// processes nothing did, and two applies interleaved leave the last saver writing a state read
|
|
// before the other acted — a hold, a firewall record, a resource just applied, lost (novox/hq
|
|
// issue 104 review). A lock on a file beside the state is what every one of them can take, however
|
|
// it was started: a `reconcile` run against a service, or against a `mesh-host run` somebody
|
|
// started by hand, waits the same way. Refusing while a named service is active would have known
|
|
// the service's name and missed the hand-started one.
|
|
//
|
|
// An advisory lock, held for the life of the open file and released by the kernel when the
|
|
// process ends, so a host that dies mid-apply leaves no lock behind for the next one to clear.
|
|
|
|
// LockName is the file the lock is taken on, beside the state.
|
|
const LockName = "state.lock"
|
|
|
|
// Lock takes the machine's apply lock and returns what releases it. When another process holds
|
|
// it, wait is told once — so a person running a command knows what they are waiting for — and
|
|
// Lock blocks until it is free.
|
|
func Lock(statePath string, wait func()) (func(), error) {
|
|
dir := filepath.Dir(statePath)
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
f, err := os.OpenFile(filepath.Join(dir, LockName), os.O_CREATE|os.O_RDWR, 0o600)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
|
}
|
|
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
|
if !errors.Is(err, syscall.EWOULDBLOCK) {
|
|
f.Close()
|
|
return nil, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
|
}
|
|
if wait != nil {
|
|
wait()
|
|
}
|
|
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
|
|
f.Close()
|
|
return nil, fmt.Errorf("waiting for this node's apply lock: %w", err)
|
|
}
|
|
}
|
|
return func() {
|
|
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
|
f.Close()
|
|
}, nil
|
|
}
|
|
|
|
// TryLockIn takes the apply lock of the state kept in dir when it is free, and never waits: false
|
|
// when another apply holds it. For one who must not start while an apply is in flight but must not
|
|
// queue behind one either — a scheduled step about to hold a container still (novox/hq issue 291).
|
|
func TryLockIn(dir string) (func(), bool, error) {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return nil, false, err
|
|
}
|
|
f, err := os.OpenFile(filepath.Join(dir, LockName), os.O_CREATE|os.O_RDWR, 0o600)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
|
}
|
|
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
|
f.Close()
|
|
if errors.Is(err, syscall.EWOULDBLOCK) {
|
|
return nil, false, nil
|
|
}
|
|
return nil, false, fmt.Errorf("cannot take this node's apply lock: %w", err)
|
|
}
|
|
return func() {
|
|
_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
|
f.Close()
|
|
}, true, nil
|
|
}
|