Lay out the assigner's wants, holds and differences in mesh-sdk, so the assigner, the watchman and the view read one layout (issue 1297, subtask B1a; ADR 0528 clauses 1 and 4)
This commit is contained in:
@@ -0,0 +1,287 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The assigner's keys (novox/hq ADR 0528 clauses 1 and 4, ADR 0429 §6, issue 1297). The assigner writes, by the six
|
||||
// rules of ADR 0429 §6, what each node should run: `want.<node>.<module>` in its bucket wants, and beside it
|
||||
// `held.<module>.<fingerprint>` for a fingerprint it holds after a failed health check. A module that leaves a node's
|
||||
// modules gets its want written removed, never deleted (ADR 0408 §4). For each node and module it compares the want with
|
||||
// what the controller sends (the build the controller's assignment names) and writes every difference under
|
||||
// `<node>.<module>` in its bucket differences, with the rule that explains it or unexplained. In release 5 wants are
|
||||
// only compared; the node agent applying them is release 6 (ADR 0429 §7). Only the assigner's credential writes these
|
||||
// buckets; the watchman, the view and the controller read them.
|
||||
//
|
||||
// **The bucket is the first word, the key the rest** (as the builder's `builds.<module>.<fp>` is the key
|
||||
// `<module>.<fp>` in its builds bucket): the differences bucket's keys are `<node>.<module>`. The wants bucket holds
|
||||
// two kinds of key, so each keeps its word: `want.<node>.<module>` and `held.<module>.<fingerprint hex>`.
|
||||
const (
|
||||
// AssignerModule is the module of the assigner part.
|
||||
AssignerModule = "mesh-delivery-assigner"
|
||||
// AssignerWantsBucket is the assigner's bucket of wants and holds, `<owner module>_<name>`.
|
||||
AssignerWantsBucket = AssignerModule + "_wants"
|
||||
// AssignerWantsHistory is the revisions the wants bucket keeps per key, so a want's history shows each move and
|
||||
// the operator's manual write (ADR 0429 §6, the one escape).
|
||||
AssignerWantsHistory = 20
|
||||
// AssignerDifferencesBucket is the assigner's bucket of differences between a want and what the controller sends.
|
||||
AssignerDifferencesBucket = AssignerModule + "_differences"
|
||||
// AssignerDifferencesHistory is the revisions the differences bucket keeps per key: acceptance test 17 of design 47
|
||||
// reads a day of them.
|
||||
AssignerDifferencesHistory = 64
|
||||
)
|
||||
|
||||
// The words that open the wants bucket's two kinds of key.
|
||||
const (
|
||||
wantWord = "want"
|
||||
heldWord = "held"
|
||||
)
|
||||
|
||||
// WantKey is the key, in AssignerWantsBucket, of a node's want for a module: `want.<node>.<module>`. A module named
|
||||
// `node` is refused, since `want.<node>.node` is the node's own key (ADR 0429 §4), as HaveKey refuses it.
|
||||
func WantKey(node, module string) (string, error) {
|
||||
have, err := HaveKey(node, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return wantWord + "." + have, nil
|
||||
}
|
||||
|
||||
// ParseWantKey is the inverse of WantKey; it refuses a held key, a have key and any key of another bucket, naming it.
|
||||
func ParseWantKey(key string) (node, module string, err error) {
|
||||
rest, ok := strings.CutPrefix(key, wantWord+".")
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a want key, want.<node>.<module>", key)
|
||||
}
|
||||
node, module, isNode, err := ParseHaveKey(rest)
|
||||
if err != nil || isNode {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a want key, want.<node>.<module>", key)
|
||||
}
|
||||
return node, module, nil
|
||||
}
|
||||
|
||||
// HeldKey is the key, in AssignerWantsBucket, of a module's fingerprint the assigner holds: `held.<module>.<hex>`,
|
||||
// the hex being the fingerprint's key part (FingerprintKeyPart).
|
||||
func HeldKey(module, fingerprint string) (string, error) {
|
||||
key, err := BuilderBuildKey(module, fingerprint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return heldWord + "." + key, nil
|
||||
}
|
||||
|
||||
// ParseHeldKey is the inverse of HeldKey: the module and the fingerprint (`src2:` and the hex); it refuses a want key
|
||||
// and any key of another bucket, naming it.
|
||||
func ParseHeldKey(key string) (module, fingerprint string, err error) {
|
||||
rest, ok := strings.CutPrefix(key, heldWord+".")
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a held key, held.<module>.<fingerprint hex>", key)
|
||||
}
|
||||
if module, fingerprint, err = ParseBuilderBuildKey(rest); err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a held key: %w", key, err)
|
||||
}
|
||||
return module, fingerprint, nil
|
||||
}
|
||||
|
||||
// DifferenceKey is the key, in AssignerDifferencesBucket, of the difference for a module on a node: `<node>.<module>`.
|
||||
func DifferenceKey(node, module string) (string, error) {
|
||||
return HaveKey(node, module)
|
||||
}
|
||||
|
||||
// ParseDifferenceKey is the inverse of DifferenceKey; it refuses a want or held key and a node's own key, naming it.
|
||||
func ParseDifferenceKey(key string) (node, module string, err error) {
|
||||
node, module, isNode, err := ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a difference key, <node>.<module>", key)
|
||||
}
|
||||
return node, module, nil
|
||||
}
|
||||
|
||||
// The six rules of ADR 0429 §6, by number, as a want or a difference names the one that holds a node back.
|
||||
const (
|
||||
RuleBuildDone = 1 // the build of F is done
|
||||
RuleNotHeld = 2 // F is not held
|
||||
RuleCanary = 3 // the node is the module's canary, or the canary runs F healthy
|
||||
RuleStep = 4 // at most step other nodes are ahead (step 0: no limit, ADR 0528 §3)
|
||||
RuleProviders = 5 // every provider the module requires runs healthy on that node, or at its canary
|
||||
RuleGrant = 6 // the module's credential for F exists (grants.<module>.<fp>)
|
||||
)
|
||||
|
||||
func checkRule(rule int, of string) error {
|
||||
if rule < RuleBuildDone || rule > RuleGrant {
|
||||
return fmt.Errorf("delivery: %s names the rule %d, which is not one of the six (1 to 6)", of, rule)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFingerprint(fp, of string) error {
|
||||
if _, err := FingerprintKeyPart(fp); err != nil {
|
||||
return fmt.Errorf("delivery: %s: %w", of, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Want is a node's want for a module: the fingerprint FP the node should run, or Removed for a module that left the
|
||||
// node's modules (then FP is empty). When the module's newest fingerprint is not yet allowed on this node, Newest names
|
||||
// it and Rule the rule that holds the node back; both are empty when FP is the newest. Words say why in a sentence.
|
||||
// The entry's own time says since when.
|
||||
type Want struct {
|
||||
V int `json:"v"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Removed bool `json:"removed,omitempty"`
|
||||
Newest string `json:"newest,omitempty"`
|
||||
Rule int `json:"rule,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
func (w Want) check() error {
|
||||
if w.Removed {
|
||||
if w.FP != "" || w.Newest != "" || w.Rule != 0 {
|
||||
return fmt.Errorf("delivery: a removed want names no fingerprint and no rule")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := checkFingerprint(w.FP, "a want's fingerprint"); err != nil {
|
||||
return err
|
||||
}
|
||||
if w.Newest == "" && w.Rule == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := checkFingerprint(w.Newest, "a want's newest fingerprint"); err != nil {
|
||||
return err
|
||||
}
|
||||
if w.Newest == w.FP {
|
||||
return fmt.Errorf("delivery: a want at its newest fingerprint names no rule holding it back")
|
||||
}
|
||||
return checkRule(w.Rule, "a want held back")
|
||||
}
|
||||
|
||||
// Held is a fingerprint the assigner holds: the node whose health check failed on it, and the check's words.
|
||||
type Held struct {
|
||||
V int `json:"v"`
|
||||
Node string `json:"node"`
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
func (h Held) check() error {
|
||||
return checkName("node", h.Node)
|
||||
}
|
||||
|
||||
// Difference is where a want and what the controller sends differ for a module on a node (ADR 0528 clause 4): the
|
||||
// want's fingerprint (Want, empty when the node has no want for the module, WantRemoved when the want is removed), the
|
||||
// fingerprint of the build the controller's assignment names (Sent, empty when it sends none or the build names no
|
||||
// fingerprint; SentBuild names that build), and the rule that explains the difference, or Unexplained.
|
||||
type Difference struct {
|
||||
V int `json:"v"`
|
||||
Want string `json:"want,omitempty"`
|
||||
WantRemoved bool `json:"want-removed,omitempty"`
|
||||
Sent string `json:"sent,omitempty"`
|
||||
SentBuild string `json:"sent-build,omitempty"`
|
||||
Rule int `json:"rule,omitempty"`
|
||||
Unexplained bool `json:"unexplained,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
func (d Difference) check() error {
|
||||
if d.WantRemoved && d.Want != "" {
|
||||
return fmt.Errorf("delivery: a difference with a removed want names no wanted fingerprint")
|
||||
}
|
||||
if d.Want != "" {
|
||||
if err := checkFingerprint(d.Want, "a difference's want"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if d.Sent != "" {
|
||||
if err := checkFingerprint(d.Sent, "a difference's sent fingerprint"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !d.WantRemoved && d.Want == d.Sent && (d.Want != "" || d.SentBuild == "") {
|
||||
return fmt.Errorf("delivery: a difference whose want and send are both %q is no difference", d.Want)
|
||||
}
|
||||
if d.Unexplained {
|
||||
if d.Rule != 0 {
|
||||
return fmt.Errorf("delivery: an unexplained difference names no rule")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return checkRule(d.Rule, "a difference")
|
||||
}
|
||||
|
||||
// EncodeWant is w as JSON at Version; it refuses a fingerprint not of the new form, a removed want naming one, and a
|
||||
// want held back without its rule.
|
||||
func EncodeWant(w Want) ([]byte, error) {
|
||||
if err := w.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.V = Version
|
||||
return json.Marshal(w)
|
||||
}
|
||||
|
||||
// DecodeWant reads a want; it refuses an unknown version (ErrUnknownVersion) and what EncodeWant refuses.
|
||||
func DecodeWant(data []byte) (Want, error) {
|
||||
var w Want
|
||||
if err := decode(data, &w); err != nil {
|
||||
return Want{}, err
|
||||
}
|
||||
if err := w.check(); err != nil {
|
||||
return Want{}, err
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// SameWant says whether two wants are the same value, so the assigner writes only when they differ.
|
||||
func SameWant(a, b Want) bool { return a == b }
|
||||
|
||||
// EncodeHeld is h as JSON at Version; it refuses a node name a key refuses.
|
||||
func EncodeHeld(h Held) ([]byte, error) {
|
||||
if err := h.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.V = Version
|
||||
return json.Marshal(h)
|
||||
}
|
||||
|
||||
// DecodeHeld reads a hold; it refuses an unknown version (ErrUnknownVersion) and what EncodeHeld refuses.
|
||||
func DecodeHeld(data []byte) (Held, error) {
|
||||
var h Held
|
||||
if err := decode(data, &h); err != nil {
|
||||
return Held{}, err
|
||||
}
|
||||
if err := h.check(); err != nil {
|
||||
return Held{}, err
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// SameHeld says whether two holds are the same value.
|
||||
func SameHeld(a, b Held) bool { return a == b }
|
||||
|
||||
// EncodeDifference is d as JSON at Version; it refuses a difference with neither a rule nor unexplained, one with both,
|
||||
// a rule outside the six, and a want equal to the send.
|
||||
func EncodeDifference(d Difference) ([]byte, error) {
|
||||
if err := d.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.V = Version
|
||||
return json.Marshal(d)
|
||||
}
|
||||
|
||||
// DecodeDifference reads a difference; it refuses an unknown version (ErrUnknownVersion) and what EncodeDifference
|
||||
// refuses.
|
||||
func DecodeDifference(data []byte) (Difference, error) {
|
||||
var d Difference
|
||||
if err := decode(data, &d); err != nil {
|
||||
return Difference{}, err
|
||||
}
|
||||
if err := d.check(); err != nil {
|
||||
return Difference{}, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// SameDifference says whether two differences are the same value.
|
||||
func SameDifference(a, b Difference) bool { return reflect.DeepEqual(a, b) }
|
||||
@@ -0,0 +1,157 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1297/B1a: a want key round-trips through its parser; a held key, a have key and a node's own key are refused,
|
||||
// naming the key.
|
||||
func TestWantKey(t *testing.T) {
|
||||
key, err := WantKey("ace", "mesh-mail")
|
||||
if err != nil || key != "want.ace.mesh-mail" {
|
||||
t.Fatalf("WantKey = %q, %v", key, err)
|
||||
}
|
||||
node, module, err := ParseWantKey(key)
|
||||
if err != nil || node != "ace" || module != "mesh-mail" {
|
||||
t.Errorf("ParseWantKey = %q, %q, %v", node, module, err)
|
||||
}
|
||||
for _, n := range [][2]string{{"ace", "node"}, {"a.b", "m"}, {"ace", ""}, {"", "m"}, {"ace", "m*"}} {
|
||||
if _, err := WantKey(n[0], n[1]); err == nil {
|
||||
t.Errorf("WantKey(%q, %q) accepted", n[0], n[1])
|
||||
}
|
||||
}
|
||||
held, _ := HeldKey("mesh-mail", fp2of('a'))
|
||||
for _, bad := range []string{"ace.mesh-mail", "want.ace.node", "want.ace", "want.ace.m.x", held, "wants.ace.m", ""} {
|
||||
if _, _, err := ParseWantKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") {
|
||||
t.Errorf("ParseWantKey(%q) = %v, want a refusal naming it", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B1a: a held key round-trips; a want key, a builder build key and a fingerprint not of the new form are refused.
|
||||
func TestHeldKey(t *testing.T) {
|
||||
fp := fp2of('b')
|
||||
key, err := HeldKey("mesh-mail", fp)
|
||||
if err != nil || key != "held.mesh-mail."+strings.Repeat("b", 64) {
|
||||
t.Fatalf("HeldKey = %q, %v", key, err)
|
||||
}
|
||||
module, back, err := ParseHeldKey(key)
|
||||
if err != nil || module != "mesh-mail" || back != fp {
|
||||
t.Errorf("ParseHeldKey = %q, %q, %v", module, back, err)
|
||||
}
|
||||
src1, _ := SourceFingerprint(SourceInputs{Module: "m", Tree: "t"})
|
||||
if _, err := HeldKey("mesh-mail", src1); err == nil {
|
||||
t.Errorf("a src1 fingerprint was held")
|
||||
}
|
||||
build, _ := BuilderBuildKey("mesh-mail", fp)
|
||||
for _, bad := range []string{build, "want.ace.mesh-mail", "held.mesh-mail.abc", "held." + strings.Repeat("b", 64), ""} {
|
||||
if _, _, err := ParseHeldKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") {
|
||||
t.Errorf("ParseHeldKey(%q) = %v, want a refusal naming it", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B1a: a difference key round-trips; a want or held key and a node's own key are refused.
|
||||
func TestDifferenceKey(t *testing.T) {
|
||||
key, err := DifferenceKey("g14", "mesh-mail")
|
||||
if err != nil || key != "g14.mesh-mail" {
|
||||
t.Fatalf("DifferenceKey = %q, %v", key, err)
|
||||
}
|
||||
node, module, err := ParseDifferenceKey(key)
|
||||
if err != nil || node != "g14" || module != "mesh-mail" {
|
||||
t.Errorf("ParseDifferenceKey = %q, %q, %v", node, module, err)
|
||||
}
|
||||
held, _ := HeldKey("mesh-mail", fp2of('a'))
|
||||
for _, bad := range []string{"want.g14.mesh-mail", held, "g14.node", "g14", ""} {
|
||||
if _, _, err := ParseDifferenceKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") {
|
||||
t.Errorf("ParseDifferenceKey(%q) = %v, want a refusal naming it", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B1a: wants, holds and differences round-trip through their encoders unchanged, at Version.
|
||||
func TestAssignerValuesRoundTrip(t *testing.T) {
|
||||
wants := []Want{
|
||||
{FP: fp2of('a')},
|
||||
{FP: fp2of('a'), Newest: fp2of('b'), Rule: RuleCanary, Words: "the canary ace does not run it healthy yet"},
|
||||
{Removed: true, Words: "the module left g14"},
|
||||
}
|
||||
for _, w := range wants {
|
||||
data, err := EncodeWant(w)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeWant(%+v): %v", w, err)
|
||||
}
|
||||
got, err := DecodeWant(data)
|
||||
w.V = Version
|
||||
if err != nil || !SameWant(got, w) {
|
||||
t.Errorf("want %+v came back %+v, %v", w, got, err)
|
||||
}
|
||||
}
|
||||
h := Held{Node: "ace", Words: "its health check failed: connection refused"}
|
||||
data, err := EncodeHeld(h)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotH, err := DecodeHeld(data)
|
||||
h.V = Version
|
||||
if err != nil || !SameHeld(gotH, h) {
|
||||
t.Errorf("held came back %+v, %v", gotH, err)
|
||||
}
|
||||
diffs := []Difference{
|
||||
{Want: fp2of('a'), Sent: fp2of('b'), Rule: RuleCanary},
|
||||
{Sent: fp2of('b'), Rule: RuleBuildDone},
|
||||
{WantRemoved: true, Sent: fp2of('b'), Unexplained: true},
|
||||
{SentBuild: "build-17", Rule: RuleGrant},
|
||||
}
|
||||
for _, d := range diffs {
|
||||
data, err := EncodeDifference(d)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeDifference(%+v): %v", d, err)
|
||||
}
|
||||
got, err := DecodeDifference(data)
|
||||
d.V = Version
|
||||
if err != nil || !SameDifference(got, d) {
|
||||
t.Errorf("difference %+v came back %+v, %v", d, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B1a: what a want, a hold and a difference may not say is refused on encode and on decode.
|
||||
func TestAssignerValuesRefused(t *testing.T) {
|
||||
src1, _ := SourceFingerprint(SourceInputs{Module: "m", Tree: "t"})
|
||||
for _, w := range []Want{
|
||||
{},
|
||||
{FP: src1},
|
||||
{Removed: true, FP: fp2of('a')},
|
||||
{FP: fp2of('a'), Newest: fp2of('b')},
|
||||
{FP: fp2of('a'), Newest: fp2of('a'), Rule: 3},
|
||||
{FP: fp2of('a'), Newest: fp2of('b'), Rule: 7},
|
||||
} {
|
||||
if _, err := EncodeWant(w); err == nil {
|
||||
t.Errorf("EncodeWant(%+v) accepted", w)
|
||||
}
|
||||
}
|
||||
if _, err := EncodeHeld(Held{Node: "a.b"}); err == nil {
|
||||
t.Errorf("a held naming a dotted node was accepted")
|
||||
}
|
||||
for _, d := range []Difference{
|
||||
{Want: fp2of('a'), Sent: fp2of('b')},
|
||||
{Want: fp2of('a'), Sent: fp2of('b'), Rule: 3, Unexplained: true},
|
||||
{Want: fp2of('a'), Sent: fp2of('a'), Rule: 3},
|
||||
{},
|
||||
{Want: fp2of('a'), Sent: fp2of('b'), Rule: 9},
|
||||
{WantRemoved: true, Want: fp2of('a'), Rule: 1},
|
||||
} {
|
||||
if _, err := EncodeDifference(d); err == nil {
|
||||
t.Errorf("EncodeDifference(%+v) accepted", d)
|
||||
}
|
||||
}
|
||||
if _, err := DecodeWant([]byte(`{"v":2,"fp":"` + fp2of('a') + `"}`)); !errors.Is(err, ErrUnknownVersion) {
|
||||
t.Errorf("a want of version 2 was read: %v", err)
|
||||
}
|
||||
if _, err := DecodeDifference([]byte(`{"v":1,"want":"` + fp2of('a') + `","sent":"` + fp2of('b') + `"}`)); err == nil {
|
||||
t.Errorf("a difference naming no rule was read")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user