// Package view derives the operator's rows from the delivery buckets. package view import ( "encoding/json" "sort" "strings" "sync" "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) // Entry is one change of a bucket's key, as the watch delivers it. type Entry struct { Bucket string Key string Value []byte At time.Time Deleted bool } // Row is one module on one node. type Row struct { Node string `json:"node"` Module string `json:"module"` Status string `json:"status"` Assigned string `json:"assigned,omitempty"` Have string `json:"have,omitempty"` Phase string `json:"phase,omitempty"` Health string `json:"health,omitempty"` Since time.Time `json:"since,omitzero"` Detail string `json:"detail,omitempty"` Waits string `json:"waits,omitempty"` // RequiredBy is what pulled the module onto the node, for an assignment that is pulled in. RequiredBy string `json:"required-by,omitempty"` Error string `json:"error,omitempty"` } // NodeRow is a node's own facts as counts. type NodeRow struct { Node string `json:"node"` Windows int `json:"windows"` Busy int `json:"busy"` Strays int `json:"strays"` Since time.Time `json:"since,omitzero"` Error string `json:"error,omitempty"` } // Snapshot is everything the view shows. type Snapshot struct { Rows []Row `json:"rows"` Nodes []NodeRow `json:"nodes"` // Offline is every node and part instance whose presence is offline (the watchman's word). Offline []OfflineRow `json:"offline"` // Stuck is every key that stopped moving and has not moved again. Stuck []StuckRow `json:"stuck"` // Unreadable is every watchman value the view could not read, named by its key: never dropped silently. Unreadable []UnreadableRow `json:"unreadable"` } // OfflineRow is a node ("node") or a part's instance ("part") that is offline, since the presence entry's time. type OfflineRow struct { Kind string `json:"kind"` Node string `json:"node,omitempty"` Part string `json:"part,omitempty"` Instance string `json:"instance,omitempty"` Since time.Time `json:"since,omitzero"` } // StuckRow is a key that stopped moving: its bucket and key, what it waited for and since when. type StuckRow struct { Bucket string `json:"bucket"` Key string `json:"key"` Waited string `json:"waited,omitempty"` Since time.Time `json:"since,omitzero"` } // UnreadableRow is a value of the watchman's buckets that did not decode, with the reader's error. type UnreadableRow struct { Bucket string `json:"bucket"` Key string `json:"key"` Error string `json:"error"` } // Model holds the rows' inputs in memory and nothing else: a restart fills it again from the buckets' last values. type Model struct { mu sync.Mutex assigns map[string]assignment builds map[string]build // by .build. newest map[string]string // module -> id haves map[string]haveEntry nodes map[string]nodeEntry // the watchman's buckets, by key presence map[string]presenceEntry stuck map[string]stuckEntry } type presenceEntry struct { value delivery.Presence err string at time.Time } type stuckEntry struct { value delivery.Stuck err string } // assignment holds only the fields of the controller's Assignment the view reads. type assignment struct { Node string `json:"node"` Module string `json:"module"` Build string `json:"build"` Commit string `json:"commit"` Since time.Time `json:"since"` Ended time.Time `json:"ended"` State string `json:"state"` PulledIn bool `json:"pulled-in"` RequiredBy string `json:"required-by"` } // open says whether the value is an assignment that stands for the module on its node, as the controller's // Assignment.Stands does. A module pulled in by another stands too. func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" } // build holds only the fields of the controller's Build record the view reads. A record with no state and no failure // is a done build. type build struct { ID string `json:"id"` Module string `json:"module"` State string `json:"state"` Commit string `json:"commit"` Failed string `json:"failed"` } func (b build) done() bool { return b.State == "" && b.Failed == "" } type haveEntry struct { have delivery.Have err string at time.Time } type nodeEntry struct { node delivery.HaveNode err string at time.Time } // New is an empty model. func New() *Model { return &Model{assigns: map[string]assignment{}, builds: map[string]build{}, newest: map[string]string{}, haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{}, presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}} } // Apply takes one change of a key. A value it cannot read is kept as an error on its row, never guessed. func (m *Model) Apply(e Entry) { m.mu.Lock() defer m.mu.Unlock() switch e.Bucket { case delivery.HaveBucket: node, module, isNode, err := delivery.ParseHaveKey(e.Key) if err != nil { return } if isNode { if e.Deleted { delete(m.nodes, node) return } n, err := delivery.DecodeHaveNode(e.Value) m.nodes[node] = nodeEntry{node: n, err: errText(err), at: e.At} return } key := node + "." + module if e.Deleted { delete(m.haves, key) return } h, err := delivery.DecodeHave(e.Value) m.haves[key] = haveEntry{have: h, err: errText(err), at: e.At} case delivery.PresenceBucket: if e.Deleted { delete(m.presence, e.Key) return } p, err := delivery.DecodePresence(e.Value) m.presence[e.Key] = presenceEntry{value: p, err: errText(err), at: e.At} case delivery.StuckBucket: if e.Deleted { delete(m.stuck, e.Key) return } st, err := delivery.DecodeStuck(e.Value) m.stuck[e.Key] = stuckEntry{value: st, err: errText(err)} case AssignmentsBucket: if e.Deleted { delete(m.assigns, e.Key) return } var a assignment if json.Unmarshal(e.Value, &a) != nil || a.Node == "" || a.Module == "" { return } m.assigns[a.Node+"."+a.Module] = a case BuildsBucket: module, rest, ok := strings.Cut(e.Key, ".") if !ok { return } switch { case rest == "newest": if e.Deleted { delete(m.newest, module) return } m.newest[module] = strings.TrimSpace(string(e.Value)) case strings.HasPrefix(rest, "build."): if e.Deleted { delete(m.builds, e.Key) return } var b build if json.Unmarshal(e.Value, &b) != nil { return } m.builds[e.Key] = b } } } func errText(err error) string { if err == nil { return "" } return err.Error() } // Snapshot derives one row per module and node with an open assignment, a row marked left behind for each have with // none, and a row per node's own key. Rows are ordered by node and module. func (m *Model) Snapshot() Snapshot { m.mu.Lock() defer m.mu.Unlock() rows := []Row{} for key, a := range m.assigns { if !a.open() { continue } r := Row{Node: a.Node, Module: a.Module, Status: StatusAssigned, Assigned: a.Build} if a.PulledIn { r.RequiredBy = a.RequiredBy } var waits []string h, has := m.haves[key] switch { case !has: waits = append(waits, "no have yet") case h.err != "": r.Since, r.Error = h.at, h.err default: r.Have, r.Phase, r.Health = h.have.Build, string(h.have.Phase), string(h.have.Health) r.Since, r.Detail = h.at, h.have.Detail if a.Build != "" && h.have.Build != a.Build { waits = append(waits, "the node has not applied "+a.Build) } } if id := m.newest[a.Module]; id != "" && a.Build != "" && id != a.Build { if b, ok := m.builds[a.Module+".build."+id]; ok && b.done() { waits = append(waits, a.Module+" has a newer build "+id+" not yet assigned") } } r.Waits = strings.Join(waits, "; ") rows = append(rows, r) } for key, h := range m.haves { if a, ok := m.assigns[key]; ok && a.open() { continue } node, module, _, _ := delivery.ParseHaveKey(key) r := Row{Node: node, Module: module, Status: StatusLeftBehind, Since: h.at, Error: h.err} if h.err == "" { r.Have, r.Phase, r.Health, r.Detail = h.have.Build, string(h.have.Phase), string(h.have.Health), h.have.Detail } rows = append(rows, r) } sort.Slice(rows, func(i, j int) bool { if rows[i].Node != rows[j].Node { return rows[i].Node < rows[j].Node } return rows[i].Module < rows[j].Module }) nodes := []NodeRow{} for name, n := range m.nodes { nodes = append(nodes, NodeRow{Node: name, Windows: count(n.node.Windows), Busy: count(n.node.Busy), Strays: count(n.node.Strays), Since: n.at, Error: n.err}) } sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node }) offline, stuck, unreadable := []OfflineRow{}, []StuckRow{}, []UnreadableRow{} for key, p := range m.presence { if p.err != "" { unreadable = append(unreadable, UnreadableRow{Bucket: delivery.PresenceBucket, Key: key, Error: p.err}) continue } if p.value.Online { continue } node, part, instance, err := delivery.ParsePresenceKey(key) if err != nil { unreadable = append(unreadable, UnreadableRow{Bucket: delivery.PresenceBucket, Key: key, Error: err.Error()}) continue } if part != "" { offline = append(offline, OfflineRow{Kind: "part", Part: part, Instance: instance, Since: p.at}) } else { offline = append(offline, OfflineRow{Kind: "node", Node: node, Since: p.at}) } } for key, st := range m.stuck { if st.err != "" { unreadable = append(unreadable, UnreadableRow{Bucket: delivery.StuckBucket, Key: key, Error: st.err}) continue } if st.value.Moved { continue } stuck = append(stuck, StuckRow{Bucket: st.value.Bucket, Key: st.value.Key, Waited: st.value.Waited, Since: st.value.Since}) } sort.Slice(offline, func(i, j int) bool { a, b := offline[i], offline[j] return a.Kind+a.Node+a.Part+a.Instance < b.Kind+b.Node+b.Part+b.Instance }) sort.Slice(stuck, func(i, j int) bool { return stuck[i].Bucket+"."+stuck[i].Key < stuck[j].Bucket+"."+stuck[j].Key }) sort.Slice(unreadable, func(i, j int) bool { return unreadable[i].Bucket+"."+unreadable[i].Key < unreadable[j].Bucket+"."+unreadable[j].Key }) return Snapshot{Rows: rows, Nodes: nodes, Offline: offline, Stuck: stuck, Unreadable: unreadable} } // count is the number of items of a carried JSON array or object, and 0 for anything else. func count(raw json.RawMessage) int { var list []json.RawMessage if json.Unmarshal(raw, &list) == nil { return len(list) } var obj map[string]json.RawMessage if json.Unmarshal(raw, &obj) == nil { return len(obj) } return 0 } // Counted is the number of rows that are modules on the nodes, not left behind. func (s Snapshot) Counted() int { n := 0 for _, r := range s.Rows { if r.Status == StatusAssigned { n++ } } return n } // The buckets the view reads besides the have bucket (which mesh-sdk's delivery package names). const ( AssignmentsBucket = "mesh-controller_assignments" BuildsBucket = "mesh-controller_builds" ) // The statuses of a row. const ( StatusAssigned = "assigned" StatusLeftBehind = "left behind" )