// Package views is the owners' side and the readers' side of a first view (novox/hq ADR 0615, ADR 0621, ADR 0622; // design 25 § 12.2): the indexer and the assigner answer a reader's first view of one of their states as a tool of // their own module (Answer, FirstView), say each change of that state as one event on their views family (Event, // Emitter), and a reader keeps its picture from the answer and the events, asking the tool again only on a gap in // revisions (Follower). The tools' read side (Read) answers the ordinary read tools of § 12.2 too. package views import ( "encoding/json" "errors" "fmt" "slices" "strings" "time" ) // Version is the version of every value this package writes and the only one it reads. const Version = 1 // The owners' views families (design 25 § 12.2; ADR 0615 §5, ADR 0619 §2): one stream each on its owner's old `streams` // section, kept by limits 7 days and at most 256 MiB, dropping its oldest when full, so a reader that missed a dropped // event sees the gap in revisions and asks once more. A stream's name is the mesh's, so each carries its owner's name. const ( IndexerStream = "indexer-views" AssignerStream = "assigner-views" StreamDays = 7 StreamMiB = 256 // EventPrefix starts each event's local name: `views.`, one event per changed key of that state. EventPrefix = "views." // EventPattern is every event of a views family, as a stream's events and a reader's consumes name it. EventPattern = EventPrefix + "**" ) // MarksState is an owner's own state of the revision each followed state was said up to (Mark), one key per state: // what a first view answers its revision from, so a reader that takes the answer misses no event after it. const MarksState = "view-marks" // MaxValueBytes is the largest value an event carries whole: under the bus's largest message (1 MiB, the server's // max_payload) with the event's own fields. A larger one is said as its revision alone (Event.Gap), and the owner // writes a finding naming the key (design 25 § 12.2). const MaxValueBytes = 960 * 1024 // PageBytes bounds the values one answer carries, so the answer fits one bus message; Answer.Next continues it. const PageBytes = 768 * 1024 // Bound is how long a reader waits for its first view before it raises the condition naming the owner and the verb // (ADR 0615 §6): longer than an owner's start, so a normal start, readers and owners coming up together, tells nobody. const Bound = 2 * time.Minute // EventName is the local name of the event saying a change of state: `views.`. func EventName(state string) string { return EventPrefix + state } // StateOfEvent is the state an event's local name (or the last tokens of its subject) says, or "". func StateOfEvent(name string) string { if i := strings.LastIndex(name, ".event."+EventPrefix); i >= 0 { name = name[i+len(".event."):] } s, ok := strings.CutPrefix(name, EventPrefix) if !ok || s == "" || strings.ContainsAny(s, ".*> ") { return "" } return s } // Event is one change of one key of an owner's state, said on its views family: the key's value whole, or a delete // mark, at the revision of the state the change was written at. Created is when the owner's state was made on the bus: // a state made again (the bus module's recreate) starts its revisions again, and a reader takes another Created as a gap. type Event struct { V int `json:"v"` Owner string `json:"owner"` State string `json:"state"` Key string `json:"key"` Revision uint64 `json:"revision"` Created time.Time `json:"created"` Deleted bool `json:"deleted,omitempty"` Value json.RawMessage `json:"value,omitempty"` // Gap says the value was too large to carry (MaxValueBytes): the reader asks its first view once. Gap bool `json:"gap,omitempty"` } // ID is the event's id on the bus, so the same change said twice (an owner's restart) is kept once. func (e Event) ID() string { return fmt.Sprintf("%s-%s-%d-%d", e.Owner, e.State, e.Created.UnixNano(), e.Revision) } // Encode writes an event. func (e Event) Encode() ([]byte, error) { e.V = Version if e.Owner == "" || e.State == "" || e.Key == "" || e.Revision == 0 { return nil, errors.New("views: an event names its owner, state, key and revision") } if e.Deleted || e.Gap { e.Value = nil } return json.Marshal(e) } // DecodeEvent reads an event of this version. func DecodeEvent(data []byte) (Event, error) { var e Event if err := decode(data, &e); err != nil { return Event{}, err } if e.State == "" || e.Key == "" || e.Revision == 0 { return Event{}, errors.New("views: an event names no state, key or revision") } return e, nil } // Mark is the revision an owner said its state up to, and when that state was made (MarksState). type Mark struct { V int `json:"v"` State string `json:"state"` Revision uint64 `json:"revision"` Created time.Time `json:"created"` At time.Time `json:"at"` // Fill is the state's bucket as the bus held it when the mark was written: what a read tool says of its fill. Fill Fill `json:"fill"` } // Fill is a state's bucket's fill against its cap, as the bus says it. type Fill struct { Keys uint64 `json:"keys"` Bytes uint64 `json:"bytes"` MaxBytes int64 `json:"max-bytes"` } // DecodeMark reads a mark of this version. func DecodeMark(data []byte) (Mark, error) { var m Mark return m, decode(data, &m) } // Answer is a first view, or one page of an ordinary read: the state's values by key, from After on, at the revision // of the state the owner said up to (Mark) before it read them; Next is the key to ask after for the rest, empty when // the answer is whole. A reader keeps the first page's revision: a value on a later page may be newer, and the events // after that revision bring every key to its newest in order. type Answer struct { V int `json:"v"` Owner string `json:"owner"` State string `json:"state"` Revision uint64 `json:"revision"` Created time.Time `json:"created"` Values map[string]json.RawMessage `json:"values"` Next string `json:"next,omitempty"` At time.Time `json:"at"` } // DecodeAnswer reads an answer of this version. func DecodeAnswer(data []byte) (Answer, error) { var a Answer if err := decode(data, &a); err != nil { return Answer{}, err } if a.Values == nil { a.Values = map[string]json.RawMessage{} } return a, nil } // Entry is one key's value and revision, as a state read gives it. type Entry struct { Value json.RawMessage Revision uint64 } // Source is the read side of one state, as a tools bundle reaches its own state (stdio.KeptState's Keys and Get). type Source interface { Keys() ([]string, error) Get(key string) (*Entry, bool, error) } // FirstView answers one page of state from its owner's source: the mark first, so no change after the revision it // answers at is missed, then the values in key order after after, until PageBytes. A state not said yet (no mark) // answers revision 0: a first view of whatever it holds, which the reader takes as a gap at the first event. func FirstView(owner, state string, values, marks Source, after string, now time.Time) (Answer, error) { a := Answer{V: Version, Owner: owner, State: state, Values: map[string]json.RawMessage{}, At: now.UTC()} if marks != nil { e, ok, err := marks.Get(state) if err != nil { return Answer{}, fmt.Errorf("the mark of %s cannot be read: %w", state, err) } if ok { m, err := DecodeMark(e.Value) if err != nil { return Answer{}, fmt.Errorf("the mark of %s: %w", state, err) } a.Revision, a.Created = m.Revision, m.Created } } keys, err := values.Keys() if err != nil { return Answer{}, fmt.Errorf("the state %s cannot be listed: %w", state, err) } slices.Sort(keys) size := 0 for _, k := range keys { if after != "" && k <= after { continue } e, ok, err := values.Get(k) if err != nil { return Answer{}, fmt.Errorf("%s.%s cannot be read: %w", state, k, err) } if !ok { continue } if size > 0 && size+len(k)+len(e.Value) > PageBytes { a.Next = lastKey(a.Values) break } a.Values[k] = e.Value size += len(k) + len(e.Value) } return a, nil } func lastKey(m map[string]json.RawMessage) string { last := "" for k := range m { if k > last { last = k } } return last } func decode(data []byte, into any) error { var probe struct { V *int `json:"v"` } if err := json.Unmarshal(data, &probe); err != nil { return fmt.Errorf("views: value does not fit its shape: %w", err) } if probe.V == nil || *probe.V != Version { return fmt.Errorf("views: unknown value version: this reader reads %d", Version) } if err := json.Unmarshal(data, into); err != nil { return fmt.Errorf("views: value does not fit its shape: %w", err) } return nil }