mesh-catalog/modules/nats/cmd/nats-tools/keep.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	occ "git.novox.be/novox/mesh-contracts/go/clients/operatorchannelclient"
	oc "git.novox.be/novox/mesh-contracts/go/seats/operatorchannel"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/handacts"
	"git.novox.be/novox/mesh-sdk/go/process"
	"git.novox.be/novox/mesh-sdk/go/seatdef"

	"nats-tools/internal/bussnap"
	"nats-tools/internal/keeper"
)

// The keeper (`nats-tools keep`): the bus module's process on its own account (own secret broker-keeper, ADR 0436 §1),
// started by the node-engine as the manifest's process keeper. It reads the fill every minute from its start, asks the
// indexer's first view once, then hears manifest-accepted and the operator's decided on its durable consumer, makes what
// each verdict asks for, and acts on the persons' requests its tools put in its own state requests (requests.go) as they
// are written or answered. It serves nothing (ADR 0436 §3).

// AcceptedOf reads a manifest-accepted body: mesh-sdk's delivery.ManifestAccepted, with the indexer's revision of the
// verdict when the event carries it (a field mesh-sdk is asked to add; 0 until then).
func AcceptedOf(body []byte) (keeper.Accepted, error) {
	var a struct {
		delivery.ManifestAccepted
		Revision uint64 `json:"revision"`
	}
	if err := json.Unmarshal(body, &a); err != nil {
		return keeper.Accepted{}, err
	}
	if a.Module == "" {
		return keeper.Accepted{}, fmt.Errorf("a manifest-accepted names no module")
	}
	return keeper.Accepted{Module: a.Module, Digest: a.Digest, Revision: a.Revision, BusState: a.BusState}, nil
}

// AcceptedSubject is the indexer's manifest-accepted on the shared events stream.
var AcceptedSubject = "mesh.mod." + keeper.FirstViewOwner + ".event." + delivery.ManifestAcceptedEvent

// The router's records of asks, read under this module's own name (ADR 0259 §3), as the mail sender reads them.
const asksBucket = "messenger_asks"

// WarrantFrom reads the operator's warrant on one of the bus module's asks from the router's own record of it, under the
// keeper's own account (read: process.Conn.Ask): the ask and its warrant, or nil while the operator has not answered.
func WarrantFrom(read func(bucket, id string) (*stdio.StateEntry, error)) func(id string) (*oc.Ask, *oc.Warrant, error) {
	return func(id string) (*oc.Ask, *oc.Warrant, error) {
		e, err := read(asksBucket, id)
		if err != nil || e == nil {
			return nil, nil, err
		}
		var rec struct {
			Ask     *oc.Ask     `json:"ask"`
			Warrant *oc.Warrant `json:"warrant"`
		}
		if err := json.Unmarshal(e.Value, &rec); err != nil {
			return nil, nil, err
		}
		return rec.Ask, rec.Warrant, nil
	}
}

// processStore is the keeper's own state through its process connection.
type processStore struct{ c *process.Conn }

func (s processStore) Get(state, key string) ([]byte, uint64, bool, error) {
	e, err := s.c.State(state).Get(key)
	if err != nil || e == nil {
		return nil, 0, false, err
	}
	return e.Value, e.Revision, true, nil
}

func (s processStore) Put(state, key string, value []byte) (uint64, error) {
	return s.c.State(state).Put(key, json.RawMessage(value))
}

func (s processStore) Delete(state, key string) error { return s.c.State(state).Delete(key) }

func (s processStore) Keys(state string) ([]string, error) { return s.c.State(state).Keys() }

func (s processStore) Create(state, key string, value []byte) (uint64, error) {
	rev, err := s.c.State(state).Create(key, json.RawMessage(value))
	if errors.Is(err, stdio.ErrChanged) {
		return 0, keeper.ErrExists
	}
	return rev, err
}

// streamEmitter says an event of the seat mesh-broker, with the SDK's headers, under its one event id (the lead's ruling,
// registry milestone, 13:0x; design 25 § 12.3): on the seat's own subject `mesh.seat.mesh-broker.event.<event>`, which the
// shared events stream keeps 7 days and every consumer of `mesh-broker.<event>` hears, like any seat's event; then on the
// bus module's own stream (its old `streams` section, 30 days, 256 MiB), which its tool `history` reads. The seat's subject
// first: a reader hears it before the history holds it, never the other way round. It says it on s.Left (every subject
// when nil) and leaves in s.Left what the bus did not take, so the keeper says that again with the same id and the bus
// keeps it once (the two publishes are not one act: a failed second one is kept and said again, never silent).
func streamEmitter(js jetstream.JetStream, a process.Account) func(*keeper.Saying) error {
	return func(s *keeper.Saying) error {
		data, err := json.Marshal(s.Body)
		if err != nil {
			return err
		}
		if s.ID == "" {
			s.ID = keeper.EventID()
		}
		at := s.At
		if at.IsZero() {
			at = time.Now().UTC()
		}
		subjects := s.Left
		if subjects == nil {
			subjects = EventSubjects(a, s.Event)
		}
		s.Said = nil
		for i, subject := range subjects {
			msg := nats.NewMsg(subject)
			msg.Header.Set("x-event-id", s.ID)
			msg.Header.Set("x-source", a.Module)
			msg.Header.Set("x-node", a.Node)
			msg.Header.Set("x-time", at.UTC().Format(time.RFC3339Nano))
			msg.Header.Set("content-type", "application/json")
			msg.Data = data
			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
			_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(s.ID))
			cancel()
			if err != nil {
				s.Left = append([]string(nil), subjects[i:]...)
				return fmt.Errorf("saying %s on %s: %w", s.Event, subject, err)
			}
			s.Said = append(s.Said, subject)
		}
		s.Left = nil
		return nil
	}
}

// EventSubjects are where one event of the seat mesh-broker is said, in order: the seat's subject, then the bus module's
// own stream.
func EventSubjects(a process.Account, event string) []string {
	return []string{seatdef.EventSubject(catalogue.BrokerSeat, event), a.StreamSubject(event)}
}

// sayer raises and clears the keeper's conditions through the condition keeper, as every module says them.
func sayer(js jetstream.JetStream) func(oc.Condition) error {
	return func(c oc.Condition) error {
		subject, err := occ.ConditionSubject(keeper.Module)
		if err != nil {
			return err
		}
		data, err := occ.EncodeCondition(c)
		if err != nil {
			return err
		}
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		_, err = js.Publish(ctx, subject, data)
		return err
	}
}

// firstView asks the indexer's indexer_bus_state as the keeper's own account.
func firstView(c *process.Conn) func() (keeper.FirstView, error) {
	return func() (keeper.FirstView, error) {
		raw, err := c.Call(keeper.FirstViewOwner+"."+keeper.FirstViewVerb, map[string]any{})
		if err != nil {
			return keeper.FirstView{}, err
		}
		return FirstViewOf(raw)
	}
}

// FirstViewOf reads indexer_bus_state's answer, as the tool runner hands it back ({result, error} or bare).
func FirstViewOf(raw []byte) (keeper.FirstView, error) {
	var wrapped struct {
		Result json.RawMessage `json:"result"`
		Error  string          `json:"error"`
	}
	if json.Unmarshal(raw, &wrapped) == nil && (wrapped.Result != nil || wrapped.Error != "") {
		if wrapped.Error != "" {
			return keeper.FirstView{}, fmt.Errorf("%s", wrapped.Error)
		}
		raw = wrapped.Result
	}
	var v keeper.FirstView
	if err := json.Unmarshal(raw, &v); err != nil {
		return keeper.FirstView{}, fmt.Errorf("the answer is not a first view: %w", err)
	}
	return v, nil
}

func percent(getenv func(string) string, name string, fallback int64) int64 {
	if n, err := strconv.ParseInt(strings.TrimSpace(getenv(name)), 10, 64); err == nil && n > 0 && n < 100 {
		return n
	}
	return fallback
}

// runKeeper is the keeper's process: it returns when ctx ends, or when it has no credential.
func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
	logger := log.New(logOut, "nats-keeper: ", log.LstdFlags)
	account, err := process.ReadBrokerAccount(getenv)
	if err != nil {
		return err
	}
	conn, err := process.Connect(account)
	if err != nil {
		return err
	}
	defer conn.Close()
	nc, js, err := account.Dial(nats.Name(account.Node + "." + account.Module + ".keeper"))
	if err != nil {
		return err
	}
	defer nc.Close()
	recorder := handacts.Recorder{Ask: conn.Call}
	k := &keeper.Keeper{
		Bus:   keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR"))},
		Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn),
		Act: func(what, why, cause string) error {
			_, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause})
			return err
		},
		Warrant: WarrantFrom(conn.Ask),
		Logf:    logger.Printf,
		Raise:   percent(getenv, "MESH_NATS_FULL_RAISE", 75), Clear: percent(getenv, "MESH_NATS_FULL_CLEAR", 70),
	}
	logger.Printf("started on the account %s: asking %s.%s for the first view", account.User, keeper.FirstViewOwner, keeper.FirstViewVerb)
	return Keep(ctx, k, hearer(js, account, logger.Printf), requestsOf(conn), logger.Printf)
}

// hearer binds the keeper's durable consumer on EVENTS (<node>_nats_keeper, filtered on manifest-accepted) and hands
// each message to handle, which acknowledges it once applied.
func hearer(js jetstream.JetStream, a process.Account, logf func(string, ...any)) func(context.Context, func(subject string, body []byte) error) error {
	return func(ctx context.Context, handle func(string, []byte) error) error {
		cons, err := js.Consumer(ctx, process.EventsStream, a.QueueConsumerName())
		if err != nil {
			return fmt.Errorf("the keeper's consumer %s on %s: %w", a.QueueConsumerName(), process.EventsStream, err)
		}
		cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
			if err := handle(m.Subject(), m.Data()); err != nil {
				logf("%v: delivered again", err)
				_ = m.NakWithDelay(30 * time.Second)
				return
			}
			_ = m.Ack()
		}, logf)
		if err != nil {
			return err
		}
		go func() { <-ctx.Done(); cc.Stop() }()
		return nil
	}
}

// FillEvery is how often the keeper's fill pass runs: a minute (a test's shorter bound beside).
var FillEvery = time.Minute

// DecidedSubject is where the operator channel says the operator's answer to one of the bus module's own asks (ADR
// 0259 §6; ADR 0620 §2), heard on the keeper's durable consumer: the warrant a recreate or a drop waits for.
var DecidedSubject = oc.SubjectDecided(keeper.Module)

// Keep is the keeper's loop, apart from the bus so a test drives it: a fill pass at once and every minute, from its
// start, so the store's one guard runs while the first view is still asked (design 25 § 12.3); the first view asked
// until it answers, on its backoff; then hear, which applies each manifest-accepted and wakes the persons' requests on
// the operator channel's decided for the bus module's own asks; the waiting entries made again on the same backoff; and
// the persons' requests acted on as they are written or answered, never on a timer. It returns when ctx ends.
func Keep(ctx context.Context, k *keeper.Keeper, hear func(context.Context, func(string, []byte) error) error,
	requests func(context.Context, func()) error, logf func(string, ...any)) error {
	tick := time.NewTicker(FillEvery)
	defer tick.Stop()
	k.FillPass()
	var wait time.Duration
	for {
		next, err := k.AskFirstView(wait)
		if err == nil {
			break
		}
		logf("the first view did not answer (%v): nothing is created or orphaned until it does; asked again in %s", err, next)
		wait = next
		again := time.After(next)
	waiting:
		for {
			select {
			case <-ctx.Done():
				return ctx.Err()
			case <-tick.C:
				k.FillPass()
			case <-again:
				break waiting
			}
		}
	}
	logf("the first view answered at revision %d", k.Status().FirstView.Revision)
	wake := func(c chan struct{}) func() {
		return func() {
			select {
			case c <- struct{}{}:
			default:
			}
		}
	}
	heard, asked := make(chan struct{}, 1), make(chan struct{}, 1)
	for backoff := 10 * time.Second; ; backoff = min(backoff*2, 5*time.Minute) {
		err := hear(ctx, func(subject string, body []byte) error {
			if subject == DecidedSubject {
				// The operator answered one of the bus module's asks: the waiting requests are read again now.
				wake(asked)()
				return nil
			}
			if subject != AcceptedSubject && !strings.HasSuffix(subject, ".event."+delivery.ManifestAcceptedEvent) {
				return nil
			}
			a, err := AcceptedOf(body)
			if err != nil {
				logf("a manifest-accepted cannot be read (%v): taken, and nothing made of it", err)
				return nil
			}
			k.Heard(a)
			wake(heard)()
			return nil
		})
		if err == nil {
			break
		}
		logf("manifest-accepted cannot be heard yet (%v): asked again in %s", err, backoff)
		// The fill pass runs on while the consumer is bound again (review round 2, running F1).
		bound := time.After(backoff)
	binding:
		for {
			select {
			case <-ctx.Done():
				return ctx.Err()
			case <-tick.C:
				k.FillPass()
			case <-bound:
				break binding
			}
		}
	}
	if requests != nil {
		if err := requests(ctx, wake(asked)); err != nil {
			logf("the persons' requests cannot be watched (%v): acted on as the operator answers", err)
		}
	}
	// What waited while the keeper was down, or before its first view, is acted on now.
	k.ActOnRequests()
	retry := 10 * time.Second
	var again <-chan time.Time
	for {
		if len(k.Waiting()) > 0 && again == nil {
			again = time.After(retry)
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-tick.C:
			k.FillPass()
		case <-asked:
			k.ActOnRequests()
		case <-heard:
		case <-again:
			again = nil
			k.CheckAll()
			if len(k.Waiting()) > 0 {
				retry = min(retry*2, 5*time.Minute)
				logf("%d entr(y/ies) wait to be made: %s; made again in %s", len(k.Waiting()), strings.Join(k.Waiting(), "; "), retry)
			} else {
				retry = 10 * time.Second
			}
		}
	}
}

// keepMain runs the keeper until a signal.
func keepMain() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()
	if err := runKeeper(ctx, os.Getenv, os.Stderr); err != nil && ctx.Err() == nil {
		fmt.Fprintln(os.Stderr, "nats-keeper:", err)
		os.Exit(1)
	}
}

// requestsOf watches the keeper's own state requests, waking act at each change.
func requestsOf(c *process.Conn) func(context.Context, func()) error {
	return func(_ context.Context, act func()) error {
		return c.State(keeper.StateRequests).Watch("", func(stdio.StateChange) error {
			act()
			return nil
		})
	}
}