mesh-controller/cmd/mesh-controller/plans_building.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// The plans building (novox/hq ADR 0483 clause 2, issue 1687).
//
// **A key per delivery plan in its build phase, so that the build seat takes no new check while a delivery builds.** The phase
// runs from the ask of the plan's first build until every build of its last tier is built, or the plan fails, stops or waits
// on the operator. It is derived from the plan alone (buildPhaseOf), so a controller that restarts finds the same answer. The
// key is put when the first build is asked (sayBuilding), kept current at every save of the plan (keepBuildingKey) and by a
// look every link.PlansBuildingRefresh while this controller holds the lease (keepingPlansBuilding), and deleted at the phase's
// end; the bucket's age removes the key of a controller that stopped, so checks are held at most that long.
//
// The layout is documented in internal/link/holds.go.

// buildPhaseOf says whether a plan is in its build phase: open, not waiting for the operator's word, with a build asked and a
// build of its tiers still to be built. A tier built and sent leaves the plan rolling while later tiers are still to be asked;
// it is in the phase still, which is what keeps a check from taking a place between two tiers.
func buildPhaseOf(p inventory.Plan) bool {
	if !p.Open() || p.Waiting() || p.Release != nil {
		return false
	}
	asked, unbuilt := false, false
	for _, tier := range p.Tiers {
		for _, m := range tier {
			st := p.Modules[m]
			if st != nil && st.State != "" {
				asked = true
			}
			if st == nil || (st.State != "built" && st.State != planDeleted) {
				unbuilt = true
			}
		}
	}
	return asked && unbuilt
}

// buildingSince is when the plan's first build was asked, or fallback when no module says.
func buildingSince(p inventory.Plan, fallback time.Time) time.Time {
	since := time.Time{}
	for _, st := range p.Modules {
		if st != nil && st.AskedAt != nil && (since.IsZero() || st.AskedAt.Before(since)) {
			since = *st.AskedAt
		}
	}
	if since.IsZero() {
		return fallback
	}
	return since
}

// buildingStore is the bucket of the plans building.
type buildingStore interface {
	Put(ctx context.Context, plan string, value []byte) error
	Delete(ctx context.Context, plan string) error
	Keys(ctx context.Context) ([]string, error)
}

// syncBuilding makes the bucket say what the plans say: a key for each plan in its build phase, put again, and none for any
// other plan, deleted. It returns the faults, each once.
func syncBuilding(ctx context.Context, store buildingStore, plans []inventory.Plan, now time.Time) []error {
	var faults []error
	live := map[string]bool{}
	for _, p := range plans {
		if !buildPhaseOf(p) {
			continue
		}
		live[p.ID] = true
		if err := putBuilding(ctx, store, p.ID, buildingSince(p, now), now); err != nil {
			faults = append(faults, err)
		}
	}
	keys, err := store.Keys(ctx)
	if err != nil {
		return append(faults, err)
	}
	for _, key := range keys {
		if live[key] {
			continue
		}
		if err := store.Delete(ctx, key); err != nil {
			faults = append(faults, fmt.Errorf("the key of %s could not be taken away: %w", key, err))
		}
	}
	return faults
}

func putBuilding(ctx context.Context, store buildingStore, plan string, since, now time.Time) error {
	body, _ := json.Marshal(link.PlanBuilding{Plan: plan, Since: since.UTC(), Seen: now.UTC()})
	if err := store.Put(ctx, plan, body); err != nil {
		return fmt.Errorf("the key of %s could not be put: %w", plan, err)
	}
	return nil
}

// busBuilding is the store on the controller's bucket.
type busBuilding struct {
	conn *nats.Conn
	mu   sync.Mutex
	kv   jetstream.KeyValue
}

func (b *busBuilding) bucket(ctx context.Context) (jetstream.KeyValue, error) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.kv != nil {
		return b.kv, nil
	}
	api, err := jetstream.New(b.conn)
	if err != nil {
		return nil, err
	}
	kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
	if err != nil {
		return nil, fmt.Errorf("the plans building: %w", err)
	}
	b.kv = kv
	return kv, nil
}

func (b *busBuilding) Put(ctx context.Context, plan string, value []byte) error {
	kv, err := b.bucket(ctx)
	if err != nil {
		return err
	}
	_, err = kv.Put(ctx, plan, value)
	return err
}

func (b *busBuilding) Delete(ctx context.Context, plan string) error {
	kv, err := b.bucket(ctx)
	if err != nil {
		return err
	}
	if err := kv.Delete(ctx, plan); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
		return err
	}
	return nil
}

func (b *busBuilding) Keys(ctx context.Context) ([]string, error) {
	kv, err := b.bucket(ctx)
	if err != nil {
		return nil, err
	}
	keys, err := kv.Keys(ctx)
	if errors.Is(err, jetstream.ErrNoKeysFound) {
		return nil, nil
	}
	return keys, err
}

// buildingKeys is the store of the serving controller; nil in a process that is not it, which says nothing.
var buildingKeys atomic.Pointer[buildingStore]

func setBuildingStore(s buildingStore) {
	if s == nil {
		buildingKeys.Store(nil)
		return
	}
	buildingKeys.Store(&s)
}

// sayBuilding puts a plan's key: its first build is being asked (or another of its builds). A key that could not be put is
// said; the look that follows puts it again.
func sayBuilding(ctx context.Context, plan string, since time.Time) {
	s := buildingKeys.Load()
	if s == nil {
		return
	}
	if err := putBuilding(ctx, *s, plan, since, time.Now()); err != nil {
		fmt.Printf("plans: %v; checks may take a place of %s until the next look\n", err, plan)
	}
}

// keepBuildingKey is told each plan this controller keeps: its key put again while it builds and taken away once it does not.
func keepBuildingKey(ctx context.Context, p inventory.Plan) {
	s := buildingKeys.Load()
	if s == nil {
		return
	}
	if buildPhaseOf(p) {
		sayBuilding(ctx, p.ID, buildingSince(p, time.Now()))
		return
	}
	if err := (*s).Delete(ctx, p.ID); err != nil {
		fmt.Printf("plans: the key of %s could not be taken away; it ages out: %v\n", p.ID, err)
	}
}

// keepingPlansBuilding looks at the plans every link.PlansBuildingRefresh for as long as this controller serves and holds the
// lease, so a key is put again before its age and the key of a plan that left the phase unseen is taken away.
func keepingPlansBuilding(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
	store := &busBuilding{conn: conn}
	setBuildingStore(store)
	defer setBuildingStore(nil)
	tick := time.NewTicker(broker.PlansBuildingRefresh)
	defer tick.Stop()
	failing := ""
	for {
		if epoch, err := theLease.epoch(ctx); err == nil && epoch != 0 {
			why := ""
			if plans, err := inv.OpenPlans(ctx); err != nil {
				why = fmt.Sprintf("the plans cannot be read: %v", err)
			} else if faults := syncBuilding(ctx, store, plans, time.Now()); len(faults) > 0 {
				why = fmt.Sprint(faults)
			}
			if why != failing {
				if why != "" {
					fmt.Printf("plans: the keys of the plans building cannot be kept: %s\n", why)
				} else {
					fmt.Printf("plans: the keys of the plans building are kept again\n")
				}
				failing = why
			}
		}
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
		}
	}
}