Files
mesh-delivery/modules/mesh-delivery-assigner/internal/assigngraph/keep.go
T
jschoubben 1e50642e3c Assigner: the pass never waits on the keeping of revisions (H-A-259)
A pass that kept a fresh assignment graph revision sat six to eight minutes
keeping it and removing the revisions not current, so the wakes of
pipeline-ended and running waited for it. The keeping now runs off the pass
loop, one at a time and folded; the removal takes each entry's module and
fingerprint from the store's listing instead of reading every entry object;
each pass and each keeping says its duration.

Builds: design 47 H H-A-259; ADR 0682
2026-11-21 12:40:15 +01:00

209 lines
7.4 KiB
Go

package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// RevisionsObjects is the local name of the kept data in the assigner's manifest's old `objects` section (design 47
// lines 1362-1366: both foundation parts keep their revisions as objects in the old form; the seat mesh-wants' state
// assignment-revisions names its key forms).
const RevisionsObjects = "assignment-revisions"
// ErrNoObject is an object the bucket does not hold.
var ErrNoObject = errors.New("no such object")
// Objects is an object bucket as the assigner keeps its revisions in it.
type Objects interface {
// Put stores data whole under name, replacing one of that name.
Put(ctx context.Context, name string, data []byte) error
// Get is the object's bytes; ErrNoObject when it holds none of that name.
Get(ctx context.Context, name string) ([]byte, error)
// Names is every object's name; none is an empty list.
Names(ctx context.Context) ([]string, error)
}
// Described is one object as the bucket's listing says it: its name, and for an entry object the module and fingerprint put beside
// its bytes (empty for one put before them).
type Described struct {
Name, Module, Fp string
}
// Describer is an object bucket whose listing says, in one read, each entry object's module and fingerprint, so the removal of the
// revisions not current reads no object per entry (design 47 H-A-259).
type Describer interface {
// PutDescribed stores data whole under name, with module and fp beside it in the listing.
PutDescribed(ctx context.Context, name string, data []byte, module, fp string) error
// Described is every object, as the listing says it.
Described(ctx context.Context) ([]Described, error)
}
// PutEntry stores the entry object e under name, with its module and fingerprint beside it where the bucket says them.
func PutEntry(ctx context.Context, o Objects, name string, data []byte, e meshwants.AssignmentEntry) error {
if d, ok := o.(Describer); ok {
return d.PutDescribed(ctx, name, data, e.Module, e.Fp)
}
return o.Put(ctx, name, data)
}
// EntryObject is the object name of the entry of that digest (the seat's key form entry.<digest>).
func EntryObject(digest string) string { return "entry." + digest }
// TrunkObject is the object name of a trunk revision (the seat's key form trunk.<revision>).
func TrunkObject(revision int64) string { return "trunk." + strconv.FormatInt(revision, 10) }
// trunkOf is the revision number a trunk object's name holds, and whether it is one.
func trunkOf(name string) (int64, bool) {
n, ok := strings.CutPrefix(name, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// Newest is the newest trunk revision kept, its list only; none kept is false.
func Newest(ctx context.Context, o Objects) (meshwants.AssignmentRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshwants.AssignmentRevision{}, false, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var newest int64
for _, n := range names {
if r, ok := trunkOf(n); ok && r > newest {
newest = r
}
}
if newest == 0 {
return meshwants.AssignmentRevision{}, false, nil
}
rev, err := ReadRevision(ctx, o, newest)
return rev, err == nil, err
}
// ReadRevision is the trunk revision of that number, its list only.
func ReadRevision(ctx context.Context, o Objects, revision int64) (meshwants.AssignmentRevision, error) {
var rev meshwants.AssignmentRevision
data, err := o.Get(ctx, TrunkObject(revision))
if err != nil {
return rev, fmt.Errorf("revision %d: %w", revision, err)
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, fmt.Errorf("revision %d: %w", revision, err)
}
return rev, nil
}
// ReadEntry is the entry kept under that digest.
func ReadEntry(ctx context.Context, o Objects, digest string) (meshwants.AssignmentEntry, error) {
var e meshwants.AssignmentEntry
data, err := o.Get(ctx, EntryObject(digest))
if err != nil {
return e, fmt.Errorf("the entry %s: %w", digest, err)
}
if err := json.Unmarshal(data, &e); err != nil {
return e, fmt.Errorf("the entry %s: %w", digest, err)
}
return e, nil
}
// Keep keeps c as the next trunk revision, numbered after the newest kept, unless the newest kept lists the same entries
// from the same inputs (Same): then it keeps nothing and answers the newest and false. Each entry object not held yet is
// written before the list, so a list is never kept whose entries are not, and an entry two revisions share is stored
// once (design 47 *An unchanged entry is stored once*).
func Keep(ctx context.Context, o Objects, c Computed, cause string, at time.Time) (meshwants.AssignmentRevision, bool, error) {
newest, has, err := Newest(ctx, o)
if err != nil {
return newest, false, err
}
if has && Same(newest, c.Revision) {
return newest, false, nil
}
rev := c.Revision
rev.Revision = newest.Revision + 1
rev.Cause = cause
t := at.UTC()
rev.At = &t
if err := writeEntries(ctx, o, c, rev); err != nil {
return newest, false, err
}
list, err := json.Marshal(rev)
if err != nil {
return newest, false, err
}
if err := o.Put(ctx, TrunkObject(rev.Revision), list); err != nil {
return newest, false, fmt.Errorf("revision %d in %s: %w", rev.Revision, RevisionsObjects, err)
}
return rev, true, nil
}
// writeEntries writes each entry object rev lists that the bucket does not hold yet, from c, before its list is kept.
func writeEntries(ctx context.Context, o Objects, c Computed, rev meshwants.AssignmentRevision) error {
names, err := o.Names(ctx)
if err != nil {
return fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
for _, ref := range rev.Entries {
name := EntryObject(ref.Digest)
if held[name] {
continue
}
e, ok := c.Entries[ref.Digest]
if !ok {
return fmt.Errorf("revision %d lists the entry of %s/%s (%s), which it does not carry", rev.Revision, ref.Node, ref.Module, ref.Digest)
}
data, err := json.Marshal(e)
if err != nil {
return err
}
if err := PutEntry(ctx, o, name, data, e); err != nil {
return fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err)
}
held[name] = true
}
return nil
}
// Event is the seat's event assignment-revision of a kept revision.
func Event(rev meshwants.AssignmentRevision) meshwants.AssignmentRevisionEvent {
return meshwants.AssignmentRevisionEvent{V: rev.V, Revision: rev.Revision, Kind: rev.Kind, Cause: rev.Cause, Group: rev.Group,
Heads: rev.Heads, Catalogs: rev.Catalogs, At: rev.At}
}
// Filler is an object bucket that says its fill: the bytes used and its cap in bytes (0: no cap known).
type Filler interface {
Fill(ctx context.Context) (used, capacity int64, err error)
}
// NewBytes is what keeping c next would add to the bucket: each entry object not held yet and the list (design 47 G §181).
func NewBytes(ctx context.Context, o Objects, c Computed) int64 {
names, err := o.Names(ctx)
if err != nil {
return 0
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
var size int64
for _, ref := range c.Revision.Entries {
if e, ok := c.Entries[ref.Digest]; ok && !held[EntryObject(ref.Digest)] {
data, _ := json.Marshal(e)
size += int64(len(data))
}
}
list, _ := json.Marshal(c.Revision)
return size + int64(len(list))
}