jschoubben
ac34bce49f
mesh/delivery merged: source.main moved to novox/mesh-controller@2bcd85d2df4c
mesh/check pass ac34bce49f58 on 2bcd85d2df4c | unit 966 run, 171 kept, 2 skipped | integration 1625 run, 232 kept, 7 skipped | over bound: integration…
A catalogue merge that leaves a module's sources alone registers it at the new commit from the same done key, whose build record stands at its first commit, so the commit had no build and the module no fingerprint of the new form: it went into the node's own composition while its want named its own, and every node declared its files twice. currentFingerprints now reads the indexer's trunk value at the commit when no build points at it (1945/AC15), and a module a want or have names by its own composition is left out of the node's own (1945/AC16).
489 lines
19 KiB
Go
489 lines
19 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/ed25519"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
|
|
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
)
|
|
|
|
// The compositions (novox/hq ADR 0539 clauses 1 to 5). Every act that sent a declaration writes compositions instead:
|
|
// the controller composes a node as it composed a send (composeSend: the same composition, the same makes), splits the
|
|
// declaration by module into one composition per module at the fingerprint the module is registered at, and the node's
|
|
// own resources, signs each with its key, and writes those that differ, by compare-and-set while it holds the lease:
|
|
// the composition into the node's bucket `composed-<node>`, which only that node reads, and its digest with the
|
|
// judgement of the move (composed_judge.go) into `mesh-controller_composed`, which the assigner reads. A file over 64
|
|
// KiB goes by digest into the node's object bucket first. Nothing is published on a node's declaration subject.
|
|
//
|
|
// **A module with no fingerprint of the new form goes with the node's own composition.** The mesh composes resources
|
|
// of modules it provides itself, built by nobody (the private network's mesh-wireguard), and a module registered
|
|
// before fingerprints of the new form has none either: no want can ever name such a module (the assigner's first rule
|
|
// asks for its done build), so its resources ride in `<node>.node`, which the node's own want names, their `module`
|
|
// taken off (a composition holds only its own module's resources); their ids still name the module.
|
|
//
|
|
// **Derived state, pruned** (clause 4): a module's compositions on a node are kept for its five newest fingerprints and
|
|
// for any a want or the node's have names; the rest are deleted with their digests.
|
|
|
|
// composedBucketsKV is what the writer needs of a key-value bucket: jetstream.KeyValue, or a test's.
|
|
type composedBucketsKV interface {
|
|
Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error)
|
|
Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error)
|
|
Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error)
|
|
Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error
|
|
ListKeys(ctx context.Context, opts ...jetstream.WatchOpt) (jetstream.KeyLister, error)
|
|
}
|
|
|
|
// composedPlace is where the writer writes a node's compositions and reads what decides them: the node's bucket, the
|
|
// digests, the node's object bucket, the wants and the haves.
|
|
type composedPlace struct {
|
|
node string
|
|
bucket composedBucketsKV
|
|
digests composedBucketsKV
|
|
objects objectStore
|
|
// wanted is the fingerprint the node's want names for a module, "" when none; ran the composition digest its have
|
|
// names (the move's judgement reads it) and the fingerprint it runs.
|
|
wanted func(ctx context.Context, module string) (string, error)
|
|
ran func(ctx context.Context, module string) (sdkdelivery.Have, bool, error)
|
|
// lastApplied is the module's slice of the declaration the node last applied, from the kept stream of declarations
|
|
// (ADR 0540 clause 9), and that declaration's digest; false when there is none to read. Nil judges without it.
|
|
lastApplied func(ctx context.Context, module string) (slice []json.RawMessage, digest string, found bool, err error)
|
|
// policy is a module's upgrade policy, where it comes from and why (catalogue.UpgradeOf), carried beside the
|
|
// judgement for the assigner's hold (ADR 0539 clause 14).
|
|
policy func(module string) (policy, from, why string)
|
|
// sign signs a composition with the controller's key.
|
|
sign func(c sdkdelivery.Composed) (sdkdelivery.Composed, error)
|
|
}
|
|
|
|
// compositionsOf splits a node's declaration body, as a send composed it, into its compositions: each module's at the
|
|
// fingerprint fps gives it, the node's own with its adoption and provisioner, and every resource of a module fps names
|
|
// no fingerprint for in the node's own, its module taken off it and named in moved. Data says the module's manifest
|
|
// declares data.
|
|
//
|
|
// **A module a want or a have names by its own composition is never in the node's own** (issue 1945): the node-engine
|
|
// runs that module's composition, and the same resources in the node's own were declared twice, which it refuses
|
|
// whole ("2 resources declare X", every node's apply failing from 2026-10-25 12:43). named says, for a module fps names
|
|
// no fingerprint of the new form for, that the node's want or have names a fingerprint of the new form; such a module's
|
|
// resources and filled paths are left out of this pass (its own composition, as written before, stays what the node
|
|
// runs) and it is named in left.
|
|
func compositionsOf(node string, body []byte, fps map[string]string, manifests map[string]catalogue.Manifest, runs []string,
|
|
named func(module string) (bool, error)) (out map[string]sdkdelivery.Composed, moved, left []string, err error) {
|
|
|
|
var envelope struct {
|
|
Resources []map[string]json.RawMessage `json:"resources"`
|
|
Adoption *sdkdecl.Adoption `json:"adoption"`
|
|
Filled []string `json:"filled_files"`
|
|
Provisioner string `json:"provisioner_account"`
|
|
}
|
|
if err := json.Unmarshal(body, &envelope); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("%s's declaration: %w", node, err)
|
|
}
|
|
inModule, ownComposition := map[string]bool{}, map[string]bool{}
|
|
resources := make([]json.RawMessage, 0, len(envelope.Resources))
|
|
for _, r := range envelope.Resources {
|
|
var module string
|
|
if raw, ok := r["module"]; ok {
|
|
_ = json.Unmarshal(raw, &module)
|
|
}
|
|
if module != "" && !strings.HasPrefix(fps[module], sdkdelivery.SourcePrefix2) {
|
|
if !inModule[module] && !ownComposition[module] {
|
|
isNamed := false
|
|
if named != nil {
|
|
if isNamed, err = named(module); err != nil {
|
|
return nil, nil, nil, fmt.Errorf("whether %s's want or have names %s's own composition: %w",
|
|
node, module, err)
|
|
}
|
|
}
|
|
if isNamed {
|
|
ownComposition[module] = true
|
|
left = append(left, module)
|
|
} else {
|
|
inModule[module] = true
|
|
moved = append(moved, module)
|
|
}
|
|
}
|
|
if ownComposition[module] {
|
|
continue
|
|
}
|
|
delete(r, "module")
|
|
}
|
|
raw, err := json.Marshal(r)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
resources = append(resources, raw)
|
|
}
|
|
// A filled file of a module moved into the node's own composition is no module's file any more, and the provisioner
|
|
// fills it all the same: the node's own composition carries its path, so the node-engine leaves the content to the
|
|
// provisioner and accepts its write. Dropping the path had the node-engine write those files empty, since their
|
|
// resources say no content (issue 1945, live on 2026-10-25: claude-code's, node-env's and power's files on ace, g14
|
|
// and shanks).
|
|
var filled, ownFilled []string
|
|
for _, p := range envelope.Filled {
|
|
switch {
|
|
case filledByMoved(envelope.Resources, p, ownComposition):
|
|
// The file of a module whose own composition the node runs: that composition names it, not this pass.
|
|
case filledByMoved(envelope.Resources, p, inModule):
|
|
ownFilled = append(ownFilled, p)
|
|
default:
|
|
filled = append(filled, p)
|
|
}
|
|
}
|
|
sort.Strings(moved)
|
|
sort.Strings(left)
|
|
out, err = sdkdelivery.SplitComposed(node, resources, fps, filled,
|
|
sdkdelivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner})
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
if len(ownFilled) > 0 {
|
|
own, err := sdkdelivery.ComposedNodeKey(node)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
c := out[own]
|
|
c.Filled = append(c.Filled, ownFilled...)
|
|
out[own] = c
|
|
}
|
|
// A module the node runs that composes no resource of its own (all its needs met by others) still has its
|
|
// composition, empty, at its fingerprint: its want names that composition (the assigner's seventh rule), and without
|
|
// one no want could ever be written.
|
|
for _, module := range runs {
|
|
fp := fps[module]
|
|
if !strings.HasPrefix(fp, sdkdelivery.SourcePrefix2) {
|
|
continue
|
|
}
|
|
key, err := sdkdelivery.ComposedKey(node, module, fp)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if _, ok := out[key]; !ok {
|
|
out[key] = sdkdelivery.Composed{Node: node, Module: module, FP: fp, Resources: []json.RawMessage{}}
|
|
}
|
|
}
|
|
for key, c := range out {
|
|
if m, ok := manifests[c.Module]; ok && c.Module != "" {
|
|
c.Data = len(m.DataItems()) > 0
|
|
out[key] = c
|
|
}
|
|
}
|
|
return out, moved, left, nil
|
|
}
|
|
|
|
// filledByMoved says a filled path is a file of a module whose resources were moved into the node's own composition.
|
|
func filledByMoved(resources []map[string]json.RawMessage, path string, moved map[string]bool) bool {
|
|
for _, r := range resources {
|
|
var kind, at, id string
|
|
_ = json.Unmarshal(r["type"], &kind)
|
|
_ = json.Unmarshal(r["path"], &at)
|
|
_ = json.Unmarshal(r["id"], &id)
|
|
if kind == string(sdkdecl.TypeFile) && at == path {
|
|
if module, ok := sdkdecl.ModuleGuess(id); ok && moved[module] {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// writeComposed writes one composition and its digest where they differ from what the buckets hold: the files by
|
|
// digest first, then the composition, then its digest and judgement, so a reader that finds the digest finds the
|
|
// composition and its files. It answers whether it wrote anything.
|
|
func writeComposed(ctx context.Context, at composedPlace, key string, c sdkdelivery.Composed) (bool, error) {
|
|
c, objects, err := c.ByDigest()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if len(objects) > 0 {
|
|
if at.objects == nil {
|
|
return false, fmt.Errorf("%s holds a file over 64 KiB and %s's object bucket is not open", key, at.node)
|
|
}
|
|
named := make(map[string][]byte, len(objects))
|
|
for digest, body := range objects {
|
|
name, err := sdkdecl.ObjectName(digest)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
named[name] = body
|
|
}
|
|
if err := putObjects(ctx, at.objects, named); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
if err := objectsHeld(ctx, at, key, c, objects); err != nil {
|
|
return false, err
|
|
}
|
|
digest, err := sdkdelivery.ComposedDigestOf(c)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
wrote := false
|
|
held, revision, err := readComposed(ctx, at.bucket, key)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if heldDigest, _ := sdkdelivery.ComposedDigestOf(held); revision == 0 || heldDigest != digest {
|
|
signed, err := at.sign(c)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
value, err := sdkdelivery.EncodeComposed(signed)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if err := casWrite(ctx, at.bucket, key, value, revision); err != nil {
|
|
return false, err
|
|
}
|
|
wrote = true
|
|
}
|
|
// **The digest kept is the digest of the composition the bucket holds** (issue 1945): another writer may write its
|
|
// composition and its digest between this one's composition and its digest, and the digest written last must not be
|
|
// of a composition the bucket no longer holds, or the assigner's want names it and the node applies nothing. After
|
|
// its digest, this writer reads the composition again and, when another holds the key, keeps that one's digest.
|
|
written := false
|
|
for follows := 0; ; follows++ {
|
|
judged, err := judgeMove(ctx, at, key, c, digest)
|
|
if composeTrace != nil {
|
|
composeTrace(fmt.Sprintf("%s: %s: written %v, judged %s %q user-list-only %v against %.19s, err %v", at.node, key,
|
|
wrote, judged.Judgement, judged.Words, judged.UserListOnly, judged.Against, err))
|
|
}
|
|
if err != nil {
|
|
return wrote || written, err
|
|
}
|
|
w, err := keepDigest(ctx, at.digests, key, judged)
|
|
written = written || w
|
|
if err != nil {
|
|
return wrote || written, err
|
|
}
|
|
now, rev, err := readComposed(ctx, at.bucket, key)
|
|
if err != nil {
|
|
return wrote || written, err
|
|
}
|
|
nowDigest, err := sdkdelivery.ComposedDigestOf(now)
|
|
if rev == 0 || err != nil || nowDigest == digest {
|
|
return wrote || written, nil
|
|
}
|
|
if follows >= 3 {
|
|
return wrote || written, fmt.Errorf("%s moved under this writer %d times; its digest is kept by the next pass",
|
|
key, follows+1)
|
|
}
|
|
c, digest = now, nowDigest
|
|
}
|
|
}
|
|
|
|
// objectsHeld refuses a composition that names a file by a digest whose object the node's bucket does not hold: a node
|
|
// that read it could not apply it (issue 1945). An object this write just put is held.
|
|
func objectsHeld(ctx context.Context, at composedPlace, key string, c sdkdelivery.Composed, put map[string][]byte) error {
|
|
for _, raw := range c.Resources {
|
|
r, err := sdkdecl.ParseResource(raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f, ok := r.(*sdkdecl.File)
|
|
if !ok || !f.IsReference() {
|
|
continue
|
|
}
|
|
if _, ok := put[f.Digest]; ok {
|
|
continue
|
|
}
|
|
name, err := sdkdecl.ObjectName(f.Digest)
|
|
if err != nil {
|
|
return fmt.Errorf("%s: file %s: %w", key, f.ID, err)
|
|
}
|
|
if at.objects == nil {
|
|
return fmt.Errorf("%s names file %s by digest and %s's object bucket is not open", key, f.ID, at.node)
|
|
}
|
|
have, _, err := at.objects.Has(ctx, name)
|
|
if err != nil {
|
|
return fmt.Errorf("%s: whether the object %s of file %s is in the bucket: %w", key, name[:12], f.ID, err)
|
|
}
|
|
if !have {
|
|
return fmt.Errorf("%s names file %s by the object %s, which %s's bucket does not hold: not written", key, f.ID,
|
|
name[:12], at.node)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// readComposed is the composition a bucket holds under key and its revision; revision 0 when there is none.
|
|
func readComposed(ctx context.Context, kv composedBucketsKV, key string) (sdkdelivery.Composed, uint64, error) {
|
|
e, err := kv.Get(ctx, key)
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
return sdkdelivery.Composed{}, 0, nil
|
|
}
|
|
if err != nil {
|
|
return sdkdelivery.Composed{}, 0, err
|
|
}
|
|
c, err := sdkdelivery.DecodeComposed(e.Value())
|
|
if err != nil {
|
|
// A value this controller cannot read is written over: it is derived, and composed again here.
|
|
return sdkdelivery.Composed{}, e.Revision(), nil
|
|
}
|
|
return c, e.Revision(), nil
|
|
}
|
|
|
|
// keepDigest writes the digest and judgement beside a composition when it differs from what the bucket holds.
|
|
func keepDigest(ctx context.Context, kv composedBucketsKV, key string, v sdkdelivery.ComposedDigest) (bool, error) {
|
|
value, err := sdkdelivery.EncodeComposedDigest(v)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
v.V = sdkdelivery.Version
|
|
var revision uint64
|
|
e, err := kv.Get(ctx, key)
|
|
switch {
|
|
case errors.Is(err, jetstream.ErrKeyNotFound):
|
|
case err != nil:
|
|
return false, err
|
|
default:
|
|
revision = e.Revision()
|
|
if held, derr := sdkdelivery.DecodeComposedDigest(e.Value()); derr == nil && sdkdelivery.SameComposedDigest(held, v) {
|
|
return false, nil
|
|
}
|
|
}
|
|
return true, casWrite(ctx, kv, key, value, revision)
|
|
}
|
|
|
|
// casWrite creates key when revision is 0 and otherwise replaces the revision read: a second writer that wrote in
|
|
// between makes this one fail, said, and the next event composes again.
|
|
func casWrite(ctx context.Context, kv composedBucketsKV, key string, value []byte, revision uint64) error {
|
|
var err error
|
|
if revision == 0 {
|
|
_, err = kv.Create(ctx, key, value)
|
|
} else {
|
|
_, err = kv.Update(ctx, key, value, revision)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("writing %s: %w", key, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// writeNodeCompositions writes every composition of a node that differs, and prunes what is no longer kept. It answers
|
|
// the keys it wrote.
|
|
func writeNodeCompositions(ctx context.Context, at composedPlace, comps map[string]sdkdelivery.Composed) ([]string, error) {
|
|
// The modules' compositions first and the node's own last, and the node's own not at all in a pass where a module's
|
|
// failed: written without the resources that moved to that module, it would have the node-engine remove them while
|
|
// the module's own composition is not there to take them (issue 1945, the review's gap 1).
|
|
own, err := sdkdelivery.ComposedNodeKey(at.node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
keys := make([]string, 0, len(comps))
|
|
for k := range comps {
|
|
if k != own {
|
|
keys = append(keys, k)
|
|
}
|
|
}
|
|
sort.Strings(keys)
|
|
var wrote []string
|
|
var errs []error
|
|
for _, key := range keys {
|
|
w, err := writeComposed(ctx, at, key, comps[key])
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
if w {
|
|
wrote = append(wrote, key)
|
|
}
|
|
}
|
|
if c, ok := comps[own]; ok {
|
|
if len(errs) > 0 {
|
|
errs = append(errs, fmt.Errorf("%s is not written: a module's composition of this pass was not", own))
|
|
} else if w, err := writeComposed(ctx, at, own, c); err != nil {
|
|
errs = append(errs, err)
|
|
} else if w {
|
|
wrote = append(wrote, own)
|
|
}
|
|
}
|
|
if err := pruneCompositions(ctx, at); err != nil {
|
|
errs = append(errs, err)
|
|
}
|
|
return wrote, errors.Join(errs...)
|
|
}
|
|
|
|
// pruneCompositions deletes, for each module of the node, the compositions outside its ComposedKeep newest fingerprints
|
|
// that neither its want nor its have names, each with its digest (ADR 0539 clause 4). Newest is by when each was last
|
|
// written: the controller writes only the fingerprint a module is registered at, so an older one is never written again.
|
|
func pruneCompositions(ctx context.Context, at composedPlace) error {
|
|
lister, err := at.bucket.ListKeys(ctx)
|
|
if err != nil {
|
|
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
type kept struct {
|
|
key, fp string
|
|
at time.Time
|
|
}
|
|
byModule := map[string][]kept{}
|
|
for key := range lister.Keys() {
|
|
node, module, fp, isNode, err := sdkdelivery.ParseComposedKey(key)
|
|
if err != nil || isNode || node != at.node {
|
|
continue
|
|
}
|
|
e, err := at.bucket.Get(ctx, key)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
byModule[module] = append(byModule[module], kept{key, fp, e.Created()})
|
|
}
|
|
var errs []error
|
|
for module, ks := range byModule {
|
|
if len(ks) <= sdkdelivery.ComposedKeep {
|
|
continue
|
|
}
|
|
sort.Slice(ks, func(i, j int) bool { return ks[i].at.After(ks[j].at) })
|
|
wanted, err := at.wanted(ctx, module)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
have, _, err := at.ran(ctx, module)
|
|
if err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
for _, k := range ks[sdkdelivery.ComposedKeep:] {
|
|
if k.fp == wanted || k.fp == have.FP {
|
|
continue
|
|
}
|
|
if err := at.bucket.Delete(ctx, k.key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
if err := at.digests.Delete(ctx, k.key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
errs = append(errs, err)
|
|
}
|
|
}
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
// controllerSigner signs a composition with the controller's active key, which its identity store keeps and never hands
|
|
// out (as it signed declarations).
|
|
func controllerSigner(ctx context.Context, open *stores) (func(sdkdelivery.Composed) (sdkdelivery.Composed, error), error) {
|
|
ident, err := open.Identity(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
key, err := ident.Active(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
public := ed25519.PublicKey(key.Public)
|
|
return func(c sdkdelivery.Composed) (sdkdelivery.Composed, error) {
|
|
return sdkdelivery.SignComposedBy(c, public, func(m []byte) ([]byte, error) { return ident.Sign(ctx, m) })
|
|
}, nil
|
|
}
|