package frombus import ( "context" "encoding/json" "fmt" "regexp" "slices" "sort" "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" ) // Answer is a node's answer to its proposal's placement candidates (novox/hq ADR 0567 §3, ADR 0559 §4): s.Node composed in shadow with // s.Placements laid over the trunk's placement (compose.CompositionsShadow, its own sealed values and boxes through r as // for its own composition), compared with what it runs (delivery.CompareComposed). It applies nothing and writes // nothing; the proposal and the revision are the candidate's, said back in the answer. // // The node's own composition is compared only where the node runs one. fps gives the build each module is composed at: what the node runs, and for a module the candidate places that the // node does not run, the build the trunk holds; a placed module with none is added, with no build of it known. What the // composition stands in for is judged: a value of a module the proposal adds (on this node or another: its credential, own // secrets and boxes are made only once it is placed, ADR 0549 §5) is not yet issued, which is no refusal; any other // stand-in is a value missing for a module the node runs, a refusal naming it ("node" when it is the node's own). func Answer(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]string, running []delivery.Composed, proposal string, revision uint64) delivery.CandidateAnswer { a := delivery.CandidateAnswer{Proposal: proposal, Revision: revision} refuse := func(err error) delivery.CandidateAnswer { a.Refused = err.Error() a.Verdict = delivery.CandidateRefused return a } trunk, err := r.Placements(ctx) if err != nil { return refuse(fmt.Errorf("frombus: placement: %w", err)) } if s.Running == nil { s.Running = running } // Every module it cannot compose is named, not the first only (issue 1126): a build it cannot read or take in, and // below, a module the composition refuses, each composed as a stub so the rest is composed and compared. failed := map[string]string{} in, err := input(ctx, r, mesh, s, func(module string, err error) { failed[module] = err.Error() }) if err != nil { return refuse(err) } asConsumer, asProvider, err := compose.CredentialsNeeded(in, s.Node) if err != nil { return refuse(err) } // What the node runs, by module. A consumer's credential is asked only for a module it runs: the vault refuses the // whole ask for a consuming module not assigned to the node (mesh-vault's own), and a module the candidate adds is // assigned only after the merge (ADR 0549 §5), so its credential is left missing, a stand-in answered as not yet // issued under Added (review of L2b, round 1, finding 1). runs := map[string]bool{} for _, c := range running { if c.Module != "" { runs[c.Module] = true } } var consumed []compose.CredentialKey for _, k := range asConsumer { if runs[k.ConsumerModule] { consumed = append(consumed, k) } } asConsumer = consumed if len(asConsumer)+len(asProvider) > 0 { if in.Credentials, in.Issued, err = r.Credentials(ctx, s.Node, asConsumer, asProvider); err != nil { return refuse(fmt.Errorf("frombus: the credentials of %s: %w", s.Node, err)) } } composed, skipped, stood, err := composeEach(in, s.Node, fps, false, s.Running, failed) if err != nil { // Not one module's: the node's own (a value served elsewhere, the mesh's network), said once as the node's. if len(failed) == 0 { return refuse(err) } failed[delivery.HaveNodeWord] = err.Error() composed, skipped, stood = nil, nil, nil } // The node's own composition is judged only where the node runs one: one that runs none applies it only once a want // names it (the node agent's own composer hands it on that want alone), so a candidate does not add it. runsOwn := false for _, c := range running { runsOwn = runsOwn || c.Module == "" } if !runsOwn { for key, c := range composed { if c.Module == "" { delete(composed, key) } } } if _, whole := failed[delivery.HaveNodeWord]; whole { // Nothing composed to compare: every refusal is said, and nothing else. for _, m := range sortedKeys(failed) { a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) } heldAsToday(&a, s.HeldToday) a.Verdict = delivery.VerdictOf(a) return a } // A module the composition folds inside the node's own is judged there: the agent also says it runs by module, from // the declaration it kept before the switch (agent.Everything), which is no removal (issue 1126, seed #531). if len(s.InOwn) > 0 { var judged []delivery.Composed for _, c := range running { if c.Module == "" || !slices.Contains(s.InOwn, c.Module) { judged = append(judged, c) } } running = judged } cmp := delivery.CompareComposedWith(composed, running, s.SameValue) // The answer rides in the node's have key: its shown bodies are bounded, the rest named only. budget := MaxShown for i := range cmp.Differs { kept := cmp.Differs[i].Bodies[:0] for _, b := range cmp.Differs[i].Bodies { if n := len(b.Composed) + len(b.Running); n <= budget { budget -= n kept = append(kept, b) } else { kept = append(kept, delivery.ResourceBodies{ID: b.ID, Why: b.Why}) a.Note = "some differing resources are named without their bodies: the answer holds at most " + fmt.Sprint(MaxShown) + " bytes of them" } } cmp.Differs[i].Bodies = kept } // A refused module was composed as a stub: what it composed is no difference; its refusal is said. notFailed := func(m string) bool { _, ok := failed[m]; return !ok } a.Same = slices.DeleteFunc(cmp.Same, func(m string) bool { return !notFailed(m) }) a.Removed = slices.DeleteFunc(cmp.Removed, func(m string) bool { return !notFailed(m) }) a.Differs = slices.DeleteFunc(cmp.Differs, func(d delivery.ModuleDiffers) bool { return !notFailed(d.Module) }) cmp.Added = slices.DeleteFunc(cmp.Added, func(d delivery.ModuleAdded) bool { return !notFailed(d.Module) }) skipped = slices.DeleteFunc(skipped, func(m string) bool { return !notFailed(m) }) // What the proposal adds, by node and module: placed by a candidate and not by the trunk on that node. added := map[string]bool{} for node, p := range s.Placements { if p == nil { continue } was := map[string]bool{} for _, m := range trunk[node].Modules { was[m.Module] = true } for _, m := range p.Modules { if !was[m.Module] { added[node+"/"+m.Module] = true } } } unissued := map[string][]string{} refusals := map[string][]string{} for _, x := range stood { node, module := standsInFor(x, s.Node) switch { case node == s.Node && runs[module]: // A value missing for a module the node runs is a refusal, whatever the trunk places (the seed's trunk is // empty, and every module counts as the group's). refusals[module] = append(refusals[module], x) case module != "" && added[node+"/"+module]: if node == s.Node { unissued[module] = append(unissued[module], x) } // another node's addition: the module here that carries it differs, which the assigner judges case module == "" || node != s.Node: // The node's own, or a value this node holds for a module elsewhere (a grant it provides): said as the node's. refusals[delivery.HaveNodeWord] = append(refusals[delivery.HaveNodeWord], x) default: refusals[module] = append(refusals[module], x) } } addedHere := map[string]delivery.ModuleAdded{} for _, m := range cmp.Added { addedHere[m.Module] = m } for _, m := range skipped { // placed, and no build given to compose it at addedHere[m] = delivery.ModuleAdded{Module: m, Unissued: []string{"no build of " + m + " is known to compose it at"}} } for m, xs := range unissued { got := addedHere[m] got.Module = m got.Unissued = append(got.Unissued, xs...) addedHere[m] = got } for _, m := range sortedKeys(addedHere) { a.Added = append(a.Added, addedHere[m]) } for _, m := range sortedKeys(refusals) { if _, isAdded := addedHere[m]; isAdded { continue // a module the candidate adds here: what it lacks is said under Added } if _, isFailed := failed[m]; isFailed { continue // said below, as why it could not be composed } a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: "stands in for " + strings.Join(refusals[m], ", ")}) } for _, m := range sortedKeys(failed) { a.Refusals = append(a.Refusals, delivery.ModuleRefused{Module: m, Why: failed[m]}) } sort.Slice(a.Refusals, func(i, j int) bool { return a.Refusals[i].Module < a.Refusals[j].Module }) heldAsToday(&a, s.HeldToday) a.Verdict = delivery.VerdictOf(a) return a } // heldAsToday moves each refusal of a module the node holds today (Scope.HeldToday) into Held: the candidate holds it as // the node does, which is no change (issue 1126: a seed reproduces today, and today holds it too). func heldAsToday(a *delivery.CandidateAnswer, today []string) { if len(today) == 0 { return } kept := a.Refusals[:0] for _, r := range a.Refusals { if slices.Contains(today, r.Module) { a.Held = append(a.Held, r) } else { kept = append(kept, r) } } a.Refusals = kept if len(a.Refusals) == 0 { a.Refusals = nil } } // maxCulprits bounds the modules one refusal is tried on, each a whole composition. const maxCulprits = 8 // MaxShown bounds the bytes of shown resource bodies in one answer (delivery.ModuleDiffers.Bodies). const MaxShown = 256 << 10 var ( grantStandIn = regexp.MustCompile(`^the grant of \S+ for (\S+) on (\S+)$`) needsStandIn = regexp.MustCompile(`^(\S+) on (\S+) needs `) ) // standsInFor is the node and module a stand-in of compose's words is for: an own secret `module/name`, a port // `port module:port`, a grant `the grant of for on `, a need ` on needs ...`; // the module empty when the words name none (the certificate, the authority). func standsInFor(x, node string) (string, string) { if m := grantStandIn.FindStringSubmatch(x); m != nil { return m[2], m[1] } if m := needsStandIn.FindStringSubmatch(x); m != nil { return m[2], m[1] } if rest, ok := strings.CutPrefix(x, "port "); ok { if module, _, ok := strings.Cut(rest, ":"); ok { return node, module } } if module, _, ok := strings.Cut(x, "/"); ok && !strings.Contains(x, " ") { return node, module } return node, "" } // composeEach is the one rule, for the node that applies and for its answer to a candidate alike, by which a module's // own refusal refuses that module and never the node (issue 1126: seed-8 fixed it in the answer alone, and the // applying path still refused all of novox for de-spiegel's unset route names). The node is composed; a refusal is laid // on the module whose stub (compose.Stub) changes it (culprit), that module is held (compose.Node.Held) and said in // failed with the refusal's words, and the rest is composed again, until it composes or the refusal is no single // module's, which is answered as the node's (err). Applied composes as the node applies (compose.Reading) and holds the // module placed here each stand-in is for; one for no module placed here (the node's own, a value this node holds for a // module elsewhere) refuses the node (compose.ErrStoodIn). Not applied, it composes in compose.Foreseeing and answers // every stand-in for the caller to judge (Answer). // // **A held module is what runs of it** (review of the apply, round 1): composed from what running holds of it // (compose.Carried of its own composition there; the node's own carries it wherever it runs there), so what the others // read of it (the machine ports its containers publish, the routes reaching it) is what runs, and none of its own // composition is in out: the node keeps what it last applied. **What depends on how it is held is held too:** a module // whose composition differs when the held ones are composed as their stubs instead is held with them, and a node's own // composition that differs (but for the held modules' own resources) refuses the node. And since a held module is // composed from its new manifest, which may not be the build's that runs, what reads that manifest is held with it // (dependents): the providers of what it contributes to (the proxy its routes), and, when it listens, the filter. func composeEach(in compose.Input, node string, fps map[string]string, applied bool, running []delivery.Composed, failed map[string]string) (out map[string]delivery.Composed, skipped, stood []string, err error) { mode := compose.Foreseeing if applied { mode = compose.Reading } runs := map[string]*delivery.Composed{} for i := range running { if m := running[i].Module; m != "" { runs[m] = &running[i] } } src := map[string]catalogue.Manifest{} // each held module's manifest before it was held bareOf := map[string]bool{} var hold func(m, why string, bare bool) hold = func(m, why string, bare bool) { if _, done := failed[m]; !done { failed[m] = why // a held module tried bare keeps the words it was first held for } first := false if _, ok := src[m]; !ok { man, ok := manifestOn(in, node, m) if !ok { return } src[m], first = man, true } bareOf[m] = bareOf[m] || bare in = heldAs(in, node, m, src[m], bareOf[m], runs[m]) if !first { return } // What the others read of its manifest (what it contributes, what it listens on) is its new manifest's, not // necessarily the build's that runs: what reads it is held with it (review of the apply, round 2). for _, r := range dependents(in, node, src[m], failed) { hold(r, "it reads what "+m+" contributes or listens on, and "+m+" is held", false) } } for _, m := range sortedKeys(failed) { hold(m, failed[m], false) } for { out, skipped, stood, err = compose.CompositionsIn(in, node, fps, mode) if err != nil { module, bare := culprit(in, node, fps, mode, failed, err) if module == "" { return nil, nil, nil, err } hold(module, err.Error(), bare) continue } again := false if applied && len(stood) > 0 { placed := map[string]bool{} for _, m := range assignedOn(in, node) { placed[m] = true } of := map[string][]string{} var nodes []string for _, x := range stood { if n, m := standsInFor(x, node); n == node && placed[m] { of[m] = append(of[m], x) } else { nodes = append(nodes, x) } } for _, m := range sortedKeys(of) { if _, done := failed[m]; !done { hold(m, "stands in for "+strings.Join(of[m], ", "), false) again = true } } if !again && len(nodes) > 0 { return nil, nil, nil, fmt.Errorf("%w on %s: %s", compose.ErrStoodIn, node, strings.Join(nodes, ", ")) } } if !again && len(failed) > 0 { dependent, own := dependsOnHeld(in, node, fps, mode, out, src, bareOf, failed) if own { return nil, nil, nil, fmt.Errorf("frombus: %s's own composition depends on how %s, held, is composed, and is not applied", node, strings.Join(sortedKeys(failed), ", ")) } for _, m := range dependent { hold(m, "what it composes of "+strings.Join(sortedKeys(failed), ", ")+", held, is not what runs of them", false) again = true } } if !again { break } } skipped = slices.DeleteFunc(skipped, func(m string) bool { _, held := failed[m]; return held }) return out, skipped, stood, nil } // dependsOnHeld composes in again with each held module as its stub rather than as what runs of it, and answers the // modules, not held, whose composition differs between the two (sorted), and own when the node's own composition does // but for the held modules' own resources. A composition of stubs that fails says nothing more: what runs composed. func dependsOnHeld(in compose.Input, node string, fps map[string]string, mode compose.Mode, asRuns map[string]delivery.Composed, src map[string]catalogue.Manifest, bareOf map[string]bool, failed map[string]string) (dependent []string, own bool) { stubs := in for _, m := range sortedKeys(failed) { if man, ok := src[m]; ok { stubs = heldAs(stubs, node, m, man, bareOf[m], nil) } } asStubs, _, _, err := compose.CompositionsIn(stubs, node, fps, mode) if err != nil { return nil, false } isHeld := func(m string) bool { _, ok := failed[m]; return ok } keys := map[string]bool{} for k := range asRuns { keys[k] = true } for k := range asStubs { keys[k] = true } for _, k := range sortedKeys(keys) { a, inA := asRuns[k] b, inB := asStubs[k] m := a.Module if !inA { m = b.Module } if m != "" && isHeld(m) { continue } if m == "" { a, b = withoutHeld(a, isHeld), withoutHeld(b, isHeld) } da, errA := delivery.ComposedDigestOf(a) db, errB := delivery.ComposedDigestOf(b) if inA == inB && errA == nil && errB == nil && da == db { continue } if m == "" { own = true continue } dependent = append(dependent, m) } return dependent, own } // withoutHeld is the node's own composition c without the resources of a held module (its id naming it) and without // its filled paths, for comparing the rest. func withoutHeld(c delivery.Composed, isHeld func(string) bool) delivery.Composed { out := c out.Filled = nil out.Resources = nil for _, raw := range c.Resources { var head struct { ID string `json:"id"` Module string `json:"module"` } _ = json.Unmarshal(raw, &head) owner := head.Module if owner == "" { owner, _ = declaration.ModuleGuess(head.ID) } if owner != "" && isHeld(owner) { continue } out.Resources = append(out.Resources, raw) } return out } // dependents is, of the modules placed on node and not held, those that read what man says to them: each that provides // what man contributes to (the proxy its routes), and, when man listens on a port, each that holds the filter (the // openings). func dependents(in compose.Input, node string, man catalogue.Manifest, failed map[string]string) []string { to := map[string]bool{} for p := range man.Contributes { to[p] = true } for p := range man.ContributesMany { to[p] = true } listens := len(man.Listens) > 0 if len(to) == 0 && !listens { return nil } var out []string for _, m := range assignedOn(in, node) { if _, held := failed[m]; held || m == man.Module { continue } pm, ok := manifestOn(in, node, m) if !ok { continue } if listens && pm.Filtering != nil { out = append(out, m) continue } for _, o := range pm.Provides { if to[o.Name] { out = append(out, m) break } } } return out } // assignedOn is the modules placed on node, in placement order. func assignedOn(in compose.Input, node string) []string { for _, n := range in.Nodes { if n.Name == node { return n.Assigned } } return nil } // manifestOn is the manifest node composes module from: its Kept, else the catalogue's. func manifestOn(in compose.Input, node, module string) (catalogue.Manifest, bool) { for _, n := range in.Nodes { if n.Name == node { if man, ok := n.Kept[module]; ok { return man, true } } } man, ok := in.Catalogue[module] return man, ok } // culprit is the module placed on node whose composition the error is of: the first, in placement order, whose stub // (compose.Stub) makes the composition in mode succeed or fail otherwise; empty when none does (the error is not one // module's). Modules already refused are passed over. A module whose stub still fails the same is tried bare (its stub // contributing nothing, bare true): a refusal of what it contributes is its own, which the stub keeps (issue 1126, // seed-8: de-spiegel's route names on novox, an unset required value, refused the whole node). func culprit(in compose.Input, node string, fps map[string]string, mode compose.Mode, failed map[string]string, err error) (string, bool) { var assigned []string for _, n := range in.Nodes { if n.Name == node { assigned = n.Assigned } } // Only the modules the error names: the words of a module's refusal name it. Each try is a whole composition, so a // refusal naming none is no module's, answered at once rather than after a composition per module (issue 1126: ace's // answer took 7m46s). var order []string for _, m := range assigned { if strings.Contains(err.Error(), m) && len(order) < maxCulprits { order = append(order, m) } } changes := func(m string, bare bool) bool { _, _, _, again := compose.CompositionsIn(stubbed(in, node, m, bare), node, fps, mode) return again == nil || again.Error() != err.Error() } for _, m := range order { if _, done := failed[m]; done { continue } if changes(m, false) { return m, false } } var kept map[string]catalogue.Manifest for _, n := range in.Nodes { if n.Name == node { kept = n.Kept } } for _, m := range order { // One already held is tried bare too, once: held for its build, its stub may still refuse (its own secret // never issued, issue 1126). if _, done := failed[m]; done && isBare(kept[m]) { continue } if changes(m, true) { return m, true } } return "", false } // isBare says a stub contributes nothing and declares no own secret (stubbed, bare). func isBare(m catalogue.Manifest) bool { return len(m.Contributes) == 0 && len(m.ContributesMany) == 0 && len(m.OwnSecrets) == 0 } // stubbed is in with module held on node as its stub alone (heldAs, nothing of it running), from the manifest node // composes it from: a culprit's try. func stubbed(in compose.Input, node, module string, bare bool) compose.Input { man, ok := manifestOn(in, node, module) if !ok { return in } return heldAs(in, node, module, man, bare, nil) } // heldAs is in with module held on node (compose.Node.Held): composed from man as what runs of it (compose.Carried of // runs; its stub, compose.Stub, when runs is nil or holds nothing of it), as node's Kept; bare, contributing nothing to // what it requires and declaring no own secret: none of a held module's own composition is applied, so what it has not // been issued (restic's repository on a node it never ran on) is no refusal of the rest. func heldAs(in compose.Input, node, module string, man catalogue.Manifest, bare bool, runs *delivery.Composed) compose.Input { out := in out.Nodes = append([]compose.Node(nil), in.Nodes...) for i := range out.Nodes { if out.Nodes[i].Name != node { continue } if bare { man.Contributes, man.ContributesMany, man.OwnSecrets = nil, nil, nil } kept := make(map[string]catalogue.Manifest, len(out.Nodes[i].Kept)+1) for k, v := range out.Nodes[i].Kept { kept[k] = v } kept[module] = compose.Carried(man, module, runs) out.Nodes[i].Kept = kept held := make(map[string]bool, len(out.Nodes[i].Held)+1) for k, v := range out.Nodes[i].Held { held[k] = v } held[module] = true out.Nodes[i].Held = held } return out }