Adoption mode: a node in use is adopted before it is converged (hq ADR 0100–0103) #44
@@ -39,6 +39,13 @@ import (
|
|||||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||||
// machines this just blocked is worth more than the milliseconds.
|
// machines this just blocked is worth more than the milliseconds.
|
||||||
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
|
func assign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||||
|
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||||
|
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||||
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
if err := open.inventory.Assign(ctx, node, module); err != nil {
|
if err := open.inventory.Assign(ctx, node, module); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -71,6 +78,11 @@ func assign(ctx context.Context, open *stores, node, module string) (string, err
|
|||||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||||
// about the command's own output would ever have said so.
|
// about the command's own output would ever have said so.
|
||||||
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
|
func unassign(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||||
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,534 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: taking a module is its cutover; converging a node is one act, previewed, and
|
||||||
|
// refused while a found container is held; returning to adopted keeps what was taken.
|
||||||
|
|
||||||
|
// anAdoptedAnchor is aMesh with the anchor adopted, running a served module the predecessor also
|
||||||
|
// runs and a module with only a file, and a filter module in the catalogue.
|
||||||
|
func anAdoptedAnchor(t *testing.T) (*stores, *[]string) {
|
||||||
|
t.Helper()
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||||
|
Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}},
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hi"},
|
||||||
|
{"id": "server", "type": "container", "name": "hello-web", "ports": []any{"8080:80"},
|
||||||
|
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||||
|
}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"},
|
||||||
|
}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "nftables", Version: "1",
|
||||||
|
Filtering: &catalogue.Filtering{Into: "/etc/nftables.conf"},
|
||||||
|
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
||||||
|
"state": "running", "restart-on": []any{"filtering"}}}})
|
||||||
|
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, m := range []string{"hello-web", "notes"} {
|
||||||
|
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sent := &[]string{}
|
||||||
|
saved := sendNodes
|
||||||
|
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
||||||
|
*sent = append(*sent, names...)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendNodes = saved })
|
||||||
|
return open, sent
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportsHolding has the anchor report, on what it was last sent, holding what is given.
|
||||||
|
func reportsHolding(t *testing.T, open *stores, held ...link.Held) {
|
||||||
|
t.Helper()
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||||
|
ContainerPort: 80},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry",
|
||||||
|
Published: true, ContainerPort: 5000},
|
||||||
|
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker",
|
||||||
|
Published: true, ContainerPort: 15672},
|
||||||
|
}, held...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportsReaching has the anchor report, on what it was last sent, what is reachable on it and
|
||||||
|
// holding what is given.
|
||||||
|
func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ...link.Held) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
body, err := composed(t, open, "anchor").Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
record, err := open.inventory.NodeByName(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||||
|
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
||||||
|
Firewall: "ufw", Held: held, Reachable: reachable,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
|
||||||
|
Target: "hello-web", Since: time.Now()}
|
||||||
|
heldFile = link.Held{ID: "notes.conf", Module: "notes", Kind: "file",
|
||||||
|
Target: "/etc/notes.conf", Since: time.Now(), Kept: "/var/lib/mesh-host/kept/abc-notes.conf"}
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
if _, err := take(t.Context(), open, "anchor", "nftables"); !errors.Is(err, inventory.ErrNotAssigned) {
|
||||||
|
t.Fatalf("taking an unassigned module gave %v", err)
|
||||||
|
}
|
||||||
|
if _, err := take(t.Context(), open, "laptop", "network"); !errors.Is(err, inventory.ErrNotAdopted) {
|
||||||
|
t.Fatalf("taking on a converged node gave %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
_, err := converge(t.Context(), open, "anchor", false, "", "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has not reported") {
|
||||||
|
t.Fatalf("a node that never reported was previewed: %v", err)
|
||||||
|
}
|
||||||
|
if len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused converge sent something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
reportsHolding(t, open, heldContainer, heldFile)
|
||||||
|
_, err := converge(t.Context(), open, "anchor", true, "", "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") {
|
||||||
|
t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(t.Context(), "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused flip changed something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTakingNamesWhatItReplaces(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
reportsHolding(t, open, heldContainer, heldFile)
|
||||||
|
said, err := take(t.Context(), open, "anchor", "hello-web")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
||||||
|
!strings.Contains(said, "push anchor") {
|
||||||
|
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"tcp/8080 hello-web (published, container port 80)",
|
||||||
|
"declared by hello-web (from anywhere)",
|
||||||
|
"WILL CLOSE — no module assigned here declares it",
|
||||||
|
// The anchor faces inward and is on the private network: the derived filter admits ssh
|
||||||
|
// from the mesh only.
|
||||||
|
"WILL CLOSE to everything outside the private network — ssh stays open from the mesh",
|
||||||
|
"notes\n replacing the found file /etc/notes.conf (notes.conf), original kept at",
|
||||||
|
"assigns nftables",
|
||||||
|
"the found firewall (ufw) is disabled, never flushed",
|
||||||
|
// What it routes is not a listener: said not to be previewed, and to be dropped.
|
||||||
|
"not previewed: traffic the machine routes that is not a published port",
|
||||||
|
"the derived filter drops it unless a module declares it",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(preview, "15672") {
|
||||||
|
t.Errorf("a loopback listener is in the preview:\n%s", preview)
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(preview, "\n") {
|
||||||
|
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
|
||||||
|
t.Errorf("an undeclared published port is not said to close: %s", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("the preview changed something")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||||
|
if n.Adopted {
|
||||||
|
t.Fatal("converge --yes left the node adopted")
|
||||||
|
}
|
||||||
|
taken, _ := open.inventory.Taken(ctx, "anchor")
|
||||||
|
if !reflect.DeepEqual(taken, []string{"hello-web", overlay.Name, "nftables", "notes"}) {
|
||||||
|
t.Fatalf("the flip took %v", taken)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(*sent, []string{"anchor"}) {
|
||||||
|
t.Fatalf("the flip sent %v", *sent)
|
||||||
|
}
|
||||||
|
declared := composed(t, open, "anchor")
|
||||||
|
if declared.Adoption != nil {
|
||||||
|
t.Fatal("a converged node is still sent an adoption envelope")
|
||||||
|
}
|
||||||
|
if !hasID(declared.Resources, "nftables.filtering") {
|
||||||
|
t.Fatal("the converged node is not declared the mesh's filter")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := adopt(ctx, open, "anchor"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := adopt(ctx, open, "anchor"); err == nil {
|
||||||
|
t.Fatal("adopting an adopted node was not refused")
|
||||||
|
}
|
||||||
|
again, _ := open.inventory.Taken(ctx, "anchor")
|
||||||
|
if !reflect.DeepEqual(again, taken) {
|
||||||
|
t.Fatalf("returning to adopted lost what was taken: %v", again)
|
||||||
|
}
|
||||||
|
declared = composed(t, open, "anchor")
|
||||||
|
if declared.Adoption == nil || len(declared.Adoption.Untaken) != 0 {
|
||||||
|
t.Fatalf("returned to adopted, the envelope is %+v", declared.Adoption)
|
||||||
|
}
|
||||||
|
if hasID(declared.Resources, "nftables.filtering") || hasID(declared.Resources, "nftables.load") {
|
||||||
|
t.Fatal("returned to adopted, the mesh's filter is still declared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasID(resources []map[string]any, id string) bool {
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// The command API refuses a flip exactly as the command line does, in the same words.
|
||||||
|
func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
reportsHolding(t, open, heldContainer)
|
||||||
|
_, direct := converge(t.Context(), open, "anchor", true, "", "")
|
||||||
|
if direct == nil {
|
||||||
|
t.Fatal("the flip was not refused")
|
||||||
|
}
|
||||||
|
got := asking(t, letIn{}, "POST", "/converge", `{"node":"anchor","yes":true}`)
|
||||||
|
if got.Code != http.StatusConflict {
|
||||||
|
t.Fatalf("got %d: %s", got.Code, got.Body.String())
|
||||||
|
}
|
||||||
|
var said map[string]any
|
||||||
|
if err := json.Unmarshal(got.Body.Bytes(), &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if said["refused"] != direct.Error() {
|
||||||
|
t.Fatalf("the API said %q and the command line %q", said["refused"], direct.Error())
|
||||||
|
}
|
||||||
|
if got := asking(t, letIn{}, "POST", "/take", `{"node":"anchor"}`); got.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("a take naming no module got %d", got.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: the preview says what the derived filter does, rendered as it is rendered. On
|
||||||
|
// a machine that faces inward, ssh is admitted from the private network only, and a port a module
|
||||||
|
// admits from the mesh only closes to everything outside it: both are said to close.
|
||||||
|
func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||||
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 5432, By: "postgres"},
|
||||||
|
{Protocol: "tcp", Address: "10.77.0.1", Port: 5432, By: "postgres"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||||
|
ContainerPort: 80},
|
||||||
|
})
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lines := map[string]string{}
|
||||||
|
for _, line := range strings.Split(preview, "\n") {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) > 1 && strings.HasPrefix(fields[0], "tcp/") {
|
||||||
|
lines[fields[0]+" "+fields[1]] += line + "\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := lines["tcp/22 sshd"]; !strings.Contains(got, "WILL CLOSE to everything outside "+
|
||||||
|
"the private network") {
|
||||||
|
t.Errorf("ssh on an inward machine is not said to close outside the mesh:\n%s", preview)
|
||||||
|
}
|
||||||
|
store := lines["tcp/5432 postgres"]
|
||||||
|
if strings.Count(store, "WILL CLOSE to everything outside the private network") != 1 ||
|
||||||
|
!strings.Contains(store, "declared by store (from mesh)") {
|
||||||
|
t.Errorf("the store's narrowing is not said to close, or its mesh address is:\n%s", preview)
|
||||||
|
}
|
||||||
|
if got := lines["tcp/8080 hello-web"]; !strings.Contains(got, "declared by hello-web (from anywhere)") {
|
||||||
|
t.Errorf("a port open to everywhere is not said to stay:\n%s", preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// digestIn is the digest a converge preview printed.
|
||||||
|
func digestIn(t *testing.T, preview string) string {
|
||||||
|
t.Helper()
|
||||||
|
for _, line := range strings.Split(preview, "\n") {
|
||||||
|
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
||||||
|
return fields[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("the preview printed no digest:\n%s", preview)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// The flip acts on the preview the operator saw: it names that preview's digest, and it is refused
|
||||||
|
// when the digest is missing, when anything the preview says has changed since, or when the node's
|
||||||
|
// account of itself is too old to be the machine as it is.
|
||||||
|
func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saw := digestIn(t, preview)
|
||||||
|
if !strings.Contains(preview, "converge anchor --yes "+saw) {
|
||||||
|
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
||||||
|
}
|
||||||
|
unchanged := func() {
|
||||||
|
t.Helper()
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused flip changed something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, "", ""); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--yes "+saw) {
|
||||||
|
t.Fatalf("a flip naming no preview was not refused: %v", err)
|
||||||
|
}
|
||||||
|
unchanged()
|
||||||
|
|
||||||
|
// Something new is reachable: the preview the operator saw is not what would happen.
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||||
|
ContainerPort: 80},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 6000, By: "something-new"},
|
||||||
|
}, heldFile)
|
||||||
|
_, err = converge(ctx, open, "anchor", true, saw, "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
||||||
|
t.Fatalf("a flip on a changed preview was not refused: %v", err)
|
||||||
|
}
|
||||||
|
unchanged()
|
||||||
|
|
||||||
|
// An account older than the flip trusts is refused, whatever digest is named.
|
||||||
|
again, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saved := reportFreshFor
|
||||||
|
reportFreshFor = time.Nanosecond
|
||||||
|
_, err = converge(ctx, open, "anchor", true, digestIn(t, again), "")
|
||||||
|
reportFreshFor = saved
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "wait for its next report") {
|
||||||
|
t.Fatalf("a flip on an old account was not refused: %v", err)
|
||||||
|
}
|
||||||
|
unchanged()
|
||||||
|
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, again), ""); err != nil {
|
||||||
|
t.Fatalf("the flip on the preview just seen was refused: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); n.Adopted {
|
||||||
|
t.Fatal("the flip did not converge the node")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
||||||
|
// composed after it — never sent after it with the node still adopted. And the send inside the
|
||||||
|
// flip is not made to wait on the flip's own hold.
|
||||||
|
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var heldElsewhere, heldHere error
|
||||||
|
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
||||||
|
// Another caller cannot hold the node while the flip sends it.
|
||||||
|
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
||||||
|
release()
|
||||||
|
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
||||||
|
}
|
||||||
|
// The flip's own send holds it without waiting on itself.
|
||||||
|
_, release, err := holdNodes(inner, open, names)
|
||||||
|
if err != nil {
|
||||||
|
heldHere = err
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if heldElsewhere != nil || heldHere != nil {
|
||||||
|
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
||||||
|
}
|
||||||
|
// And given back once it is done.
|
||||||
|
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||||
|
// not only its files, and the digest changes when any of them does.
|
||||||
|
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
since := time.Now()
|
||||||
|
held := []link.Held{heldFile,
|
||||||
|
{ID: "notes.data", Module: "notes", Kind: "directory", Target: "/var/lib/notes", Since: since},
|
||||||
|
{ID: "notes.daemon", Module: "notes", Kind: "service", Target: "notes.service", Since: since},
|
||||||
|
{ID: "notes.seed", Module: "notes", Kind: "archive", Target: "/srv/notes", Since: since},
|
||||||
|
{ID: "notes.worker", Module: "notes", Kind: "process", Target: "notes-worker", Since: since},
|
||||||
|
{ID: "notes.account", Module: "notes", Kind: "user", Target: "notes", Since: since},
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, held...)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"replacing the found directory /var/lib/notes (notes.data)",
|
||||||
|
"replacing the found service notes.service (notes.daemon)",
|
||||||
|
"replacing the found archive /srv/notes (notes.seed)",
|
||||||
|
"replacing the found process notes-worker (notes.worker)",
|
||||||
|
"replacing the found user notes (notes.account)",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, held[:len(held)-1]...)
|
||||||
|
fewer, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if digestIn(t, fewer) == digestIn(t, preview) {
|
||||||
|
t.Fatal("the digest does not change with what is held")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing
|
||||||
|
// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors
|
||||||
|
// did not, and flipping would close ports the preview never named.
|
||||||
|
func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Only a loopback listener: nothing off the machine, which is the same silence.
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
||||||
|
}, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(preview, "this account looks partial") {
|
||||||
|
t.Errorf("the preview does not mark a partial account:\n%s", preview)
|
||||||
|
}
|
||||||
|
_, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") {
|
||||||
|
t.Fatalf("the flip was not refused on an account naming nothing: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused flip changed something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
||||||
|
// holds the node, so it waits for whatever is converging it.
|
||||||
|
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||||
|
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
||||||
|
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
||||||
|
|
||||||
|
var whileFlipping error
|
||||||
|
sendNodes = func(context.Context, *stores, []string) error {
|
||||||
|
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
||||||
|
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,557 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A node is adopted or converged (novox/hq ADR 0100), and it is said to be adopted wherever the
|
||||||
|
// mesh reports a node's state: node list, node show, status and the board.
|
||||||
|
|
||||||
|
// showMode is the node show lines about a node's mode and what was taken on it.
|
||||||
|
func showMode(ctx context.Context, inv *inventory.Inventory, node inventory.Node) error {
|
||||||
|
if !node.Adopted {
|
||||||
|
fmt.Printf(" mode converged\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf(" mode adopted since %s\n",
|
||||||
|
node.AdoptedSince.Local().Format(time.DateTime))
|
||||||
|
taken, err := inv.Taken(ctx, node.Name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(taken) == 0 {
|
||||||
|
fmt.Printf(" taken nothing yet\n")
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" taken %s\n", strings.Join(taken, ", "))
|
||||||
|
}
|
||||||
|
said, err := inv.AdoptionOf(ctx, node.Name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if said.At.IsZero() {
|
||||||
|
fmt.Printf(" it has not yet said what it found\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf(" firewall found %s\n", orNone(said.Firewall))
|
||||||
|
if len(said.Held) == 0 {
|
||||||
|
fmt.Printf(" holding nothing found\n")
|
||||||
|
}
|
||||||
|
for _, h := range said.Held {
|
||||||
|
// A held thing that changed is how a predecessor still writing is caught: said first.
|
||||||
|
line := fmt.Sprintf(" holds %-11s %s %s, for %s", h.Kind, h.Target, h.ID, h.Module)
|
||||||
|
if h.Changed != "" {
|
||||||
|
line += " — " + strings.ToUpper(h.Changed) + " by something other than the mesh"
|
||||||
|
}
|
||||||
|
fmt.Println(line)
|
||||||
|
if h.Kept != "" {
|
||||||
|
fmt.Printf(" %-17s original kept at %s\n", "", h.Kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf(" as of %s\n", said.At.Local().Format(time.DateTime))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNone(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "none reported"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptedNodes are the names of every adopted node, in the order given.
|
||||||
|
func adoptedNodes(nodes []inventory.Node) []string {
|
||||||
|
var out []string
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.Adopted {
|
||||||
|
out = append(out, n.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// The operator's acts on an adopted node (novox/hq ADR 0100). Called by the command line and the
|
||||||
|
// command API alike, so a refusal is the same refusal in the same words at both (ADR 0035).
|
||||||
|
|
||||||
|
// sendNodes sends the named machines what they should be now. A variable so a test can see what
|
||||||
|
// an act would send without a broker.
|
||||||
|
var sendNodes = sendTo
|
||||||
|
|
||||||
|
// DefaultFilter is the module converging a node assigns to load the mesh's derived filter.
|
||||||
|
const DefaultFilter = "nftables"
|
||||||
|
|
||||||
|
// take is a module's cutover on an adopted node: the operator's act, done when that module's data
|
||||||
|
// has moved. From the next push its resources converge there like any other, replacing what the
|
||||||
|
// node found and holds for it.
|
||||||
|
func take(ctx context.Context, open *stores, node, module string) (string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
assigned, err := inv.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if !slices.Contains(assigned, module) {
|
||||||
|
if plan, _, err := planFor(ctx, open, node); err == nil {
|
||||||
|
if why, runs := plan.Because[module]; runs {
|
||||||
|
return "", fmt.Errorf("%w: %s runs on %s because %s — assign it to %s to take it",
|
||||||
|
inventory.ErrNotAssigned, module, node, why, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.Take(ctx, node, module); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
said := fmt.Sprintf("%s is taken on %s", module, node)
|
||||||
|
reported, err := inv.AdoptionOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var replaces []string
|
||||||
|
for _, h := range reported.Held {
|
||||||
|
if h.Module == module {
|
||||||
|
replaces = append(replaces, " "+heldLine(h))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(replaces) > 0 {
|
||||||
|
said += "; the next push replaces what the node found and holds for it:\n" +
|
||||||
|
strings.Join(replaces, "\n")
|
||||||
|
}
|
||||||
|
return said + fmt.Sprintf("\n run `push %s` to cut it over", node), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportFreshFor is how old a node's account of itself may be for the flip to act on it. A
|
||||||
|
// variable so a test can age a report without waiting.
|
||||||
|
var reportFreshFor = 15 * time.Minute
|
||||||
|
|
||||||
|
// converge previews, and with yes makes, the flip of an adopted node to converged: every module it
|
||||||
|
// runs is taken, the filter module is assigned to load the mesh's derived filter in place of the
|
||||||
|
// guard, and the found firewall is retired — disabled, never flushed — by the host.
|
||||||
|
//
|
||||||
|
// Refused while an assigned module still holds a found container: each service is taken on its
|
||||||
|
// own, when its data has moved, never by the flip. And refused on a preview that would be stale:
|
||||||
|
// what is reachable is the node's last account, so that account must be of what it was last sent.
|
||||||
|
//
|
||||||
|
// **The flip acts on the preview the operator saw** and on nothing else. The preview ends with a
|
||||||
|
// short digest of what it said — every reachable thing and its fate, the modules the flip takes and
|
||||||
|
// the filter — and yes must name that digest: if anything the preview would say has changed since,
|
||||||
|
// the flip is refused rather than done on a preview nobody read. And it is refused on an account
|
||||||
|
// older than reportFreshFor: what was reachable then is not evidence of what is reachable now.
|
||||||
|
func converge(ctx context.Context, open *stores, node string, yes bool, digest string,
|
||||||
|
filter string) (string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
if filter == "" {
|
||||||
|
filter = DefaultFilter
|
||||||
|
}
|
||||||
|
if yes {
|
||||||
|
// Held from the checks to the send, so no push composed before the flip is sent after it
|
||||||
|
// and returns the node to adopted.
|
||||||
|
held, release, err := holdNodes(ctx, open, []string{node})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
ctx = held
|
||||||
|
}
|
||||||
|
record, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if !record.Adopted {
|
||||||
|
return "", fmt.Errorf("%s is converged already; there is nothing to flip", node)
|
||||||
|
}
|
||||||
|
reports, err := inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
current := false
|
||||||
|
for _, r := range reports {
|
||||||
|
if r.Node == node {
|
||||||
|
current = r.Current
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reported, err := inv.AdoptionOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if !current || reported.At.IsZero() {
|
||||||
|
return "", fmt.Errorf("%s has not reported on the declaration it was last sent, so what it "+
|
||||||
|
"says is reachable may not be the machine as it is: run `push %s --wait 2m` and "+
|
||||||
|
"converge once it has applied", node, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
assignedWhenPreviewed, err := inv.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
plan, settings, err := planFor(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
runs := map[string]bool{}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
runs[m.Module] = true
|
||||||
|
}
|
||||||
|
var holding []string
|
||||||
|
for _, h := range reported.Held {
|
||||||
|
if h.Kind == "container" && runs[h.Module] {
|
||||||
|
holding = append(holding, fmt.Sprintf(" %s holds the found container %s — take %s %s "+
|
||||||
|
"once its data has moved", h.Module, h.Target, node, h.Module))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(holding) > 0 {
|
||||||
|
sort.Strings(holding)
|
||||||
|
return "", fmt.Errorf("%s still holds what it found, and a service is taken on its own, "+
|
||||||
|
"never by the flip:\n%s", node, strings.Join(holding, "\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
filterModule, known := shelf[filter]
|
||||||
|
if !known {
|
||||||
|
return "", fmt.Errorf("%w: %s — converging assigns it to load the mesh's filter; "+
|
||||||
|
"name another with --filter", inventory.ErrNoSuchModule, filter)
|
||||||
|
}
|
||||||
|
if filterModule.Filtering == nil {
|
||||||
|
return "", fmt.Errorf("%s loads no filter of the mesh's; name a module that does with --filter",
|
||||||
|
filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
gens, err := generators(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
rules, err := plan.Rules(with)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
taken, err := inv.Taken(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
derived := derivedFilter{rules: rules, foundation: with.Foundation, mesh: with.Mesh,
|
||||||
|
outward: plan.PublicDomain != ""}
|
||||||
|
preview, saw := previewOf(node, reported, derived, plan, taken, filter, runs[filter])
|
||||||
|
preview += "\n\n preview " + saw
|
||||||
|
if !yes {
|
||||||
|
return preview + fmt.Sprintf("\n\nNothing has changed. Run `converge %s --yes %s` to do "+
|
||||||
|
"it.", node, saw), nil
|
||||||
|
}
|
||||||
|
// An account naming nothing reachable is not an account of a machine: every machine answers
|
||||||
|
// on ssh, and the host's collectors failing — `ss` refusing, or the container runtime not
|
||||||
|
// answering, which drops every published port at once — leaves exactly this. Flipping on it
|
||||||
|
// would close ports the preview never named.
|
||||||
|
if yes && countReachable(reported) == 0 {
|
||||||
|
return preview, fmt.Errorf("%s says nothing is reachable on it, which no machine that is "+
|
||||||
|
"up ever is: its account looks partial — whatever reads what is listening, or what "+
|
||||||
|
"the container runtime publishes, did not answer. Fix that on the machine and run "+
|
||||||
|
"`push %s --wait 2m`, then preview again", node, node)
|
||||||
|
}
|
||||||
|
if age := time.Since(reported.At); age > reportFreshFor {
|
||||||
|
return preview, fmt.Errorf("%s last said what is reachable on it %s ago, and the flip acts "+
|
||||||
|
"only on an account newer than %s: wait for its next report, or run `push %s --wait 2m`, "+
|
||||||
|
"then preview again", node, age.Round(time.Second), reportFreshFor, node)
|
||||||
|
}
|
||||||
|
if digest == "" {
|
||||||
|
return preview, fmt.Errorf("converging %s acts on the preview you saw: name its digest, "+
|
||||||
|
"`converge %s --yes %s`, once you have read it", node, node, saw)
|
||||||
|
}
|
||||||
|
if digest != saw {
|
||||||
|
return preview, fmt.Errorf("what converging %s would do has changed since preview %s "+
|
||||||
|
"(it is now %s): read the preview above, and run `converge %s --yes %s` if it is "+
|
||||||
|
"what you want", node, digest, saw, node, saw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The flip. The filter first, and only kept if the node still resolves with it: a node that
|
||||||
|
// cannot be worked out would be sent nothing, and would sit with its guard and no filter.
|
||||||
|
assigned, err := inv.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// And nothing assigned since the preview was composed: the flip takes every module the node
|
||||||
|
// runs, and one assigned in between would be taken without ever having been previewed.
|
||||||
|
if !slices.Equal(assigned, assignedWhenPreviewed) {
|
||||||
|
return preview, fmt.Errorf("what %s runs changed while this was converging (it is now %s): "+
|
||||||
|
"the flip takes every module on the node, so read the preview again", node,
|
||||||
|
strings.Join(assigned, ", "))
|
||||||
|
}
|
||||||
|
if !slices.Contains(assigned, filter) {
|
||||||
|
if err := inv.Assign(ctx, node, filter); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if _, _, err := planFor(ctx, open, node); err != nil {
|
||||||
|
_ = inv.Unassign(ctx, node, filter)
|
||||||
|
return "", fmt.Errorf("%s cannot run %s, so it was not converged: %w", node, filter, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
took, err := inv.Converge(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
said := preview + fmt.Sprintf("\n\n%s is converged", node)
|
||||||
|
if len(took) > 0 {
|
||||||
|
said += "; took " + strings.Join(took, ", ")
|
||||||
|
}
|
||||||
|
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||||
|
return said + "\n and it could not be sent: run `push " + node + "`", err
|
||||||
|
}
|
||||||
|
return said + "\n sent: the host loads the mesh's filter and disables the firewall it found", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// previewOf is what converging a node will change, before it changes it, and a short digest of
|
||||||
|
// what it said: every reachable thing and its fate, the modules the flip takes and the filter. The
|
||||||
|
// digest is what the flip is asked to act on, so it changes whenever any of those would.
|
||||||
|
func previewOf(node string, reported inventory.Adoption, derived derivedFilter,
|
||||||
|
plan catalogue.Resolution, taken []string, filter string, filterAssigned bool) (string, string) {
|
||||||
|
var said []string
|
||||||
|
var b strings.Builder
|
||||||
|
fmt.Fprintf(&b, "converging %s\n", node)
|
||||||
|
fmt.Fprintf(&b, "\n reachable on the machine now, as it reported at %s:\n",
|
||||||
|
reported.At.Local().Format(time.DateTime))
|
||||||
|
for _, r := range reported.Reachable {
|
||||||
|
if loopback(r.Address) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
what := fmt.Sprintf("%s/%d", r.Protocol, r.Port)
|
||||||
|
if r.By != "" {
|
||||||
|
what += " " + r.By
|
||||||
|
}
|
||||||
|
if r.Published {
|
||||||
|
what += fmt.Sprintf(" (published, container port %d)", r.ContainerPort)
|
||||||
|
}
|
||||||
|
fate := derived.fate(r)
|
||||||
|
fmt.Fprintf(&b, " %-44s %s\n", what, fate)
|
||||||
|
said = append(said, fmt.Sprintf("reach %s %s %s", r.Address, what, fate))
|
||||||
|
}
|
||||||
|
if countReachable(reported) == 0 {
|
||||||
|
// Said as what it is: no machine that is up is reachable on nothing, so this is an
|
||||||
|
// account that did not come back, not a machine with nothing on it.
|
||||||
|
b.WriteString(" nothing reported — this account looks partial, and the flip is " +
|
||||||
|
"refused on it\n")
|
||||||
|
said = append(said, "reach nothing reported")
|
||||||
|
}
|
||||||
|
// What the machine routes for others is not a listener and not a published port, so nothing
|
||||||
|
// above can show it; the derived filter's forward chain drops it all the same.
|
||||||
|
b.WriteString(" not previewed: traffic the machine routes that is not a published port " +
|
||||||
|
"(a tunnel, NAT in the found firewall) — the derived filter drops it unless a module " +
|
||||||
|
"declares it\n")
|
||||||
|
|
||||||
|
isTaken := map[string]bool{}
|
||||||
|
for _, m := range taken {
|
||||||
|
isTaken[m] = true
|
||||||
|
}
|
||||||
|
var takes []string
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
if !isTaken[m.Module] {
|
||||||
|
takes = append(takes, m.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !filterAssigned && !isTaken[filter] {
|
||||||
|
takes = append(takes, filter)
|
||||||
|
}
|
||||||
|
sort.Strings(takes)
|
||||||
|
b.WriteString("\n the flip takes:\n")
|
||||||
|
if len(takes) == 0 {
|
||||||
|
b.WriteString(" nothing — every module is taken already\n")
|
||||||
|
}
|
||||||
|
for _, m := range takes {
|
||||||
|
fmt.Fprintf(&b, " %s\n", m)
|
||||||
|
said = append(said, "take "+m)
|
||||||
|
// Every kind it holds — a directory, a service, an archive, a process, a user as well as a
|
||||||
|
// file (novox/hq ADR 0103) — each said, and each part of what the flip is asked to act on.
|
||||||
|
for _, h := range reported.Held {
|
||||||
|
if h.Module != m {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " replacing the found %s", heldLine(h))
|
||||||
|
if h.Kept != "" {
|
||||||
|
fmt.Fprintf(&b, ", original kept at %s", h.Kept)
|
||||||
|
}
|
||||||
|
b.WriteString("\n")
|
||||||
|
said = append(said, "replace "+m+" "+heldLine(h)+" "+h.Kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !filterAssigned {
|
||||||
|
fmt.Fprintf(&b, "\n and assigns %s, which loads the mesh's filter in place of its guard\n", filter)
|
||||||
|
}
|
||||||
|
fw := reported.Firewall
|
||||||
|
if fw == "" || fw == "none" {
|
||||||
|
b.WriteString(" no firewall was found on the machine; the mesh's filter is its first\n")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(&b, " the found firewall (%s) is disabled, never flushed: its configuration stays on disk\n", fw)
|
||||||
|
}
|
||||||
|
said = append(said, fmt.Sprintf("filter %s assigned=%t firewall=%s", filter, filterAssigned, fw))
|
||||||
|
// Sorted: the same account, reported in another order, is the same preview.
|
||||||
|
sort.Strings(said)
|
||||||
|
sum := sha256.Sum256([]byte(strings.Join(said, "\n")))
|
||||||
|
return strings.TrimRight(b.String(), "\n"), hex.EncodeToString(sum[:])[:12]
|
||||||
|
}
|
||||||
|
|
||||||
|
// derivedFilter is what the filter the flip loads is rendered from, as AsNftables renders it.
|
||||||
|
type derivedFilter struct {
|
||||||
|
rules []catalogue.Rule
|
||||||
|
foundation []int
|
||||||
|
// mesh is every address on the private network; outward says the machine faces outside.
|
||||||
|
mesh []string
|
||||||
|
outward bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// closesOutside is what a narrowing from everywhere to the private network is called: it closes.
|
||||||
|
const closesOutside = "WILL CLOSE to everything outside the private network"
|
||||||
|
|
||||||
|
// fate is what the derived filter does to one reachable thing: which module declares it and from
|
||||||
|
// where, or that it will close — wholly, or to everything outside the private network. Rendered
|
||||||
|
// exactly as AsNftables admits it, ssh included.
|
||||||
|
func (d derivedFilter) fate(r inventory.Reach) string {
|
||||||
|
// Bound to an address on the private network, it was never reachable from outside it, so
|
||||||
|
// admitting it from the mesh narrows nothing.
|
||||||
|
onMesh := slices.Contains(d.mesh, strings.Trim(r.Address, "[]"))
|
||||||
|
if r.Protocol == "tcp" && r.Port == catalogue.SSHPort {
|
||||||
|
// From everywhere only when the machine faces outward or the mesh has no addresses to
|
||||||
|
// narrow it to; otherwise from the private network only.
|
||||||
|
if d.outward || len(d.mesh) == 0 || onMesh {
|
||||||
|
return "stays open — ssh is never closed"
|
||||||
|
}
|
||||||
|
return closesOutside + " — ssh stays open from the mesh, never closed there"
|
||||||
|
}
|
||||||
|
for _, port := range d.foundation {
|
||||||
|
if r.Protocol == "tcp" && r.Port == port {
|
||||||
|
return "stays open — the mesh's own, from anywhere"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, rule := range d.rules {
|
||||||
|
if rule.Port != r.Port || rule.Protocol != r.Protocol {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
by := strings.Join(rule.Because, ", ")
|
||||||
|
switch rule.From {
|
||||||
|
case catalogue.FromMachine:
|
||||||
|
return fmt.Sprintf("WILL CLOSE to the network — declared by %s for this machine only", by)
|
||||||
|
case catalogue.FromMesh:
|
||||||
|
if len(d.mesh) == 0 {
|
||||||
|
return fmt.Sprintf("WILL CLOSE — declared by %s from the mesh, and this node "+
|
||||||
|
"knows no mesh addresses", by)
|
||||||
|
}
|
||||||
|
if !onMesh {
|
||||||
|
return fmt.Sprintf("%s — declared by %s from the mesh only", closesOutside, by)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("declared by %s (from %s)", by, rule.From)
|
||||||
|
}
|
||||||
|
return "WILL CLOSE — no module assigned here declares it"
|
||||||
|
}
|
||||||
|
|
||||||
|
// countReachable is how much of a node's account of itself names something off the machine.
|
||||||
|
// Loopback is left out for the same reason the preview leaves it out: nothing outside reaches it,
|
||||||
|
// so a report of loopback alone says nothing about what the filter would close.
|
||||||
|
func countReachable(reported inventory.Adoption) int {
|
||||||
|
n := 0
|
||||||
|
for _, r := range reported.Reachable {
|
||||||
|
if !loopback(r.Address) {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldLine is one thing a node holds as found, as take and the converge preview both say it.
|
||||||
|
func heldLine(h inventory.Held) string {
|
||||||
|
return fmt.Sprintf("%s %s (%s)", h.Kind, h.Target, h.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loopback is an address nothing off the machine reaches.
|
||||||
|
func loopback(address string) bool {
|
||||||
|
a := strings.Trim(address, "[]")
|
||||||
|
return strings.HasPrefix(a, "127.") || a == "::1" || a == "localhost"
|
||||||
|
}
|
||||||
|
|
||||||
|
// adopt returns a converged node to adopted: the mesh's filter is unloaded, the guard restored,
|
||||||
|
// the found firewall enabled again and the openings converged through it once more. What was
|
||||||
|
// taken stays taken.
|
||||||
|
func adopt(ctx context.Context, open *stores, node string) (string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
record, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if record.Adopted {
|
||||||
|
return "", fmt.Errorf("%s is adopted already", node)
|
||||||
|
}
|
||||||
|
held, release, err := holdNodes(ctx, open, []string{node})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
ctx = held
|
||||||
|
if err := inv.SetAdopted(ctx, node, true); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
said := fmt.Sprintf("%s is adopted; what was taken on it stays taken", node)
|
||||||
|
if err := sendNodes(ctx, open, []string{node}); err != nil {
|
||||||
|
return said + "\n and it could not be sent: run `push " + node + "`", err
|
||||||
|
}
|
||||||
|
return said + "\n sent: the host unloads the mesh's filter and enables the firewall it found", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeCommand, convergeCommand and adoptCommand are the command line's adapters to the acts above.
|
||||||
|
func takeCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 2 {
|
||||||
|
return errors.New("take <node> <module>")
|
||||||
|
}
|
||||||
|
return runAct(ctx, func(open *stores) (string, error) { return take(ctx, open, args[0], args[1]) })
|
||||||
|
}
|
||||||
|
|
||||||
|
func convergeCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("converge", flag.ContinueOnError)
|
||||||
|
yes := set.String("yes", "", "do it, naming the digest the preview printed; without it, only "+
|
||||||
|
"the preview")
|
||||||
|
filter := set.String("filter", DefaultFilter, "the module that loads the mesh's filter")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("converge <node> [--yes <digest>] [--filter nftables]")
|
||||||
|
}
|
||||||
|
return runAct(ctx, func(open *stores) (string, error) {
|
||||||
|
return converge(ctx, open, positionals[0], *yes != "", *yes, *filter)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func adoptCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("adopt <node>")
|
||||||
|
}
|
||||||
|
return runAct(ctx, func(open *stores) (string, error) { return adopt(ctx, open, args[0]) })
|
||||||
|
}
|
||||||
|
|
||||||
|
func runAct(ctx context.Context, act func(*stores) (string, error)) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
said, err := act(open)
|
||||||
|
if said != "" {
|
||||||
|
fmt.Println(said)
|
||||||
|
}
|
||||||
|
if err != nil && said != "" {
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -94,19 +94,29 @@ func (n notYet) Who(*http.Request) (string, error) {
|
|||||||
func commands(who Authenticator) http.Handler {
|
func commands(who Authenticator) http.Handler {
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
mux.HandleFunc("POST /assign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) {
|
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
return assign(ctx, open, in.Node, in.Module)
|
return assign(ctx, open, in.Node, in.Module)
|
||||||
}))
|
}))
|
||||||
mux.HandleFunc("POST /unassign", acting(who, func(ctx context.Context, open *stores, in request) (string, error) {
|
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
return unassign(ctx, open, in.Node, in.Module)
|
return unassign(ctx, open, in.Node, in.Module)
|
||||||
}))
|
}))
|
||||||
|
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||||
|
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return take(ctx, open, in.Node, in.Module)
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter)
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return adopt(ctx, open, in.Node)
|
||||||
|
}))
|
||||||
|
|
||||||
// Anything else is said plainly, because a command surface answering 404 to a verb somebody
|
// Anything else is said plainly, because a command surface answering 404 to a verb somebody
|
||||||
// expected is indistinguishable from one that is down.
|
// expected is indistinguishable from one that is down.
|
||||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
refuse(w, http.StatusNotFound, fmt.Errorf(
|
refuse(w, http.StatusNotFound, fmt.Errorf(
|
||||||
"%s %s is not something this mesh can be asked; it accepts POST /assign and "+
|
"%s %s is not something this mesh can be asked; it accepts POST /assign, "+
|
||||||
"POST /unassign", r.Method, r.URL.Path))
|
"POST /unassign, POST /take, POST /converge and POST /adopt", r.Method, r.URL.Path))
|
||||||
})
|
})
|
||||||
return mux
|
return mux
|
||||||
}
|
}
|
||||||
@@ -114,11 +124,17 @@ func commands(who Authenticator) http.Handler {
|
|||||||
type request struct {
|
type request struct {
|
||||||
Node string `json:"node"`
|
Node string `json:"node"`
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
|
// Yes, Digest and Filter are converge's: do it rather than preview it, the digest of the
|
||||||
|
// preview it acts on, and which module loads the mesh's filter.
|
||||||
|
Yes bool `json:"yes,omitempty"`
|
||||||
|
Digest string `json:"digest,omitempty"`
|
||||||
|
Filter string `json:"filter,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// acting is the shape every route shares: authenticate, read, act, answer.
|
// acting is the shape every route shares: authenticate, read, act, answer.
|
||||||
func acting(
|
func acting(
|
||||||
who Authenticator,
|
who Authenticator,
|
||||||
|
needsModule bool,
|
||||||
do func(context.Context, *stores, request) (string, error),
|
do func(context.Context, *stores, request) (string, error),
|
||||||
) http.HandlerFunc {
|
) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -131,8 +147,12 @@ func acting(
|
|||||||
refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err))
|
refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if in.Node == "" || in.Module == "" {
|
if in.Node == "" || (needsModule && in.Module == "") {
|
||||||
|
if needsModule {
|
||||||
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
|
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
|
||||||
|
} else {
|
||||||
|
refuse(w, http.StatusBadRequest, errors.New(`"node" is needed`))
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -137,6 +137,9 @@ type view struct {
|
|||||||
Unresolved []blockedMachine
|
Unresolved []blockedMachine
|
||||||
// Network is why the private network could not be computed, when it could not.
|
// Network is why the private network could not be computed, when it could not.
|
||||||
Network string
|
Network string
|
||||||
|
// Adopted is every node still adopted (novox/hq ADR 0100). Not broken: nothing forces the
|
||||||
|
// flip, so a node left adopted is shown rather than read as converged.
|
||||||
|
Adopted []string
|
||||||
At string
|
At string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,7 +184,7 @@ type staleModule struct {
|
|||||||
|
|
||||||
func viewOf(asked answers) view {
|
func viewOf(asked answers) view {
|
||||||
out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
|
out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
|
||||||
Network: asked.network}
|
Network: asked.network, Adopted: adoptedNodes(asked.nodes)}
|
||||||
var blocked []string
|
var blocked []string
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
blocked = append(blocked, name)
|
blocked = append(blocked, name)
|
||||||
@@ -311,6 +314,10 @@ new, switched off, or unreachable.</p>
|
|||||||
<p class="quiet">Never told is not out of date: nobody has asked that machine to be anything yet.
|
<p class="quiet">Never told is not out of date: nobody has asked that machine to be anything yet.
|
||||||
Both are sent by <code>push --behind</code>.</p>
|
Both are sent by <code>push --behind</code>.</p>
|
||||||
{{else}}<p class="quiet">Every machine is running what the mesh would send it.</p>{{end}}
|
{{else}}<p class="quiet">Every machine is running what the mesh would send it.</p>{{end}}
|
||||||
|
{{if .Adopted}}
|
||||||
|
<h2>Which machines are adopted?</h2>
|
||||||
|
<ul>{{range .Adopted}}<li><strong>{{.}}</strong> <span class="quiet">adopted — what was found on it is kept until each module is taken</span></li>{{end}}</ul>
|
||||||
|
{{end}}
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
<footer>Read at {{.At}}. This page holds nothing and changes nothing.</footer>
|
<footer>Read at {{.At}}. This page holds nothing and changes nothing.</footer>
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
)
|
||||||
|
|
||||||
|
// heldKey carries the nodes this call already holds, so an act that holds a node and then sends
|
||||||
|
// through sendTo does not wait on itself.
|
||||||
|
type heldKey struct{}
|
||||||
|
|
||||||
|
// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR
|
||||||
|
// 0100), skipping any the context already holds, and returns a context that says it holds them.
|
||||||
|
// Release gives back only what this call took.
|
||||||
|
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||||
|
already, _ := ctx.Value(heldKey{}).(map[string]bool)
|
||||||
|
var take []string
|
||||||
|
for _, n := range names {
|
||||||
|
if !already[n] {
|
||||||
|
take = append(take, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(take) == 0 {
|
||||||
|
return ctx, func() {}, nil
|
||||||
|
}
|
||||||
|
release, err := open.inventory.HoldNodes(ctx, take)
|
||||||
|
if err != nil {
|
||||||
|
return ctx, nil, err
|
||||||
|
}
|
||||||
|
held := map[string]bool{}
|
||||||
|
for n := range already {
|
||||||
|
held[n] = true
|
||||||
|
}
|
||||||
|
for _, n := range take {
|
||||||
|
held[n] = true
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, heldKey{}, held), release, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled
|
||||||
|
// and a send that fails leave nobody waiting for the node.
|
||||||
|
func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
unmarshallable := func(context.Context, string) (sendable, error) {
|
||||||
|
return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil
|
||||||
|
}
|
||||||
|
plain := func(context.Context, string) (sendable, error) {
|
||||||
|
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||||
|
}
|
||||||
|
failing := func(readyNode, []byte) error { return errors.New("the broker went away") }
|
||||||
|
fine := func(readyNode, []byte) error { return nil }
|
||||||
|
|
||||||
|
for name, round := range map[string]func() error{
|
||||||
|
"a body that cannot be marshalled": func() error {
|
||||||
|
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
"a send that fails": func() error {
|
||||||
|
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
if err := round(); err == nil {
|
||||||
|
t.Fatalf("%s was not an error", name)
|
||||||
|
}
|
||||||
|
waiting, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||||
|
release, err := open.inventory.HoldNodes(waiting, []string{"anchor"})
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("after %s the node is still held: %v", name, err)
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -98,6 +98,12 @@ func run() error {
|
|||||||
return moduleCommand(ctx, args[1:])
|
return moduleCommand(ctx, args[1:])
|
||||||
case "assign", "unassign":
|
case "assign", "unassign":
|
||||||
return assignCommand(ctx, args[0], args[1:])
|
return assignCommand(ctx, args[0], args[1:])
|
||||||
|
case "take":
|
||||||
|
return takeCommand(ctx, args[1:])
|
||||||
|
case "converge":
|
||||||
|
return convergeCommand(ctx, args[1:])
|
||||||
|
case "adopt":
|
||||||
|
return adoptCommand(ctx, args[1:])
|
||||||
case "settings":
|
case "settings":
|
||||||
return settingsCommand(ctx, args[1:])
|
return settingsCommand(ctx, args[1:])
|
||||||
case "secret":
|
case "secret":
|
||||||
@@ -126,7 +132,7 @@ func usage() {
|
|||||||
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||||
|
|
||||||
migrate bring each context's schema up to date
|
migrate bring each context's schema up to date
|
||||||
node add <name> create a node record
|
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
||||||
node list the nodes this mesh knows about
|
node list the nodes this mesh knows about
|
||||||
node show <name> what one machine reported it can do, and why
|
node show <name> what one machine reported it can do, and why
|
||||||
node public-domain <name> the domain it composes its routed names under
|
node public-domain <name> the domain it composes its routed names under
|
||||||
@@ -134,6 +140,7 @@ func usage() {
|
|||||||
node public-domain <name> --clear ...it faces the outside no longer
|
node public-domain <name> --clear ...it faces the outside no longer
|
||||||
token issue --node <name> a one-time right to join, for an existing record
|
token issue --node <name> a one-time right to join, for an existing record
|
||||||
token issue --new <name> create the record and issue for it
|
token issue --new <name> create the record and issue for it
|
||||||
|
token issue ... --adopted ...for a machine in use, which joins adopted
|
||||||
identity show this control plane's signing key
|
identity show this control plane's signing key
|
||||||
broker show where the broker is, and what to expect there
|
broker show where the broker is, and what to expect there
|
||||||
serve consume what nodes say, and answer
|
serve consume what nodes say, and answer
|
||||||
@@ -154,6 +161,9 @@ func usage() {
|
|||||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||||
assign <node> <module> put a module on a node
|
assign <node> <module> put a module on a node
|
||||||
unassign <node> <module> take it off
|
unassign <node> <module> take it off
|
||||||
|
take <node> <module> cut a module over on an adopted node, once its data has moved
|
||||||
|
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||||
|
adopt <node> return a converged node to adopted; what was taken stays taken
|
||||||
settings set <module> <file> what a module's config should say, for the whole mesh
|
settings set <module> <file> what a module's config should say, for the whole mesh
|
||||||
settings set <module> <file> --node <n> ...or for one machine
|
settings set <module> <file> --node <n> ...or for one machine
|
||||||
settings clear <module> [--node <n>] take a layer away
|
settings clear <module> [--node <n>] take a layer away
|
||||||
|
|||||||
@@ -38,15 +38,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
return showNode(ctx, inv, args[1])
|
return showNode(ctx, inv, args[1])
|
||||||
case "add":
|
case "add":
|
||||||
if len(args) != 2 {
|
return addNode(ctx, inv, args[1:])
|
||||||
return errors.New("node add <name>")
|
|
||||||
}
|
|
||||||
node, err := inv.AddNode(ctx, args[1])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf("added %s (%s)\n", node.Name, node.ID)
|
|
||||||
return nil
|
|
||||||
|
|
||||||
case "list":
|
case "list":
|
||||||
nodes, err := inv.Nodes(ctx)
|
nodes, err := inv.Nodes(ctx)
|
||||||
@@ -61,7 +53,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
for _, n := range nodes {
|
for _, n := range nodes {
|
||||||
fmt.Printf("%-20s %-14s %s\n", n.Name, heardFrom(n), n.ID)
|
fmt.Printf("%-20s %-14s %-9s %s\n", n.Name, heardFrom(n), modeOf(n), n.ID)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
@@ -80,6 +72,39 @@ func nodeCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||||
|
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||||
|
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||||
|
adopted := set.Bool("adopted", false,
|
||||||
|
"the machine is in use: keep what is found on it until each module is taken")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("node add <name> [--adopted]")
|
||||||
|
}
|
||||||
|
node, err := inv.AddNodeAs(ctx, positionals[0], *adopted)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("added %s (%s)", node.Name, node.ID)
|
||||||
|
if node.Adopted {
|
||||||
|
fmt.Print(", adopted")
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// modeOf is a node's mode as a word (novox/hq ADR 0100): an adopted node is said to be adopted
|
||||||
|
// wherever the mesh reports a node's state.
|
||||||
|
func modeOf(n inventory.Node) string {
|
||||||
|
if n.Adopted {
|
||||||
|
return "adopted"
|
||||||
|
}
|
||||||
|
return "converged"
|
||||||
|
}
|
||||||
|
|
||||||
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
|
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
|
||||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||||
"<name> <domain> to set it; <name> --clear to take it away"
|
"<name> <domain> to set it; <name> --clear to take it away"
|
||||||
@@ -153,6 +178,8 @@ func tokenCommand(ctx context.Context, args []string) error {
|
|||||||
existing := set.String("node", "", "issue for a node record that already exists")
|
existing := set.String("node", "", "issue for a node record that already exists")
|
||||||
fresh := set.String("new", "", "create the node record, then issue for it")
|
fresh := set.String("new", "", "create the node record, then issue for it")
|
||||||
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
||||||
|
adopted := set.Bool("adopted", false,
|
||||||
|
"the machine joining is in use: it is adopted, and keeps what is found on it")
|
||||||
if err := set.Parse(args[1:]); err != nil {
|
if err := set.Parse(args[1:]); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -171,16 +198,7 @@ func tokenCommand(ctx context.Context, args []string) error {
|
|||||||
defer open.Close()
|
defer open.Close()
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
|
|
||||||
name := *existing
|
issued, err := issueFor(ctx, inv, *existing, *fresh, *adopted, *validFor)
|
||||||
if *fresh != "" {
|
|
||||||
node, err := inv.AddNode(ctx, *fresh)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
name = node.Name
|
|
||||||
}
|
|
||||||
|
|
||||||
issued, err := inv.IssueToken(ctx, name, *validFor)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -211,7 +229,8 @@ func tokenCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret}
|
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret,
|
||||||
|
Adopted: issued.Node.Adopted}
|
||||||
|
|
||||||
// Absent is a state, not a failure: a control plane can hold records and a key before it has
|
// Absent is a state, not a failure: a control plane can hold records and a key before it has
|
||||||
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
|
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
|
||||||
@@ -228,8 +247,12 @@ func tokenCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("token for %s, usable once, until %s\n\n %s\n\n",
|
joins := ""
|
||||||
issued.Node.Name, issued.Expires.Format(time.RFC3339), encoded)
|
if made.Adopted {
|
||||||
|
joins = ", joining adopted"
|
||||||
|
}
|
||||||
|
fmt.Printf("token for %s%s, usable once, until %s\n\n %s\n\n",
|
||||||
|
issued.Node.Name, joins, issued.Expires.Format(time.RFC3339), encoded)
|
||||||
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
|
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
|
||||||
|
|
||||||
if missing := made.Missing(); len(missing) > 0 {
|
if missing := made.Missing(); len(missing) > 0 {
|
||||||
@@ -243,6 +266,38 @@ func tokenCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
|
||||||
|
// when the operator says so, and the one-time secret for it. The node in what it returns carries
|
||||||
|
// its mode, which is what the token says.
|
||||||
|
func issueFor(ctx context.Context, inv *inventory.Inventory, existing, fresh string, adopted bool,
|
||||||
|
validFor time.Duration) (inventory.Issued, error) {
|
||||||
|
name := existing
|
||||||
|
if fresh != "" {
|
||||||
|
node, err := inv.AddNodeAs(ctx, fresh, adopted)
|
||||||
|
if err != nil {
|
||||||
|
return inventory.Issued{}, err
|
||||||
|
}
|
||||||
|
name = node.Name
|
||||||
|
}
|
||||||
|
// Not saying adopted leaves the node as it is: re-issuing a token for an adopted node does not
|
||||||
|
// converge it — converging is its own act, previewed (novox/hq ADR 0100). And saying it for a
|
||||||
|
// node already converged is refused rather than done quietly: returning a node to adopted is
|
||||||
|
// its own act too, which unloads the mesh's filter and enables the found firewall again.
|
||||||
|
if adopted && fresh == "" {
|
||||||
|
node, err := inv.NodeByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return inventory.Issued{}, err
|
||||||
|
}
|
||||||
|
if !node.Adopted {
|
||||||
|
return inventory.Issued{}, fmt.Errorf("%s is converged, and a token does not change "+
|
||||||
|
"that: run `adopt %s` to return it to adopted, then issue the token without "+
|
||||||
|
"--adopted", name, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return inv.IssueToken(ctx, name, validFor)
|
||||||
|
}
|
||||||
|
|
||||||
func identityCommand(ctx context.Context, args []string) error {
|
func identityCommand(ctx context.Context, args []string) error {
|
||||||
if len(args) == 0 || args[0] != "show" {
|
if len(args) == 0 || args[0] != "show" {
|
||||||
return errors.New("identity show")
|
return errors.New("identity show")
|
||||||
@@ -334,6 +389,9 @@ func showNode(ctx context.Context, inv *inventory.Inventory, name string) error
|
|||||||
}
|
}
|
||||||
fmt.Printf("%s\n", node.Name)
|
fmt.Printf("%s\n", node.Name)
|
||||||
fmt.Printf(" last heard from %s\n", heardFrom(node))
|
fmt.Printf(" last heard from %s\n", heardFrom(node))
|
||||||
|
if err := showMode(ctx, inv, node); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
|
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
|
||||||
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
|
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// **A read-shaped invocation is never a destructive write.**
|
// **A read-shaped invocation is never a destructive write.**
|
||||||
@@ -97,3 +98,57 @@ func TestAskingAboutAMachineTheMeshHasNeverHeardOfIsRefused(t *testing.T) {
|
|||||||
t.Fatal("a name the mesh does not know was answered as if it were a machine")
|
t.Fatal("a name the mesh does not know was answered as if it were a machine")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: the operator says a node is adopted — `node add --adopted` for a record, and
|
||||||
|
// the token for a machine joining.
|
||||||
|
func TestANodeAddedAdoptedIsAdopted(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := addNode(ctx, open.inventory, []string{"joiner", "--adopted"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n, err := open.inventory.NodeByName(ctx, "joiner")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !n.Adopted {
|
||||||
|
t.Fatal("node add --adopted made a converged node")
|
||||||
|
}
|
||||||
|
if err := addNode(ctx, open.inventory, []string{"plain"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "plain"); n.Adopted {
|
||||||
|
t.Fatal("node add without --adopted made an adopted node")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATokenIssuedAdoptedSaysSoAndReissuingDoesNotConverge(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
issued, err := issueFor(ctx, open.inventory, "", "joiner", true, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !issued.Node.Adopted {
|
||||||
|
t.Fatal("a token issued --adopted is for a node that is not adopted")
|
||||||
|
}
|
||||||
|
again, err := issueFor(ctx, open.inventory, "joiner", "", false, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !again.Node.Adopted {
|
||||||
|
t.Fatal("re-issuing without --adopted converged the node; converging is its own act")
|
||||||
|
}
|
||||||
|
// --adopted for a node already converged is refused, and points at the act that does it.
|
||||||
|
if _, err := issueFor(ctx, open.inventory, "laptop", "", true, time.Hour); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "adopt laptop") {
|
||||||
|
t.Fatalf("--adopted on a converged node was not refused: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "laptop"); n.Adopted {
|
||||||
|
t.Fatal("a refused token flipped the node to adopted")
|
||||||
|
}
|
||||||
|
// And said for a node that is adopted already, it is the ordinary re-issue.
|
||||||
|
if _, err := issueFor(ctx, open.inventory, "joiner", "", true, time.Hour); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+112
-22
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -232,12 +233,19 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.World{}, err
|
return catalogue.World{}, err
|
||||||
}
|
}
|
||||||
serves := catalogue.ServedOn(m, name, assigned)
|
|
||||||
if len(serves) > 0 {
|
|
||||||
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
|
layers, err := inv.SettingsFor(ctx, o.node.Name, m.Module)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.World{}, err
|
return catalogue.World{}, err
|
||||||
}
|
}
|
||||||
|
// A port that node was given is where its consumers reach it (novox/hq ADR
|
||||||
|
// 0100). Unreadable given ports are that node's refusal to report, not this one's.
|
||||||
|
if given, err := catalogue.GivenPorts(m, layers); err == nil {
|
||||||
|
for wanted, at := range given {
|
||||||
|
assigned[wanted] = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serves := catalogue.ServedOn(m, name, assigned)
|
||||||
|
if len(serves) > 0 {
|
||||||
serves, err = catalogue.Settle(serves, layers)
|
serves, err = catalogue.Settle(serves, layers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return catalogue.World{}, err
|
return catalogue.World{}, err
|
||||||
@@ -271,10 +279,10 @@ func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
|
|||||||
// silently produced a declaration missing them — a difference between what `plan` showed and what
|
// silently produced a declaration missing them — a difference between what `plan` showed and what
|
||||||
// `plan --json` handed to anything reading it.
|
// `plan --json` handed to anything reading it.
|
||||||
func declarationFor(ctx context.Context, open *stores, node string,
|
func declarationFor(ctx context.Context, open *stores, node string,
|
||||||
plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) {
|
plan catalogue.Resolution, settings catalogue.SettingsBy) (sendable, error) {
|
||||||
gens, err := generators(ctx, open)
|
gens, err := generators(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return sendable{}, err
|
||||||
}
|
}
|
||||||
// **Allocating, because `plan` is the send without the sending.** It is one machine, named by
|
// **Allocating, because `plan` is the send without the sending.** It is one machine, named by
|
||||||
// a person, who is asking what a push would do — so the port it shows and the secret it seals
|
// a person, who is asking what a push would do — so the port it shows and the secret it seals
|
||||||
@@ -316,11 +324,31 @@ const (
|
|||||||
|
|
||||||
func declarationWith(ctx context.Context, open *stores, node string,
|
func declarationWith(ctx context.Context, open *stores, node string,
|
||||||
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||||
gens map[string]catalogue.Generator, choosing Choosing) ([]map[string]any, error) {
|
gens map[string]catalogue.Generator, choosing Choosing) (sendable, error) {
|
||||||
|
with, record, err := renderingFor(ctx, open, node, plan, settings, gens, choosing)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
composed, err := plan.Compose(with)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
// And what was taken on it, said in every declaration it is sent from this one place.
|
||||||
|
adoption, err := adoptionOf(ctx, open.inventory, record, plan, composed)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
|
}
|
||||||
|
return sendable{Resources: composed.Resources, Adoption: adoption}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderingFor is everything a node's declaration is composed with, and the node's record.
|
||||||
|
func renderingFor(ctx context.Context, open *stores, node string,
|
||||||
|
plan catalogue.Resolution, settings catalogue.SettingsBy,
|
||||||
|
gens map[string]catalogue.Generator, choosing Choosing) (catalogue.Rendering, inventory.Node, error) {
|
||||||
inv := open.inventory
|
inv := open.inventory
|
||||||
grants, err := grantsFor(ctx, open, node)
|
grants, err := grantsFor(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
// Where this machine puts what each module needs reachable (novox/hq ADR 0038).
|
// Where this machine puts what each module needs reachable (novox/hq ADR 0038).
|
||||||
//
|
//
|
||||||
@@ -333,7 +361,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
if choosing == Reading {
|
if choosing == Reading {
|
||||||
held, err := inv.PortsFor(ctx, node)
|
held, err := inv.PortsFor(ctx, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
for _, a := range held {
|
for _, a := range held {
|
||||||
if already[a.Module] == nil {
|
if already[a.Module] == nil {
|
||||||
@@ -343,9 +371,44 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The machine ports this node was given for its modules (novox/hq ADR 0100): the foundation's
|
||||||
|
// ports, as genesis chose them. A given port wins over anything assigned and over a manifest's
|
||||||
|
// own long-form mapping.
|
||||||
|
given := map[string]map[int]int{}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
g, err := catalogue.GivenPorts(m, settings[m.Module])
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
if g != nil {
|
||||||
|
given[m.Module] = g
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And one holder per machine port across the node's modules: settings written before this was
|
||||||
|
// refused where they are set are refused here rather than composed into two containers on
|
||||||
|
// one port.
|
||||||
|
holders := map[int]string{}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
for _, at := range given[m.Module] {
|
||||||
|
if other, twice := holders[at]; twice && other != m.Module {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf("%w: %s and %s are "+
|
||||||
|
"both given machine port %d on %s", inventory.ErrPortTaken, other, m.Module,
|
||||||
|
at, node)
|
||||||
|
}
|
||||||
|
holders[at] = m.Module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ports := map[string]map[int]int{}
|
ports := map[string]map[int]int{}
|
||||||
for _, m := range plan.Modules {
|
for _, m := range plan.Modules {
|
||||||
for _, l := range m.Listens {
|
for _, l := range m.Listens {
|
||||||
|
if at, isGiven := given[m.Module][l.Port]; isGiven {
|
||||||
|
if ports[m.Module] == nil {
|
||||||
|
ports[m.Module] = map[int]int{}
|
||||||
|
}
|
||||||
|
ports[m.Module][l.Port] = at
|
||||||
|
continue
|
||||||
|
}
|
||||||
// **Only a port the module actually publishes is the mesh's to move.** A container's
|
// **Only a port the module actually publishes is the mesh's to move.** A container's
|
||||||
// mapping is the thing that translates; without one the software binds what it binds,
|
// mapping is the thing that translates; without one the software binds what it binds,
|
||||||
// and an assignment would not move the service — it would open the wrong number in the
|
// and an assignment would not move the service — it would open the wrong number in the
|
||||||
@@ -357,7 +420,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
case mayAssign && choosing == Allocating:
|
case mayAssign && choosing == Allocating:
|
||||||
at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed)
|
at, err := inv.PortFor(ctx, node, m.Module, l.Port, l.Fixed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf(
|
return catalogue.Rendering{}, inventory.Node{}, fmt.Errorf(
|
||||||
"%s needs %d reachable on %s and it could not be assigned: %w",
|
"%s needs %d reachable on %s and it could not be assigned: %w",
|
||||||
m.Module, l.Port, node, err)
|
m.Module, l.Port, node, err)
|
||||||
}
|
}
|
||||||
@@ -398,7 +461,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
if needed[m.Module] == nil {
|
if needed[m.Module] == nil {
|
||||||
needed[m.Module] = map[string]string{}
|
needed[m.Module] = map[string]string{}
|
||||||
@@ -416,7 +479,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
}
|
}
|
||||||
issued, meshCA, err := certificateFor(ctx, open, node)
|
issued, meshCA, err := certificateFor(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
certificate, authority = issued, meshCA
|
certificate, authority = issued, meshCA
|
||||||
break
|
break
|
||||||
@@ -429,11 +492,11 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
// One reading of the catalogue for the three questions below that resolve the whole mesh.
|
// One reading of the catalogue for the three questions below that resolve the whole mesh.
|
||||||
shelf, err := inv.Catalogue(ctx)
|
shelf, err := inv.Catalogue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
private, err := onThePrivateNetwork(ctx, inv, shelf)
|
private, err := onThePrivateNetwork(ctx, inv, shelf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// And every machine's name, so a container can reach one. The same set that writes the
|
// And every machine's name, so a container can reach one. The same set that writes the
|
||||||
@@ -441,7 +504,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
// about where another machine is.
|
// about where another machine is.
|
||||||
names, err := namesInTheMesh(ctx, inv, shelf)
|
names, err := namesInTheMesh(ctx, inv, shelf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
// And every routed name → the node that serves it (novox/hq ADR 0066). Alongside the
|
||||||
@@ -450,7 +513,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
// to serve and knows nothing about what they mean.
|
// to serve and knows nothing about what they mean.
|
||||||
routes, err := routeNamesInTheMesh(ctx, open)
|
routes, err := routeNamesInTheMesh(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
for name, at := range routes {
|
for name, at := range routes {
|
||||||
names[name] = at
|
names[name] = at
|
||||||
@@ -479,15 +542,36 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if kept, err = inv.OperatorExport(ctx); err != nil {
|
if kept, err = inv.OperatorExport(ctx); err != nil {
|
||||||
return nil, err
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
return plan.Declaration(catalogue.Rendering{
|
// Whether this node is adopted (novox/hq ADR 0100): then the found firewall stays in force, and
|
||||||
|
// the declaration carries openings and the mesh's guard in place of a filter.
|
||||||
|
record, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
// And, on an adopted node, which modules were taken there: the guard is derived from those
|
||||||
|
// only (novox/hq ADR 0103).
|
||||||
|
var taken map[string]bool
|
||||||
|
if record.Adopted {
|
||||||
|
list, err := inv.Taken(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Rendering{}, inventory.Node{}, err
|
||||||
|
}
|
||||||
|
taken = map[string]bool{}
|
||||||
|
for _, m := range list {
|
||||||
|
taken[m] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return catalogue.Rendering{
|
||||||
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
|
||||||
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
Certificate: certificate, Authority: authority, Mesh: private, Names: names,
|
||||||
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept})
|
Suffix: overlay.Suffix(), Foundation: foundation, Kept: kept, Adopted: record.Adopted,
|
||||||
|
Given: given, Taken: taken,
|
||||||
|
}, record, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
|
// routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq
|
||||||
@@ -736,16 +820,21 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if *asJSON {
|
if *asJSON {
|
||||||
resources, err := declarationFor(ctx, open, args[0], plan, settings)
|
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
body, err := json.MarshalIndent(
|
// The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an
|
||||||
map[string]any{"declaration": 1, "resources": resources}, "", " ")
|
// envelope other than the one sent.
|
||||||
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Println(string(body))
|
var indented bytes.Buffer
|
||||||
|
if err := json.Indent(&indented, body, "", " "); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(indented.String())
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -769,10 +858,11 @@ func planCommand(ctx context.Context, args []string) error {
|
|||||||
for _, n := range plan.Needs {
|
for _, n := range plan.Needs {
|
||||||
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
||||||
}
|
}
|
||||||
resources, err := declarationFor(ctx, open, args[0], plan, settings)
|
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
resources := declared.Resources
|
||||||
for module, layers := range settings {
|
for module, layers := range settings {
|
||||||
for _, layer := range layers {
|
for _, layer := range layers {
|
||||||
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
||||||
|
|||||||
+77
-35
@@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -283,10 +282,16 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
asked = append(asked, n.Name)
|
asked = append(asked, n.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
sending, refusals := composeEach(asked, func(node string) ([]map[string]any, error) {
|
// Held from composing to sending, so a converge on one of them cannot send between the two
|
||||||
plan, settings, err := planFor(ctx, open, node)
|
// and be overtaken by what was composed before it (novox/hq ADR 0100).
|
||||||
|
held, release, err := holdNodes(ctx, open, asked)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return err
|
||||||
|
}
|
||||||
|
sending, refusals := composeEach(asked, func(node string) (sendable, error) {
|
||||||
|
plan, settings, err := planFor(held, open, node)
|
||||||
|
if err != nil {
|
||||||
|
return sendable{}, err
|
||||||
}
|
}
|
||||||
// A module assigned here that this machine cannot host is said and left out, not fatal: the
|
// A module assigned here that this machine cannot host is said and left out, not fatal: the
|
||||||
// healthy modules beside it are still resolved and sent. Reported so it is not silently
|
// healthy modules beside it are still resolved and sent. Reported so it is not silently
|
||||||
@@ -295,12 +300,13 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
// The private network is in here with everything else. It used to be composed separately
|
// The private network is in here with everything else. It used to be composed separately
|
||||||
// and prepended, which meant every machine with an address was on it and no machine could
|
// and prepended, which meant every machine with an address was on it and no machine could
|
||||||
// be kept off. It is a module now, so it arrives the way a module does.
|
// be kept off. It is a module now, so it arrives the way a module does.
|
||||||
return declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
return declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||||
})
|
})
|
||||||
|
|
||||||
sentDigest := map[string]string{}
|
sentDigest := map[string]string{}
|
||||||
|
defer release()
|
||||||
for _, s := range sending {
|
for _, s := range sending {
|
||||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
body, err := s.declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -318,8 +324,9 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
sentDigest[s.node] = digest
|
sentDigest[s.node] = digest
|
||||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources))
|
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||||
}
|
}
|
||||||
|
release()
|
||||||
fmt.Printf("\n%d node(s) told\n", len(sending))
|
fmt.Printf("\n%d node(s) told\n", len(sending))
|
||||||
|
|
||||||
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
// **A named push leaves the mesh consistent, not just the machine it named** (novox/hq
|
||||||
@@ -374,21 +381,20 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
|
// (novox/hq ADR 0066). The earlier cut routed these through sendTo, which is
|
||||||
// all-or-nothing — so one swept machine's compose error failed the operator's named
|
// all-or-nothing — so one swept machine's compose error failed the operator's named
|
||||||
// push and skipped its --wait, the very intolerance the main path exists to avoid.
|
// push and skipped its --wait, the very intolerance the main path exists to avoid.
|
||||||
sending, refused := composeEach(also, func(node string) ([]map[string]any, error) {
|
// Held for this round only, and after the last round's were given back, so two pushes
|
||||||
plan, settings, err := planFor(ctx, open, node)
|
// cascading into each other's machines never each wait on the other.
|
||||||
|
refused, err := sendRound(ctx, open, also,
|
||||||
|
func(held context.Context, node string) (sendable, error) {
|
||||||
|
plan, settings, err := planFor(held, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return sendable{}, err
|
||||||
}
|
}
|
||||||
reportUnhostable(node, plan)
|
reportUnhostable(node, plan)
|
||||||
return declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
return declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||||
})
|
},
|
||||||
refusals = append(refusals, refused...)
|
func(s readyNode, body []byte) error {
|
||||||
for _, s := range sending {
|
if err := link.Declare(ctx, server.Channel(), ident, s.node, body,
|
||||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
15*time.Second); err != nil {
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
record, err := inv.NodeByName(ctx, s.node)
|
record, err := inv.NodeByName(ctx, s.node)
|
||||||
@@ -398,7 +404,12 @@ func pushCommand(ctx context.Context, args []string) error {
|
|||||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.resources))
|
fmt.Printf("sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
refusals = append(refusals, refused...)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
// Every candidate this round is marked handled — the sent ones so they are not
|
// Every candidate this round is marked handled — the sent ones so they are not
|
||||||
// re-listed, and the refused ones so a machine that cannot be composed does not make
|
// re-listed, and the refused ones so a machine that cannot be composed does not make
|
||||||
@@ -459,7 +470,7 @@ func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest
|
|||||||
// readyNode is one machine and the declaration it would be sent.
|
// readyNode is one machine and the declaration it would be sent.
|
||||||
type readyNode struct {
|
type readyNode struct {
|
||||||
node string
|
node string
|
||||||
resources []map[string]any
|
declared sendable
|
||||||
}
|
}
|
||||||
|
|
||||||
// composeEach works out what each named machine should be, and never lets one machine's answer
|
// composeEach works out what each named machine should be, and never lets one machine's answer
|
||||||
@@ -480,25 +491,52 @@ type readyNode struct {
|
|||||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||||
// across two machines that must agree — and dropped here, where it never did.
|
// across two machines that must agree — and dropped here, where it never did.
|
||||||
func composeEach(names []string,
|
func composeEach(names []string,
|
||||||
compose func(node string) ([]map[string]any, error)) ([]readyNode, []string) {
|
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||||
|
|
||||||
var sending []readyNode
|
var sending []readyNode
|
||||||
var refusals []string
|
var refusals []string
|
||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
resources, err := compose(name)
|
declared, err := compose(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if len(resources) == 0 {
|
if len(declared.Resources) == 0 {
|
||||||
fmt.Printf("%s is assigned nothing — skipped\n", name)
|
fmt.Printf("%s is assigned nothing — skipped\n", name)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sending = append(sending, readyNode{name, resources})
|
sending = append(sending, readyNode{name, declared})
|
||||||
}
|
}
|
||||||
return sending, refusals
|
return sending, refusals
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold
|
||||||
|
// back on every way out — a body that cannot be marshalled and a send that fails included
|
||||||
|
// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are
|
||||||
|
// still sent.
|
||||||
|
func sendRound(ctx context.Context, open *stores, names []string,
|
||||||
|
compose func(held context.Context, node string) (sendable, error),
|
||||||
|
send func(s readyNode, body []byte) error) ([]string, error) {
|
||||||
|
held, release, err := holdNodes(ctx, open, names)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
sending, refused := composeEach(names, func(node string) (sendable, error) {
|
||||||
|
return compose(held, node)
|
||||||
|
})
|
||||||
|
for _, s := range sending {
|
||||||
|
body, err := s.declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
return refused, err
|
||||||
|
}
|
||||||
|
if err := send(s, body); err != nil {
|
||||||
|
return refused, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return refused, nil
|
||||||
|
}
|
||||||
|
|
||||||
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
|
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
|
||||||
//
|
//
|
||||||
// **After the rest have been sent, never instead of sending them.** It is still an error, because
|
// **After the rest have been sent, never instead of sending them.** It is still an error, because
|
||||||
@@ -533,11 +571,15 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
type ready struct {
|
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||||
node string
|
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||||
resources []map[string]any
|
ctx, release, err := holdNodes(ctx, open, names)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
var sending []ready
|
defer release()
|
||||||
|
|
||||||
|
var sending []readyNode
|
||||||
var refusals []string
|
var refusals []string
|
||||||
for _, name := range names {
|
for _, name := range names {
|
||||||
plan, settings, err := planFor(ctx, open, name)
|
plan, settings, err := planFor(ctx, open, name)
|
||||||
@@ -546,12 +588,12 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
reportUnhostable(name, plan)
|
reportUnhostable(name, plan)
|
||||||
resources, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating)
|
declared, err := declarationWith(ctx, open, name, plan, settings, gens, Allocating)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
sending = append(sending, ready{name, resources})
|
sending = append(sending, readyNode{name, declared})
|
||||||
}
|
}
|
||||||
if len(refusals) > 0 {
|
if len(refusals) > 0 {
|
||||||
return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||||
@@ -565,7 +607,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
|||||||
defer server.Close()
|
defer server.Close()
|
||||||
|
|
||||||
for _, s := range sending {
|
for _, s := range sending {
|
||||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
body, err := s.declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -579,7 +621,7 @@ func sendTo(ctx context.Context, open *stores, names []string) error {
|
|||||||
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
if err := inv.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources))
|
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.declared.Resources))
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -610,11 +652,11 @@ func wouldSend(ctx context.Context, open *stores,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
resources, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
declared, err := declarationWith(ctx, open, n.Name, plan, settings, gens, Reading)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": resources})
|
body, err := declared.Body()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,11 +18,11 @@ import (
|
|||||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||||
sending, refusals := composeEach(
|
sending, refusals := composeEach(
|
||||||
[]string{"anchor", "home-server", "laptop"},
|
[]string{"anchor", "home-server", "laptop"},
|
||||||
func(node string) ([]map[string]any, error) {
|
func(node string) (sendable, error) {
|
||||||
if node == "anchor" {
|
if node == "anchor" {
|
||||||
return nil, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||||
}
|
}
|
||||||
return []map[string]any{{"id": node + ".thing"}}, nil
|
return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil
|
||||||
})
|
})
|
||||||
|
|
||||||
var told []string
|
var told []string
|
||||||
@@ -42,7 +42,7 @@ func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
|||||||
// And a machine assigned nothing is neither sent nor a refusal — it is nothing to say.
|
// And a machine assigned nothing is neither sent nor a refusal — it is nothing to say.
|
||||||
func TestAMachineAssignedNothingIsNotARefusal(t *testing.T) {
|
func TestAMachineAssignedNothingIsNotARefusal(t *testing.T) {
|
||||||
sending, refusals := composeEach([]string{"spare"},
|
sending, refusals := composeEach([]string{"spare"},
|
||||||
func(string) ([]map[string]any, error) { return nil, nil })
|
func(string) (sendable, error) { return sendable{}, nil })
|
||||||
if len(sending) != 0 || len(refusals) != 0 {
|
if len(sending) != 0 || len(refusals) != 0 {
|
||||||
t.Errorf("a machine assigned nothing was treated as something: %v / %v", sending, refusals)
|
t.Errorf("a machine assigned nothing was treated as something: %v / %v", sending, refusals)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ type meshStatus struct {
|
|||||||
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
|
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
|
||||||
// "none at all".
|
// "none at all".
|
||||||
Machines int `json:"machines"`
|
Machines int `json:"machines"`
|
||||||
|
// Adopted is every node still adopted (novox/hq ADR 0100); absent when none is.
|
||||||
|
Adopted []string `json:"adopted,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type machineUnresolved struct {
|
type machineUnresolved struct {
|
||||||
@@ -133,7 +135,7 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
|||||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||||
Network: asked.network}
|
Network: asked.network, Adopted: adoptedNodes(nodes)}
|
||||||
for name := range asked.refused {
|
for name := range asked.refused {
|
||||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
Node: name, Problem: asked.refused[name]})
|
Node: name, Problem: asked.refused[name]})
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sendable is a declaration as a machine is sent it: its resources and, for an adopted node, its
|
||||||
|
// mode and which modules were taken on it (novox/hq ADR 0100).
|
||||||
|
//
|
||||||
|
// **Body is the only place the envelope is marshalled.** It was written by hand at every send site,
|
||||||
|
// in the digest the mesh compares, and in `plan --json`; a key added at one and not another would
|
||||||
|
// make a machine look out of date for ever, or send something `plan` never showed.
|
||||||
|
type sendable struct {
|
||||||
|
Resources []map[string]any
|
||||||
|
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||||
|
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||||
|
Adoption *adoptionEnvelope
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
|
||||||
|
// it that it runs; Untaken is, for every module it runs that is not taken, the ids of every one of
|
||||||
|
// that module's resources — what the host keeps as found until the module is taken (ADR 0103). Ids
|
||||||
|
// rather than a rule to split them by, because a module's name may contain a dot.
|
||||||
|
type adoptionEnvelope struct {
|
||||||
|
Taken []string `json:"taken"`
|
||||||
|
Untaken map[string][]string `json:"untaken,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Body is the declaration's bytes, as sent and as digested.
|
||||||
|
func (s sendable) Body() ([]byte, error) {
|
||||||
|
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||||
|
if s.Adoption != nil {
|
||||||
|
envelope["adoption"] = s.Adoption
|
||||||
|
}
|
||||||
|
return json.Marshal(envelope)
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptionOf is the envelope for a node, nil when it is converged.
|
||||||
|
func adoptionOf(ctx context.Context, inv *inventory.Inventory, record inventory.Node,
|
||||||
|
plan catalogue.Resolution, composed catalogue.Composed) (*adoptionEnvelope, error) {
|
||||||
|
if !record.Adopted {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
taken, err := inv.Taken(ctx, record.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return adoptionFor(plan, taken, composed), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptionFor is the envelope computed from what was taken and who owns each resource.
|
||||||
|
//
|
||||||
|
// Every module the node runs that is not taken is untaken — including one pulled in by another
|
||||||
|
// rather than assigned: what is found is kept until its module is taken, whoever put it there.
|
||||||
|
func adoptionFor(plan catalogue.Resolution, taken []string,
|
||||||
|
composed catalogue.Composed) *adoptionEnvelope {
|
||||||
|
isTaken := map[string]bool{}
|
||||||
|
for _, m := range taken {
|
||||||
|
isTaken[m] = true
|
||||||
|
}
|
||||||
|
out := &adoptionEnvelope{Taken: []string{}}
|
||||||
|
runs := map[string]bool{}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
runs[m.Module] = true
|
||||||
|
if isTaken[m.Module] {
|
||||||
|
out.Taken = append(out.Taken, m.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out.Taken)
|
||||||
|
for _, r := range composed.Resources {
|
||||||
|
id, _ := r["id"].(string)
|
||||||
|
module, owned := composed.Owner[id]
|
||||||
|
// Every kind, not only files and containers (novox/hq ADR 0103): a directory, a service, a
|
||||||
|
// container mounting what was found and an action run in a held container all reach what
|
||||||
|
// the machine already has. What the mesh declares of its own is never held.
|
||||||
|
if !owned || !runs[module] || isTaken[module] ||
|
||||||
|
strings.HasPrefix(id, catalogue.AdoptionPrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if out.Untaken == nil {
|
||||||
|
out.Untaken = map[string][]string{}
|
||||||
|
}
|
||||||
|
out.Untaken[module] = append(out.Untaken[module], id)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules
|
||||||
|
// were taken on it; a converged node's declaration is byte for byte what it was.
|
||||||
|
|
||||||
|
// helloWeb is a module the predecessor also runs: a page and a server under names it uses.
|
||||||
|
func helloWeb() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hello"},
|
||||||
|
{"id": "server", "type": "container", "name": "hello-web",
|
||||||
|
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||||
|
{"id": "served", "type": "directory", "path": "/var/lib/hello-web"},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composed is what node would be sent now, as push composes it.
|
||||||
|
func composed(t *testing.T, open *stores, node string) sendable {
|
||||||
|
t.Helper()
|
||||||
|
plan, settings, err := planFor(t.Context(), open, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared, err := declarationFor(t.Context(), open, node, plan, settings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return declared
|
||||||
|
}
|
||||||
|
|
||||||
|
// convergedBefore is the envelope a converged node was sent before adoption existed, captured by
|
||||||
|
// running this same composition at the commit this branch left main (0a39b7d). The mesh here is
|
||||||
|
// aMesh's laptop with the private network taken off it, so nothing in the declaration is random:
|
||||||
|
// what changes this string is a change to what a converged machine is sent, which is the thing an
|
||||||
|
// older host would refuse.
|
||||||
|
const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"hosts":["anchor.internal:10.77.0.1"],"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}`
|
||||||
|
|
||||||
|
func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, helloWeb())
|
||||||
|
if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared := composed(t, open, "laptop")
|
||||||
|
if declared.Adoption != nil {
|
||||||
|
t.Fatal("a converged node was given an adoption envelope")
|
||||||
|
}
|
||||||
|
body, err := declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(body) != convergedBefore {
|
||||||
|
t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s",
|
||||||
|
body, convergedBefore)
|
||||||
|
}
|
||||||
|
if bytes.Contains(body, []byte(`"adoption"`)) {
|
||||||
|
t.Fatal("a converged declaration names adoption; an older host would refuse it")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, helloWeb())
|
||||||
|
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
declared := composed(t, open, "anchor")
|
||||||
|
if declared.Adoption == nil {
|
||||||
|
t.Fatal("an adopted node's declaration does not say it is adopted")
|
||||||
|
}
|
||||||
|
untaken := declared.Adoption.Untaken["hello-web"]
|
||||||
|
// Every kind — its directory too (novox/hq ADR 0103).
|
||||||
|
if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server",
|
||||||
|
"hello-web.served"}) {
|
||||||
|
t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption)
|
||||||
|
}
|
||||||
|
if len(declared.Adoption.Taken) != 0 {
|
||||||
|
t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var envelope map[string]any
|
||||||
|
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adoption, _ := envelope["adoption"].(map[string]any)
|
||||||
|
if _, ok := adoption["taken"].([]any); !ok {
|
||||||
|
t.Fatalf("taken is not a list on the wire, even empty: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The digest the mesh compares is the digest of what is sent: status and push agree.
|
||||||
|
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if would["anchor"] != digestOf(body) {
|
||||||
|
t.Fatal("the digest the mesh compares is not of the declaration push sends")
|
||||||
|
}
|
||||||
|
|
||||||
|
// plan --json prints that same envelope.
|
||||||
|
printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) })
|
||||||
|
var compact bytes.Buffer
|
||||||
|
if err := json.Compact(&compact, []byte(printed)); err != nil {
|
||||||
|
t.Fatalf("plan --json is not JSON: %v\n%s", err, printed)
|
||||||
|
}
|
||||||
|
if digestOf(compact.Bytes()) != digestOf(body) {
|
||||||
|
t.Fatalf("plan --json shows something other than what push sends:\n%s", printed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Taking the module moves its resources out of untaken.
|
||||||
|
if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared = composed(t, open, "anchor")
|
||||||
|
if _, still := declared.Adoption.Untaken["hello-web"]; still {
|
||||||
|
t.Fatalf("a taken module is still untaken: %v", declared.Adoption)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) {
|
||||||
|
t.Fatalf("taken is %v", declared.Adoption.Taken)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustNodes(t *testing.T, open *stores) []inventory.Node {
|
||||||
|
t.Helper()
|
||||||
|
nodes, err := open.inventory.Nodes(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return nodes
|
||||||
|
}
|
||||||
|
|
||||||
|
// stdoutOf is what run printed.
|
||||||
|
func stdoutOf(t *testing.T, run func() error) string {
|
||||||
|
t.Helper()
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saved := os.Stdout
|
||||||
|
os.Stdout = w
|
||||||
|
done := make(chan string)
|
||||||
|
go func() {
|
||||||
|
all, _ := io.ReadAll(r)
|
||||||
|
done <- string(all)
|
||||||
|
}()
|
||||||
|
runErr := run()
|
||||||
|
os.Stdout = saved
|
||||||
|
w.Close()
|
||||||
|
out := <-done
|
||||||
|
if runErr != nil {
|
||||||
|
t.Fatalf("%v\n%s", runErr, out)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: a port a node was given for the store is where its consumers on other
|
||||||
|
// machines are told to reach it, and a port given for the whole mesh is refused.
|
||||||
|
func TestConsumersAreToldTheGivenPort(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}},
|
||||||
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||||
|
Guards: []int{5432},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||||
|
"ports": []any{"5432:5432"},
|
||||||
|
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetSettings(ctx, "anchor", "store",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
plan, _, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var told any
|
||||||
|
for _, n := range plan.Needs {
|
||||||
|
if n.Name == "database" {
|
||||||
|
told = n.Serves["port"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if told != 5433 {
|
||||||
|
t.Fatalf("the consumer is told the store is on %v", told)
|
||||||
|
}
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) {
|
||||||
|
t.Fatalf("the store publishes %v", r["ports"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A port for the whole mesh is refused where it is set, not stored to refuse every node's
|
||||||
|
// declaration afterwards.
|
||||||
|
if err := open.inventory.SetSettings(ctx, "", "store",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "per node") {
|
||||||
|
t.Fatalf("a port given for the whole mesh was not refused: %v", err)
|
||||||
|
}
|
||||||
|
plan, settings, err := planFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil {
|
||||||
|
t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0103: the guard an adopted node is sent follows what was taken there. A store
|
||||||
|
// assigned but not taken is not guarded — its port may still be the predecessor's — and taking it
|
||||||
|
// guards it from the next declaration.
|
||||||
|
func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||||
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||||
|
Guards: []int{5432},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||||
|
"ports": []any{"5432:5432"},
|
||||||
|
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||||
|
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) {
|
||||||
|
t.Fatal("an untaken store is guarded")
|
||||||
|
}
|
||||||
|
if err := open.inventory.Take(ctx, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == catalogue.GuardID() {
|
||||||
|
if r["content"] != catalogue.AsGuard([]int{5432}) {
|
||||||
|
t.Fatalf("the taken store's guard is:\n%s", r["content"])
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("a taken store is not guarded")
|
||||||
|
}
|
||||||
@@ -171,6 +171,13 @@ func statusCommand(ctx context.Context, args []string) error {
|
|||||||
fmt.Printf("\n `push --behind` sends them\n\n")
|
fmt.Printf("\n `push --behind` sends them\n\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||||
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||||
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||||
|
fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", "))
|
||||||
|
fmt.Printf("\n `converge <node>` previews the flip\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
|
if len(wrong) == 0 && len(quiet) == 0 && len(behind) == 0 && len(asked.waiting) == 0 &&
|
||||||
len(asked.refused) == 0 && asked.network == "" {
|
len(asked.refused) == 0 && asked.network == "" {
|
||||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||||
|
|||||||
@@ -0,0 +1,237 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What an adopted node is declared in place of a filter (novox/hq ADR 0100).
|
||||||
|
//
|
||||||
|
// On an adopted node the firewall found on the machine stays in force: the mesh loads no table
|
||||||
|
// that drops by default or holds an accept. What the mesh needs reachable is declared as
|
||||||
|
// openings, which the host converges through the found firewall in its own terms; and the mesh
|
||||||
|
// guards its own foundation ports itself, in a table that only refuses.
|
||||||
|
|
||||||
|
// AdoptionPrefix is the id prefix of what the mesh declares of its own on an adopted node. It is
|
||||||
|
// never a module's, so none of it is ever held as found.
|
||||||
|
const AdoptionPrefix = "adoption."
|
||||||
|
|
||||||
|
// Where an opening admits from, on the wire.
|
||||||
|
const (
|
||||||
|
OpeningFromEverywhere = "everywhere"
|
||||||
|
OpeningFromMesh = "mesh"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The two paths a packet reaches a port by: received by the machine, or forwarded to a
|
||||||
|
// container that publishes it.
|
||||||
|
const (
|
||||||
|
PathIncoming = "incoming"
|
||||||
|
PathForwarded = "forwarded"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Guard resources: the refusal-only table, the unit that loads it, and that unit running.
|
||||||
|
const (
|
||||||
|
GuardPath = "/etc/mesh/guard.nft"
|
||||||
|
GuardUnit = "mesh-guard.service"
|
||||||
|
// GuardUnitPath is where the unit is written.
|
||||||
|
GuardUnitPath = "/etc/systemd/system/" + GuardUnit
|
||||||
|
)
|
||||||
|
|
||||||
|
// GuardID, GuardUnitID and GuardRunningID are the guard's resource identities. The installer
|
||||||
|
// raises the same three on an adopted genesis, so the first push finds them already there.
|
||||||
|
func GuardID() string { return AdoptionPrefix + "guard" }
|
||||||
|
func GuardUnitID() string { return AdoptionPrefix + "guard-unit" }
|
||||||
|
func GuardRunningID() string { return AdoptionPrefix + "guard-running" }
|
||||||
|
|
||||||
|
// GuardPackageID is the tool that loads the guard, declared first: a node joining adopted has
|
||||||
|
// no filter module and may have no nft at all, and a table nothing can load guards nothing.
|
||||||
|
func GuardPackageID() string { return AdoptionPrefix + "guard-package" }
|
||||||
|
|
||||||
|
// GuardPackage is the package that carries nft.
|
||||||
|
const GuardPackage = "nftables"
|
||||||
|
|
||||||
|
// OpeningID is an opening's resource identity: its protocol, port and path say what it is.
|
||||||
|
func OpeningID(protocol string, port int, path string) string {
|
||||||
|
return fmt.Sprintf("%sopening-%s-%d-%s", AdoptionPrefix, protocol, port, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Openings are what the mesh needs reachable on an adopted node, from the same inputs as the
|
||||||
|
// filter it would load were the node converged, each from where that filter would admit it.
|
||||||
|
//
|
||||||
|
// `rules` is Filtering's answer — every module's listens, the hub's port, the per-node exposure —
|
||||||
|
// and `foundation` is the ports the mesh itself needs, from everywhere. A rule for this machine
|
||||||
|
// only opens nothing. `published` maps a machine port a container publishes to the container's
|
||||||
|
// port: a published port is forwarded, not received, so its opening names the forwarded path and
|
||||||
|
// the port the packet is forwarded to.
|
||||||
|
func Openings(rules []Rule, foundation []int, published map[string]map[int]int) []map[string]any {
|
||||||
|
type key struct {
|
||||||
|
protocol string
|
||||||
|
port int
|
||||||
|
}
|
||||||
|
from := map[key]string{}
|
||||||
|
var order []key
|
||||||
|
widen := func(k key, f string) {
|
||||||
|
was, seen := from[k]
|
||||||
|
if !seen {
|
||||||
|
order = append(order, k)
|
||||||
|
}
|
||||||
|
if !seen || was != OpeningFromEverywhere {
|
||||||
|
from[k] = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, rule := range rules {
|
||||||
|
switch rule.From {
|
||||||
|
case FromEverywhere:
|
||||||
|
widen(key{rule.Protocol, rule.Port}, OpeningFromEverywhere)
|
||||||
|
case FromMesh:
|
||||||
|
widen(key{rule.Protocol, rule.Port}, OpeningFromMesh)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, port := range foundation {
|
||||||
|
widen(key{"tcp", port}, OpeningFromEverywhere)
|
||||||
|
}
|
||||||
|
sort.Slice(order, func(a, b int) bool {
|
||||||
|
if order[a].port != order[b].port {
|
||||||
|
return order[a].port < order[b].port
|
||||||
|
}
|
||||||
|
return order[a].protocol < order[b].protocol
|
||||||
|
})
|
||||||
|
out := make([]map[string]any, 0, len(order))
|
||||||
|
for _, k := range order {
|
||||||
|
opening := map[string]any{"type": "opening", "port": k.port, "protocol": k.protocol,
|
||||||
|
"from": from[k]}
|
||||||
|
if to, forwarded := published[k.protocol][k.port]; forwarded {
|
||||||
|
opening["id"] = OpeningID(k.protocol, k.port, PathForwarded)
|
||||||
|
opening["path"] = PathForwarded
|
||||||
|
opening["to"] = to
|
||||||
|
} else {
|
||||||
|
opening["id"] = OpeningID(k.protocol, k.port, PathIncoming)
|
||||||
|
opening["path"] = PathIncoming
|
||||||
|
}
|
||||||
|
out = append(out, opening)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Published is every port the given containers publish on the machine, by protocol and machine
|
||||||
|
// port, mapped to the container's own port. A mapping bound to loopback is left out: nothing off
|
||||||
|
// the machine reaches it, forwarded or not.
|
||||||
|
func Published(resources []map[string]any) map[string]map[int]int {
|
||||||
|
out := map[string]map[int]int{}
|
||||||
|
for _, r := range resources {
|
||||||
|
if fmt.Sprint(r["type"]) != "container" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
listed, _ := r["ports"].([]any)
|
||||||
|
for _, entry := range listed {
|
||||||
|
written := strings.TrimSpace(fmt.Sprint(entry))
|
||||||
|
protocol := "tcp"
|
||||||
|
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||||
|
protocol = written[cut+1:]
|
||||||
|
}
|
||||||
|
// Indexed from the end, so an IPv6 address's own colons never shift the ports.
|
||||||
|
outer, inner, address, ok := mapping(written)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch strings.Trim(address, "[]") {
|
||||||
|
case "127.0.0.1", "localhost", "::1":
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if out[protocol] == nil {
|
||||||
|
out[protocol] = map[int]int{}
|
||||||
|
}
|
||||||
|
out[protocol][outer] = inner
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// AsGuard renders the mesh's refusal-only table for the given machine ports.
|
||||||
|
//
|
||||||
|
// It passes everything by default and holds nothing but a refusal, so it cannot close anything
|
||||||
|
// the machine serves; and it is the mesh's own table, so the found firewall reloading does not
|
||||||
|
// touch it. It refuses only packets addressed to this machine, and the ports except from the
|
||||||
|
// machine itself — its loopback and the container runtime's own networks — and from the private
|
||||||
|
// network, known by the interface a packet arrives on and never by its source address. At
|
||||||
|
// prerouting, ahead of the runtime's destination translation, so it matches the port the packet
|
||||||
|
// was sent to; in the inet family, so both address families.
|
||||||
|
//
|
||||||
|
// **The machine's own interfaces are named, where the derived filter names address ranges.** The
|
||||||
|
// filter accepts the container runtime's networks by CIDR; this excludes its bridges by name — lo,
|
||||||
|
// docker0, the br-* a compose network gets, and the mesh's own mesh0. A runtime whose bridge is
|
||||||
|
// named anything else (a podman or libvirt bridge, or a docker network created with a fixed name)
|
||||||
|
// would have its containers' traffic to a guarded port refused, which reads as the port being
|
||||||
|
// down. Names rather than addresses is deliberate: a source address can be claimed by whoever
|
||||||
|
// sends the packet, and this table exists to refuse what the found firewall never sees. Widening
|
||||||
|
// it means adding names here and in the installer's copy together, which the golden test holds to
|
||||||
|
// one text.
|
||||||
|
//
|
||||||
|
// The same text the installer raises on an adopted genesis; a test holds both to it.
|
||||||
|
func AsGuard(ports []int) string {
|
||||||
|
sorted := append([]int{}, ports...)
|
||||||
|
sort.Ints(sorted)
|
||||||
|
listed := make([]string, len(sorted))
|
||||||
|
for i, p := range sorted {
|
||||||
|
listed[i] = strconv.Itoa(p)
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("table inet mesh_guard {}\n")
|
||||||
|
b.WriteString("delete table inet mesh_guard\n")
|
||||||
|
b.WriteString("table inet mesh_guard {\n")
|
||||||
|
b.WriteString("\tchain prerouting {\n")
|
||||||
|
b.WriteString("\t\ttype filter hook prerouting priority raw; policy accept;\n")
|
||||||
|
// Only packets addressed to this machine: traffic it routes for others — a predecessor's hub,
|
||||||
|
// say — is never the guard's business (novox/hq ADR 0103).
|
||||||
|
fmt.Fprintf(&b, "\t\tfib daddr type local iifname != \"lo\" iifname != \"docker0\" "+
|
||||||
|
"iifname != \"br-*\" iifname != \"mesh0\" tcp dport { %s } drop\n",
|
||||||
|
strings.Join(listed, ", "))
|
||||||
|
b.WriteString("\t}\n")
|
||||||
|
b.WriteString("}\n")
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// GuardUnitText is the unit that loads the guard. Stopping it deletes only its own table: never
|
||||||
|
// a flush, which would take the container runtime's rules and the found firewall with it.
|
||||||
|
func GuardUnitText() string {
|
||||||
|
return "[Unit]\n" +
|
||||||
|
"Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)\n" +
|
||||||
|
// Early, before the network is up, and without the default dependencies that would
|
||||||
|
// order it after the network; stopped at shutdown like any unit.
|
||||||
|
"DefaultDependencies=no\n" +
|
||||||
|
"Wants=network-pre.target\n" +
|
||||||
|
"Before=network-pre.target shutdown.target\n" +
|
||||||
|
"Conflicts=shutdown.target\n" +
|
||||||
|
"\n" +
|
||||||
|
"[Service]\n" +
|
||||||
|
"Type=oneshot\n" +
|
||||||
|
"RemainAfterExit=yes\n" +
|
||||||
|
"ExecStart=nft -f " + GuardPath + "\n" +
|
||||||
|
"ExecReload=nft -f " + GuardPath + "\n" +
|
||||||
|
"ExecStop=nft delete table inet mesh_guard\n" +
|
||||||
|
"\n" +
|
||||||
|
"[Install]\n" +
|
||||||
|
"WantedBy=multi-user.target\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// GuardResources are the guard as four resources of the existing kinds: the tool that loads it,
|
||||||
|
// the table, the unit, and the unit running — reloaded when the table changes, so the new table
|
||||||
|
// replaces the old in one `nft -f` through the unit's ExecReload with no moment unguarded, and
|
||||||
|
// restarted only when the unit itself changes. Nothing when there is nothing to guard: an empty
|
||||||
|
// set is not a table nft loads.
|
||||||
|
func GuardResources(ports []int) []map[string]any {
|
||||||
|
if len(ports) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return []map[string]any{
|
||||||
|
{"id": GuardPackageID(), "type": "package", "package": GuardPackage},
|
||||||
|
{"id": GuardID(), "type": "file", "path": GuardPath, "content": AsGuard(ports),
|
||||||
|
"mode": "0644"},
|
||||||
|
{"id": GuardUnitID(), "type": "file", "path": GuardUnitPath, "content": GuardUnitText(),
|
||||||
|
"mode": "0644"},
|
||||||
|
{"id": GuardRunningID(), "type": "service", "unit": GuardUnit, "state": "running",
|
||||||
|
"boot": "enabled", "restart-on": []any{GuardUnitID()}, "reload-on": []any{GuardID()}},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,431 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: on an adopted node the found firewall stays in force. The mesh declares
|
||||||
|
// openings where it would have loaded a filter, and guards its own ports in a table that only
|
||||||
|
// refuses.
|
||||||
|
|
||||||
|
// hub is the private network's generator on the hub: it opens the hub's port from anywhere.
|
||||||
|
type hub struct{}
|
||||||
|
|
||||||
|
func (hub) Resources(string) ([]map[string]any, bool, error) {
|
||||||
|
return []map[string]any{{"id": "config", "type": "file", "path": "/etc/wireguard/mesh0.conf",
|
||||||
|
"content": "[Interface]\n"}}, true, nil
|
||||||
|
}
|
||||||
|
func (hub) Listens(string) ([]Listening, error) {
|
||||||
|
return []Listening{{Port: 51820, Protocol: "udp", From: FromEverywhere}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// anAdoptedAnchor is the control-node's set: the store, the bus, the registry, the private network,
|
||||||
|
// a served module and the filter module.
|
||||||
|
func anAdoptedAnchor() Resolution {
|
||||||
|
return Resolution{Node: "anchor", Modules: []Manifest{
|
||||||
|
{Module: "network", Computed: "overlay"},
|
||||||
|
{Module: "postgres", Guards: []int{5432},
|
||||||
|
Listens: []Listening{{Port: 5432, From: FromMesh}},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||||
|
"ports": []any{"5432:5432"}}}},
|
||||||
|
{Module: "lavinmq", Guards: []int{15672},
|
||||||
|
Listens: []Listening{{Port: 5671, From: FromMesh}, {Port: 5672, From: FromMesh}},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||||
|
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}}}},
|
||||||
|
{Module: "distribution",
|
||||||
|
Listens: []Listening{{Port: 5000, From: FromMesh}},
|
||||||
|
Resources: []map[string]any{{"id": "store", "type": "container", "name": "registry",
|
||||||
|
"ports": []any{"5000"}}}},
|
||||||
|
{Module: "hello-web",
|
||||||
|
Listens: []Listening{{Port: 8080, From: FromEverywhere}},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "hello-web",
|
||||||
|
"ports": []any{"8080"}}}},
|
||||||
|
{Module: "helper", Listens: []Listening{{Port: 9000, From: FromMachine}}},
|
||||||
|
{Module: "nftables", Filtering: &Filtering{Into: "/etc/nftables.conf"},
|
||||||
|
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
||||||
|
"state": "running", "restart-on": []any{"filtering"}}}},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func anchorRendering(adopted bool) Rendering {
|
||||||
|
return Rendering{
|
||||||
|
Generators: map[string]Generator{"overlay": hub{}},
|
||||||
|
Ports: map[string]map[int]int{"distribution": {5000: 5000}, "hello-web": {8080: 20001}},
|
||||||
|
Settings: SettingsBy{"distribution": {{From: "node anchor",
|
||||||
|
Values: map[string]any{ExposeSetting: map[string]any{"5000": FromEverywhere}}}}},
|
||||||
|
Mesh: []string{"10.42.0.1"},
|
||||||
|
Foundation: []int{5671},
|
||||||
|
Adopted: adopted,
|
||||||
|
// Genesis takes the foundation's modules.
|
||||||
|
Taken: map[string]bool{"postgres": true, "lavinmq": true},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func byID(resources []map[string]any) map[string]map[string]any {
|
||||||
|
out := map[string]map[string]any{}
|
||||||
|
for _, r := range resources {
|
||||||
|
out[r["id"].(string)] = r
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdoptedNodeIsDeclaredOpeningsFromTheSameInputsAsTheFilter(t *testing.T) {
|
||||||
|
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := byID(composed.Resources)
|
||||||
|
want := map[string]map[string]any{
|
||||||
|
// The hub's port, from anywhere, received.
|
||||||
|
"adoption.opening-udp-51820-incoming": {"port": 51820, "protocol": "udp",
|
||||||
|
"from": "everywhere", "path": "incoming"},
|
||||||
|
// The store's port from the private network only, and forwarded: a container publishes it.
|
||||||
|
"adoption.opening-tcp-5432-forwarded": {"port": 5432, "protocol": "tcp", "from": "mesh",
|
||||||
|
"path": "forwarded", "to": 5432},
|
||||||
|
// The bus from anywhere: a node enrols over it before it has a private address.
|
||||||
|
"adoption.opening-tcp-5671-forwarded": {"port": 5671, "protocol": "tcp",
|
||||||
|
"from": "everywhere", "path": "forwarded", "to": 5671},
|
||||||
|
"adoption.opening-tcp-5672-forwarded": {"port": 5672, "protocol": "tcp", "from": "mesh",
|
||||||
|
"path": "forwarded", "to": 5672},
|
||||||
|
// The registry from anywhere, by its node's exposure setting.
|
||||||
|
"adoption.opening-tcp-5000-forwarded": {"port": 5000, "protocol": "tcp",
|
||||||
|
"from": "everywhere", "path": "forwarded", "to": 5000},
|
||||||
|
// A published port names the machine port and the container port it is forwarded to.
|
||||||
|
"adoption.opening-tcp-20001-forwarded": {"port": 20001, "protocol": "tcp",
|
||||||
|
"from": "everywhere", "path": "forwarded", "to": 8080},
|
||||||
|
}
|
||||||
|
for id, fields := range want {
|
||||||
|
opening, ok := got[id]
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("no %s among %v", id, keys(got))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if opening["type"] != "opening" {
|
||||||
|
t.Errorf("%s is a %v", id, opening["type"])
|
||||||
|
}
|
||||||
|
for k, v := range fields {
|
||||||
|
if opening[k] != v {
|
||||||
|
t.Errorf("%s: %s is %v, want %v", id, k, opening[k], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for id := range got {
|
||||||
|
if strings.HasPrefix(id, "adoption.opening-") && want[id] == nil {
|
||||||
|
t.Errorf("an opening nothing asked for: %s", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A port for this machine only opens nothing, and the management port is not opened at all.
|
||||||
|
for id := range got {
|
||||||
|
if strings.Contains(id, "-9000-") || strings.Contains(id, "-15672-") {
|
||||||
|
t.Errorf("%s is opened", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And openings come first, in the order the machine applies them.
|
||||||
|
if !strings.HasPrefix(composed.Resources[0]["id"].(string), "adoption.opening-") {
|
||||||
|
t.Errorf("openings are not first: %v", composed.Resources[0]["id"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdoptedNodeLoadsNoFilterOfTheMeshs(t *testing.T) {
|
||||||
|
composed, err := anAdoptedAnchor().Compose(anchorRendering(true))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, r := range composed.Resources {
|
||||||
|
if r["path"] == "/etc/nftables.conf" || strings.HasPrefix(r["id"].(string), "nftables.") {
|
||||||
|
t.Fatalf("an adopted node is declared the filter module's %v", r["id"])
|
||||||
|
}
|
||||||
|
if content, _ := r["content"].(string); strings.Contains(content, "policy drop") {
|
||||||
|
t.Fatalf("an adopted node is declared a table that drops by default: %v", r["id"])
|
||||||
|
}
|
||||||
|
// Nothing but refusals: the only accept in the guard is its policy.
|
||||||
|
if content, _ := r["content"].(string); r["id"] == GuardID() &&
|
||||||
|
strings.Count(content, "accept") != 1 {
|
||||||
|
t.Fatalf("the guard holds an accept:\n%s", content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got := byID(composed.Resources)
|
||||||
|
guard := got[GuardID()]
|
||||||
|
if guard == nil || got[GuardUnitID()] == nil || got[GuardRunningID()] == nil {
|
||||||
|
t.Fatalf("no guard: %v", keys(got))
|
||||||
|
}
|
||||||
|
if guard["content"] != AsGuard([]int{5432, 5672, 15672}) {
|
||||||
|
t.Fatalf("the guard does not guard the store, the broker and its management port:\n%s",
|
||||||
|
guard["content"])
|
||||||
|
}
|
||||||
|
// The tool that loads it comes first, and the table after it: a node joining adopted has no
|
||||||
|
// filter module and may have no nft.
|
||||||
|
pkg, table := -1, -1
|
||||||
|
for i, r := range composed.Resources {
|
||||||
|
switch r["id"] {
|
||||||
|
case GuardPackageID():
|
||||||
|
pkg = i
|
||||||
|
if r["type"] != "package" || r["package"] != "nftables" {
|
||||||
|
t.Fatalf("the guard's package is %v", r)
|
||||||
|
}
|
||||||
|
case GuardID():
|
||||||
|
table = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pkg < 0 || pkg > table {
|
||||||
|
t.Fatalf("nftables is not declared before the guard's table (%d, %d)", pkg, table)
|
||||||
|
}
|
||||||
|
// A changed table is reloaded — one `nft -f`, atomic — never restarted, which would delete the
|
||||||
|
// table and leave the ports unguarded until it is loaded again. Only a changed unit restarts.
|
||||||
|
if !reflect.DeepEqual(got[GuardRunningID()]["reload-on"], []any{GuardID()}) ||
|
||||||
|
!reflect.DeepEqual(got[GuardRunningID()]["restart-on"], []any{GuardUnitID()}) {
|
||||||
|
t.Fatalf("the guard is not reloaded on its table and restarted on its unit: %v",
|
||||||
|
got[GuardRunningID()])
|
||||||
|
}
|
||||||
|
// Nothing of the mesh's own is anybody's to hold.
|
||||||
|
for id, module := range composed.Owner {
|
||||||
|
if strings.HasPrefix(id, AdoptionPrefix) {
|
||||||
|
t.Fatalf("%s is owned by %s", id, module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAConvergedNodeIsDeclaredItsFilterAndNoOpenings(t *testing.T) {
|
||||||
|
composed, err := anAdoptedAnchor().Compose(anchorRendering(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := byID(composed.Resources)
|
||||||
|
if got["nftables.filtering"] == nil || got["nftables.load"] == nil {
|
||||||
|
t.Fatalf("a converged node lost its filter: %v", keys(got))
|
||||||
|
}
|
||||||
|
for id := range got {
|
||||||
|
if strings.HasPrefix(id, AdoptionPrefix) {
|
||||||
|
t.Fatalf("a converged node is declared %s", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plain, err := anAdoptedAnchor().Declaration(anchorRendering(false))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, _ := json.Marshal(plain)
|
||||||
|
b, _ := json.Marshal(composed.Resources)
|
||||||
|
if string(a) != string(b) {
|
||||||
|
t.Fatal("Compose and Declaration disagree on a converged node")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The table the installer raises and the controller declares, character for character.
|
||||||
|
func TestTheGuardIsExactlyThisTable(t *testing.T) {
|
||||||
|
const golden = `table inet mesh_guard {}
|
||||||
|
delete table inet mesh_guard
|
||||||
|
table inet mesh_guard {
|
||||||
|
chain prerouting {
|
||||||
|
type filter hook prerouting priority raw; policy accept;
|
||||||
|
fib daddr type local iifname != "lo" iifname != "docker0" iifname != "br-*" iifname != "mesh0" tcp dport { 5432, 15672 } drop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`
|
||||||
|
if got := AsGuard([]int{15672, 5432}); got != golden {
|
||||||
|
t.Fatalf("the guard changed:\n%s", got)
|
||||||
|
}
|
||||||
|
const unit = `[Unit]
|
||||||
|
Description=The mesh's guard: refuses its own ports from outside (novox/hq ADR 0100)
|
||||||
|
DefaultDependencies=no
|
||||||
|
Wants=network-pre.target
|
||||||
|
Before=network-pre.target shutdown.target
|
||||||
|
Conflicts=shutdown.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
RemainAfterExit=yes
|
||||||
|
ExecStart=nft -f /etc/mesh/guard.nft
|
||||||
|
ExecReload=nft -f /etc/mesh/guard.nft
|
||||||
|
ExecStop=nft delete table inet mesh_guard
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
`
|
||||||
|
if got := GuardUnitText(); got != unit {
|
||||||
|
t.Fatalf("the guard's unit changed:\n%s", got)
|
||||||
|
}
|
||||||
|
if GuardResources(nil) != nil {
|
||||||
|
t.Fatal("a guard with nothing to guard is an empty set nft refuses to load")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGuardedPortMustBeAPort(t *testing.T) {
|
||||||
|
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[5432]}`)); err != nil {
|
||||||
|
t.Fatalf("guards is refused: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := ParseManifest([]byte(`{"module":"postgres","guards":[0]}`)); err == nil {
|
||||||
|
t.Fatal("guarding port 0 was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func keys[V any](m map[string]V) []string {
|
||||||
|
return sortedKeys(m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: the foundation's ports are the node's. Given 5433 for the store, every place
|
||||||
|
// that uses the port reads it from there: the container, the filter, the openings, the guard.
|
||||||
|
func TestAGivenPortIsUsedEverywhereThePortIs(t *testing.T) {
|
||||||
|
given := map[string]map[int]int{"postgres": {5432: 5433}, "lavinmq": {15672: 15673}}
|
||||||
|
for _, adopted := range []bool{true, false} {
|
||||||
|
with := anchorRendering(adopted)
|
||||||
|
with.Given = given
|
||||||
|
with.Ports["postgres"] = map[int]int{5432: 5433}
|
||||||
|
composed, err := anAdoptedAnchor().Compose(with)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := byID(composed.Resources)
|
||||||
|
if ports := got["postgres.server"]["ports"]; !reflect.DeepEqual(ports, []any{"5433:5432"}) {
|
||||||
|
t.Fatalf("the store's container publishes %v", ports)
|
||||||
|
}
|
||||||
|
if ports := got["lavinmq.server"]["ports"]; !reflect.DeepEqual(ports,
|
||||||
|
[]any{"5671:5671", "5672:5672", "127.0.0.1:15673:15672"}) {
|
||||||
|
t.Fatalf("the broker's container publishes %v", ports)
|
||||||
|
}
|
||||||
|
if !adopted {
|
||||||
|
filter, _ := got["nftables.filtering"]["content"].(string)
|
||||||
|
if !strings.Contains(filter, "tcp dport 5433 accept") || strings.Contains(filter, "5432") {
|
||||||
|
t.Fatalf("the filter does not use the given port:\n%s", filter)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if o := got["adoption.opening-tcp-5433-forwarded"]; o == nil || o["to"] != 5432 {
|
||||||
|
t.Fatalf("no opening for the given port: %v", keys(got))
|
||||||
|
}
|
||||||
|
if guard := got[GuardID()]["content"]; guard != AsGuard([]int{5433, 5672, 15673}) {
|
||||||
|
t.Fatalf("the guard does not guard the given ports:\n%s", guard)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGivenPortIsTheNodesAndReachesSomething(t *testing.T) {
|
||||||
|
store := anAdoptedAnchor().Modules[1]
|
||||||
|
node := func(v any) []Layer {
|
||||||
|
return []Layer{{From: "anchor", Values: map[string]any{PortsSetting: v}}}
|
||||||
|
}
|
||||||
|
if got, err := GivenPorts(store, node(map[string]any{"5432": float64(5433)})); err != nil ||
|
||||||
|
got[5432] != 5433 {
|
||||||
|
t.Fatalf("a node's given port was not read: %v %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := GivenPorts(store, []Layer{{From: MeshWideLayer,
|
||||||
|
Values: map[string]any{PortsSetting: map[string]any{"5432": float64(5433)}}}}); err == nil {
|
||||||
|
t.Fatal("a port given for the whole mesh was accepted")
|
||||||
|
}
|
||||||
|
if _, err := GivenPorts(store, node(map[string]any{"6000": float64(6001)})); err == nil {
|
||||||
|
t.Fatal("a port the module neither listens on, publishes nor guards was given")
|
||||||
|
}
|
||||||
|
if _, err := GivenPorts(store, node(map[string]any{"5432": float64(70000)})); err == nil {
|
||||||
|
t.Fatal("a machine port that is not a port was given")
|
||||||
|
}
|
||||||
|
if _, err := GivenPorts(store, node(map[string]any{"5432": float64(22)})); err == nil {
|
||||||
|
t.Fatal("ssh's port was given")
|
||||||
|
}
|
||||||
|
broker := anAdoptedAnchor().Modules[2]
|
||||||
|
if _, err := GivenPorts(broker, node(map[string]any{"5671": float64(5700),
|
||||||
|
"5672": float64(5700)})); err == nil {
|
||||||
|
t.Fatal("one machine port was given for two of the module's ports")
|
||||||
|
}
|
||||||
|
if stray := UnusedSettings(store, node(map[string]any{"5432": float64(5433)})); len(stray) != 0 {
|
||||||
|
t.Fatalf("a given port is called stray: %v", stray)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0103: the guard is derived, and from taken modules only — every machine port a taken
|
||||||
|
// module publishes that the filter admits from the private network only, and the ports its
|
||||||
|
// manifest guards. A module assigned but not taken is not guarded: its port may still be the
|
||||||
|
// predecessor's.
|
||||||
|
func TestTheGuardIsDerivedFromTakenModulesOnly(t *testing.T) {
|
||||||
|
guardOf := func(with Rendering) string {
|
||||||
|
t.Helper()
|
||||||
|
composed, err := anAdoptedAnchor().Compose(with)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
content, _ := byID(composed.Resources)[GuardID()]["content"].(string)
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
|
||||||
|
// The broker taken, the store not: the broker's plain port follows from its listens (from
|
||||||
|
// the mesh, published), its management port from its manifest; the store is not guarded, and
|
||||||
|
// neither is the bus, which the mesh needs from everywhere.
|
||||||
|
with := anchorRendering(true)
|
||||||
|
with.Taken = map[string]bool{"lavinmq": true}
|
||||||
|
if got := guardOf(with); got != AsGuard([]int{5672, 15672}) {
|
||||||
|
t.Fatalf("the guard is not the taken broker's ports:\n%s", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A taken module publishing a port admitted from everywhere is not guarded; one admitted from
|
||||||
|
// the mesh is. The registry is exposed everywhere on this node, and hello-web listens from
|
||||||
|
// everywhere.
|
||||||
|
with.Taken = map[string]bool{"distribution": true, "hello-web": true}
|
||||||
|
if got := guardOf(with); got != "" {
|
||||||
|
t.Fatalf("a port admitted from everywhere is guarded:\n%s", got)
|
||||||
|
}
|
||||||
|
with.Settings = nil
|
||||||
|
if got := guardOf(with); got != AsGuard([]int{5000}) {
|
||||||
|
t.Fatalf("the registry, from the mesh only, is not guarded:\n%s", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing taken, nothing guarded — and no guard at all rather than an empty set.
|
||||||
|
with = anchorRendering(true)
|
||||||
|
with.Taken = nil
|
||||||
|
if got := guardOf(with); got != "" {
|
||||||
|
t.Fatalf("an untaken store is guarded:\n%s", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A given port is followed: where the machine put it is what is refused.
|
||||||
|
with = anchorRendering(true)
|
||||||
|
with.Given = map[string]map[int]int{"lavinmq": {5672: 5682, 15672: 15673}}
|
||||||
|
with.Ports["lavinmq"] = map[int]int{5671: 5671, 5672: 5682}
|
||||||
|
if got := guardOf(with); got != AsGuard([]int{5432, 5682, 15673}) {
|
||||||
|
t.Fatalf("the guard does not follow the given ports:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A mapping bound to loopback is not published to anything off the machine — in either address
|
||||||
|
// family — and an address's own colons never shift the ports.
|
||||||
|
func TestPublishedLeavesOutLoopbackInBothFamilies(t *testing.T) {
|
||||||
|
got := Published([]map[string]any{{"type": "container", "ports": []any{
|
||||||
|
"127.0.0.1:15672:15672", "[::1]:8080:80", "localhost:9090:90",
|
||||||
|
"[::]:8443:443", "0.0.0.0:5000:5000", "5353:53/udp"}}})
|
||||||
|
want := map[string]map[int]int{"tcp": {8443: 443, 5000: 5000}, "udp": {5353: 53}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("published is %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0038: only a published port is the mesh's to move. A module that binds the machine
|
||||||
|
// itself listens where its software was told to, so giving it a machine port is refused.
|
||||||
|
func TestAGivenPortIsRefusedForAPortNoContainerPublishes(t *testing.T) {
|
||||||
|
onTheMachine := Manifest{Module: "daemon",
|
||||||
|
Listens: []Listening{{Port: 9000, From: FromMesh}}, Guards: []int{9000}}
|
||||||
|
layers := []Layer{{From: "node anchor",
|
||||||
|
Values: map[string]any{PortsSetting: map[string]any{"9000": float64(9100)}}}}
|
||||||
|
_, err := GivenPorts(onTheMachine, layers)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "does not publish") {
|
||||||
|
t.Fatalf("a port no container publishes was given: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0103: a guarded port this node is told to open to everyone is opened, not guarded.
|
||||||
|
// An opening from everywhere beside a guard dropping it is one statement refusing the other.
|
||||||
|
func TestAGuardedPortOpenedToEveryoneIsNotGuarded(t *testing.T) {
|
||||||
|
with := anchorRendering(true)
|
||||||
|
with.Settings["postgres"] = []Layer{{From: "node anchor",
|
||||||
|
Values: map[string]any{ExposeSetting: map[string]any{"5432": FromEverywhere}}}}
|
||||||
|
composed, err := anAdoptedAnchor().Compose(with)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := byID(composed.Resources)
|
||||||
|
if o := got["adoption.opening-tcp-5432-forwarded"]; o == nil || o["from"] != OpeningFromEverywhere {
|
||||||
|
t.Fatalf("the store's port is not opened to everyone: %v", o)
|
||||||
|
}
|
||||||
|
if guard, _ := got[GuardID()]["content"].(string); guard != AsGuard([]int{5672, 15672}) {
|
||||||
|
t.Fatalf("a port opened to everyone is still guarded:\n%s", guard)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package catalogue
|
package catalogue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -157,3 +158,47 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reloading answers the runtime's trust as the networking module does (novox/hq ADR 0102): a file
|
||||||
|
// written into, and the runtime reloaded on it.
|
||||||
|
type reloading struct{}
|
||||||
|
|
||||||
|
func (reloading) Resources(node string) ([]map[string]any, bool, error) {
|
||||||
|
return []map[string]any{
|
||||||
|
{"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json",
|
||||||
|
"merge": MergeJSON, "into": "json", "content": `{"insecure-registries":["r:5000"]}`},
|
||||||
|
{"id": "registry-trust-reload", "type": "service", "unit": "docker.service",
|
||||||
|
"state": "running", "reload-on": []string{"registry-trust"}},
|
||||||
|
}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWhatAServiceIsReloadedOnIsNamedAsTheHostWillSeeIt(t *testing.T) {
|
||||||
|
// Ids are prefixed with their module on the way out. An unprefixed reload-on would name a
|
||||||
|
// resource the host never sees, and the runtime would never be reloaded for its trust.
|
||||||
|
got, err := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), World{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out, err := got.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": reloading{}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var file, service map[string]any
|
||||||
|
for _, r := range out {
|
||||||
|
switch r["type"] {
|
||||||
|
case "file":
|
||||||
|
file = r
|
||||||
|
case "service":
|
||||||
|
service = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if file == nil || service == nil {
|
||||||
|
t.Fatalf("got %v", out)
|
||||||
|
}
|
||||||
|
if file["into"] != "json" {
|
||||||
|
t.Errorf("into did not reach the host: %v", file)
|
||||||
|
}
|
||||||
|
if want := "[" + file["id"].(string) + "]"; fmt.Sprint(service["reload-on"]) != want {
|
||||||
|
t.Errorf("reload-on names %v, the file is %v", service["reload-on"], file["id"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -128,12 +128,30 @@ type Rendering struct {
|
|||||||
// set, for what a consumer is told, and for what the runtime publishes — and nothing checked
|
// set, for what a consumer is told, and for what the runtime publishes — and nothing checked
|
||||||
// that the three agreed. They are all derived from this.
|
// that the three agreed. They are all derived from this.
|
||||||
Ports map[string]map[int]int
|
Ports map[string]map[int]int
|
||||||
|
|
||||||
|
// Adopted says the node is adopted (novox/hq ADR 0100): the firewall found on it stays in
|
||||||
|
// force, so no module that loads a filter is declared there, and what the mesh needs
|
||||||
|
// reachable is declared as openings, with its own ports guarded by a table that only refuses.
|
||||||
|
Adopted bool
|
||||||
|
|
||||||
|
// Given is the machine ports this node was given for its modules' ports, by module and by the
|
||||||
|
// port the software uses (novox/hq ADR 0100) — the foundation's ports, as genesis chose them.
|
||||||
|
// They win over anything the mesh would assign and over a manifest's own long-form mapping.
|
||||||
|
Given map[string]map[int]int
|
||||||
|
|
||||||
|
// Taken is the modules taken on this adopted node (novox/hq ADR 0100). The guard is derived
|
||||||
|
// from these only (ADR 0103): a port of a module assigned but not taken may still be the
|
||||||
|
// predecessor's.
|
||||||
|
Taken map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
|
// machinePort is where a module's port lives on this machine, or the port itself when the mesh has
|
||||||
// not been asked. Unassigned is not an error here: a module with no `listens` never needed one,
|
// not been asked. Unassigned is not an error here: a module with no `listens` never needed one,
|
||||||
// and a caller composing a declaration without a store still gets something coherent.
|
// and a caller composing a declaration without a store still gets something coherent.
|
||||||
func (r Rendering) machinePort(module string, wanted int) int {
|
func (r Rendering) machinePort(module string, wanted int) int {
|
||||||
|
if at, given := r.Given[module][wanted]; given {
|
||||||
|
return at
|
||||||
|
}
|
||||||
if at, known := r.Ports[module][wanted]; known {
|
if at, known := r.Ports[module][wanted]; known {
|
||||||
return at
|
return at
|
||||||
}
|
}
|
||||||
@@ -146,6 +164,34 @@ func (r Rendering) machinePort(module string, wanted int) int {
|
|||||||
// both call something "config", and without this the second would silently replace the first —
|
// both call something "config", and without this the second would silently replace the first —
|
||||||
// the node applying one of them and reporting success.
|
// the node applying one of them and reporting success.
|
||||||
func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
||||||
|
composed, err := r.Compose(with)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return composed.Resources, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Composed is a declaration's resources and which module each came from.
|
||||||
|
//
|
||||||
|
// Owner is kept beside the resources because a resource id cannot be split back into its module:
|
||||||
|
// a module's name may itself contain a dot. What the mesh adds of its own — an opening, the guard —
|
||||||
|
// has no owner.
|
||||||
|
type Composed struct {
|
||||||
|
Resources []map[string]any
|
||||||
|
Owner map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compose is Declaration with the owner of every resource said.
|
||||||
|
func (r Resolution) Compose(with Rendering) (Composed, error) {
|
||||||
|
owner := map[string]string{}
|
||||||
|
resources, err := r.compose(with, owner)
|
||||||
|
if err != nil {
|
||||||
|
return Composed{}, err
|
||||||
|
}
|
||||||
|
return Composed{Resources: resources, Owner: owner}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r Resolution) compose(with Rendering, owner map[string]string) ([]map[string]any, error) {
|
||||||
// Where each provision's credentials land, so a contribution can name the file rather than
|
// Where each provision's credentials land, so a contribution can name the file rather than
|
||||||
// carry a value the mesh does not have.
|
// carry a value the mesh does not have.
|
||||||
directories := map[string]string{}
|
directories := map[string]string{}
|
||||||
@@ -211,17 +257,7 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
|||||||
// Once, from every module's listens -- not per module. A module receiving only its own ports
|
// Once, from every module's listens -- not per module. A module receiving only its own ports
|
||||||
// would write a rule set that closed every other module on the machine. Each module's per-node
|
// would write a rule set that closed every other module on the machine. Each module's per-node
|
||||||
// exposure settings override its listens' source first (novox/hq ADR 0046).
|
// exposure settings override its listens' source first (novox/hq ADR 0046).
|
||||||
exposure := map[string]map[int]string{}
|
rules, err := r.Rules(with)
|
||||||
for _, m := range r.Modules {
|
|
||||||
e, err := Exposure(m, with.Settings[m.Module])
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if e != nil {
|
|
||||||
exposure[m.Module] = e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rules, err := r.Filtering(with.Generators, with.Ports, exposure)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -229,6 +265,13 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
|||||||
|
|
||||||
var out []map[string]any
|
var out []map[string]any
|
||||||
for _, m := range r.Modules {
|
for _, m := range r.Modules {
|
||||||
|
if with.Adopted && m.Filtering != nil {
|
||||||
|
// Nothing of a module that loads a filter, on an adopted node: its table would drop
|
||||||
|
// by default and hold accepts, and the found firewall stays in force. Every resource,
|
||||||
|
// not only the rule set — its service must not run, and a node returned to adopted
|
||||||
|
// stops it by the ordinary removal of what is no longer declared.
|
||||||
|
continue
|
||||||
|
}
|
||||||
resources := m.Resources
|
resources := m.Resources
|
||||||
|
|
||||||
// What the mesh computes for this module goes FIRST, before the module's own resources.
|
// What the mesh computes for this module goes FIRST, before the module's own resources.
|
||||||
@@ -521,6 +564,12 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
|||||||
if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil {
|
if renamed := reflectsRenamed(m.Module, resource["restart-on"]); renamed != nil {
|
||||||
copied["restart-on"] = renamed
|
copied["restart-on"] = renamed
|
||||||
}
|
}
|
||||||
|
// And what it is reloaded on, by the same rule (novox/hq ADR 0102): an id left
|
||||||
|
// unprefixed matches nothing, and the service is never reloaded.
|
||||||
|
if renamed := reflectsRenamed(m.Module, resource["reload-on"]); renamed != nil {
|
||||||
|
copied["reload-on"] = renamed
|
||||||
|
}
|
||||||
|
owner[fmt.Sprint(copied["id"])] = m.Module
|
||||||
out = append(out, copied)
|
out = append(out, copied)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -534,12 +583,138 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
|
|||||||
}
|
}
|
||||||
for _, fact := range given {
|
for _, fact := range given {
|
||||||
fact["id"] = m.Module + "." + fmt.Sprint(fact["id"])
|
fact["id"] = m.Module + "." + fmt.Sprint(fact["id"])
|
||||||
|
owner[fmt.Sprint(fact["id"])] = m.Module
|
||||||
out = append(out, fact)
|
out = append(out, fact)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if with.Adopted {
|
||||||
|
// First, before anything a module declares: what the mesh needs reachable, then its guard.
|
||||||
|
// The order a machine applies is the order written here.
|
||||||
|
ours := Openings(rules, with.Foundation, Published(out))
|
||||||
|
ours = append(ours, GuardResources(r.guarded(out, owner, rules, with))...)
|
||||||
|
out = append(ours, out...)
|
||||||
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// guarded is what the mesh's guard refuses on an adopted node (novox/hq ADR 0103): derived, and
|
||||||
|
// for taken modules only.
|
||||||
|
//
|
||||||
|
// For each taken module, every machine port its containers publish that the filter would admit
|
||||||
|
// from the private network only — a published port is forwarded, not received, so a found
|
||||||
|
// firewall filtering only what it receives never sees it — together with the ports the module's
|
||||||
|
// manifest guards explicitly (the store's port, the broker's management port), wherever the
|
||||||
|
// machine put them. A port of a module assigned but not taken is not guarded: it may still be the
|
||||||
|
// predecessor's, serving the predecessor's other machines. The foundation's ports are admitted
|
||||||
|
// from everywhere and are never guarded.
|
||||||
|
func (r Resolution) guarded(out []map[string]any, owner map[string]string, rules []Rule,
|
||||||
|
with Rendering) []int {
|
||||||
|
meshOnly := map[int]bool{}
|
||||||
|
fromEverywhere := map[int]bool{}
|
||||||
|
for _, rule := range rules {
|
||||||
|
if rule.Protocol != "tcp" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch rule.From {
|
||||||
|
case FromMesh:
|
||||||
|
meshOnly[rule.Port] = true
|
||||||
|
case FromEverywhere:
|
||||||
|
fromEverywhere[rule.Port] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, port := range with.Foundation {
|
||||||
|
delete(meshOnly, port)
|
||||||
|
fromEverywhere[port] = true
|
||||||
|
}
|
||||||
|
seen := map[int]bool{}
|
||||||
|
var ports []int
|
||||||
|
guard := func(at int) {
|
||||||
|
if !seen[at] {
|
||||||
|
seen[at] = true
|
||||||
|
ports = append(ports, at)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, m := range r.Modules {
|
||||||
|
if !with.Taken[m.Module] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var mine []map[string]any
|
||||||
|
for _, resource := range out {
|
||||||
|
if owner[fmt.Sprint(resource["id"])] == m.Module {
|
||||||
|
mine = append(mine, resource)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for outer := range Published(mine)["tcp"] {
|
||||||
|
if meshOnly[outer] {
|
||||||
|
guard(outer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, want := range m.Guards {
|
||||||
|
at := with.machinePort(m.Module, want)
|
||||||
|
for _, resource := range mine {
|
||||||
|
if fmt.Sprint(resource["type"]) != "container" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
listed, _ := resource["ports"].([]any)
|
||||||
|
for _, entry := range listed {
|
||||||
|
outer, inner, _, ok := mapping(fmt.Sprint(entry))
|
||||||
|
if ok && inner == want {
|
||||||
|
at = outer
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Not what this node is told to open to everyone: a per-node exposure setting that
|
||||||
|
// widens a guarded port is the operator saying so, and declaring an opening for it
|
||||||
|
// and a guard dropping it would be one statement refusing the other.
|
||||||
|
if !fromEverywhere[at] {
|
||||||
|
guard(at)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Ints(ports)
|
||||||
|
return ports
|
||||||
|
}
|
||||||
|
|
||||||
|
// mapping reads a container's port mapping — `[address:]outer:inner[/protocol]`, the address
|
||||||
|
// possibly an IPv6 one in brackets — indexing from the end, so an address's own colons never
|
||||||
|
// shift the ports. Not ok for a short form or anything that is not a mapping.
|
||||||
|
func mapping(written string) (outer, inner int, address string, ok bool) {
|
||||||
|
written = strings.TrimSpace(written)
|
||||||
|
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||||
|
written = written[:cut]
|
||||||
|
}
|
||||||
|
parts := strings.Split(written, ":")
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return 0, 0, "", false
|
||||||
|
}
|
||||||
|
inner, err := strconv.Atoi(parts[len(parts)-1])
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, "", false
|
||||||
|
}
|
||||||
|
outer, err = strconv.Atoi(parts[len(parts)-2])
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, "", false
|
||||||
|
}
|
||||||
|
return outer, inner, strings.Join(parts[:len(parts)-2], ":"), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rules is the rule set this node's filter is derived from: every module's listens, what was
|
||||||
|
// computed for this machine, and each module's per-node exposure. The same answer whether the node
|
||||||
|
// is adopted or converged — the one loads it as a filter, the other declares it as openings.
|
||||||
|
func (r Resolution) Rules(with Rendering) ([]Rule, error) {
|
||||||
|
exposure := map[string]map[int]string{}
|
||||||
|
for _, m := range r.Modules {
|
||||||
|
e, err := Exposure(m, with.Settings[m.Module])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if e != nil {
|
||||||
|
exposure[m.Module] = e
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r.Filtering(with.Generators, with.Ports, exposure)
|
||||||
|
}
|
||||||
|
|
||||||
// Contribution is one module telling the answer to a requirement what it needs from it.
|
// Contribution is one module telling the answer to a requirement what it needs from it.
|
||||||
type Contribution struct {
|
type Contribution struct {
|
||||||
// From is the module that said it, so the provider and a person reading the file can tell
|
// From is the module that said it, so the provider and a person reading the file can tell
|
||||||
@@ -1094,7 +1269,11 @@ func publishedOn(resource map[string]any, module string, with Rendering) {
|
|||||||
for _, entry := range listed {
|
for _, entry := range listed {
|
||||||
written := fmt.Sprint(entry)
|
written := fmt.Sprint(entry)
|
||||||
if strings.Contains(written, ":") {
|
if strings.Contains(written, ":") {
|
||||||
out = append(out, written)
|
// Written the long way, and left alone — unless this node was given a machine port for
|
||||||
|
// it (novox/hq ADR 0100): the foundation's ports are the node's, and a manifest's
|
||||||
|
// number is only the default. The outer port only; an address and the software's
|
||||||
|
// port stay as written.
|
||||||
|
out = append(out, givenOuter(written, with.Given[module]))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
wanted, err := strconv.Atoi(strings.TrimSpace(written))
|
wanted, err := strconv.Atoi(strings.TrimSpace(written))
|
||||||
@@ -1109,6 +1288,29 @@ func publishedOn(resource map[string]any, module string, with Rendering) {
|
|||||||
resource["ports"] = out
|
resource["ports"] = out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// givenOuter rewrites the machine side of a long-form mapping to the port this node was given for
|
||||||
|
// its software side, when it was given one.
|
||||||
|
func givenOuter(written string, given map[int]int) string {
|
||||||
|
if len(given) == 0 {
|
||||||
|
return written
|
||||||
|
}
|
||||||
|
mapping, protocol := written, ""
|
||||||
|
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||||
|
mapping, protocol = written[:cut], written[cut:]
|
||||||
|
}
|
||||||
|
parts := strings.Split(mapping, ":")
|
||||||
|
inner, err := strconv.Atoi(strings.TrimSpace(parts[len(parts)-1]))
|
||||||
|
if err != nil {
|
||||||
|
return written
|
||||||
|
}
|
||||||
|
at, ok := given[inner]
|
||||||
|
if !ok {
|
||||||
|
return written
|
||||||
|
}
|
||||||
|
parts[len(parts)-2] = strconv.Itoa(at)
|
||||||
|
return strings.Join(parts, ":") + protocol
|
||||||
|
}
|
||||||
|
|
||||||
// ServedOn is what a provider tells a consumer, with the port that machine actually uses.
|
// ServedOn is what a provider tells a consumer, with the port that machine actually uses.
|
||||||
//
|
//
|
||||||
// **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three
|
// **The module writes the port once, in `listens`** (novox/hq ADR 0038). It used to write it three
|
||||||
|
|||||||
@@ -457,3 +457,108 @@ func byFamily(addresses []string) (four []string, six []string) {
|
|||||||
}
|
}
|
||||||
return four, six
|
return four, six
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PortsSetting is the settings key that gives a module's port a machine port on one node (novox/hq
|
||||||
|
// ADR 0100):
|
||||||
|
//
|
||||||
|
// {"ports": {"5432": 5433}}
|
||||||
|
//
|
||||||
|
// puts what the software calls 5432 on the machine's 5433. The foundation's ports are the node's:
|
||||||
|
// every one is an input to genesis, checked free there, and becomes that node's setting for the
|
||||||
|
// foundation's modules — the catalogue's numbers are only their defaults. Keyed by the port the
|
||||||
|
// software uses, like expose; the value is where the machine puts it.
|
||||||
|
const PortsSetting = "ports"
|
||||||
|
|
||||||
|
// MeshWideLayer is what a layer set for the whole mesh is called, rather than for one node.
|
||||||
|
const MeshWideLayer = "the mesh"
|
||||||
|
|
||||||
|
// GivenPorts reads a module's given machine ports from its settings: software port → machine port.
|
||||||
|
//
|
||||||
|
// Refused from a mesh-wide layer — a port is a fact about one machine, and one number for every
|
||||||
|
// machine is the collision this exists to avoid — and for a port the module's containers do not
|
||||||
|
// publish.
|
||||||
|
//
|
||||||
|
// **Only a published port is the mesh's to move** (novox/hq ADR 0038). A container's mapping is
|
||||||
|
// what translates; a module binding the machine's network directly binds the number its software
|
||||||
|
// was configured with, and moving that number would put it in the filter, in the openings and in
|
||||||
|
// what consumers are told while the software still listens on the old one — a port that reads as
|
||||||
|
// moved and is not.
|
||||||
|
func GivenPorts(m Manifest, layers []Layer) (map[int]int, error) {
|
||||||
|
known := map[int]bool{}
|
||||||
|
for _, p := range containerPorts(m) {
|
||||||
|
known[p] = true
|
||||||
|
}
|
||||||
|
out := map[int]int{}
|
||||||
|
for _, layer := range layers {
|
||||||
|
raw, ok := layer.Values[PortsSetting]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if layer.From == MeshWideLayer {
|
||||||
|
return nil, fmt.Errorf("%s: %s is given per node — a port is a fact about one "+
|
||||||
|
"machine; set it with --node", m.Module, PortsSetting)
|
||||||
|
}
|
||||||
|
entries, ok := raw.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%s: %s is a { port: machine-port } map, and %q set it to "+
|
||||||
|
"something else", m.Module, PortsSetting, layer.From)
|
||||||
|
}
|
||||||
|
for portText, value := range entries {
|
||||||
|
port, err := strconv.Atoi(portText)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%s gives %q a port, which is not a port", m.Module, portText)
|
||||||
|
}
|
||||||
|
if !known[port] {
|
||||||
|
return nil, fmt.Errorf("%s gives port %d a machine port, and no container of its "+
|
||||||
|
"publishes %d — the mesh cannot move a port the module does not publish; the "+
|
||||||
|
"software would go on listening where it was told to", m.Module, port, port)
|
||||||
|
}
|
||||||
|
at, ok := asPort(value)
|
||||||
|
if !ok || at < 1 || at > 65535 {
|
||||||
|
return nil, fmt.Errorf("%s gives port %d the machine port %v, which is not a port",
|
||||||
|
m.Module, port, value)
|
||||||
|
}
|
||||||
|
if at == SSHPort {
|
||||||
|
return nil, fmt.Errorf("%s gives port %d the machine port %d, which is ssh's — the "+
|
||||||
|
"one port a machine may never lose", m.Module, port, at)
|
||||||
|
}
|
||||||
|
out[port] = at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// One holder per machine port, within the module too.
|
||||||
|
holder := map[int]int{}
|
||||||
|
for port, at := range out {
|
||||||
|
if other, twice := holder[at]; twice {
|
||||||
|
return nil, fmt.Errorf("%s gives machine port %d to both its %d and its %d", m.Module,
|
||||||
|
at, min(port, other), max(port, other))
|
||||||
|
}
|
||||||
|
holder[at] = port
|
||||||
|
}
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// containerPorts are the software ports a module's containers publish, whichever form they are
|
||||||
|
// written in.
|
||||||
|
func containerPorts(m Manifest) []int {
|
||||||
|
var out []int
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if fmt.Sprint(r["type"]) != "container" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
listed, _ := r["ports"].([]any)
|
||||||
|
for _, entry := range listed {
|
||||||
|
written := strings.TrimSpace(fmt.Sprint(entry))
|
||||||
|
if cut := strings.LastIndex(written, "/"); cut >= 0 {
|
||||||
|
written = written[:cut]
|
||||||
|
}
|
||||||
|
parts := strings.Split(written, ":")
|
||||||
|
if n, err := strconv.Atoi(parts[len(parts)-1]); err == nil {
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
package catalogue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The catalogue's foundation modules as they are, parsed by the real parser (novox/hq ADR 0100):
|
||||||
|
// the store and the broker say which of their ports the mesh guards on an adopted node, and the
|
||||||
|
// filter module loads its table through a unit of its own whose stop deletes only that table.
|
||||||
|
func catalogueManifest(t *testing.T, module string) Manifest {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile("../../../mesh-catalog/modules/" + module + "/module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||||
|
}
|
||||||
|
m, err := ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s does not parse:\n%v", module, err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheStoreAndTheBrokerSayWhatTheMeshGuards(t *testing.T) {
|
||||||
|
if got := catalogueManifest(t, "postgres").Guards; !reflect.DeepEqual(got, []int{5432}) {
|
||||||
|
t.Errorf("postgres guards %v; the store's port must be refused from outside", got)
|
||||||
|
}
|
||||||
|
if got := catalogueManifest(t, "lavinmq").Guards; !reflect.DeepEqual(got, []int{15672}) {
|
||||||
|
t.Errorf("lavinmq guards %v; the management port must be refused from outside", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheFilterModuleNeverFlushesTheRuleset(t *testing.T) {
|
||||||
|
m := catalogueManifest(t, "nftables")
|
||||||
|
var unit, stock, load map[string]any
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
switch r["id"] {
|
||||||
|
case "unit":
|
||||||
|
unit = r
|
||||||
|
case "stock-unit-stop":
|
||||||
|
stock = r
|
||||||
|
case "load":
|
||||||
|
load = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if load == nil || load["unit"] != "mesh-filter.service" {
|
||||||
|
t.Fatalf("the filter is not loaded by its own unit: %v", load)
|
||||||
|
}
|
||||||
|
content, _ := unit["content"].(string)
|
||||||
|
if unit == nil || unit["path"] != "/etc/systemd/system/mesh-filter.service" {
|
||||||
|
t.Fatalf("the filter's unit is not written: %v", unit)
|
||||||
|
}
|
||||||
|
if strings.Contains(content, "flush") {
|
||||||
|
t.Fatalf("stopping the filter flushes the whole ruleset — the runtime's and the found "+
|
||||||
|
"firewall's with it:\n%s", content)
|
||||||
|
}
|
||||||
|
if !strings.Contains(content, "ExecStop=nft delete table inet mesh\n") ||
|
||||||
|
!strings.Contains(content, "ExecStart=nft -f "+m.Filtering.Into+"\n") {
|
||||||
|
t.Fatalf("the unit does not load the computed rule set and delete only its own table:\n%s",
|
||||||
|
content)
|
||||||
|
}
|
||||||
|
// A node converged before the filter had its own unit still has the stock nftables.service
|
||||||
|
// enabled, whose stop flushes the whole ruleset: a drop-in makes it delete only the mesh's
|
||||||
|
// table, and the load is restarted on it so the host reloads units and the drop-in is read.
|
||||||
|
if stock == nil || stock["path"] != "/etc/systemd/system/nftables.service.d/mesh.conf" ||
|
||||||
|
!strings.HasSuffix(fmt.Sprint(stock["content"]),
|
||||||
|
"[Service]\nExecStop=\nExecStop=nft delete table inet mesh\n") {
|
||||||
|
t.Fatalf("the stock unit's stop is not replaced with deleting the mesh's table: %v", stock)
|
||||||
|
}
|
||||||
|
// A changed rule set is RELOADED — ExecReload replaces the table in one `nft -f`, so the node
|
||||||
|
// is never unfiltered — and only the units themselves restart it, which is the one change a
|
||||||
|
// reload cannot carry.
|
||||||
|
if !reflect.DeepEqual(load["reload-on"], []any{"filtering"}) {
|
||||||
|
t.Fatalf("the filter is restarted rather than reloaded when its rules change, leaving the "+
|
||||||
|
"node unfiltered in between: %v", load)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(load["restart-on"], []any{"unit", "stock-unit-stop"}) {
|
||||||
|
t.Fatalf("the filter is not restarted when its unit or the stock unit's drop-in changes: %v",
|
||||||
|
load["restart-on"])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -346,6 +346,14 @@ type Manifest struct {
|
|||||||
// that could only see its own ports would write a rule set that closed everything else.
|
// that could only see its own ports would write a rule set that closed everything else.
|
||||||
Filtering *Filtering `json:"filtering,omitempty"`
|
Filtering *Filtering `json:"filtering,omitempty"`
|
||||||
|
|
||||||
|
// Guards are ports of this module's the mesh refuses on an adopted node except from the
|
||||||
|
// private network and from the machine itself (novox/hq ADR 0100) — the store's port and the
|
||||||
|
// broker's management port. The ports the software uses; the mesh guards where the machine
|
||||||
|
// publishes them. On an adopted node the found firewall stays in force and the mesh loads no
|
||||||
|
// filter of its own, so this is what keeps them unreachable from outside whatever that
|
||||||
|
// firewall does. Ignored on a converged node, whose derived filter already closes them.
|
||||||
|
Guards []int `json:"guards,omitempty"`
|
||||||
|
|
||||||
// Facts are things only the mesh knows, written where this module asks for them.
|
// Facts are things only the mesh knows, written where this module asks for them.
|
||||||
//
|
//
|
||||||
// **The graph is the control plane's; how a machine uses it is the module's.** The mesh knows
|
// **The graph is the control plane's; how a machine uses it is the module's.** The mesh knows
|
||||||
@@ -950,6 +958,12 @@ func ParseManifest(raw []byte) (Manifest, error) {
|
|||||||
"%s listens on %d over %q, which is tcp or udp", m.Module, l.Port, p))
|
"%s listens on %d over %q, which is tcp or udp", m.Module, l.Port, p))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, port := range m.Guards {
|
||||||
|
if port < 1 || port > 65535 {
|
||||||
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
"%s guards port %d, which is not a port", m.Module, port))
|
||||||
|
}
|
||||||
|
}
|
||||||
if c := m.Certificate; c != nil {
|
if c := m.Certificate; c != nil {
|
||||||
if !strings.HasPrefix(c.Into, "/") {
|
if !strings.HasPrefix(c.Into, "/") {
|
||||||
problems = append(problems, fmt.Sprintf(
|
problems = append(problems, fmt.Sprintf(
|
||||||
|
|||||||
@@ -101,6 +101,11 @@ func settle(base map[string]any, layers []Layer, protected map[string]bool, what
|
|||||||
merged := deepCopy(base)
|
merged := deepCopy(base)
|
||||||
for _, layer := range layers {
|
for _, layer := range layers {
|
||||||
for key, value := range layer.Values {
|
for key, value := range layer.Values {
|
||||||
|
if key == PortsSetting {
|
||||||
|
// Where the machine puts a port is the mesh's to apply, not a value for a file or
|
||||||
|
// for what a consumer is told (novox/hq ADR 0100); it reaches both as the port.
|
||||||
|
continue
|
||||||
|
}
|
||||||
if protected[key] {
|
if protected[key] {
|
||||||
// The module said it must own this one. Refused rather than ignored: a setting
|
// The module said it must own this one. Refused rather than ignored: a setting
|
||||||
// that is quietly dropped is somebody believing they changed something.
|
// that is quietly dropped is somebody believing they changed something.
|
||||||
@@ -176,6 +181,11 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
|
|||||||
if key == ExposeSetting && len(m.Listens) > 0 {
|
if key == ExposeSetting && len(m.Listens) > 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// `ports` gives a module's port a machine port on one node (novox/hq ADR 0100),
|
||||||
|
// validated in GivenPorts, so it is not stray here either.
|
||||||
|
if key == PortsSetting {
|
||||||
|
continue
|
||||||
|
}
|
||||||
unused = append(unused, fmt.Sprintf(
|
unused = append(unused, fmt.Sprintf(
|
||||||
"%s sets %q, and %s has no file or contribution to merge it into",
|
"%s sets %q, and %s has no file or contribution to merge it into",
|
||||||
layer.From, key, m.Module))
|
layer.From, key, m.Module))
|
||||||
|
|||||||
@@ -0,0 +1,246 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A node is adopted or converged (novox/hq ADR 0100).
|
||||||
|
//
|
||||||
|
// Adopted: what is found on the machine is kept until its module is taken, and the firewall found
|
||||||
|
// there stays in force. Converged: the machine is what the mesh declares, as every node was before
|
||||||
|
// adoption existed. The controller is authoritative, and every declaration it sends says which.
|
||||||
|
|
||||||
|
// ErrNotAdopted is taking a module on a node that is converged. On a converged node every assigned
|
||||||
|
// module converges already; there is nothing to take.
|
||||||
|
var ErrNotAdopted = errors.New("the node is converged, so every module on it is taken already")
|
||||||
|
|
||||||
|
// ErrNotAssigned is taking a module that is not on the node. Taking is the cutover of a module
|
||||||
|
// the node runs; one it does not run has nothing to cut over.
|
||||||
|
var ErrNotAssigned = errors.New("that module is not assigned to the node")
|
||||||
|
|
||||||
|
// SetAdopted makes a node adopted or converged. Becoming adopted stamps when; converging stamps
|
||||||
|
// when too. Neither touches what was taken: what was taken stays taken when a node returns to
|
||||||
|
// adopted, and converging takes the rest by its own act.
|
||||||
|
func (i *Inventory) SetAdopted(ctx context.Context, name string, adopted bool) error {
|
||||||
|
node, err := i.NodeByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if node.Adopted == adopted {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if adopted {
|
||||||
|
_, err = i.store.Pool().Exec(ctx,
|
||||||
|
`update node set adopted = true, adopted_since = now() where id = $1`, node.ID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = i.store.Pool().Exec(ctx,
|
||||||
|
`update node set adopted = false, adopted_since = null, converged_at = now() where id = $1`,
|
||||||
|
node.ID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Take records that a module has been taken on an adopted node: its cutover. From then on the
|
||||||
|
// module's resources converge on that node like any other, replacing what was found.
|
||||||
|
//
|
||||||
|
// Refused on a converged node and for a module not assigned there. Taking again is not an error;
|
||||||
|
// the first time it was taken is kept.
|
||||||
|
func (i *Inventory) Take(ctx context.Context, nodeName, module string) error {
|
||||||
|
node, err := i.NodeByName(ctx, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !node.Adopted {
|
||||||
|
return fmt.Errorf("%w: %s", ErrNotAdopted, nodeName)
|
||||||
|
}
|
||||||
|
return i.take(ctx, node, module)
|
||||||
|
}
|
||||||
|
|
||||||
|
// take is Take without the adopted check, for converging, which takes every assigned module in
|
||||||
|
// the same act that makes the node converged.
|
||||||
|
func (i *Inventory) take(ctx context.Context, node Node, module string) error {
|
||||||
|
var assigned bool
|
||||||
|
if err := i.store.Pool().QueryRow(ctx,
|
||||||
|
`select exists (select 1 from assignment where node = $1 and module = $2)`,
|
||||||
|
node.ID, module).Scan(&assigned); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !assigned {
|
||||||
|
return fmt.Errorf("%w: %s is not on %s; assign it first", ErrNotAssigned, module, node.Name)
|
||||||
|
}
|
||||||
|
_, err := i.store.Pool().Exec(ctx,
|
||||||
|
`insert into taken (node, module) values ($1, $2) on conflict do nothing`, node.ID, module)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Converge makes an adopted node converged in one act: every module assigned there is taken, and
|
||||||
|
// the node is recorded converged. Returned is what this act took, in name order.
|
||||||
|
func (i *Inventory) Converge(ctx context.Context, nodeName string) ([]string, error) {
|
||||||
|
node, err := i.NodeByName(ctx, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if !node.Adopted {
|
||||||
|
return nil, fmt.Errorf("%s is converged already", nodeName)
|
||||||
|
}
|
||||||
|
tx, err := i.store.Pool().Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||||
|
rows, err := tx.Query(ctx,
|
||||||
|
`insert into taken (node, module)
|
||||||
|
select node, module from assignment where node = $1
|
||||||
|
on conflict do nothing
|
||||||
|
returning module`, node.ID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var took []string
|
||||||
|
for rows.Next() {
|
||||||
|
var m string
|
||||||
|
if err := rows.Scan(&m); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
took = append(took, m)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx,
|
||||||
|
`update node set adopted = false, adopted_since = null, converged_at = now(),
|
||||||
|
held = null, reachable = null, firewall = null, adoption_reported = null
|
||||||
|
where id = $1`,
|
||||||
|
node.ID); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(took)
|
||||||
|
return took, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Taken is every module taken on a node, in name order — including one no longer assigned there:
|
||||||
|
// unassigning does not un-take.
|
||||||
|
func (i *Inventory) Taken(ctx context.Context, nodeName string) ([]string, error) {
|
||||||
|
node, err := i.NodeByName(ctx, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select module from taken where node = $1 order by module`, node.ID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []string
|
||||||
|
for rows.Next() {
|
||||||
|
var m string
|
||||||
|
if err := rows.Scan(&m); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held is one file or container an adopted node found and keeps as it was until its module is
|
||||||
|
// taken. The node's own account, kept as it said it.
|
||||||
|
type Held struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Target string `json:"target"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
Changed string `json:"changed,omitempty"`
|
||||||
|
Kept string `json:"kept,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reach is one thing reachable on an adopted node: a listening socket or a published port.
|
||||||
|
type Reach struct {
|
||||||
|
Protocol string `json:"protocol"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
Port int `json:"port"`
|
||||||
|
By string `json:"by,omitempty"`
|
||||||
|
Published bool `json:"published,omitempty"`
|
||||||
|
ContainerPort int `json:"container-port,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adoption is what an adopted node last said about adoption, and when.
|
||||||
|
type Adoption struct {
|
||||||
|
Held []Held
|
||||||
|
Firewall string
|
||||||
|
Reachable []Reach
|
||||||
|
// At is when it said so; zero when it never has.
|
||||||
|
At time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordAdoption keeps what a node last reported about adoption, replacing what was there: the
|
||||||
|
// question is the machine as it is now.
|
||||||
|
func (i *Inventory) RecordAdoption(ctx context.Context, node string, held []Held, firewall string,
|
||||||
|
reachable []Reach) error {
|
||||||
|
heldRaw, err := json.Marshal(nonNil(held))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
reachRaw, err := json.Marshal(nonNil(reachable))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = i.store.Pool().Exec(ctx,
|
||||||
|
`update node set held = $2, firewall = nullif($3, ''), reachable = $4,
|
||||||
|
adoption_reported = now(), last_seen = now()
|
||||||
|
where id = $1`, node, heldRaw, firewall, reachRaw)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func nonNil[T any](s []T) []T {
|
||||||
|
if s == nil {
|
||||||
|
return []T{}
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// AdoptionOf is what a node last reported about adoption.
|
||||||
|
func (i *Inventory) AdoptionOf(ctx context.Context, name string) (Adoption, error) {
|
||||||
|
var heldRaw, reachRaw []byte
|
||||||
|
var firewall *string
|
||||||
|
var at *time.Time
|
||||||
|
err := i.store.Pool().QueryRow(ctx,
|
||||||
|
`select held, firewall, reachable, adoption_reported from node where name = $1`, name).
|
||||||
|
Scan(&heldRaw, &firewall, &reachRaw, &at)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return Adoption{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return Adoption{}, err
|
||||||
|
}
|
||||||
|
var out Adoption
|
||||||
|
if firewall != nil {
|
||||||
|
out.Firewall = *firewall
|
||||||
|
}
|
||||||
|
if at != nil {
|
||||||
|
out.At = *at
|
||||||
|
}
|
||||||
|
if len(heldRaw) > 0 {
|
||||||
|
if err := json.Unmarshal(heldRaw, &out.Held); err != nil {
|
||||||
|
return Adoption{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(reachRaw) > 0 {
|
||||||
|
if err := json.Unmarshal(reachRaw, &out.Reachable); err != nil {
|
||||||
|
return Adoption{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,161 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: a node is adopted or converged, and the controller records which.
|
||||||
|
|
||||||
|
func TestANodeAddedWithoutSayingIsConverged(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
if _, err := inv.AddNode(t.Context(), "anchor"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n, err := inv.NodeByName(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n.Adopted || !n.AdoptedSince.IsZero() {
|
||||||
|
t.Fatalf("a node nobody said anything about reads as adopted: %+v", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdoptedNodeRoundTripsThroughEveryReading(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
made, err := inv.AddNodeAs(t.Context(), "anchor", true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !made.Adopted || made.AdoptedSince.IsZero() {
|
||||||
|
t.Fatalf("added adopted, got %+v", made)
|
||||||
|
}
|
||||||
|
byName, err := inv.NodeByName(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
all, err := inv.Nodes(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !byName.Adopted || len(all) != 1 || !all[0].Adopted {
|
||||||
|
t.Fatalf("adoption did not survive reading back: %+v %+v", byName, all)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And through a token: enrolment reads the node from the token it spends.
|
||||||
|
issued, err := inv.IssueToken(t.Context(), "anchor", 60e9)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
claimed, err := inv.Claim(t.Context(), issued.Secret, "a-key", false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !claimed.Adopted {
|
||||||
|
t.Fatal("the node a token claims lost its mode")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTakingIsRefusedOnAConvergedNodeAndForAnUnassignedModule(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
if err := inv.RegisterModule(t.Context(), manifest("hello-web", nil, nil), Source{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.AddNode(t.Context(), "converged"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.Assign(t.Context(), "converged", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.Take(t.Context(), "converged", "hello-web"); !errors.Is(err, ErrNotAdopted) {
|
||||||
|
t.Fatalf("taking on a converged node gave %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := inv.AddNodeAs(t.Context(), "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.Take(t.Context(), "anchor", "hello-web"); !errors.Is(err, ErrNotAssigned) {
|
||||||
|
t.Fatalf("taking an unassigned module gave %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATakenModuleOutlivesItsAssignmentAndReturningToAdopted(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
for _, m := range []string{"hello-web", "postgres"} {
|
||||||
|
if err := inv.RegisterModule(t.Context(), manifest(m, nil, nil), Source{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := inv.AddNodeAs(t.Context(), "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, m := range []string{"hello-web", "postgres"} {
|
||||||
|
if err := inv.Assign(t.Context(), "anchor", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := inv.Take(t.Context(), "anchor", "postgres"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.Take(t.Context(), "anchor", "postgres"); err != nil {
|
||||||
|
t.Fatalf("taking twice is not an error: %v", err)
|
||||||
|
}
|
||||||
|
if err := inv.Unassign(t.Context(), "anchor", "postgres"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
taken, err := inv.Taken(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(taken, []string{"postgres"}) {
|
||||||
|
t.Fatalf("unassigning un-took it: %v", taken)
|
||||||
|
}
|
||||||
|
|
||||||
|
took, err := inv.Converge(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(took, []string{"hello-web"}) {
|
||||||
|
t.Fatalf("converging took %v; it takes every assigned module not yet taken", took)
|
||||||
|
}
|
||||||
|
n, _ := inv.NodeByName(t.Context(), "anchor")
|
||||||
|
if n.Adopted {
|
||||||
|
t.Fatal("converged node still reads adopted")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := inv.SetAdopted(t.Context(), "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
taken, _ = inv.Taken(t.Context(), "anchor")
|
||||||
|
if !reflect.DeepEqual(taken, []string{"hello-web", "postgres"}) {
|
||||||
|
t.Fatalf("returning to adopted lost what was taken: %v", taken)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Converging clears the whole of a node's account of itself: what it held, what was reachable, the
|
||||||
|
// firewall it found and when it said so. Keeping any of it would have `node show` report an
|
||||||
|
// adopted machine's account of a converged one.
|
||||||
|
func TestConvergingClearsTheAccountTheNodeGave(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
made, err := inv.AddNodeAs(ctx, "anchor", true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordAdoption(ctx, made.ID,
|
||||||
|
[]Held{{ID: "notes.conf", Module: "notes", Kind: "file", Target: "/etc/notes.conf"}},
|
||||||
|
"ufw", []Reach{{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := inv.Converge(ctx, "anchor"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err := inv.AdoptionOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(said.Held) != 0 || len(said.Reachable) != 0 || said.Firewall != "" || !said.At.IsZero() {
|
||||||
|
t.Fatalf("converging kept the adopted machine's account: %+v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
@@ -583,6 +585,17 @@ func (i *Inventory) SetSettings(ctx context.Context, nodeName, module string, va
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if nodeName == "" {
|
if nodeName == "" {
|
||||||
|
// A port is a fact about one machine (novox/hq ADR 0100). Refused here, in composition's
|
||||||
|
// words: stored, it refuses every node running the module at composition, and the mesh
|
||||||
|
// cannot be pushed at all until somebody finds the layer that did it.
|
||||||
|
given, err := givenIn(module, raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(given) > 0 {
|
||||||
|
return fmt.Errorf("%s: %s is given per node — a port is a fact about one machine; "+
|
||||||
|
"set it with --node", module, catalogue.PortsSetting)
|
||||||
|
}
|
||||||
_, err = i.store.Pool().Exec(ctx,
|
_, err = i.store.Pool().Exec(ctx,
|
||||||
`insert into settings (node, module, values) values (null, $1, $2)
|
`insert into settings (node, module, values) values (null, $1, $2)
|
||||||
on conflict (module) where node is null
|
on conflict (module) where node is null
|
||||||
@@ -593,11 +606,157 @@ func (i *Inventory) SetSettings(ctx context.Context, nodeName, module string, va
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
_, err = i.store.Pool().Exec(ctx,
|
given, err := givenIn(module, raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tx, err := i.store.Pool().Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||||
|
if len(given) > 0 {
|
||||||
|
if err := refuseGivenCollisions(ctx, tx, node.ID, nodeName, module, given); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(ctx,
|
||||||
`insert into settings (node, module, values) values ($1, $2, $3)
|
`insert into settings (node, module, values) values ($1, $2, $3)
|
||||||
on conflict (node, module) where node is not null
|
on conflict (node, module) where node is not null
|
||||||
do update set values = excluded.values, set_at = now()`, node.ID, module, raw)
|
do update set values = excluded.values, set_at = now()`, node.ID, module, raw)
|
||||||
|
if err != nil {
|
||||||
return wrapModule(err, module)
|
return wrapModule(err, module)
|
||||||
|
}
|
||||||
|
// A given port replaces what the mesh assigned for that port: the assignment is given back,
|
||||||
|
// so the number is free for the next module rather than held for ever in the name of a port
|
||||||
|
// that now lives elsewhere.
|
||||||
|
for wanted := range given {
|
||||||
|
if _, err := tx.Exec(ctx,
|
||||||
|
`delete from port_assignment where node = $1 and module = $2 and wanted = $3`,
|
||||||
|
node.ID, module, wanted); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tx.Commit(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// givenIn is the machine ports a node-level settings layer gives a module, software port →
|
||||||
|
// machine port (novox/hq ADR 0100). Nothing when the layer gives none; what is not a port is left
|
||||||
|
// for composition to refuse in its own words.
|
||||||
|
func givenIn(module string, raw []byte) (map[int]int, error) {
|
||||||
|
var layer map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &layer); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
entries, ok := layer[catalogue.PortsSetting].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
out := map[int]int{}
|
||||||
|
by := map[int]int{}
|
||||||
|
for text, value := range entries {
|
||||||
|
wanted, err := strconv.Atoi(text)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at, ok := value.(float64)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
machine := int(at)
|
||||||
|
if machine == catalogue.SSHPort {
|
||||||
|
return nil, fmt.Errorf("%s cannot be given port %d for its %d: that is ssh's, the one "+
|
||||||
|
"port a machine may never lose", module, machine, wanted)
|
||||||
|
}
|
||||||
|
if other, twice := by[machine]; twice {
|
||||||
|
return nil, fmt.Errorf("%s gives machine port %d to both its %d and its %d; a machine "+
|
||||||
|
"port has one holder", module, machine, min(other, wanted), max(other, wanted))
|
||||||
|
}
|
||||||
|
by[machine] = wanted
|
||||||
|
out[wanted] = machine
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// refuseGivenCollisions refuses a given machine port another module on the node already has —
|
||||||
|
// assigned by the mesh or given by its own setting — or that this module has for another of its
|
||||||
|
// ports. A port it was assigned for the same software port is not a collision: the given one
|
||||||
|
// replaces it.
|
||||||
|
func refuseGivenCollisions(ctx context.Context, tx pgx.Tx, nodeID any, node, module string,
|
||||||
|
given map[int]int) error {
|
||||||
|
type holder struct {
|
||||||
|
module string
|
||||||
|
wanted int
|
||||||
|
}
|
||||||
|
held := map[int]holder{}
|
||||||
|
rows, err := tx.Query(ctx,
|
||||||
|
`select machine, module, wanted from port_assignment where node = $1`, nodeID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var h holder
|
||||||
|
var machine int
|
||||||
|
if err := rows.Scan(&machine, &h.module, &h.wanted); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
held[machine] = h
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows, err = tx.Query(ctx,
|
||||||
|
`select module, values->'ports' from settings
|
||||||
|
where node = $1 and module <> $2 and jsonb_typeof(values->'ports') = 'object'`,
|
||||||
|
nodeID, module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var other string
|
||||||
|
var raw []byte
|
||||||
|
if err := rows.Scan(&other, &raw); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var theirs map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &theirs); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for text, v := range theirs {
|
||||||
|
wanted, _ := strconv.Atoi(text)
|
||||||
|
if at, ok := v.(float64); ok {
|
||||||
|
held[int(at)] = holder{module: other, wanted: wanted}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
wanted := make([]int, 0, len(given))
|
||||||
|
for w := range given {
|
||||||
|
wanted = append(wanted, w)
|
||||||
|
}
|
||||||
|
sort.Ints(wanted)
|
||||||
|
for _, w := range wanted {
|
||||||
|
machine := given[w]
|
||||||
|
h, taken := held[machine]
|
||||||
|
if !taken || (h.module == module && h.wanted == w) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, moving := given[h.wanted]; h.module == module && moving {
|
||||||
|
// Its own port for another of its software ports, which this same layer moves away.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%w: %s cannot be given %d on %s for its %d — %s already has it for "+
|
||||||
|
"its %d", ErrPortTaken, module, machine, node, w, h.module, h.wanted)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func wrapModule(err error, module string) error {
|
func wrapModule(err error, module string) error {
|
||||||
@@ -654,7 +813,7 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([
|
|||||||
}
|
}
|
||||||
from := nodeName
|
from := nodeName
|
||||||
if meshWide {
|
if meshWide {
|
||||||
from = "the mesh"
|
from = catalogue.MeshWideLayer
|
||||||
}
|
}
|
||||||
layers = append(layers, catalogue.Layer{From: from, Values: values})
|
layers = append(layers, catalogue.Layer{From: from, Values: values})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrNodeBusy is a node another act kept holding for longer than a caller waits.
|
||||||
|
var ErrNodeBusy = errors.New("another act is composing or sending that node's declaration")
|
||||||
|
|
||||||
|
// HoldWaitFor is how long HoldNodes waits for a node another act holds, and HoldPoll how often it
|
||||||
|
// looks again. Variables so a test need not wait minutes.
|
||||||
|
var (
|
||||||
|
HoldWaitFor = 2 * time.Minute
|
||||||
|
HoldPoll = 250 * time.Millisecond
|
||||||
|
)
|
||||||
|
|
||||||
|
// HoldNodes serialises composing and sending a declaration per node (novox/hq ADR 0100): while
|
||||||
|
// one caller holds a node, another asking for it waits. Without it a push that composed a node as
|
||||||
|
// adopted could send that declaration after `converge --yes` sent the converged one, and the node
|
||||||
|
// would return to adopted with nobody having asked.
|
||||||
|
//
|
||||||
|
// Session-level advisory locks on one connection, all or none: a set not wholly free is given back
|
||||||
|
// at once, so two callers holding overlapping sets never each wait on the other. **A waiter pins
|
||||||
|
// no connection.** It looks again every HoldPoll with a connection borrowed for the look, and gives
|
||||||
|
// up after HoldWaitFor with ErrNodeBusy naming the node — so a stuck holder costs the pool one
|
||||||
|
// connection, never one per caller queued behind it. Release gives every one back, and may be
|
||||||
|
// called more than once.
|
||||||
|
func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), error) {
|
||||||
|
sorted := slices.Clone(names)
|
||||||
|
slices.Sort(sorted)
|
||||||
|
sorted = slices.Compact(sorted)
|
||||||
|
deadline := time.Now().Add(HoldWaitFor)
|
||||||
|
for {
|
||||||
|
release, busy, err := i.tryHold(ctx, sorted)
|
||||||
|
if err != nil || busy == "" {
|
||||||
|
return release, err
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return nil, fmt.Errorf("%w: %s has been held for over %s — try again once it is done",
|
||||||
|
ErrNodeBusy, busy, HoldWaitFor)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(HoldPoll):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tryHold takes every named node's lock or none, and says which node was busy when it took none.
|
||||||
|
func (i *Inventory) tryHold(ctx context.Context, sorted []string) (func(), string, error) {
|
||||||
|
conn, err := i.store.Pool().Acquire(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
var once sync.Once
|
||||||
|
release := func() {
|
||||||
|
once.Do(func() {
|
||||||
|
// Unlocking all of this session's advisory locks, then handing the connection back: a
|
||||||
|
// connection returned still holding one would hold it for whoever borrows it next.
|
||||||
|
_, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`)
|
||||||
|
if err != nil {
|
||||||
|
// The session's locks die with the session: close it rather than return it.
|
||||||
|
_ = conn.Conn().Close(context.WithoutCancel(ctx))
|
||||||
|
}
|
||||||
|
conn.Release()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, name := range sorted {
|
||||||
|
var took bool
|
||||||
|
if err := conn.QueryRow(ctx,
|
||||||
|
`select pg_try_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`,
|
||||||
|
name).Scan(&took); err != nil {
|
||||||
|
release()
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
if !took {
|
||||||
|
release()
|
||||||
|
return nil, strings.TrimSpace(name), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return release, "", nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package inventory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Composing and sending one node's declaration is serialised: a second holder waits for the first.
|
||||||
|
func TestHoldingANodeMakesTheNextHolderWait(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
release, err := inv.HoldNodes(ctx, []string{"anchor", "laptop"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := make(chan func(), 1)
|
||||||
|
go func() {
|
||||||
|
second, err := inv.HoldNodes(ctx, []string{"laptop"})
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
got <- func() {}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
got <- second
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-got:
|
||||||
|
t.Fatal("a node held by one caller was held by another at the same time")
|
||||||
|
case <-time.After(300 * time.Millisecond):
|
||||||
|
}
|
||||||
|
// Another node is not held up.
|
||||||
|
other, err := inv.HoldNodes(ctx, []string{"joiner"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
other()
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case second := <-got:
|
||||||
|
second()
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("releasing the node did not let the next holder in")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A waiter pins no pool connection while it waits, and gives up after a bounded wait saying which
|
||||||
|
// node is busy.
|
||||||
|
func TestAWaiterPinsNoConnectionAndGivesUp(t *testing.T) {
|
||||||
|
inv := fresh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
release, err := inv.HoldNodes(ctx, []string{"anchor"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
savedWait, savedPoll := HoldWaitFor, HoldPoll
|
||||||
|
HoldWaitFor, HoldPoll = 1500*time.Millisecond, 50*time.Millisecond
|
||||||
|
defer func() { HoldWaitFor, HoldPoll = savedWait, savedPoll }()
|
||||||
|
|
||||||
|
pool := inv.store.Pool()
|
||||||
|
base := pool.Stat().AcquiredConns()
|
||||||
|
const waiters = 3
|
||||||
|
done := make(chan error, waiters)
|
||||||
|
for range waiters {
|
||||||
|
go func() {
|
||||||
|
_, err := inv.HoldNodes(ctx, []string{"anchor"})
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
// While they wait, the pool lends nothing to them for longer than a look.
|
||||||
|
pinned := 0
|
||||||
|
for range 10 {
|
||||||
|
time.Sleep(60 * time.Millisecond)
|
||||||
|
if n := int(pool.Stat().AcquiredConns() - base); n > pinned {
|
||||||
|
pinned = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pinned >= waiters {
|
||||||
|
t.Fatalf("%d connections were held by %d waiters", pinned, waiters)
|
||||||
|
}
|
||||||
|
for range waiters {
|
||||||
|
if err := <-done; !errors.Is(err, ErrNodeBusy) || !strings.Contains(err.Error(), "anchor") {
|
||||||
|
t.Fatalf("a waiter did not give up naming the busy node: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
-- A node is adopted or converged, and the mesh records which (novox/hq ADR 0100).
|
||||||
|
--
|
||||||
|
-- A machine already serving a predecessor's services is adopted: what is found on it is kept
|
||||||
|
-- until its module is taken, and the firewall found on it stays in force. It stays adopted until
|
||||||
|
-- the operator converges it. False by default, so every node that exists is converged, as it was.
|
||||||
|
|
||||||
|
alter table node add column adopted boolean not null default false;
|
||||||
|
-- When it was last made adopted, and when it last converged. Both kept: a node returned to adopted
|
||||||
|
-- after converging is a different history from one that never converged.
|
||||||
|
alter table node add column adopted_since timestamptz;
|
||||||
|
alter table node add column converged_at timestamptz;
|
||||||
|
|
||||||
|
-- The modules taken on a node: its cutover, the operator's act, done when the module's data has
|
||||||
|
-- moved. A row per node and module, and it outlives the assignment on purpose -- unassigning a
|
||||||
|
-- module does not un-take it, and what was taken stays taken when a node returns to adopted.
|
||||||
|
create table taken (
|
||||||
|
node uuid not null references node(id) on delete cascade,
|
||||||
|
module text not null references module(name) on delete cascade,
|
||||||
|
taken_at timestamptz not null default now(),
|
||||||
|
primary key (node, module)
|
||||||
|
);
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
-- What an adopted node last reported about adoption (novox/hq ADR 0100): the files and containers
|
||||||
|
-- it found and holds until their module is taken, the firewall it found, and what is reachable on
|
||||||
|
-- the machine now — which converging it previews, so nothing closes without being named first.
|
||||||
|
--
|
||||||
|
-- The last report, replaced, like what a node owns: the question is the machine as it is now.
|
||||||
|
-- Kept apart from node_report because a node reports it on its own schedule, when what it holds
|
||||||
|
-- changes, and not only after an apply.
|
||||||
|
|
||||||
|
alter table node add column held jsonb;
|
||||||
|
alter table node add column firewall text;
|
||||||
|
alter table node add column reachable jsonb;
|
||||||
|
alter table node add column adoption_reported timestamptz;
|
||||||
+46
-20
@@ -49,6 +49,12 @@ type Node struct {
|
|||||||
// month's assignments, and until this existed those looked the same as a node that is
|
// month's assignments, and until this existed those looked the same as a node that is
|
||||||
// current (novox/hq 09-the-node-lifecycle).
|
// current (novox/hq 09-the-node-lifecycle).
|
||||||
LastSeen time.Time
|
LastSeen time.Time
|
||||||
|
|
||||||
|
// Adopted is whether this node is adopted rather than converged (novox/hq ADR 0100): what is
|
||||||
|
// found on it is kept until its module is taken, and the firewall found on it stays in force.
|
||||||
|
// AdoptedSince is when it last became so; zero for a converged node.
|
||||||
|
Adopted bool
|
||||||
|
AdoptedSince time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// Silent is how long since this node was last heard from, and whether it ever was.
|
// Silent is how long since this node was last heard from, and whether it ever was.
|
||||||
@@ -74,28 +80,59 @@ var ErrNameTaken = errors.New("a node of that name already exists")
|
|||||||
// (novox/hq 09-the-node-lifecycle), so the record is what a token binds to and must exist before
|
// (novox/hq 09-the-node-lifecycle), so the record is what a token binds to and must exist before
|
||||||
// there is anything to join.
|
// there is anything to join.
|
||||||
func (i *Inventory) AddNode(ctx context.Context, name string) (Node, error) {
|
func (i *Inventory) AddNode(ctx context.Context, name string) (Node, error) {
|
||||||
|
return i.AddNodeAs(ctx, name, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddNodeAs creates a node record, adopted or converged (novox/hq ADR 0100). The operator says
|
||||||
|
// which; a node added without saying is converged, as every node was before adoption existed.
|
||||||
|
func (i *Inventory) AddNodeAs(ctx context.Context, name string, adopted bool) (Node, error) {
|
||||||
name = strings.TrimSpace(name)
|
name = strings.TrimSpace(name)
|
||||||
if name == "" {
|
if name == "" {
|
||||||
return Node{}, errors.New("a node needs a name: it is how a token is issued for it")
|
return Node{}, errors.New("a node needs a name: it is how a token is issued for it")
|
||||||
}
|
}
|
||||||
|
|
||||||
var n Node
|
var n Node
|
||||||
|
var since *time.Time
|
||||||
err := i.store.Pool().QueryRow(ctx,
|
err := i.store.Pool().QueryRow(ctx,
|
||||||
`insert into node (name) values ($1) returning id, name, created`,
|
`insert into node (name, adopted, adopted_since)
|
||||||
name).Scan(&n.ID, &n.Name, &n.Created)
|
values ($1, $2, case when $2 then now() end)
|
||||||
|
returning id, name, created, adopted, adopted_since`,
|
||||||
|
name, adopted).Scan(&n.ID, &n.Name, &n.Created, &n.Adopted, &since)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "node_name_key") {
|
if strings.Contains(err.Error(), "node_name_key") {
|
||||||
return Node{}, fmt.Errorf("%w: %s", ErrNameTaken, name)
|
return Node{}, fmt.Errorf("%w: %s", ErrNameTaken, name)
|
||||||
}
|
}
|
||||||
return Node{}, err
|
return Node{}, err
|
||||||
}
|
}
|
||||||
|
if since != nil {
|
||||||
|
n.AdoptedSince = *since
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeColumns and scanNode are the one reading of a node row, so every way of finding a node
|
||||||
|
// says whether it is adopted.
|
||||||
|
const nodeColumns = `id, name, created, last_seen, adopted, adopted_since`
|
||||||
|
|
||||||
|
func scanNode(row pgx.Row) (Node, error) {
|
||||||
|
var n Node
|
||||||
|
var seen, since *time.Time
|
||||||
|
if err := row.Scan(&n.ID, &n.Name, &n.Created, &seen, &n.Adopted, &since); err != nil {
|
||||||
|
return Node{}, err
|
||||||
|
}
|
||||||
|
if seen != nil {
|
||||||
|
n.LastSeen = *seen
|
||||||
|
}
|
||||||
|
if since != nil {
|
||||||
|
n.AdoptedSince = *since
|
||||||
|
}
|
||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Nodes are every node record, oldest first.
|
// Nodes are every node record, oldest first.
|
||||||
func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) {
|
func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) {
|
||||||
rows, err := i.store.Pool().Query(ctx,
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
`select id, name, created, last_seen from node order by created, name`)
|
`select `+nodeColumns+` from node order by created, name`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -103,14 +140,10 @@ func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) {
|
|||||||
|
|
||||||
var nodes []Node
|
var nodes []Node
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var n Node
|
n, err := scanNode(rows)
|
||||||
var seen *time.Time
|
if err != nil {
|
||||||
if err := rows.Scan(&n.ID, &n.Name, &n.Created, &seen); err != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if seen != nil {
|
|
||||||
n.LastSeen = *seen
|
|
||||||
}
|
|
||||||
nodes = append(nodes, n)
|
nodes = append(nodes, n)
|
||||||
}
|
}
|
||||||
return nodes, rows.Err()
|
return nodes, rows.Err()
|
||||||
@@ -118,9 +151,8 @@ func (i *Inventory) Nodes(ctx context.Context) ([]Node, error) {
|
|||||||
|
|
||||||
// NodeByName finds one node record.
|
// NodeByName finds one node record.
|
||||||
func (i *Inventory) NodeByName(ctx context.Context, name string) (Node, error) {
|
func (i *Inventory) NodeByName(ctx context.Context, name string) (Node, error) {
|
||||||
var n Node
|
n, err := scanNode(i.store.Pool().QueryRow(ctx,
|
||||||
err := i.store.Pool().QueryRow(ctx,
|
`select `+nodeColumns+` from node where name = $1`, name))
|
||||||
`select id, name, created from node where name = $1`, name).Scan(&n.ID, &n.Name, &n.Created)
|
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return Node{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
return Node{}, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
|
||||||
}
|
}
|
||||||
@@ -248,10 +280,7 @@ func (i *Inventory) Claim(ctx context.Context, secret, by string, again bool) (N
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Node{}, err
|
return Node{}, err
|
||||||
}
|
}
|
||||||
var n Node
|
return scanNode(i.store.Pool().QueryRow(ctx, `select `+nodeColumns+` from node where id = $1`, id))
|
||||||
err = i.store.Pool().QueryRow(ctx,
|
|
||||||
`select id, name, created from node where id = $1`, id).Scan(&n.ID, &n.Name, &n.Created)
|
|
||||||
return n, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Spend makes a claimed token used, only for the presenter holding the claim. The last write to the
|
// Spend makes a claimed token used, only for the presenter holding the claim. The last write to the
|
||||||
@@ -293,10 +322,7 @@ func (i *Inventory) Redeem(ctx context.Context, secret string) (Node, error) {
|
|||||||
return Node{}, err
|
return Node{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var n Node
|
return scanNode(i.store.Pool().QueryRow(ctx, `select `+nodeColumns+` from node where id = $1`, id))
|
||||||
err = i.store.Pool().QueryRow(ctx,
|
|
||||||
`select id, name, created from node where id = $1`, id).Scan(&n.ID, &n.Name, &n.Created)
|
|
||||||
return n, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecordProfile keeps the last thing a node said about what it can do.
|
// RecordProfile keeps the last thing a node said about what it can do.
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package inventory
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
@@ -132,6 +133,17 @@ func (i *Inventory) assignPort(
|
|||||||
taken[port] = "something this machine already runs"
|
taken[port] = "something this machine already runs"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// And every port this machine was given for a module (novox/hq ADR 0100): the foundation's
|
||||||
|
// ports, as genesis chose them, are the node's settings and never the mesh's to hand out.
|
||||||
|
given, err := i.givenOn(ctx, nodeID)
|
||||||
|
if err != nil {
|
||||||
|
return Assigned{}, err
|
||||||
|
}
|
||||||
|
for port, by := range given {
|
||||||
|
if _, mine := taken[port]; !mine {
|
||||||
|
taken[port] = by
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
machine := wanted
|
machine := wanted
|
||||||
if !fixed {
|
if !fixed {
|
||||||
@@ -258,3 +270,33 @@ func (i *Inventory) ReleasePorts(ctx context.Context, node, module string) error
|
|||||||
`delete from port_assignment where node = $1 and module = $2`, record.ID, module)
|
`delete from port_assignment where node = $1 and module = $2`, record.ID, module)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// givenOn is every machine port a module was given on this node by its `ports` setting, and which
|
||||||
|
// module it was given to.
|
||||||
|
func (i *Inventory) givenOn(ctx context.Context, nodeID any) (map[int]string, error) {
|
||||||
|
rows, err := i.store.Pool().Query(ctx,
|
||||||
|
`select module, values->'ports' from settings
|
||||||
|
where node = $1 and jsonb_typeof(values->'ports') = 'object'`, nodeID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[int]string{}
|
||||||
|
for rows.Next() {
|
||||||
|
var module string
|
||||||
|
var raw []byte
|
||||||
|
if err := rows.Scan(&module, &raw); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var given map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &given); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, v := range given {
|
||||||
|
if at, ok := v.(float64); ok {
|
||||||
|
out[int(at)] = module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package inventory
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/novox/mesh-controller/internal/catalogue"
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
@@ -236,3 +237,110 @@ func TestUnassigningReleasesTheModulesPorts(t *testing.T) {
|
|||||||
t.Fatalf("port 25 is still held in the name of a module that was unassigned: %v", err)
|
t.Fatalf("port 25 is still held in the name of a module that was unassigned: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: a port a node was given for a module is the node's, and the mesh never hands
|
||||||
|
// it to another.
|
||||||
|
func TestAGivenPortIsNeverAssigned(t *testing.T) {
|
||||||
|
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := inv.SetSettings(ctx, node, "postgres",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 20000}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := inv.PortFor(ctx, node, "web", 8080, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Machine == 20000 {
|
||||||
|
t.Fatal("a port given to postgres was assigned to web")
|
||||||
|
}
|
||||||
|
if _, err := inv.PortFor(ctx, node, "web", 20000, true); !errors.Is(err, ErrPortTaken) {
|
||||||
|
t.Fatalf("a fixed port given to another module was handed over: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: a given machine port has one holder. It is refused when another module has it,
|
||||||
|
// assigned or given, when the same layer gives it twice, and when it is ssh's; and when it replaces
|
||||||
|
// what the mesh assigned for that port, the assignment is given back.
|
||||||
|
func TestAGivenPortHasOneHolderAndReplacesTheAssignment(t *testing.T) {
|
||||||
|
inv, node := aNodeWithModules(t, "postgres", "web", "cache")
|
||||||
|
ctx := t.Context()
|
||||||
|
give := func(module string, ports map[string]any) error {
|
||||||
|
return inv.SetSettings(ctx, node, module, map[string]any{catalogue.PortsSetting: ports})
|
||||||
|
}
|
||||||
|
|
||||||
|
web, err := inv.PortFor(ctx, node, "web", 8080, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := give("postgres", map[string]any{"5432": web.Machine}); !errors.Is(err, ErrPortTaken) {
|
||||||
|
t.Fatalf("a port the mesh assigned to web was given to postgres: %v", err)
|
||||||
|
}
|
||||||
|
if err := give("postgres", map[string]any{"5432": 22}); err == nil {
|
||||||
|
t.Fatal("ssh's port was given")
|
||||||
|
}
|
||||||
|
if err := give("postgres", map[string]any{"5432": 5433, "5433": 5433}); err == nil {
|
||||||
|
t.Fatal("one machine port was given for two ports")
|
||||||
|
}
|
||||||
|
if err := give("cache", map[string]any{"6379": 6380}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := give("postgres", map[string]any{"5432": 6380}); !errors.Is(err, ErrPortTaken) {
|
||||||
|
t.Fatalf("a port given to cache was given to postgres: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Postgres was assigned a port for 5432; given one, the assignment is released and the number
|
||||||
|
// is free again.
|
||||||
|
assigned, err := inv.PortFor(ctx, node, "postgres", 5432, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := give("postgres", map[string]any{"5432": 5433}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Given again, the same: its own given port is not a collision with itself.
|
||||||
|
if err := give("postgres", map[string]any{"5432": 5433}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
held, err := inv.PortsFor(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, a := range held {
|
||||||
|
if a.Module == "postgres" {
|
||||||
|
t.Fatalf("the assignment a given port replaced is still held: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
other, err := inv.PortFor(ctx, node, "cache", 11211, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if other.Machine != assigned.Machine {
|
||||||
|
t.Fatalf("the released port %d was not free again (got %d)", assigned.Machine, other.Machine)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: a port is a fact about one machine, so a layer for the whole mesh cannot give
|
||||||
|
// one. Refused where it is set — stored, it refuses every node running the module at composition,
|
||||||
|
// and the mesh cannot be pushed until somebody finds the layer that did it.
|
||||||
|
func TestAPortGivenForTheWholeMeshIsRefusedWhereItIsSet(t *testing.T) {
|
||||||
|
inv, node := aNodeWithModules(t, "postgres")
|
||||||
|
ctx := t.Context()
|
||||||
|
err := inv.SetSettings(ctx, "", "postgres",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "per node") {
|
||||||
|
t.Fatalf("a port given for the whole mesh was accepted: %v", err)
|
||||||
|
}
|
||||||
|
layers, err := inv.SettingsFor(ctx, node, "postgres")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(layers) != 0 {
|
||||||
|
t.Fatalf("the refused layer was stored: %v", layers)
|
||||||
|
}
|
||||||
|
// The same values for one machine are the ordinary setting.
|
||||||
|
if err := inv.SetSettings(ctx, node, "postgres",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -199,6 +199,27 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (err error) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What an adopted node holds, which firewall it found, and what is reachable on it (novox/hq
|
||||||
|
// ADR 0100). Recorded whenever a report carries any of it — a node reports these on its own
|
||||||
|
// schedule, when what it holds changes, not only after an apply — and never cleared by a
|
||||||
|
// report that carries none, which is every bare word that the node is there. An adopted node
|
||||||
|
// always names its firewall, so a report from one replaces all three, emptied held included.
|
||||||
|
if len(report.Held) > 0 || report.Firewall != "" || len(report.Reachable) > 0 {
|
||||||
|
held := make([]inventory.Held, 0, len(report.Held))
|
||||||
|
for _, h := range report.Held {
|
||||||
|
held = append(held, inventory.Held{ID: h.ID, Module: h.Module, Kind: h.Kind,
|
||||||
|
Target: h.Target, Since: h.Since, Changed: h.Changed, Kept: h.Kept})
|
||||||
|
}
|
||||||
|
reachable := make([]inventory.Reach, 0, len(report.Reachable))
|
||||||
|
for _, r := range report.Reachable {
|
||||||
|
reachable = append(reachable, inventory.Reach{Protocol: r.Protocol, Address: r.Address,
|
||||||
|
Port: r.Port, By: r.By, Published: r.Published, ContainerPort: r.ContainerPort})
|
||||||
|
}
|
||||||
|
if err := e.Inventory.RecordAdoption(ctx, node.ID, held, report.Firewall, reachable); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// A bare word that a node is there is not an account of what the machine did or holds: it
|
// A bare word that a node is there is not an account of what the machine did or holds: it
|
||||||
// moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery),
|
// moves last_seen and touches nothing else. This arrives every minute (link.AliveEvery),
|
||||||
// while a real report is rare, so recording it as one would overwrite the node's last real
|
// while a real report is rare, so recording it as one would overwrite the node's last real
|
||||||
|
|||||||
@@ -175,3 +175,46 @@ func TestAFailureDoesNotBecomeTheAccountOfWhatTheMachineHolds(t *testing.T) {
|
|||||||
t.Fatalf("a partial report replaced the account of what the machine holds: %v", owned)
|
t.Fatalf("a partial report replaced the account of what the machine holds: %v", owned)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: what an adopted node holds, the firewall it found and what is reachable on it
|
||||||
|
// are kept from the report that carries them, and a bare word that the node is there wipes none.
|
||||||
|
func TestWhatAnAdoptedNodeHoldsIsKeptAndAnAliveWordDoesNotWipeIt(t *testing.T) {
|
||||||
|
inv, _, _ := heardFrom(t, link.Report{
|
||||||
|
Node: "anchor", Applied: []string{"hello-web.served"}, Firewall: "ufw",
|
||||||
|
Held: []link.Held{{ID: "hello-web.page", Module: "hello-web", Kind: "file",
|
||||||
|
Target: "/var/lib/hello-web/index.html", Changed: "rewritten", Kept: "/var/lib/mesh/kept/x"}},
|
||||||
|
Reachable: []link.Reach{{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web",
|
||||||
|
Published: true, ContainerPort: 80}},
|
||||||
|
})
|
||||||
|
ctx := context.Background()
|
||||||
|
check := func(when string) {
|
||||||
|
t.Helper()
|
||||||
|
got, err := inv.AdoptionOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.Firewall != "ufw" || len(got.Held) != 1 || got.Held[0].Changed != "rewritten" ||
|
||||||
|
len(got.Reachable) != 1 || got.Reachable[0].ContainerPort != 80 || got.At.IsZero() {
|
||||||
|
t.Fatalf("%s: what the node said is not what was kept: %+v", when, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check("after the report")
|
||||||
|
|
||||||
|
if err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{Node: "anchor"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check("after an alive word")
|
||||||
|
|
||||||
|
// A reconcile report carrying only adoption is recorded, though it applied nothing.
|
||||||
|
if err := (link.Enrolment{Inventory: inv}).Heard(ctx, link.Report{Node: "anchor",
|
||||||
|
Firewall: "ufw"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := inv.AdoptionOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got.Held) != 0 || got.Firewall != "ufw" {
|
||||||
|
t.Fatalf("a report from an adopted node holding nothing did not empty held: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,7 +6,10 @@
|
|||||||
// traffic between them, each receiving half of what it expects. That has happened here before.
|
// traffic between them, each receiving half of what it expects. That has happened here before.
|
||||||
package link
|
package link
|
||||||
|
|
||||||
import "encoding/base64"
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
// Exchange is where nodes publish everything they have to say.
|
// Exchange is where nodes publish everything they have to say.
|
||||||
const Exchange = "mesh"
|
const Exchange = "mesh"
|
||||||
@@ -116,6 +119,43 @@ type Report struct {
|
|||||||
// Declared is the digest of the declaration this report is about — the same bytes, hashed
|
// Declared is the digest of the declaration this report is about — the same bytes, hashed
|
||||||
// the same way, as the `sent` digest the mesh recorded. Which declaration, not when.
|
// the same way, as the `sent` digest the mesh recorded. Which declaration, not when.
|
||||||
Declared string `json:"declared,omitempty"`
|
Declared string `json:"declared,omitempty"`
|
||||||
|
|
||||||
|
// Held is what an adopted node found and is keeping as it was until its module is taken
|
||||||
|
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
|
||||||
|
Held []Held `json:"held,omitempty"`
|
||||||
|
// Firewall is the firewall found on the machine — "ufw" or "none" — and empty on a node that
|
||||||
|
// was never asked, which is every converged one.
|
||||||
|
Firewall string `json:"firewall,omitempty"`
|
||||||
|
// Reachable is what can be reached on the machine now: every listening socket and every
|
||||||
|
// published container port. Only an adopted node reports it; it is what converging previews.
|
||||||
|
Reachable []Reach `json:"reachable,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held is one file or container found on an adopted node and kept as it was.
|
||||||
|
type Held struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Target string `json:"target"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
// Changed is what something other than the mesh did to it since — rewritten, stopped,
|
||||||
|
// replaced or gone — and empty while it is as found.
|
||||||
|
Changed string `json:"changed,omitempty"`
|
||||||
|
// Kept is where a file's original was kept.
|
||||||
|
Kept string `json:"kept,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reach is one thing reachable on the machine: a listening socket, or a published container port.
|
||||||
|
type Reach struct {
|
||||||
|
Protocol string `json:"protocol"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
Port int `json:"port"`
|
||||||
|
// By is what holds it — a process, or a container's name.
|
||||||
|
By string `json:"by,omitempty"`
|
||||||
|
// Published is a container port the runtime publishes, reached on the forwarded path; its
|
||||||
|
// container's own port is ContainerPort.
|
||||||
|
Published bool `json:"published,omitempty"`
|
||||||
|
ContainerPort int `json:"container-port,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnrolReply is what the mesh says back.
|
// EnrolReply is what the mesh says back.
|
||||||
|
|||||||
@@ -16,6 +16,13 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
|||||||
{Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}},
|
{Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}},
|
||||||
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
|
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
|
||||||
[]string{"node", "applied", "failed", "refused"}},
|
[]string{"node", "applied", "failed", "refused"}},
|
||||||
|
{Report{Node: "n", Held: []Held{{ID: "m.f"}}, Firewall: "ufw", Reachable: []Reach{{Port: 1}}},
|
||||||
|
[]string{"node", "held", "firewall", "reachable"}},
|
||||||
|
{Held{ID: "m.f", Module: "m", Kind: "file", Target: "/f", Changed: "rewritten", Kept: "/k"},
|
||||||
|
[]string{"id", "module", "kind", "target", "since", "changed", "kept"}},
|
||||||
|
{Reach{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||||
|
ContainerPort: 80},
|
||||||
|
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
||||||
{EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")},
|
{EnrolRequest{Node: "n", Secret: "s", PublicKey: []byte("k")},
|
||||||
[]string{"node", "secret", "public_key"}},
|
[]string{"node", "secret", "public_key"}},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -140,18 +140,20 @@ func (g *Generator) Resources(node string) ([]map[string]any, bool, error) {
|
|||||||
}
|
}
|
||||||
resources = append(resources,
|
resources = append(resources,
|
||||||
map[string]any{
|
map[string]any{
|
||||||
// Merged, not owned: the runtime's daemon file is the machine's, and this states
|
// Written into, not over (novox/hq ADR 0102): the runtime's daemon file is the
|
||||||
// one fact into it. The registry speaks plain HTTP because every path to it is
|
// machine's — its data directory, its logging, whatever a predecessor set — and
|
||||||
// already inside the overlay's encryption (ADR 0082) — this line is the runtime
|
// this states one fact in it. The host sets this key and keeps every other.
|
||||||
// being told what the mesh already means.
|
// ("merge" is the operator's settings merged into this content; "into" is the
|
||||||
|
// content written into the machine's file.) The registry speaks plain HTTP
|
||||||
|
// because every path to it is already inside the overlay's encryption (ADR 0082).
|
||||||
"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json",
|
"id": "registry-trust", "type": "file", "path": "/etc/docker/daemon.json",
|
||||||
"content": string(trust) + "\n", "mode": "0644", "merge": "json",
|
"content": string(trust) + "\n", "mode": "0644", "merge": "json", "into": "json",
|
||||||
},
|
},
|
||||||
map[string]any{
|
map[string]any{
|
||||||
// The runtime reloads nothing for this setting, so it is restarted when the fact
|
// Reloaded, not restarted: the runtime re-reads its trusted registries on a reload,
|
||||||
// changes — once, at joining, before the machine runs anything that would mind.
|
// and a restart stops every container on the machine (measured; ADR 0102).
|
||||||
"id": "registry-trust-reload", "type": "service", "unit": "docker.service",
|
"id": "registry-trust-reload", "type": "service", "unit": "docker.service",
|
||||||
"state": "running", "restart-on": []string{"registry-trust"},
|
"state": "running", "reload-on": []string{"registry-trust"},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return resources, true, nil
|
return resources, true, nil
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package overlay
|
package overlay
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
@@ -44,13 +45,20 @@ func TestTheNetworkCarriesRegistryTrust(t *testing.T) {
|
|||||||
if file == nil || service == nil {
|
if file == nil || service == nil {
|
||||||
t.Fatalf("the trust file or its reload is missing: %v", trusted)
|
t.Fatalf("the trust file or its reload is missing: %v", trusted)
|
||||||
}
|
}
|
||||||
if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" {
|
if file["path"] != "/etc/docker/daemon.json" || file["merge"] != "json" || file["into"] != "json" {
|
||||||
t.Fatalf("the trust is not a merged daemon.json: %v", file)
|
t.Fatalf("the trust is not written into daemon.json (ADR 0102): %v", file)
|
||||||
}
|
}
|
||||||
if content, _ := file["content"].(string); !strings.Contains(content, `"anchor.internal:5000"`) {
|
if content, _ := file["content"].(string); !strings.Contains(content, `"anchor.internal:5000"`) {
|
||||||
t.Fatalf("the trust does not name the store: %v", file["content"])
|
t.Fatalf("the trust does not name the store: %v", file["content"])
|
||||||
}
|
}
|
||||||
if service["unit"] != "docker.service" {
|
if service["unit"] != "docker.service" {
|
||||||
t.Fatalf("the reload does not restart the runtime: %v", service)
|
t.Fatalf("the reload is not the runtime's: %v", service)
|
||||||
|
}
|
||||||
|
// Reloaded, never restarted: a restart stops every container on the machine (ADR 0102).
|
||||||
|
if _, restarts := service["restart-on"]; restarts {
|
||||||
|
t.Fatalf("the runtime is restarted for its trust: %v", service)
|
||||||
|
}
|
||||||
|
if fmt.Sprint(service["reload-on"]) != "[registry-trust]" {
|
||||||
|
t.Fatalf("the runtime is not reloaded for its trust: %v", service)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,6 +50,11 @@ type Token struct {
|
|||||||
|
|
||||||
// Secret is the one-time right to join. Useless once used, useless after it expires.
|
// Secret is the one-time right to join. Useless once used, useless after it expires.
|
||||||
Secret string `json:"secret"`
|
Secret string `json:"secret"`
|
||||||
|
|
||||||
|
// Adopted says the node joins adopted (novox/hq ADR 0100): the host checks, before enrolling,
|
||||||
|
// that it speaks the firewall found on the machine, because an adopted node keeps that firewall
|
||||||
|
// in force. Absent for a converged node, so a converged token is byte for byte what it was.
|
||||||
|
Adopted bool `json:"adopted,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Missing names the parts that are not filled in.
|
// Missing names the parts that are not filled in.
|
||||||
|
|||||||
@@ -140,6 +140,22 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
|||||||
if len(fields) != 6 {
|
if len(fields) != 6 {
|
||||||
t.Errorf("the token has %d fields, expected 6: %v", len(fields), fields)
|
t.Errorf("the token has %d fields, expected 6: %v", len(fields), fields)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An adopted node's token says so, under exactly this name, and a converged one does not
|
||||||
|
// carry it at all (novox/hq ADR 0100).
|
||||||
|
adopted := complete(t)
|
||||||
|
adopted.Adopted = true
|
||||||
|
raw, err = json.Marshal(adopted)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fields = nil
|
||||||
|
if err := json.Unmarshal(raw, &fields); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fields["adopted"] != true || len(fields) != 7 {
|
||||||
|
t.Errorf("an adopted token does not carry \"adopted\": true: %v", fields)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestATokenWithNoNameIsRefused(t *testing.T) {
|
func TestATokenWithNoNameIsRefused(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user